Lines Matching full:mod
118 static void mod_update_bounds(struct module *mod) in mod_update_bounds() argument
121 struct module_memory *mod_mem = &mod->mem[type]; in mod_update_bounds()
154 static inline int strong_try_module_get(struct module *mod) in strong_try_module_get() argument
156 BUG_ON(mod && mod->state == MODULE_STATE_UNFORMED); in strong_try_module_get()
157 if (mod && mod->state == MODULE_STATE_COMING) in strong_try_module_get()
159 if (try_module_get(mod)) in strong_try_module_get()
165 static inline void add_taint_module(struct module *mod, unsigned flag, in add_taint_module() argument
169 set_bit(flag, &mod->taints); in add_taint_module()
176 void __noreturn __module_put_and_kthread_exit(struct module *mod, long code) in __module_put_and_kthread_exit() argument
178 module_put(mod); in __module_put_and_kthread_exit()
345 struct module *mod; in find_symbol() local
354 list_for_each_entry_rcu(mod, &modules, list, in find_symbol()
357 { mod->syms, mod->syms + mod->num_syms, mod->crcs, in find_symbol()
359 { mod->gpl_syms, mod->gpl_syms + mod->num_gpl_syms, in find_symbol()
360 mod->gpl_crcs, in find_symbol()
364 if (mod->state == MODULE_STATE_UNFORMED) in find_symbol()
368 if (find_exported_symbol_in_section(&arr[i], mod, fsa)) in find_symbol()
383 struct module *mod; in find_module_all() local
387 list_for_each_entry_rcu(mod, &modules, list, in find_module_all()
389 if (!even_unformed && mod->state == MODULE_STATE_UNFORMED) in find_module_all()
391 if (strlen(mod->name) == len && !memcmp(mod->name, name, len)) in find_module_all()
392 return mod; in find_module_all()
404 static inline void __percpu *mod_percpu(struct module *mod) in mod_percpu() argument
406 return mod->percpu; in mod_percpu()
409 static int percpu_modalloc(struct module *mod, struct load_info *info) in percpu_modalloc() argument
419 mod->name, align, PAGE_SIZE); in percpu_modalloc()
423 mod->percpu = __alloc_reserved_percpu(pcpusec->sh_size, align); in percpu_modalloc()
424 if (!mod->percpu) { in percpu_modalloc()
426 mod->name, (unsigned long)pcpusec->sh_size); in percpu_modalloc()
429 mod->percpu_size = pcpusec->sh_size; in percpu_modalloc()
433 static void percpu_modfree(struct module *mod) in percpu_modfree() argument
435 free_percpu(mod->percpu); in percpu_modfree()
443 static void percpu_modcopy(struct module *mod, in percpu_modcopy() argument
449 memcpy(per_cpu_ptr(mod->percpu, cpu), from, size); in percpu_modcopy()
454 struct module *mod; in __is_module_percpu_address() local
459 list_for_each_entry_rcu(mod, &modules, list) { in __is_module_percpu_address()
460 if (mod->state == MODULE_STATE_UNFORMED) in __is_module_percpu_address()
462 if (!mod->percpu_size) in __is_module_percpu_address()
465 void *start = per_cpu_ptr(mod->percpu, cpu); in __is_module_percpu_address()
468 if (va >= start && va < start + mod->percpu_size) { in __is_module_percpu_address()
472 per_cpu_ptr(mod->percpu, in __is_module_percpu_address()
500 static inline void __percpu *mod_percpu(struct module *mod) in mod_percpu() argument
504 static int percpu_modalloc(struct module *mod, struct load_info *info) in percpu_modalloc() argument
511 static inline void percpu_modfree(struct module *mod) in percpu_modfree() argument
518 static inline void percpu_modcopy(struct module *mod, in percpu_modcopy() argument
537 static void setup_modinfo_##field(struct module *mod, const char *s) \
539 mod->field = kstrdup(s, GFP_KERNEL); \
544 return scnprintf(buffer, PAGE_SIZE, "%s\n", mk->mod->field); \
546 static int modinfo_##field##_exists(struct module *mod) \
548 return mod->field != NULL; \
550 static void free_modinfo_##field(struct module *mod) \
552 kfree(mod->field); \
553 mod->field = NULL; \
579 static int module_unload_init(struct module *mod) in module_unload_init() argument
585 atomic_set(&mod->refcnt, MODULE_REF_BASE); in module_unload_init()
587 INIT_LIST_HEAD(&mod->source_list); in module_unload_init()
588 INIT_LIST_HEAD(&mod->target_list); in module_unload_init()
591 atomic_inc(&mod->refcnt); in module_unload_init()
654 static void module_unload_free(struct module *mod) in module_unload_free() argument
659 list_for_each_entry_safe(use, tmp, &mod->target_list, target_list) { in module_unload_free()
661 pr_debug("%s unusing %s\n", mod->name, i->name); in module_unload_free()
686 static int try_release_module_ref(struct module *mod) in try_release_module_ref() argument
691 ret = atomic_sub_return(MODULE_REF_BASE, &mod->refcnt); in try_release_module_ref()
695 ret = atomic_add_unless(&mod->refcnt, MODULE_REF_BASE, 0); in try_release_module_ref()
700 static int try_stop_module(struct module *mod, int flags, int *forced) in try_stop_module() argument
703 if (try_release_module_ref(mod) != 0) { in try_stop_module()
710 mod->state = MODULE_STATE_GOING; in try_stop_module()
717 * @mod: the module we're checking
723 int module_refcount(struct module *mod) in module_refcount() argument
725 return atomic_read(&mod->refcnt) - MODULE_REF_BASE; in module_refcount()
730 static void free_module(struct module *mod);
735 struct module *mod; in SYSCALL_DEFINE2() local
752 mod = find_module(name); in SYSCALL_DEFINE2()
753 if (!mod) { in SYSCALL_DEFINE2()
758 if (!list_empty(&mod->source_list)) { in SYSCALL_DEFINE2()
765 if (mod->state != MODULE_STATE_LIVE) { in SYSCALL_DEFINE2()
767 pr_debug("%s already dying\n", mod->name); in SYSCALL_DEFINE2()
773 if (mod->init && !mod->exit) { in SYSCALL_DEFINE2()
782 ret = try_stop_module(mod, flags, &forced); in SYSCALL_DEFINE2()
788 if (mod->exit != NULL) in SYSCALL_DEFINE2()
789 mod->exit(); in SYSCALL_DEFINE2()
791 MODULE_STATE_GOING, mod); in SYSCALL_DEFINE2()
792 klp_module_going(mod); in SYSCALL_DEFINE2()
793 ftrace_release_mod(mod); in SYSCALL_DEFINE2()
798 strscpy(last_unloaded_module.name, mod->name, sizeof(last_unloaded_module.name)); in SYSCALL_DEFINE2()
799 …strscpy(last_unloaded_module.taints, module_flags(mod, buf, false), sizeof(last_unloaded_module.ta… in SYSCALL_DEFINE2()
801 free_module(mod); in SYSCALL_DEFINE2()
848 return sprintf(buffer, "%i\n", module_refcount(mk->mod)); in show_refcnt()
892 static inline void module_unload_free(struct module *mod) in module_unload_free() argument
901 static inline int module_unload_init(struct module *mod) in module_unload_init() argument
925 switch (mk->mod->state) { in show_initstate()
960 unsigned int size = mk->mod->mem[MOD_TEXT].size; in show_coresize()
964 size += mk->mod->mem[type].size; in show_coresize()
979 size += mk->mod->mem[type].size; in show_datasize()
993 size += mk->mod->mem[type].size; in show_initsize()
1005 l = module_flags_taint(mk->mod->taints, buffer); in show_taint()
1034 int try_to_force_load(struct module *mod, const char *reason) in try_to_force_load() argument
1038 pr_warn("%s: %s: kernel tainted.\n", mod->name, reason); in try_to_force_load()
1039 add_taint_module(mod, TAINT_FORCED_MODULE, LOCKDEP_NOW_UNRELIABLE); in try_to_force_load()
1098 struct module *mod) in verify_namespace_is_imported() argument
1115 mod->name, kernel_symbol_name(sym), namespace); in verify_namespace_is_imported()
1123 static bool inherit_taint(struct module *mod, struct module *owner, const char *name) in inherit_taint() argument
1128 if (mod->using_gplonly_symbols) { in inherit_taint()
1130 mod->name, name, owner->name); in inherit_taint()
1134 if (!test_bit(TAINT_PROPRIETARY_MODULE, &mod->taints)) { in inherit_taint()
1136 mod->name, name, owner->name); in inherit_taint()
1137 set_bit(TAINT_PROPRIETARY_MODULE, &mod->taints); in inherit_taint()
1143 static const struct kernel_symbol *resolve_symbol(struct module *mod, in resolve_symbol() argument
1150 .gplok = !(mod->taints & (1 << TAINT_PROPRIETARY_MODULE)), in resolve_symbol()
1166 mod->using_gplonly_symbols = true; in resolve_symbol()
1168 if (!inherit_taint(mod, fsa.owner, name)) { in resolve_symbol()
1173 if (!check_version(info, name, mod, fsa.crc)) { in resolve_symbol()
1178 err = verify_namespace_is_imported(info, fsa.sym, mod); in resolve_symbol()
1184 err = ref_module(mod, fsa.owner); in resolve_symbol()
1199 resolve_symbol_wait(struct module *mod, in resolve_symbol_wait() argument
1207 !IS_ERR(ksym = resolve_symbol(mod, info, name, owner)) in resolve_symbol_wait()
1211 mod->name, owner); in resolve_symbol_wait()
1216 void __weak module_arch_cleanup(struct module *mod) in module_arch_cleanup() argument
1220 void __weak module_arch_freeing_init(struct module *mod) in module_arch_freeing_init() argument
1224 void *__module_writable_address(struct module *mod, void *loc) in __module_writable_address() argument
1227 struct module_memory *mem = &mod->mem[type]; in __module_writable_address()
1236 static int module_memory_alloc(struct module *mod, enum mod_mem_type type) in module_memory_alloc() argument
1238 unsigned int size = PAGE_ALIGN(mod->mem[type].size); in module_memory_alloc()
1242 mod->mem[type].size = size; in module_memory_alloc()
1253 mod->mem[type].base = ptr; in module_memory_alloc()
1259 execmem_free(mod->mem[type].base); in module_memory_alloc()
1263 mod->mem[type].rw_copy = ptr; in module_memory_alloc()
1264 mod->mem[type].is_rox = true; in module_memory_alloc()
1266 mod->mem[type].rw_copy = mod->mem[type].base; in module_memory_alloc()
1267 memset(mod->mem[type].base, 0, size); in module_memory_alloc()
1286 static void module_memory_free(struct module *mod, enum mod_mem_type type) in module_memory_free() argument
1288 struct module_memory *mem = &mod->mem[type]; in module_memory_free()
1296 static void free_mod_mem(struct module *mod) in free_mod_mem() argument
1299 struct module_memory *mod_mem = &mod->mem[type]; in free_mod_mem()
1307 module_memory_free(mod, type); in free_mod_mem()
1310 /* MOD_DATA hosts mod, so free it at last */ in free_mod_mem()
1311 lockdep_free_key_range(mod->mem[MOD_DATA].base, mod->mem[MOD_DATA].size); in free_mod_mem()
1312 module_memory_free(mod, MOD_DATA); in free_mod_mem()
1316 static void free_module(struct module *mod) in free_module() argument
1318 trace_module_free(mod); in free_module()
1320 codetag_unload_module(mod); in free_module()
1322 mod_sysfs_teardown(mod); in free_module()
1329 mod->state = MODULE_STATE_UNFORMED; in free_module()
1333 module_arch_cleanup(mod); in free_module()
1336 module_unload_free(mod); in free_module()
1339 destroy_params(mod->kp, mod->num_kp); in free_module()
1341 if (is_livepatch_module(mod)) in free_module()
1342 free_module_elf(mod); in free_module()
1347 list_del_rcu(&mod->list); in free_module()
1348 mod_tree_remove(mod); in free_module()
1350 module_bug_cleanup(mod); in free_module()
1351 /* Wait for RCU-sched synchronizing before releasing mod->list and buglist. */ in free_module()
1353 if (try_add_tainted_module(mod)) in free_module()
1355 mod->name); in free_module()
1359 module_arch_freeing_init(mod); in free_module()
1360 kfree(mod->args); in free_module()
1361 percpu_modfree(mod); in free_module()
1363 free_mod_mem(mod); in free_module()
1398 static int verify_exported_symbols(struct module *mod) in verify_exported_symbols() argument
1406 { mod->syms, mod->num_syms }, in verify_exported_symbols()
1407 { mod->gpl_syms, mod->num_gpl_syms }, in verify_exported_symbols()
1419 mod->name, kernel_symbol_name(s), in verify_exported_symbols()
1444 static int simplify_symbols(struct module *mod, const struct load_info *info) in simplify_symbols() argument
1468 mod->name); in simplify_symbols()
1483 ksym = resolve_symbol_wait(mod, info, name); in simplify_symbols()
1498 mod->name, name, ret); in simplify_symbols()
1504 secbase = (unsigned long)mod_percpu(mod); in simplify_symbols()
1515 static int apply_relocations(struct module *mod, const struct load_info *info) in apply_relocations() argument
1533 err = klp_apply_section_relocs(mod, info->sechdrs, in apply_relocations()
1540 info->index.sym, i, mod); in apply_relocations()
1543 info->index.sym, i, mod); in apply_relocations()
1551 unsigned int __weak arch_mod_section_prepend(struct module *mod, in arch_mod_section_prepend() argument
1558 long module_get_offset_and_type(struct module *mod, enum mod_mem_type type, in module_get_offset_and_type() argument
1564 mod->mem[type].size += arch_mod_section_prepend(mod, section); in module_get_offset_and_type()
1565 offset = ALIGN(mod->mem[type].size, sechdr->sh_addralign ?: 1); in module_get_offset_and_type()
1566 mod->mem[type].size = offset + sechdr->sh_size; in module_get_offset_and_type()
1581 static void __layout_sections(struct module *mod, struct load_info *info, bool is_init) in __layout_sections() argument
1632 if (codetag_needs_module_section(mod, sname, s->sh_size)) { in __layout_sections()
1642 s->sh_entsize = module_get_offset_and_type(mod, type, s, i); in __layout_sections()
1654 static void layout_sections(struct module *mod, struct load_info *info) in layout_sections() argument
1661 pr_debug("Core section allocation order for %s:\n", mod->name); in layout_sections()
1662 __layout_sections(mod, info, false); in layout_sections()
1664 pr_debug("Init section allocation order for %s:\n", mod->name); in layout_sections()
1665 __layout_sections(mod, info, true); in layout_sections()
1668 static void module_license_taint_check(struct module *mod, const char *license) in module_license_taint_check() argument
1676 mod->name, license); in module_license_taint_check()
1677 add_taint_module(mod, TAINT_PROPRIETARY_MODULE, in module_license_taint_check()
1682 static void setup_modinfo(struct module *mod, struct load_info *info) in setup_modinfo() argument
1689 attr->setup(mod, get_modinfo(info, attr->attr.name)); in setup_modinfo()
1693 static void free_modinfo(struct module *mod) in free_modinfo() argument
1700 attr->free(mod); in free_modinfo()
1964 * modules's *.mod.c file. If the struct module of the kernel changes a full
1976 * If all checks pass, the index will be cached in &load_info->index.mod
2012 info->index.mod = mod_idx; in elf_validity_cache_index_mod()
2266 * As a last step we set info->mod to the temporary copy of the module in
2288 /* This is temporary: point mod into copy of data. */ in elf_validity_cache_copy()
2289 info->mod = (void *)info->hdr + info->sechdrs[info->index.mod].sh_offset; in elf_validity_cache_copy()
2293 * on-disk struct mod 'name' field. in elf_validity_cache_copy()
2296 info->name = info->mod->name; in elf_validity_cache_copy()
2318 static int check_modinfo_livepatch(struct module *mod, struct load_info *info) in check_modinfo_livepatch() argument
2324 if (set_livepatch_module(mod)) in check_modinfo_livepatch()
2328 mod->name); in check_modinfo_livepatch()
2332 static void check_modinfo_retpoline(struct module *mod, struct load_info *info) in check_modinfo_retpoline() argument
2338 mod->name); in check_modinfo_retpoline()
2421 static void module_augment_kernel_taints(struct module *mod, struct load_info *info) in module_augment_kernel_taints() argument
2429 mod->name); in module_augment_kernel_taints()
2430 add_taint_module(mod, TAINT_OOT_MODULE, LOCKDEP_STILL_OK); in module_augment_kernel_taints()
2433 check_modinfo_retpoline(mod, info); in module_augment_kernel_taints()
2436 add_taint_module(mod, TAINT_CRAP, LOCKDEP_STILL_OK); in module_augment_kernel_taints()
2438 "is unknown, you have been warned.\n", mod->name); in module_augment_kernel_taints()
2441 if (is_livepatch_module(mod)) { in module_augment_kernel_taints()
2442 add_taint_module(mod, TAINT_LIVEPATCH, LOCKDEP_STILL_OK); in module_augment_kernel_taints()
2444 mod->name); in module_augment_kernel_taints()
2447 module_license_taint_check(mod, get_modinfo(info, "license")); in module_augment_kernel_taints()
2452 mod->name); in module_augment_kernel_taints()
2453 add_taint_module(mod, TAINT_TEST, LOCKDEP_STILL_OK); in module_augment_kernel_taints()
2456 mod->sig_ok = info->sig_ok; in module_augment_kernel_taints()
2457 if (!mod->sig_ok) { in module_augment_kernel_taints()
2460 "kernel\n", mod->name); in module_augment_kernel_taints()
2461 add_taint_module(mod, TAINT_UNSIGNED_MODULE, LOCKDEP_STILL_OK); in module_augment_kernel_taints()
2470 if (strcmp(mod->name, "ndiswrapper") == 0) in module_augment_kernel_taints()
2474 if (strcmp(mod->name, module_license_offenders[i]) == 0) in module_augment_kernel_taints()
2475 add_taint_module(mod, TAINT_PROPRIETARY_MODULE, in module_augment_kernel_taints()
2480 pr_warn("%s: module license taints kernel.\n", mod->name); in module_augment_kernel_taints()
2484 static int check_modinfo(struct module *mod, struct load_info *info, int flags) in check_modinfo() argument
2494 err = try_to_force_load(mod, "bad vermagic"); in check_modinfo()
2503 err = check_modinfo_livepatch(mod, info); in check_modinfo()
2510 static int find_module_sections(struct module *mod, struct load_info *info) in find_module_sections() argument
2512 mod->kp = section_objs(info, "__param", in find_module_sections()
2513 sizeof(*mod->kp), &mod->num_kp); in find_module_sections()
2514 mod->syms = section_objs(info, "__ksymtab", in find_module_sections()
2515 sizeof(*mod->syms), &mod->num_syms); in find_module_sections()
2516 mod->crcs = section_addr(info, "__kcrctab"); in find_module_sections()
2517 mod->gpl_syms = section_objs(info, "__ksymtab_gpl", in find_module_sections()
2518 sizeof(*mod->gpl_syms), in find_module_sections()
2519 &mod->num_gpl_syms); in find_module_sections()
2520 mod->gpl_crcs = section_addr(info, "__kcrctab_gpl"); in find_module_sections()
2523 mod->ctors = section_objs(info, ".ctors", in find_module_sections()
2524 sizeof(*mod->ctors), &mod->num_ctors); in find_module_sections()
2525 if (!mod->ctors) in find_module_sections()
2526 mod->ctors = section_objs(info, ".init_array", in find_module_sections()
2527 sizeof(*mod->ctors), &mod->num_ctors); in find_module_sections()
2534 mod->name); in find_module_sections()
2539 mod->noinstr_text_start = section_objs(info, ".noinstr.text", 1, in find_module_sections()
2540 &mod->noinstr_text_size); in find_module_sections()
2543 mod->tracepoints_ptrs = section_objs(info, "__tracepoints_ptrs", in find_module_sections()
2544 sizeof(*mod->tracepoints_ptrs), in find_module_sections()
2545 &mod->num_tracepoints); in find_module_sections()
2548 mod->srcu_struct_ptrs = section_objs(info, "___srcu_struct_ptrs", in find_module_sections()
2549 sizeof(*mod->srcu_struct_ptrs), in find_module_sections()
2550 &mod->num_srcu_structs); in find_module_sections()
2553 mod->bpf_raw_events = section_objs(info, "__bpf_raw_tp_map", in find_module_sections()
2554 sizeof(*mod->bpf_raw_events), in find_module_sections()
2555 &mod->num_bpf_raw_events); in find_module_sections()
2558 mod->btf_data = any_section_objs(info, ".BTF", 1, &mod->btf_data_size); in find_module_sections()
2559 mod->btf_base_data = any_section_objs(info, ".BTF.base", 1, in find_module_sections()
2560 &mod->btf_base_data_size); in find_module_sections()
2563 mod->jump_entries = section_objs(info, "__jump_table", in find_module_sections()
2564 sizeof(*mod->jump_entries), in find_module_sections()
2565 &mod->num_jump_entries); in find_module_sections()
2568 mod->trace_events = section_objs(info, "_ftrace_events", in find_module_sections()
2569 sizeof(*mod->trace_events), in find_module_sections()
2570 &mod->num_trace_events); in find_module_sections()
2571 mod->trace_evals = section_objs(info, "_ftrace_eval_map", in find_module_sections()
2572 sizeof(*mod->trace_evals), in find_module_sections()
2573 &mod->num_trace_evals); in find_module_sections()
2576 mod->trace_bprintk_fmt_start = section_objs(info, "__trace_printk_fmt", in find_module_sections()
2577 sizeof(*mod->trace_bprintk_fmt_start), in find_module_sections()
2578 &mod->num_trace_bprintk_fmt); in find_module_sections()
2582 mod->ftrace_callsites = section_objs(info, FTRACE_CALLSITE_SECTION, in find_module_sections()
2583 sizeof(*mod->ftrace_callsites), in find_module_sections()
2584 &mod->num_ftrace_callsites); in find_module_sections()
2587 mod->ei_funcs = section_objs(info, "_error_injection_whitelist", in find_module_sections()
2588 sizeof(*mod->ei_funcs), in find_module_sections()
2589 &mod->num_ei_funcs); in find_module_sections()
2592 mod->kprobes_text_start = section_objs(info, ".kprobes.text", 1, in find_module_sections()
2593 &mod->kprobes_text_size); in find_module_sections()
2594 mod->kprobe_blacklist = section_objs(info, "_kprobe_blacklist", in find_module_sections()
2596 &mod->num_kprobe_blacklist); in find_module_sections()
2599 mod->printk_index_start = section_objs(info, ".printk_index", in find_module_sections()
2600 sizeof(*mod->printk_index_start), in find_module_sections()
2601 &mod->printk_index_size); in find_module_sections()
2604 mod->static_call_sites = section_objs(info, ".static_call_sites", in find_module_sections()
2605 sizeof(*mod->static_call_sites), in find_module_sections()
2606 &mod->num_static_call_sites); in find_module_sections()
2609 mod->kunit_suites = section_objs(info, ".kunit_test_suites", in find_module_sections()
2610 sizeof(*mod->kunit_suites), in find_module_sections()
2611 &mod->num_kunit_suites); in find_module_sections()
2612 mod->kunit_init_suites = section_objs(info, ".kunit_init_test_suites", in find_module_sections()
2613 sizeof(*mod->kunit_init_suites), in find_module_sections()
2614 &mod->num_kunit_init_suites); in find_module_sections()
2617 mod->extable = section_objs(info, "__ex_table", in find_module_sections()
2618 sizeof(*mod->extable), &mod->num_exentries); in find_module_sections()
2621 pr_warn("%s: Ignoring obsolete parameters\n", mod->name); in find_module_sections()
2624 mod->dyndbg_info.descs = section_objs(info, "__dyndbg", in find_module_sections()
2625 sizeof(*mod->dyndbg_info.descs), in find_module_sections()
2626 &mod->dyndbg_info.num_descs); in find_module_sections()
2627 mod->dyndbg_info.classes = section_objs(info, "__dyndbg_classes", in find_module_sections()
2628 sizeof(*mod->dyndbg_info.classes), in find_module_sections()
2629 &mod->dyndbg_info.num_classes); in find_module_sections()
2635 static int move_module(struct module *mod, struct load_info *info) in move_module() argument
2643 if (!mod->mem[type].size) { in move_module()
2644 mod->mem[type].base = NULL; in move_module()
2645 mod->mem[type].rw_copy = NULL; in move_module()
2649 ret = module_memory_alloc(mod, type); in move_module()
2657 pr_debug("Final section addresses for %s:\n", mod->name); in move_module()
2672 if (codetag_needs_module_section(mod, sname, shdr->sh_size)) { in move_module()
2673 dest = codetag_alloc_module_section(mod, sname, shdr->sh_size, in move_module()
2674 arch_mod_section_prepend(mod, i), shdr->sh_addralign); in move_module()
2689 addr = (unsigned long)mod->mem[type].base + offset; in move_module()
2690 dest = mod->mem[type].rw_copy + offset; in move_module()
2700 if (i == info->index.mod && in move_module()
2721 module_memory_free(mod, t); in move_module()
2723 codetag_free_module_sections(mod); in move_module()
2728 static int check_export_symbol_versions(struct module *mod) in check_export_symbol_versions() argument
2731 if ((mod->num_syms && !mod->crcs) || in check_export_symbol_versions()
2732 (mod->num_gpl_syms && !mod->gpl_crcs)) { in check_export_symbol_versions()
2733 return try_to_force_load(mod, in check_export_symbol_versions()
2740 static void flush_module_icache(const struct module *mod) in flush_module_icache() argument
2748 const struct module_memory *mod_mem = &mod->mem[type]; in flush_module_icache()
2765 struct module *mod) in module_frob_arch_sections() argument
2793 struct module *mod; in layout_and_allocate() local
2799 info->secstrings, info->mod); in layout_and_allocate()
2804 info->secstrings, info->mod); in layout_and_allocate()
2834 layout_sections(info->mod, info); in layout_and_allocate()
2835 layout_symtab(info->mod, info); in layout_and_allocate()
2838 err = move_module(info->mod, info); in layout_and_allocate()
2843 mod = (void *)info->sechdrs[info->index.mod].sh_addr; in layout_and_allocate()
2844 kmemleak_load_module(mod, info); in layout_and_allocate()
2845 codetag_module_replaced(info->mod, mod); in layout_and_allocate()
2847 return mod; in layout_and_allocate()
2850 /* mod is no longer valid after this! */
2851 static void module_deallocate(struct module *mod, struct load_info *info) in module_deallocate() argument
2853 percpu_modfree(mod); in module_deallocate()
2854 module_arch_freeing_init(mod); in module_deallocate()
2856 free_mod_mem(mod); in module_deallocate()
2873 static int post_relocation(struct module *mod, const struct load_info *info) in post_relocation() argument
2878 sort_extable(mod->extable, mod->extable + mod->num_exentries); in post_relocation()
2881 percpu_modcopy(mod, (void *)info->sechdrs[info->index.pcpu].sh_addr, in post_relocation()
2885 add_kallsyms(mod, info); in post_relocation()
2888 ret = module_finalize(info->hdr, info->sechdrs, mod); in post_relocation()
2893 struct module_memory *mem = &mod->mem[type]; in post_relocation()
2905 return module_post_finalize(info->hdr, info->sechdrs, mod); in post_relocation()
2909 static void do_mod_ctors(struct module *mod) in do_mod_ctors() argument
2914 for (i = 0; i < mod->num_ctors; i++) in do_mod_ctors()
2915 mod->ctors[i](); in do_mod_ctors()
2962 static noinline int do_init_module(struct module *mod) in do_init_module() argument
2970 const struct module_memory *mod_mem = &mod->mem[type]; in do_init_module()
2984 freeinit->init_text = mod->mem[MOD_INIT_TEXT].base; in do_init_module()
2985 freeinit->init_data = mod->mem[MOD_INIT_DATA].base; in do_init_module()
2986 freeinit->init_rodata = mod->mem[MOD_INIT_RODATA].base; in do_init_module()
2988 do_mod_ctors(mod); in do_init_module()
2990 if (mod->init != NULL) in do_init_module()
2991 ret = do_one_initcall(mod->init); in do_init_module()
2999 __func__, mod->name, ret, __func__); in do_init_module()
3004 mod->state = MODULE_STATE_LIVE; in do_init_module()
3006 MODULE_STATE_LIVE, mod); in do_init_module()
3009 kobject_uevent(&mod->mkobj.kobj, KOBJ_ADD); in do_init_module()
3019 if (!mod->async_probe_requested) in do_init_module()
3022 ftrace_free_mem(mod, mod->mem[MOD_INIT_TEXT].base, in do_init_module()
3023 mod->mem[MOD_INIT_TEXT].base + mod->mem[MOD_INIT_TEXT].size); in do_init_module()
3026 module_put(mod); in do_init_module()
3027 trim_init_extable(mod); in do_init_module()
3030 rcu_assign_pointer(mod->kallsyms, &mod->core_kallsyms); in do_init_module()
3032 ret = module_enable_rodata_ro_after_init(mod); in do_init_module()
3036 mod->name, ret); in do_init_module()
3038 mod_tree_remove_init(mod); in do_init_module()
3039 module_arch_freeing_init(mod); in do_init_module()
3041 mod->mem[type].base = NULL; in do_init_module()
3042 mod->mem[type].size = 0; in do_init_module()
3047 mod->btf_data = NULL; in do_init_module()
3048 mod->btf_base_data = NULL; in do_init_module()
3080 mod->state = MODULE_STATE_GOING; in do_init_module()
3082 module_put(mod); in do_init_module()
3084 MODULE_STATE_GOING, mod); in do_init_module()
3085 klp_module_going(mod); in do_init_module()
3086 ftrace_release_mod(mod); in do_init_module()
3087 free_module(mod); in do_init_module()
3104 struct module *mod; in finished_loading() local
3114 mod = find_module_all(name, strlen(name), true); in finished_loading()
3115 ret = !mod || mod->state == MODULE_STATE_LIVE in finished_loading()
3116 || mod->state == MODULE_STATE_GOING; in finished_loading()
3167 static int add_unformed_module(struct module *mod) in add_unformed_module() argument
3171 mod->state = MODULE_STATE_UNFORMED; in add_unformed_module()
3174 err = module_patient_check_exists(mod->name, FAIL_DUP_MOD_LOAD); in add_unformed_module()
3178 mod_update_bounds(mod); in add_unformed_module()
3179 list_add_rcu(&mod->list, &modules); in add_unformed_module()
3180 mod_tree_insert(mod); in add_unformed_module()
3188 static int complete_formation(struct module *mod, struct load_info *info) in complete_formation() argument
3195 err = verify_exported_symbols(mod); in complete_formation()
3200 module_bug_finalize(info->hdr, info->sechdrs, mod); in complete_formation()
3201 module_cfi_finalize(info->hdr, info->sechdrs, mod); in complete_formation()
3203 err = module_enable_rodata_ro(mod); in complete_formation()
3206 err = module_enable_data_nx(mod); in complete_formation()
3209 err = module_enable_text_rox(mod); in complete_formation()
3217 mod->state = MODULE_STATE_COMING; in complete_formation()
3223 module_bug_cleanup(mod); in complete_formation()
3229 static int prepare_coming_module(struct module *mod) in prepare_coming_module() argument
3233 ftrace_module_enable(mod); in prepare_coming_module()
3234 err = klp_module_coming(mod); in prepare_coming_module()
3239 MODULE_STATE_COMING, MODULE_STATE_GOING, mod); in prepare_coming_module()
3242 klp_module_going(mod); in prepare_coming_module()
3250 struct module *mod = arg; in unknown_module_param_cb() local
3254 if (kstrtobool(val, &mod->async_probe_requested)) in unknown_module_param_cb()
3255 mod->async_probe_requested = true; in unknown_module_param_cb()
3285 if (!check_modstruct_version(info, info->mod)) in early_mod_check()
3288 err = check_modinfo(info->mod, info, flags); in early_mod_check()
3293 err = module_patient_check_exists(info->mod->name, FAIL_DUP_MOD_BECOMING); in early_mod_check()
3306 struct module *mod; in load_module() local
3330 * info->mod to the userspace passed struct module. in load_module()
3341 mod = layout_and_allocate(info, flags); in load_module()
3342 if (IS_ERR(mod)) { in load_module()
3343 err = PTR_ERR(mod); in load_module()
3349 audit_log_kern_module(mod->name); in load_module()
3352 err = add_unformed_module(mod); in load_module()
3360 module_augment_kernel_taints(mod, info); in load_module()
3363 err = percpu_modalloc(mod, info); in load_module()
3368 err = module_unload_init(mod); in load_module()
3372 init_param_lock(mod); in load_module()
3378 err = find_module_sections(mod, info); in load_module()
3382 err = check_export_symbol_versions(mod); in load_module()
3387 setup_modinfo(mod, info); in load_module()
3390 err = simplify_symbols(mod, info); in load_module()
3394 err = apply_relocations(mod, info); in load_module()
3398 err = post_relocation(mod, info); in load_module()
3402 flush_module_icache(mod); in load_module()
3405 mod->args = strndup_user(uargs, ~0UL >> 1); in load_module()
3406 if (IS_ERR(mod->args)) { in load_module()
3407 err = PTR_ERR(mod->args); in load_module()
3411 init_build_id(mod, info); in load_module()
3414 ftrace_module_init(mod); in load_module()
3417 err = complete_formation(mod, info); in load_module()
3421 err = prepare_coming_module(mod); in load_module()
3425 mod->async_probe_requested = async_probe; in load_module()
3428 after_dashes = parse_args(mod->name, mod->args, mod->kp, mod->num_kp, in load_module()
3429 -32768, 32767, mod, in load_module()
3436 mod->name, after_dashes); in load_module()
3440 err = mod_sysfs_setup(mod, info, mod->kp, mod->num_kp); in load_module()
3444 if (is_livepatch_module(mod)) { in load_module()
3445 err = copy_module_elf(mod, info); in load_module()
3453 codetag_load_module(mod); in load_module()
3456 trace_module_load(mod); in load_module()
3458 return do_init_module(mod); in load_module()
3461 mod_sysfs_teardown(mod); in load_module()
3463 mod->state = MODULE_STATE_GOING; in load_module()
3464 destroy_params(mod->kp, mod->num_kp); in load_module()
3466 MODULE_STATE_GOING, mod); in load_module()
3467 klp_module_going(mod); in load_module()
3469 mod->state = MODULE_STATE_GOING; in load_module()
3472 module_bug_cleanup(mod); in load_module()
3476 ftrace_release_mod(mod); in load_module()
3478 kfree(mod->args); in load_module()
3480 module_arch_cleanup(mod); in load_module()
3482 free_modinfo(mod); in load_module()
3484 module_unload_free(mod); in load_module()
3488 list_del_rcu(&mod->list); in load_module()
3489 mod_tree_remove(mod); in load_module()
3491 /* Wait for RCU-sched synchronizing before releasing mod->list. */ in load_module()
3498 lockdep_free_key_range(mod->mem[type].base, in load_module()
3499 mod->mem[type].size); in load_module()
3502 module_deallocate(mod, info); in load_module()
3689 char *module_flags(struct module *mod, char *buf, bool show_state) in module_flags() argument
3693 BUG_ON(mod->state == MODULE_STATE_UNFORMED); in module_flags()
3694 if (!mod->taints && !show_state) in module_flags()
3696 if (mod->taints || in module_flags()
3697 mod->state == MODULE_STATE_GOING || in module_flags()
3698 mod->state == MODULE_STATE_COMING) { in module_flags()
3700 bx += module_flags_taint(mod->taints, buf + bx); in module_flags()
3702 if (mod->state == MODULE_STATE_GOING && show_state) in module_flags()
3705 if (mod->state == MODULE_STATE_COMING && show_state) in module_flags()
3719 struct module *mod; in search_module_extables() local
3722 mod = __module_address(addr); in search_module_extables()
3723 if (!mod) in search_module_extables()
3726 if (!mod->num_exentries) in search_module_extables()
3729 e = search_extable(mod->extable, in search_module_extables()
3730 mod->num_exentries, in search_module_extables()
3769 struct module *mod; in __module_address() local
3784 mod = mod_find(addr, &mod_tree); in __module_address()
3785 if (mod) { in __module_address()
3786 BUG_ON(!within_module(addr, mod)); in __module_address()
3787 if (mod->state == MODULE_STATE_UNFORMED) in __module_address()
3788 mod = NULL; in __module_address()
3790 return mod; in __module_address()
3821 struct module *mod = __module_address(addr); in __module_text_address() local
3822 if (mod) { in __module_text_address()
3824 if (!within_module_mem_type(addr, mod, MOD_TEXT) && in __module_text_address()
3825 !within_module_mem_type(addr, mod, MOD_INIT_TEXT)) in __module_text_address()
3826 mod = NULL; in __module_text_address()
3828 return mod; in __module_text_address()
3834 struct module *mod; in print_modules() local
3840 list_for_each_entry_rcu(mod, &modules, list) { in print_modules()
3841 if (mod->state == MODULE_STATE_UNFORMED) in print_modules()
3843 pr_cont(" %s%s", mod->name, module_flags(mod, buf, true)); in print_modules()