Lines Matching refs:mod
177 static void mod_tree_insert(struct module *mod)
179 mod->core_layout.mtn.mod = mod;
180 mod->init_layout.mtn.mod = mod;
182 __mod_tree_insert(&mod->core_layout.mtn);
183 if (mod->init_layout.size)
184 __mod_tree_insert(&mod->init_layout.mtn);
187 static void mod_tree_remove_init(struct module *mod)
189 if (mod->init_layout.size)
190 __mod_tree_remove(&mod->init_layout.mtn);
193 static void mod_tree_remove(struct module *mod)
195 __mod_tree_remove(&mod->core_layout.mtn);
196 mod_tree_remove_init(mod);
207 return container_of(ltn, struct mod_tree_node, node)->mod;
214 static void mod_tree_insert(struct module *mod) { }
215 static void mod_tree_remove_init(struct module *mod) { }
216 static void mod_tree_remove(struct module *mod) { }
220 struct module *mod;
222 list_for_each_entry_rcu(mod, &modules, list,
224 if (within_module(addr, mod))
225 return mod;
248 static void mod_update_bounds(struct module *mod)
250 __mod_update_bounds(mod->core_layout.base, mod->core_layout.size);
251 if (mod->init_layout.size)
252 __mod_update_bounds(mod->init_layout.base, mod->init_layout.size);
323 static inline int strong_try_module_get(struct module *mod)
325 BUG_ON(mod && mod->state == MODULE_STATE_UNFORMED);
326 if (mod && mod->state == MODULE_STATE_COMING)
328 if (try_module_get(mod))
334 static inline void add_taint_module(struct module *mod, unsigned flag,
338 set_bit(flag, &mod->taints);
345 void __noreturn __module_put_and_exit(struct module *mod, long code)
347 module_put(mod);
436 struct module *mod;
461 list_for_each_entry_rcu(mod, &modules, list,
464 { mod->syms, mod->syms + mod->num_syms, mod->crcs,
466 { mod->gpl_syms, mod->gpl_syms + mod->num_gpl_syms,
467 mod->gpl_crcs,
469 { mod->gpl_future_syms,
470 mod->gpl_future_syms + mod->num_gpl_future_syms,
471 mod->gpl_future_crcs,
474 { mod->unused_syms,
475 mod->unused_syms + mod->num_unused_syms,
476 mod->unused_crcs,
478 { mod->unused_gpl_syms,
479 mod->unused_gpl_syms + mod->num_unused_gpl_syms,
480 mod->unused_gpl_crcs,
485 if (mod->state == MODULE_STATE_UNFORMED)
488 if (each_symbol_in_section(arr, ARRAY_SIZE(arr), mod, fn, data))
629 struct module *mod;
633 list_for_each_entry_rcu(mod, &modules, list,
635 if (!even_unformed && mod->state == MODULE_STATE_UNFORMED)
637 if (strlen(mod->name) == len && !memcmp(mod->name, name, len))
638 return mod;
652 static inline void __percpu *mod_percpu(struct module *mod)
654 return mod->percpu;
657 static int percpu_modalloc(struct module *mod, struct load_info *info)
667 mod->name, align, PAGE_SIZE);
671 mod->percpu = __alloc_reserved_percpu(pcpusec->sh_size, align);
672 if (!mod->percpu) {
674 mod->name, (unsigned long)pcpusec->sh_size);
677 mod->percpu_size = pcpusec->sh_size;
681 static void percpu_modfree(struct module *mod)
683 free_percpu(mod->percpu);
691 static void percpu_modcopy(struct module *mod,
697 memcpy(per_cpu_ptr(mod->percpu, cpu), from, size);
702 struct module *mod;
707 list_for_each_entry_rcu(mod, &modules, list) {
708 if (mod->state == MODULE_STATE_UNFORMED)
710 if (!mod->percpu_size)
713 void *start = per_cpu_ptr(mod->percpu, cpu);
716 if (va >= start && va < start + mod->percpu_size) {
720 per_cpu_ptr(mod->percpu,
749 static inline void __percpu *mod_percpu(struct module *mod)
753 static int percpu_modalloc(struct module *mod, struct load_info *info)
760 static inline void percpu_modfree(struct module *mod)
767 static inline void percpu_modcopy(struct module *mod,
786 static void setup_modinfo_##field(struct module *mod, const char *s) \
788 mod->field = kstrdup(s, GFP_KERNEL); \
793 return scnprintf(buffer, PAGE_SIZE, "%s\n", mk->mod->field); \
795 static int modinfo_##field##_exists(struct module *mod) \
797 return mod->field != NULL; \
799 static void free_modinfo_##field(struct module *mod) \
801 kfree(mod->field); \
802 mod->field = NULL; \
825 static int module_unload_init(struct module *mod)
831 atomic_set(&mod->refcnt, MODULE_REF_BASE);
833 INIT_LIST_HEAD(&mod->source_list);
834 INIT_LIST_HEAD(&mod->target_list);
837 atomic_inc(&mod->refcnt);
902 static void module_unload_free(struct module *mod)
907 list_for_each_entry_safe(use, tmp, &mod->target_list, target_list) {
909 pr_debug("%s unusing %s\n", mod->name, i->name);
934 static int try_release_module_ref(struct module *mod)
939 ret = atomic_sub_return(MODULE_REF_BASE, &mod->refcnt);
943 ret = atomic_add_unless(&mod->refcnt, MODULE_REF_BASE, 0);
948 static int try_stop_module(struct module *mod, int flags, int *forced)
951 if (try_release_module_ref(mod) != 0) {
958 mod->state = MODULE_STATE_GOING;
966 * @mod: the module we're checking
972 int module_refcount(struct module *mod)
974 return atomic_read(&mod->refcnt) - MODULE_REF_BASE;
979 static void free_module(struct module *mod);
984 struct module *mod;
1000 mod = find_module(name);
1001 if (!mod) {
1006 if (!list_empty(&mod->source_list)) {
1013 if (mod->state != MODULE_STATE_LIVE) {
1015 pr_debug("%s already dying\n", mod->name);
1021 if (mod->init && !mod->exit) {
1031 ret = try_stop_module(mod, flags, &forced);
1037 if (mod->exit != NULL)
1038 mod->exit();
1040 MODULE_STATE_GOING, mod);
1041 klp_module_going(mod);
1042 ftrace_release_mod(mod);
1047 strlcpy(last_unloaded_module, mod->name, sizeof(last_unloaded_module));
1049 free_module(mod);
1058 static inline void print_unload_info(struct seq_file *m, struct module *mod)
1063 seq_printf(m, " %i ", module_refcount(mod));
1069 list_for_each_entry(use, &mod->source_list, source_list) {
1074 if (mod->init != NULL && mod->exit == NULL) {
1119 return sprintf(buffer, "%i\n", module_refcount(mk->mod));
1170 static inline void print_unload_info(struct seq_file *m, struct module *mod)
1176 static inline void module_unload_free(struct module *mod)
1185 static inline int module_unload_init(struct module *mod)
1191 static size_t module_flags_taint(struct module *mod, char *buf)
1197 if (taint_flags[i].module && test_bit(i, &mod->taints))
1209 switch (mk->mod->state) {
1244 return sprintf(buffer, "%u\n", mk->mod->core_layout.size);
1253 return sprintf(buffer, "%u\n", mk->mod->init_layout.size);
1264 l = module_flags_taint(mk->mod, buffer);
1288 static int try_to_force_load(struct module *mod, const char *reason)
1292 pr_warn("%s: %s: kernel tainted.\n", mod->name, reason);
1293 add_taint_module(mod, TAINT_FORCED_MODULE, LOCKDEP_NOW_UNRELIABLE);
1309 struct module *mod,
1323 return try_to_force_load(mod, symname) == 0;
1357 struct module *mod)
1371 return check_version(info, "module_layout", mod, crc);
1387 struct module *mod,
1394 struct module *mod)
1412 struct module *mod)
1432 mod->name, kernel_symbol_name(sym), namespace);
1440 static bool inherit_taint(struct module *mod, struct module *owner)
1445 if (mod->using_gplonly_symbols) {
1447 mod->name, owner->name);
1451 if (!test_bit(TAINT_PROPRIETARY_MODULE, &mod->taints)) {
1453 mod->name, owner->name);
1454 set_bit(TAINT_PROPRIETARY_MODULE, &mod->taints);
1460 static const struct kernel_symbol *resolve_symbol(struct module *mod,
1479 !(mod->taints & (1 << TAINT_PROPRIETARY_MODULE)), true);
1484 mod->using_gplonly_symbols = true;
1486 if (!inherit_taint(mod, owner)) {
1491 if (!check_version(info, name, mod, crc)) {
1496 err = verify_namespace_is_imported(info, sym, mod);
1502 err = ref_module(mod, owner);
1517 resolve_symbol_wait(struct module *mod,
1525 !IS_ERR(ksym = resolve_symbol(mod, info, name, owner))
1529 mod->name, owner);
1596 static void add_sect_attrs(struct module *mod, const struct load_info *info)
1639 if (sysfs_create_group(&mod->mkobj.kobj, §_attrs->grp))
1642 mod->sect_attrs = sect_attrs;
1648 static void remove_sect_attrs(struct module *mod)
1650 if (mod->sect_attrs) {
1651 sysfs_remove_group(&mod->mkobj.kobj,
1652 &mod->sect_attrs->grp);
1655 free_sect_attrs(mod->sect_attrs);
1656 mod->sect_attrs = NULL;
1693 static void add_notes_attrs(struct module *mod, const struct load_info *info)
1700 if (!mod->sect_attrs)
1725 nattr->attr.name = mod->sect_attrs->attrs[loaded].battr.attr.name;
1735 notes_attrs->dir = kobject_create_and_add("notes", &mod->mkobj.kobj);
1744 mod->notes_attrs = notes_attrs;
1751 static void remove_notes_attrs(struct module *mod)
1753 if (mod->notes_attrs)
1754 free_notes_attrs(mod->notes_attrs, mod->notes_attrs->notes);
1759 static inline void add_sect_attrs(struct module *mod,
1764 static inline void remove_sect_attrs(struct module *mod)
1768 static inline void add_notes_attrs(struct module *mod,
1773 static inline void remove_notes_attrs(struct module *mod)
1778 static void del_usage_links(struct module *mod)
1784 list_for_each_entry(use, &mod->target_list, target_list)
1785 sysfs_remove_link(use->target->holders_dir, mod->name);
1790 static int add_usage_links(struct module *mod)
1797 list_for_each_entry(use, &mod->target_list, target_list) {
1799 &mod->mkobj.kobj, mod->name);
1805 del_usage_links(mod);
1810 static void module_remove_modinfo_attrs(struct module *mod, int end);
1812 static int module_add_modinfo_attrs(struct module *mod)
1819 mod->modinfo_attrs = kzalloc((sizeof(struct module_attribute) *
1822 if (!mod->modinfo_attrs)
1825 temp_attr = mod->modinfo_attrs;
1827 if (!attr->test || attr->test(mod)) {
1830 error = sysfs_create_file(&mod->mkobj.kobj,
1842 module_remove_modinfo_attrs(mod, --i);
1844 kfree(mod->modinfo_attrs);
1848 static void module_remove_modinfo_attrs(struct module *mod, int end)
1853 for (i = 0; (attr = &mod->modinfo_attrs[i]); i++) {
1859 sysfs_remove_file(&mod->mkobj.kobj, &attr->attr);
1861 attr->free(mod);
1863 kfree(mod->modinfo_attrs);
1866 static void mod_kobject_put(struct module *mod)
1869 mod->mkobj.kobj_completion = &c;
1870 kobject_put(&mod->mkobj.kobj);
1874 static int mod_sysfs_init(struct module *mod)
1880 pr_err("%s: module sysfs not initialized\n", mod->name);
1885 kobj = kset_find_obj(module_kset, mod->name);
1887 pr_err("%s: module is already loaded\n", mod->name);
1893 mod->mkobj.mod = mod;
1895 memset(&mod->mkobj.kobj, 0, sizeof(mod->mkobj.kobj));
1896 mod->mkobj.kobj.kset = module_kset;
1897 err = kobject_init_and_add(&mod->mkobj.kobj, &module_ktype, NULL,
1898 "%s", mod->name);
1900 mod_kobject_put(mod);
1906 static int mod_sysfs_setup(struct module *mod,
1913 err = mod_sysfs_init(mod);
1917 mod->holders_dir = kobject_create_and_add("holders", &mod->mkobj.kobj);
1918 if (!mod->holders_dir) {
1923 err = module_param_sysfs_setup(mod, kparam, num_params);
1927 err = module_add_modinfo_attrs(mod);
1931 err = add_usage_links(mod);
1935 add_sect_attrs(mod, info);
1936 add_notes_attrs(mod, info);
1941 module_remove_modinfo_attrs(mod, -1);
1943 module_param_sysfs_remove(mod);
1945 kobject_put(mod->holders_dir);
1947 mod_kobject_put(mod);
1952 static void mod_sysfs_fini(struct module *mod)
1954 remove_notes_attrs(mod);
1955 remove_sect_attrs(mod);
1956 mod_kobject_put(mod);
1959 static void init_param_lock(struct module *mod)
1961 mutex_init(&mod->param_lock);
1965 static int mod_sysfs_setup(struct module *mod,
1973 static void mod_sysfs_fini(struct module *mod)
1977 static void module_remove_modinfo_attrs(struct module *mod, int end)
1981 static void del_usage_links(struct module *mod)
1985 static void init_param_lock(struct module *mod)
1990 static void mod_sysfs_teardown(struct module *mod)
1992 del_usage_links(mod);
1993 module_remove_modinfo_attrs(mod, -1);
1994 module_param_sysfs_remove(mod);
1995 kobject_put(mod->mkobj.drivers_dir);
1996 kobject_put(mod->holders_dir);
1997 mod_sysfs_fini(mod);
2030 static void module_enable_x(const struct module *mod)
2032 frob_text(&mod->core_layout, set_memory_x);
2033 frob_text(&mod->init_layout, set_memory_x);
2036 static void module_enable_x(const struct module *mod) { }
2070 static void module_enable_ro(const struct module *mod, bool after_init)
2075 set_vm_flush_reset_perms(mod->core_layout.base);
2076 set_vm_flush_reset_perms(mod->init_layout.base);
2077 frob_text(&mod->core_layout, set_memory_ro);
2079 frob_rodata(&mod->core_layout, set_memory_ro);
2080 frob_text(&mod->init_layout, set_memory_ro);
2081 frob_rodata(&mod->init_layout, set_memory_ro);
2084 frob_ro_after_init(&mod->core_layout, set_memory_ro);
2087 static void module_enable_nx(const struct module *mod)
2089 frob_rodata(&mod->core_layout, set_memory_nx);
2090 frob_ro_after_init(&mod->core_layout, set_memory_nx);
2091 frob_writable_data(&mod->core_layout, set_memory_nx);
2092 frob_rodata(&mod->init_layout, set_memory_nx);
2093 frob_writable_data(&mod->init_layout, set_memory_nx);
2097 char *secstrings, struct module *mod)
2105 mod->name, secstrings + sechdrs[i].sh_name, i);
2114 static void module_enable_nx(const struct module *mod) { }
2115 static void module_enable_ro(const struct module *mod, bool after_init) {}
2117 char *secstrings, struct module *mod)
2127 * index from info to mod->klp_info.
2129 static int copy_module_elf(struct module *mod, struct load_info *info)
2134 size = sizeof(*mod->klp_info);
2135 mod->klp_info = kmalloc(size, GFP_KERNEL);
2136 if (mod->klp_info == NULL)
2140 size = sizeof(mod->klp_info->hdr);
2141 memcpy(&mod->klp_info->hdr, info->hdr, size);
2145 mod->klp_info->sechdrs = kmemdup(info->sechdrs, size, GFP_KERNEL);
2146 if (mod->klp_info->sechdrs == NULL) {
2153 mod->klp_info->secstrings = kmemdup(info->secstrings, size, GFP_KERNEL);
2154 if (mod->klp_info->secstrings == NULL) {
2161 mod->klp_info->symndx = symndx;
2169 mod->klp_info->sechdrs[symndx].sh_addr = \
2170 (unsigned long) mod->core_kallsyms.symtab;
2175 kfree(mod->klp_info->sechdrs);
2177 kfree(mod->klp_info);
2181 static void free_module_elf(struct module *mod)
2183 kfree(mod->klp_info->sechdrs);
2184 kfree(mod->klp_info->secstrings);
2185 kfree(mod->klp_info);
2188 static int copy_module_elf(struct module *mod, struct load_info *info)
2193 static void free_module_elf(struct module *mod)
2208 void __weak module_arch_cleanup(struct module *mod)
2212 void __weak module_arch_freeing_init(struct module *mod)
2216 static void cfi_cleanup(struct module *mod);
2219 static void free_module(struct module *mod)
2221 trace_module_free(mod);
2223 mod_sysfs_teardown(mod);
2228 mod->state = MODULE_STATE_UNFORMED;
2232 ddebug_remove_module(mod->name);
2235 module_arch_cleanup(mod);
2238 module_unload_free(mod);
2241 destroy_params(mod->kp, mod->num_kp);
2243 if (is_livepatch_module(mod))
2244 free_module_elf(mod);
2249 list_del_rcu(&mod->list);
2250 mod_tree_remove(mod);
2252 module_bug_cleanup(mod);
2253 /* Wait for RCU-sched synchronizing before releasing mod->list and buglist. */
2258 cfi_cleanup(mod);
2261 module_arch_freeing_init(mod);
2262 module_memfree(mod->init_layout.base);
2263 kfree(mod->args);
2264 percpu_modfree(mod);
2267 lockdep_free_key_range(mod->core_layout.base, mod->core_layout.size);
2270 module_memfree(mod->core_layout.base);
2314 static int verify_exported_symbols(struct module *mod)
2323 { mod->syms, mod->num_syms },
2324 { mod->gpl_syms, mod->num_gpl_syms },
2325 { mod->gpl_future_syms, mod->num_gpl_future_syms },
2327 { mod->unused_syms, mod->num_unused_syms },
2328 { mod->unused_gpl_syms, mod->num_unused_gpl_syms },
2338 mod->name, kernel_symbol_name(s),
2363 static int simplify_symbols(struct module *mod, const struct load_info *info)
2385 mod->name);
2400 ksym = resolve_symbol_wait(mod, info, name);
2415 mod->name, name, ret);
2421 secbase = (unsigned long)mod_percpu(mod);
2432 static int apply_relocations(struct module *mod, const struct load_info *info)
2450 err = klp_apply_section_relocs(mod, info->sechdrs,
2457 info->index.sym, i, mod);
2460 info->index.sym, i, mod);
2468 unsigned int __weak arch_mod_section_prepend(struct module *mod,
2476 static long get_offset(struct module *mod, unsigned int *size,
2481 *size += arch_mod_section_prepend(mod, section);
2491 static void layout_sections(struct module *mod, struct load_info *info)
2519 s->sh_entsize = get_offset(mod, &mod->core_layout.size, s, i);
2524 mod->core_layout.size = debug_align(mod->core_layout.size);
2525 mod->core_layout.text_size = mod->core_layout.size;
2528 mod->core_layout.size = debug_align(mod->core_layout.size);
2529 mod->core_layout.ro_size = mod->core_layout.size;
2532 mod->core_layout.size = debug_align(mod->core_layout.size);
2533 mod->core_layout.ro_after_init_size = mod->core_layout.size;
2536 mod->core_layout.size = debug_align(mod->core_layout.size);
2552 s->sh_entsize = (get_offset(mod, &mod->init_layout.size, s, i)
2558 mod->init_layout.size = debug_align(mod->init_layout.size);
2559 mod->init_layout.text_size = mod->init_layout.size;
2562 mod->init_layout.size = debug_align(mod->init_layout.size);
2563 mod->init_layout.ro_size = mod->init_layout.size;
2570 mod->init_layout.ro_after_init_size = mod->init_layout.ro_size;
2573 mod->init_layout.size = debug_align(mod->init_layout.size);
2579 static void set_license(struct module *mod, const char *license)
2587 mod->name, license);
2588 add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
2643 static void setup_modinfo(struct module *mod, struct load_info *info)
2650 attr->setup(mod, get_modinfo(info, attr->attr.name));
2654 static void free_modinfo(struct module *mod)
2661 attr->free(mod);
2677 const struct module *mod)
2680 if (!mod)
2683 ks = lookup_exported_symbol(name, mod->syms, mod->syms + mod->num_syms);
2762 static void layout_symtab(struct module *mod, struct load_info *info)
2771 symsect->sh_entsize = get_offset(mod, &mod->init_layout.size, symsect,
2780 if (i == 0 || is_livepatch_module(mod) ||
2789 info->symoffs = ALIGN(mod->core_layout.size, symsect->sh_addralign ?: 1);
2790 info->stroffs = mod->core_layout.size = info->symoffs + ndst * sizeof(Elf_Sym);
2791 mod->core_layout.size += strtab_size;
2792 info->core_typeoffs = mod->core_layout.size;
2793 mod->core_layout.size += ndst * sizeof(char);
2794 mod->core_layout.size = debug_align(mod->core_layout.size);
2798 strsect->sh_entsize = get_offset(mod, &mod->init_layout.size, strsect,
2803 mod->init_layout.size = ALIGN(mod->init_layout.size,
2805 info->mod_kallsyms_init_off = mod->init_layout.size;
2806 mod->init_layout.size += sizeof(struct mod_kallsyms);
2807 info->init_typeoffs = mod->init_layout.size;
2808 mod->init_layout.size += nsrc * sizeof(char);
2809 mod->init_layout.size = debug_align(mod->init_layout.size);
2817 static void add_kallsyms(struct module *mod, const struct load_info *info)
2826 mod->kallsyms = mod->init_layout.base + info->mod_kallsyms_init_off;
2828 mod->kallsyms->symtab = (void *)symsec->sh_addr;
2829 mod->kallsyms->num_symtab = symsec->sh_size / sizeof(Elf_Sym);
2831 mod->kallsyms->strtab = (void *)info->sechdrs[info->index.str].sh_addr;
2832 mod->kallsyms->typetab = mod->init_layout.base + info->init_typeoffs;
2838 mod->core_kallsyms.symtab = dst = mod->core_layout.base + info->symoffs;
2839 mod->core_kallsyms.strtab = s = mod->core_layout.base + info->stroffs;
2840 mod->core_kallsyms.typetab = mod->core_layout.base + info->core_typeoffs;
2841 src = mod->kallsyms->symtab;
2842 for (ndst = i = 0; i < mod->kallsyms->num_symtab; i++) {
2843 mod->kallsyms->typetab[i] = elf_type(src + i, info);
2844 if (i == 0 || is_livepatch_module(mod) ||
2847 mod->core_kallsyms.typetab[ndst] =
2848 mod->kallsyms->typetab[i];
2850 dst[ndst++].st_name = s - mod->core_kallsyms.strtab;
2851 s += strlcpy(s, &mod->kallsyms->strtab[src[i].st_name],
2855 mod->core_kallsyms.num_symtab = ndst;
2858 static inline void layout_symtab(struct module *mod, struct load_info *info)
2862 static void add_kallsyms(struct module *mod, const struct load_info *info)
2867 static void dynamic_debug_setup(struct module *mod, struct _ddebug *debug, unsigned int num)
2871 ddebug_add_module(debug, num, mod->name);
2874 static void dynamic_debug_remove(struct module *mod, struct _ddebug *debug)
2877 ddebug_remove_module(mod->name);
2898 static void kmemleak_load_module(const struct module *mod,
2904 kmemleak_scan_area(mod, sizeof(struct module), GFP_KERNEL);
2918 static inline void kmemleak_load_module(const struct module *mod,
2930 const void *mod = info->hdr;
2938 memcmp(mod + info->len - markerlen, MODULE_SIG_STRING, markerlen) == 0) {
2941 err = mod_verify_sig(mod, info);
3114 static int check_modinfo_livepatch(struct module *mod, struct load_info *info)
3117 mod->klp = true;
3118 add_taint_module(mod, TAINT_LIVEPATCH, LOCKDEP_STILL_OK);
3120 mod->name);
3126 static int check_modinfo_livepatch(struct module *mod, struct load_info *info)
3130 mod->name);
3138 static void check_modinfo_retpoline(struct module *mod, struct load_info *info)
3144 mod->name);
3213 * Set info->mod to the temporary copy of the module in info->hdr. The final one
3242 info->index.mod = find_sec(info, ".gnu.linkonce.this_module");
3243 if (!info->index.mod) {
3248 /* This is temporary: point mod into copy of data. */
3249 info->mod = (void *)info->hdr + info->sechdrs[info->index.mod].sh_offset;
3253 * on-disk struct mod 'name' field.
3256 info->name = info->mod->name;
3268 static int check_modinfo(struct module *mod, struct load_info *info, int flags)
3278 err = try_to_force_load(mod, "bad vermagic");
3290 mod->name);
3291 add_taint_module(mod, TAINT_OOT_MODULE, LOCKDEP_STILL_OK);
3294 check_modinfo_retpoline(mod, info);
3297 add_taint_module(mod, TAINT_CRAP, LOCKDEP_STILL_OK);
3299 "is unknown, you have been warned.\n", mod->name);
3302 err = check_modinfo_livepatch(mod, info);
3307 set_license(mod, get_modinfo(info, "license"));
3312 static int find_module_sections(struct module *mod, struct load_info *info)
3314 mod->kp = section_objs(info, "__param",
3315 sizeof(*mod->kp), &mod->num_kp);
3316 mod->syms = section_objs(info, "__ksymtab",
3317 sizeof(*mod->syms), &mod->num_syms);
3318 mod->crcs = section_addr(info, "__kcrctab");
3319 mod->gpl_syms = section_objs(info, "__ksymtab_gpl",
3320 sizeof(*mod->gpl_syms),
3321 &mod->num_gpl_syms);
3322 mod->gpl_crcs = section_addr(info, "__kcrctab_gpl");
3323 mod->gpl_future_syms = section_objs(info,
3325 sizeof(*mod->gpl_future_syms),
3326 &mod->num_gpl_future_syms);
3327 mod->gpl_future_crcs = section_addr(info, "__kcrctab_gpl_future");
3330 mod->unused_syms = section_objs(info, "__ksymtab_unused",
3331 sizeof(*mod->unused_syms),
3332 &mod->num_unused_syms);
3333 mod->unused_crcs = section_addr(info, "__kcrctab_unused");
3334 mod->unused_gpl_syms = section_objs(info, "__ksymtab_unused_gpl",
3335 sizeof(*mod->unused_gpl_syms),
3336 &mod->num_unused_gpl_syms);
3337 mod->unused_gpl_crcs = section_addr(info, "__kcrctab_unused_gpl");
3340 mod->ctors = section_objs(info, ".ctors",
3341 sizeof(*mod->ctors), &mod->num_ctors);
3342 if (!mod->ctors)
3343 mod->ctors = section_objs(info, ".init_array",
3344 sizeof(*mod->ctors), &mod->num_ctors);
3351 mod->name);
3356 mod->noinstr_text_start = section_objs(info, ".noinstr.text", 1,
3357 &mod->noinstr_text_size);
3360 mod->tracepoints_ptrs = section_objs(info, "__tracepoints_ptrs",
3361 sizeof(*mod->tracepoints_ptrs),
3362 &mod->num_tracepoints);
3365 mod->srcu_struct_ptrs = section_objs(info, "___srcu_struct_ptrs",
3366 sizeof(*mod->srcu_struct_ptrs),
3367 &mod->num_srcu_structs);
3370 mod->bpf_raw_events = section_objs(info, "__bpf_raw_tp_map",
3371 sizeof(*mod->bpf_raw_events),
3372 &mod->num_bpf_raw_events);
3375 mod->jump_entries = section_objs(info, "__jump_table",
3376 sizeof(*mod->jump_entries),
3377 &mod->num_jump_entries);
3380 mod->trace_events = section_objs(info, "_ftrace_events",
3381 sizeof(*mod->trace_events),
3382 &mod->num_trace_events);
3383 mod->trace_evals = section_objs(info, "_ftrace_eval_map",
3384 sizeof(*mod->trace_evals),
3385 &mod->num_trace_evals);
3388 mod->trace_bprintk_fmt_start = section_objs(info, "__trace_printk_fmt",
3389 sizeof(*mod->trace_bprintk_fmt_start),
3390 &mod->num_trace_bprintk_fmt);
3394 mod->ftrace_callsites = section_objs(info, FTRACE_CALLSITE_SECTION,
3395 sizeof(*mod->ftrace_callsites),
3396 &mod->num_ftrace_callsites);
3399 mod->ei_funcs = section_objs(info, "_error_injection_whitelist",
3400 sizeof(*mod->ei_funcs),
3401 &mod->num_ei_funcs);
3404 mod->kprobes_text_start = section_objs(info, ".kprobes.text", 1,
3405 &mod->kprobes_text_size);
3406 mod->kprobe_blacklist = section_objs(info, "_kprobe_blacklist",
3408 &mod->num_kprobe_blacklist);
3411 mod->static_call_sites = section_objs(info, ".static_call_sites",
3412 sizeof(*mod->static_call_sites),
3413 &mod->num_static_call_sites);
3415 mod->extable = section_objs(info, "__ex_table",
3416 sizeof(*mod->extable), &mod->num_exentries);
3419 pr_warn("%s: Ignoring obsolete parameters\n", mod->name);
3427 static int move_module(struct module *mod, struct load_info *info)
3433 ptr = module_alloc(mod->core_layout.size);
3443 memset(ptr, 0, mod->core_layout.size);
3444 mod->core_layout.base = ptr;
3446 if (mod->init_layout.size) {
3447 ptr = module_alloc(mod->init_layout.size);
3456 module_memfree(mod->core_layout.base);
3459 memset(ptr, 0, mod->init_layout.size);
3460 mod->init_layout.base = ptr;
3462 mod->init_layout.base = NULL;
3474 dest = mod->init_layout.base
3477 dest = mod->core_layout.base + shdr->sh_entsize;
3490 static int check_module_license_and_versions(struct module *mod)
3499 if (strcmp(mod->name, "ndiswrapper") == 0)
3503 if (strcmp(mod->name, "driverloader") == 0)
3504 add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
3508 if (strcmp(mod->name, "lve") == 0)
3509 add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
3513 pr_warn("%s: module license taints kernel.\n", mod->name);
3516 if ((mod->num_syms && !mod->crcs)
3517 || (mod->num_gpl_syms && !mod->gpl_crcs)
3518 || (mod->num_gpl_future_syms && !mod->gpl_future_crcs)
3520 || (mod->num_unused_syms && !mod->unused_crcs)
3521 || (mod->num_unused_gpl_syms && !mod->unused_gpl_crcs)
3524 return try_to_force_load(mod,
3531 static void flush_module_icache(const struct module *mod)
3538 if (mod->init_layout.base)
3539 flush_icache_range((unsigned long)mod->init_layout.base,
3540 (unsigned long)mod->init_layout.base
3541 + mod->init_layout.size);
3542 flush_icache_range((unsigned long)mod->core_layout.base,
3543 (unsigned long)mod->core_layout.base + mod->core_layout.size);
3549 struct module *mod)
3577 struct module *mod;
3581 err = check_modinfo(info->mod, info, flags);
3587 info->secstrings, info->mod);
3592 info->secstrings, info->mod);
3620 layout_sections(info->mod, info);
3621 layout_symtab(info->mod, info);
3624 err = move_module(info->mod, info);
3629 mod = (void *)info->sechdrs[info->index.mod].sh_addr;
3630 kmemleak_load_module(mod, info);
3631 return mod;
3634 /* mod is no longer valid after this! */
3635 static void module_deallocate(struct module *mod, struct load_info *info)
3637 percpu_modfree(mod);
3638 module_arch_freeing_init(mod);
3639 module_memfree(mod->init_layout.base);
3640 module_memfree(mod->core_layout.base);
3650 static int post_relocation(struct module *mod, const struct load_info *info)
3653 sort_extable(mod->extable, mod->extable + mod->num_exentries);
3656 percpu_modcopy(mod, (void *)info->sechdrs[info->index.pcpu].sh_addr,
3660 add_kallsyms(mod, info);
3663 return module_finalize(info->hdr, info->sechdrs, mod);
3669 struct module *mod;
3679 mod = find_module_all(name, strlen(name), true);
3680 ret = !mod || mod->state == MODULE_STATE_LIVE
3681 || mod->state == MODULE_STATE_GOING;
3688 static void do_mod_ctors(struct module *mod)
3693 for (i = 0; i < mod->num_ctors; i++)
3694 mod->ctors[i]();
3726 static noinline int do_init_module(struct module *mod)
3736 freeinit->module_init = mod->init_layout.base;
3738 do_mod_ctors(mod);
3740 if (mod->init != NULL)
3741 ret = do_one_initcall(mod->init);
3749 __func__, mod->name, ret, __func__);
3754 mod->state = MODULE_STATE_LIVE;
3756 MODULE_STATE_LIVE, mod);
3759 kobject_uevent(&mod->mkobj.kobj, KOBJ_ADD);
3769 if (!mod->async_probe_requested)
3772 ftrace_free_mem(mod, mod->init_layout.base, mod->init_layout.base +
3773 mod->init_layout.size);
3776 module_put(mod);
3777 trim_init_extable(mod);
3780 rcu_assign_pointer(mod->kallsyms, &mod->core_kallsyms);
3782 module_enable_ro(mod, true);
3783 mod_tree_remove_init(mod);
3784 module_arch_freeing_init(mod);
3785 mod->init_layout.base = NULL;
3786 mod->init_layout.size = 0;
3787 mod->init_layout.ro_size = 0;
3788 mod->init_layout.ro_after_init_size = 0;
3789 mod->init_layout.text_size = 0;
3815 mod->state = MODULE_STATE_GOING;
3817 module_put(mod);
3819 MODULE_STATE_GOING, mod);
3820 klp_module_going(mod);
3821 ftrace_release_mod(mod);
3822 free_module(mod);
3840 static int add_unformed_module(struct module *mod)
3845 mod->state = MODULE_STATE_UNFORMED;
3848 old = find_module_all(mod->name, strlen(mod->name), true);
3855 finished_loading(mod->name));
3861 old = find_module_all(mod->name, strlen(mod->name),
3878 mod_update_bounds(mod);
3879 list_add_rcu(&mod->list, &modules);
3880 mod_tree_insert(mod);
3889 static int complete_formation(struct module *mod, struct load_info *info)
3896 err = verify_exported_symbols(mod);
3901 module_bug_finalize(info->hdr, info->sechdrs, mod);
3903 module_enable_ro(mod, false);
3904 module_enable_nx(mod);
3905 module_enable_x(mod);
3909 mod->state = MODULE_STATE_COMING;
3919 static int prepare_coming_module(struct module *mod)
3923 ftrace_module_enable(mod);
3924 err = klp_module_coming(mod);
3929 MODULE_STATE_COMING, MODULE_STATE_GOING, mod);
3932 klp_module_going(mod);
3940 struct module *mod = arg;
3944 mod->async_probe_requested = true;
3955 static void cfi_init(struct module *mod);
3962 struct module *mod;
4015 if (!check_modstruct_version(info, info->mod)) {
4021 mod = layout_and_allocate(info, flags);
4022 if (IS_ERR(mod)) {
4023 err = PTR_ERR(mod);
4027 audit_log_kern_module(mod->name);
4030 err = add_unformed_module(mod);
4035 mod->sig_ok = info->sig_ok;
4036 if (!mod->sig_ok) {
4039 "kernel\n", mod->name);
4040 add_taint_module(mod, TAINT_UNSIGNED_MODULE, LOCKDEP_STILL_OK);
4045 err = percpu_modalloc(mod, info);
4050 err = module_unload_init(mod);
4054 init_param_lock(mod);
4058 err = find_module_sections(mod, info);
4062 err = check_module_license_and_versions(mod);
4067 setup_modinfo(mod, info);
4070 err = simplify_symbols(mod, info);
4074 err = apply_relocations(mod, info);
4078 err = post_relocation(mod, info);
4082 flush_module_icache(mod);
4085 cfi_init(mod);
4088 mod->args = strndup_user(uargs, ~0UL >> 1);
4089 if (IS_ERR(mod->args)) {
4090 err = PTR_ERR(mod->args);
4094 dynamic_debug_setup(mod, info->debug, info->num_debug);
4097 ftrace_module_init(mod);
4100 err = complete_formation(mod, info);
4104 err = prepare_coming_module(mod);
4109 after_dashes = parse_args(mod->name, mod->args, mod->kp, mod->num_kp,
4110 -32768, 32767, mod,
4117 mod->name, after_dashes);
4121 err = mod_sysfs_setup(mod, info, mod->kp, mod->num_kp);
4125 if (is_livepatch_module(mod)) {
4126 err = copy_module_elf(mod, info);
4135 trace_module_load(mod);
4137 return do_init_module(mod);
4140 mod_sysfs_teardown(mod);
4142 mod->state = MODULE_STATE_GOING;
4143 destroy_params(mod->kp, mod->num_kp);
4145 MODULE_STATE_GOING, mod);
4146 klp_module_going(mod);
4148 mod->state = MODULE_STATE_GOING;
4151 module_bug_cleanup(mod);
4155 ftrace_release_mod(mod);
4156 dynamic_debug_remove(mod, info->debug);
4158 kfree(mod->args);
4160 cfi_cleanup(mod);
4161 module_arch_cleanup(mod);
4163 free_modinfo(mod);
4165 module_unload_free(mod);
4169 list_del_rcu(&mod->list);
4170 mod_tree_remove(mod);
4172 /* Wait for RCU-sched synchronizing before releasing mod->list. */
4177 lockdep_free_key_range(mod->core_layout.base, mod->core_layout.size);
4179 module_deallocate(mod, info);
4258 static const char *find_kallsyms_symbol(struct module *mod,
4265 struct mod_kallsyms *kallsyms = rcu_dereference_sched(mod->kallsyms);
4268 if (within_module_init(addr, mod))
4269 nextval = (unsigned long)mod->init_layout.base+mod->init_layout.text_size;
4271 nextval = (unsigned long)mod->core_layout.base+mod->core_layout.text_size;
4309 void * __weak dereference_module_function_descriptor(struct module *mod,
4324 struct module *mod;
4327 mod = __module_address(addr);
4328 if (mod) {
4330 *modname = mod->name;
4332 ret = find_kallsyms_symbol(mod, addr, size, offset);
4346 struct module *mod;
4349 list_for_each_entry_rcu(mod, &modules, list) {
4350 if (mod->state == MODULE_STATE_UNFORMED)
4352 if (within_module(addr, mod)) {
4355 sym = find_kallsyms_symbol(mod, addr, NULL, NULL);
4372 struct module *mod;
4375 list_for_each_entry_rcu(mod, &modules, list) {
4376 if (mod->state == MODULE_STATE_UNFORMED)
4378 if (within_module(addr, mod)) {
4381 sym = find_kallsyms_symbol(mod, addr, size, offset);
4385 strlcpy(modname, mod->name, MODULE_NAME_LEN);
4400 struct module *mod;
4403 list_for_each_entry_rcu(mod, &modules, list) {
4406 if (mod->state == MODULE_STATE_UNFORMED)
4408 kallsyms = rcu_dereference_sched(mod->kallsyms);
4415 strlcpy(module_name, mod->name, MODULE_NAME_LEN);
4416 *exported = is_exported(name, *value, mod);
4427 static unsigned long find_kallsyms_symbol_value(struct module *mod, const char *name)
4430 struct mod_kallsyms *kallsyms = rcu_dereference_sched(mod->kallsyms);
4445 struct module *mod;
4452 if ((mod = find_module_all(name, colon - name, false)) != NULL)
4453 ret = find_kallsyms_symbol_value(mod, colon+1);
4455 list_for_each_entry_rcu(mod, &modules, list) {
4456 if (mod->state == MODULE_STATE_UNFORMED)
4458 if ((ret = find_kallsyms_symbol_value(mod, name)) != 0)
4470 struct module *mod;
4476 list_for_each_entry(mod, &modules, list) {
4478 struct mod_kallsyms *kallsyms = mod->kallsyms;
4480 if (mod->state == MODULE_STATE_UNFORMED)
4489 mod, kallsyms_symbol_value(sym));
4498 static void cfi_init(struct module *mod)
4505 mod->cfi_check = (cfi_check_fn)
4506 find_kallsyms_symbol_value(mod, "__cfi_check");
4508 find_kallsyms_symbol_value(mod, "__cfi_jt_init_module");
4510 find_kallsyms_symbol_value(mod, "__cfi_jt_cleanup_module");
4515 mod->init = *init;
4518 mod->exit = *exit;
4521 cfi_module_add(mod, module_addr_min);
4525 static void cfi_cleanup(struct module *mod)
4528 cfi_module_remove(mod, module_addr_min);
4536 static char *module_flags(struct module *mod, char *buf)
4540 BUG_ON(mod->state == MODULE_STATE_UNFORMED);
4541 if (mod->taints ||
4542 mod->state == MODULE_STATE_GOING ||
4543 mod->state == MODULE_STATE_COMING) {
4545 bx += module_flags_taint(mod, buf + bx);
4547 if (mod->state == MODULE_STATE_GOING)
4550 if (mod->state == MODULE_STATE_COMING)
4579 struct module *mod = list_entry(p, struct module, list);
4584 if (mod->state == MODULE_STATE_UNFORMED)
4588 mod->name, mod->init_layout.size + mod->core_layout.size);
4589 print_unload_info(m, mod);
4593 mod->state == MODULE_STATE_GOING ? "Unloading" :
4594 mod->state == MODULE_STATE_COMING ? "Loading" :
4597 value = m->private ? NULL : mod->core_layout.base;
4601 if (mod->taints)
4602 seq_printf(m, " %s", module_flags(mod, buf));
4659 struct module *mod;
4662 mod = __module_address(addr);
4663 if (!mod)
4666 if (!mod->num_exentries)
4669 e = search_extable(mod->extable,
4670 mod->num_exentries,
4709 struct module *mod;
4716 mod = mod_find(addr);
4717 if (mod) {
4718 BUG_ON(!within_module(addr, mod));
4719 if (mod->state == MODULE_STATE_UNFORMED)
4720 mod = NULL;
4722 return mod;
4753 struct module *mod = __module_address(addr);
4754 if (mod) {
4756 if (!within(addr, mod->init_layout.base, mod->init_layout.text_size)
4757 && !within(addr, mod->core_layout.base, mod->core_layout.text_size))
4758 mod = NULL;
4760 return mod;
4766 struct module *mod;
4772 list_for_each_entry_rcu(mod, &modules, list) {
4773 if (mod->state == MODULE_STATE_UNFORMED)
4775 pr_cont(" %s%s", mod->name, module_flags(mod, buf));
4786 void module_layout(struct module *mod,