[PATCH v3 bpf-next 5/7] btf: Relocate and expose module inline BTF

@@ -6912,15 +6912,137 @@ __u32 btf_relocate_id(const struct btf *btf, __u32 id)
 
 #ifdef CONFIG_DEBUG_INFO_BTF_MODULES
 
+static struct btf *btf_parse_split(struct btf_verifier_env *env,
+				   const char *name, const void *data,
+				   unsigned int data_size, struct btf *base_btf)
+{
+	struct btf *btf;
+	int err;
+
+	btf = kzalloc_obj(*btf, GFP_KERNEL | __GFP_NOWARN);
+	if (!btf)
+		return ERR_PTR(-ENOMEM);
+	env->btf = btf;
+
+	btf_set_base_btf(btf, base_btf);
+	btf->kernel_btf = true;
+	btf->named_start_id = 0;
+	strscpy(btf->name, name);
+
+	btf->data = kvmemdup(data, data_size, GFP_KERNEL | __GFP_NOWARN);
+	if (!btf->data) {
+		err = -ENOMEM;
+		goto errout;
+	}
+	btf->data_size = data_size;
+
+	err = btf_parse_hdr(env);
+	if (err)
+		goto errout;
+
+	btf->nohdr_data = btf->data + btf->hdr.hdr_len;
+
+	err = btf_parse_str_sec(env);
+	if (err)
+		goto errout;
+
+	err = btf_check_all_metas(env);
+	if (err)
+		goto errout;
+
+	err = btf_check_modifier_chain_length(env, btf, btf_nr_types(base_btf));
+	if (err)
+		goto errout;
+
+	return btf;
+
+errout:
+	btf_free(btf);
+	return ERR_PTR(err);
+}
+
+static int btf_rebase_inline(struct btf *inline_btf,
+			     const struct btf *module_btf,
+			     const u32 *module_id_map,
+			     const u32 *module_str_map,
+			     u32 old_module_type_cnt)
+{
+	u32 old_start_id = inline_btf->start_id;
+	u32 old_start_str_off = inline_btf->start_str_off;
+	u32 old_module_start_str_off = old_start_str_off - module_btf->hdr.str_len;
+	u32 new_start_id = btf_nr_types(module_btf);
+	u32 new_start_str_off = module_btf->start_str_off + module_btf->hdr.str_len;
+	s64 id_delta = (s64)new_start_id - old_start_id;
+	s64 str_delta = (s64)new_start_str_off - old_start_str_off;
+	u32 i;
+
+	/*
+	 * The inline BTF was parsed relative to the original module BTF. Its
+	 * base IDs must therefore use the map generated when that BTF was
+	 * relocated, while IDs for inline-local types only move by the change
+	 * in the module BTF's starting ID.
+	 */
+	for (i = 0; i < inline_btf->nr_types; i++) {
+		struct btf_field_iter it;
+		struct btf_type *t = inline_btf->types[i];
+		u32 *id, *str_off;
+		int err;
+
+		err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS);
+		if (err)
+			return err;
+		while ((id = btf_field_iter_next(&it))) {
+			if (!*id)
+				continue;
+			if (*id < old_module_type_cnt) {
+				if (module_id_map)
+					*id = module_id_map[*id];
+			} else if (*id >= old_start_id) {
+				*id += id_delta;
+			} else {
+				return -EINVAL;
+			}
+		}
+
+		err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS);
+		if (err)
+			return err;
+		while ((str_off = btf_field_iter_next(&it))) {
+			if (!*str_off)
+				continue;
+			if (*str_off < old_module_start_str_off) {
+				/* Vmlinux strings retain their offsets for in-tree modules. */
+				if (!module_id_map)
+					continue;
+				if (!module_str_map || !module_str_map[*str_off])
+					return -EINVAL;
+				*str_off = module_str_map[*str_off];
+				continue;
+			}
+			*str_off += str_delta;
+		}
+	}
+
+	btf_set_base_btf(inline_btf, module_btf);
+	btf_check_sorted(inline_btf);
+	return 0;
+}
+
 static struct btf *btf_parse_module(const char *module_name, const void *data,
 				    unsigned int data_size, void *base_data,
-				    unsigned int base_data_size)
+				    unsigned int base_data_size,
+				    const void *inline_data,
+				    unsigned int inline_data_size,
+				    void **relocated_inline_data)
 {
-	struct btf *btf = NULL, *vmlinux_btf, *base_btf = NULL;
+	struct btf *btf = NULL, *inline_btf = NULL, *vmlinux_btf, *base_btf = NULL;
 	struct btf_verifier_env *env = NULL;
 	struct bpf_verifier_log *log;
+	u32 old_module_type_cnt;
+	u32 *module_str_map = NULL;
 	int err = 0;
 
+	*relocated_inline_data = NULL;
 	vmlinux_btf = bpf_get_btf_vmlinux();
 	if (IS_ERR(vmlinux_btf))
 		return vmlinux_btf;

@@ -6944,67 +7066,66 @@ static struct btf *btf_parse_module(const char *module_name, const void *data,
 		base_btf = vmlinux_btf;
 	}
 
-	btf = kzalloc_obj(*btf, GFP_KERNEL | __GFP_NOWARN);
-	if (!btf) {
-		err = -ENOMEM;
+	btf = btf_parse_split(env, module_name, data, data_size, base_btf);
+	if (IS_ERR(btf)) {
+		err = PTR_ERR(btf);
+		btf = NULL;
 		goto errout;
 	}
-	env->btf = btf;
-
-	btf->base_btf = base_btf;
-	btf->start_id = base_btf->nr_types;
-	btf->start_str_off = base_btf->hdr.str_len;
-	btf->kernel_btf = true;
-	btf->named_start_id = 0;
-	strscpy(btf->name, module_name);
 
-	btf->data = kvmemdup(data, data_size, GFP_KERNEL | __GFP_NOWARN);
-	if (!btf->data) {
-		err = -ENOMEM;
-		goto errout;
+	if (inline_data_size) {
+		inline_btf = btf_parse_split(env, module_name, inline_data,
+					     inline_data_size, btf);
+		if (IS_ERR(inline_btf)) {
+			pr_warn("failed to validate module [%s] inline BTF: %ld\n",
+				module_name, PTR_ERR(inline_btf));
+			inline_btf = NULL;
+		}
 	}
-	btf->data_size = data_size;
-
-	err = btf_parse_hdr(env);
-	if (err)
-		goto errout;
-
-	btf->nohdr_data = btf->data + btf->hdr.hdr_len;
-
-	err = btf_parse_str_sec(env);
-	if (err)
-		goto errout;
-
-	err = btf_check_all_metas(env);
-	if (err)
-		goto errout;
-
-	err = btf_check_modifier_chain_length(env, btf, btf_nr_types(base_btf));
-	if (err)
-		goto errout;
 
+	old_module_type_cnt = btf_nr_types(btf);
 	if (base_btf != vmlinux_btf) {
-		err = btf_relocate(btf, vmlinux_btf, &btf->base_id_map, NULL);
+		err = btf_relocate(btf, vmlinux_btf, &btf->base_id_map,
+				   &module_str_map);
 		if (err)
 			goto errout;
 		btf_free(base_btf);
 		base_btf = vmlinux_btf;
 	}
 
-	btf_verifier_env_free(env);
+	if (inline_btf) {
+		err = btf_rebase_inline(inline_btf, btf, btf->base_id_map,
+					module_str_map, old_module_type_cnt);
+		if (err) {
+			pr_warn("failed to relocate module [%s] inline BTF: %d\n",
+				module_name, err);
+			btf_free(inline_btf);
+		} else {
+			*relocated_inline_data = inline_btf->data;
+			inline_btf->data = NULL;
+			btf_free(inline_btf);
+		}
+	}
+
+	/*
+	 * With a distilled base, btf_relocate() replaces the base BTF and
+	 * rewrites string offsets. Check ordering only after that final BTF
+	 * view has been established, so named_start_id describes the BTF used
+	 * by name lookups.
+	 */
 	btf_check_sorted(btf);
+	btf_verifier_env_free(env);
+	kvfree(module_str_map);
 	refcount_set(&btf->refcnt, 1);
 	return btf;
 
 errout:
+	kvfree(module_str_map);
 	btf_verifier_env_free(env);
+	btf_free(inline_btf);
 	if (!IS_ERR(base_btf) && base_btf != vmlinux_btf)
 		btf_free(base_btf);
-	if (btf) {
-		kvfree(btf->data);
-		kvfree(btf->types);
-		kfree(btf);
-	}
+	btf_free(btf);
 	return ERR_PTR(err);
 }
 

@@ -8998,6 +9119,8 @@ struct btf_module {
 	struct module *module;
 	struct btf *btf;
 	struct bin_attribute *sysfs_attr;
+	struct bin_attribute *sysfs_inline_attr;
+	void *btf_inline_data;
 	int flags;
 };
 

@@ -9072,7 +9195,8 @@ static int btf_module_handle_vmlinux_inline(const struct module *mod)
 {
 	void *data = NULL;
 
-	if (!vmlinux_inline_link || strcmp(mod->name, vmlinux_inline_link->module_name))
+	if (!vmlinux_inline_link ||
+	    strcmp(mod->name, vmlinux_inline_link->module_name))
 		return -ENOENT;
 
 	data = vmalloc_user(mod->btf_inline_data_size);

@@ -9107,6 +9231,8 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op,
 	struct module *mod = module;
 	struct bin_attribute *attr;
 	struct btf *btf;
+	void *inline_data = NULL, *relocated_inline_data = NULL;
+	unsigned int inline_data_size = 0;
 	int err = 0;
 
 	if (op != MODULE_STATE_COMING && op != MODULE_STATE_LIVE &&

@@ -9120,6 +9246,12 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op,
 			err = 0;
 			break;
 		}
+#if IS_ENABLED(CONFIG_DEBUG_INFO_BTF_INLINE)
+		if (mod->btf_inline_data_size) {
+			inline_data = mod->btf_inline_data;
+			inline_data_size = mod->btf_inline_data_size;
+		}
+#endif
 		if (!mod->btf_data_size)
 			break;
 		btf_mod = kzalloc_obj(*btf_mod);

@@ -9128,7 +9260,9 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op,
 			goto out;
 		}
 		btf = btf_parse_module(mod->name, mod->btf_data, mod->btf_data_size,
-				       mod->btf_base_data, mod->btf_base_data_size);
+				       mod->btf_base_data, mod->btf_base_data_size,
+				       inline_data, inline_data_size,
+				       &relocated_inline_data);
 		if (IS_ERR(btf)) {
 			kfree(btf_mod);
 			if (!IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH)) {

@@ -9143,6 +9277,7 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op,
 		err = btf_alloc_id(btf);
 		if (err) {
 			btf_free(btf);
+			kvfree(relocated_inline_data);
 			kfree(btf_mod);
 			goto out;
 		}

@@ -9151,6 +9286,7 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op,
 		mutex_lock(&btf_module_mutex);
 		btf_mod->module = module;
 		btf_mod->btf = btf;
+		btf_mod->btf_inline_data = relocated_inline_data;
 		list_add(&btf_mod->list, &btf_modules);
 		mutex_unlock(&btf_module_mutex);
 

@@ -9160,6 +9296,21 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op,
 			goto out;
 		}
 		btf_mod->sysfs_attr = attr;
+
+		if (relocated_inline_data) {
+			char name[MODULE_NAME_LEN + sizeof(".inline")];
+
+			snprintf(name, sizeof(name), "%s.inline", mod->name);
+			attr = sysfs_btf_add(name, relocated_inline_data,
+					     inline_data_size, false, NULL);
+			if (IS_ERR(attr)) {
+				kvfree(relocated_inline_data);
+				btf_mod->btf_inline_data = NULL;
+				err = 0;
+				break;
+			}
+			btf_mod->sysfs_inline_attr = attr;
+		}
 		break;
 	case MODULE_STATE_LIVE:
 		if (!mod->btf_data_size)

@@ -9192,6 +9343,9 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op,
 			list_del(&btf_mod->list);
 			if (btf_mod->sysfs_attr)
 				sysfs_btf_remove(btf_mod->sysfs_attr);
+			if (btf_mod->sysfs_inline_attr)
+				sysfs_btf_remove(btf_mod->sysfs_inline_attr);
+			kvfree(btf_mod->btf_inline_data);
 			purge_cand_cache(btf_mod->btf);
 			btf_put(btf_mod->btf);
 			kfree(btf_mod);

@@ -68,10 +68,7 @@ esac
 
 gen_btf_data()
 {
-	inline=""
-	if [ -n "${BTF_INLINE}" ] && [ -z "${BTF_BASE}" ]; then
-		inline="--inline"
-	fi
+	inline=${BTF_INLINE:+--inline}
 	btf1="${ELF_FILE}.BTF.1"
 	${PAHOLE} -J ${PAHOLE_FLAGS}			\
 		${BTF_BASE:+--btf_base ${BTF_BASE}}	\

@@ -130,7 +127,7 @@ embed_btf_data()
 	if [ -f "${btf_base}" ]; then
 		${OBJCOPY} --add-section .BTF.base=${btf_base} ${ELF_FILE}
 	fi
-	btf_inline=""
+	btf_inline=${ELF_FILE}.BTF.inline
 	case "${ELF_FILE}" in
 	*/btf_vmlinux_inline.ko)
 		btf_inline=${BTF_BASE}.BTF.inline