diff --git a/include/linux/bpf-cgroup-defs.h b/include/linux/bpf-cgroup-defs.h
index 695d1224a71b..6c661b4df9fa 100644
--- a/include/linux/bpf-cgroup-defs.h
+++ b/include/linux/bpf-cgroup-defs.h
@@ -10,6 +10,8 @@
struct bpf_prog_array;
+#define CGROUP_LSM_NUM 211 /* will be addressed in the next patch */
+
enum cgroup_bpf_attach_type {
CGROUP_BPF_ATTACH_TYPE_INVALID = -1,
CGROUP_INET_INGRESS = 0,@@ -35,6 +37,10 @@ enum cgroup_bpf_attach_type {
CGROUP_INET4_GETSOCKNAME,
CGROUP_INET6_GETSOCKNAME,
CGROUP_INET_SOCK_RELEASE,
+#ifdef CONFIG_BPF_LSM
+ CGROUP_LSM_START,
+ CGROUP_LSM_END = CGROUP_LSM_START + CGROUP_LSM_NUM - 1,
+#endif
MAX_CGROUP_BPF_ATTACH_TYPE
};
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 487aba40ce52..17bbe2f7b2be 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -807,6 +807,9 @@ static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func(
#ifdef CONFIG_BPF_JIT
int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
+int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
+ struct bpf_attach_target_info *tgt_info);
+void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog);
struct bpf_trampoline *bpf_trampoline_get(u64 key,
struct bpf_attach_target_info *tgt_info);
void bpf_trampoline_put(struct bpf_trampoline *tr);
@@ -865,6 +868,14 @@ static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog,
{
return -ENOTSUPP;
}
+static inline int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
+ struct bpf_attach_target_info *tgt_info)
+{
+ return -EOPNOTSUPP;
+}
+static inline void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog)
+{
+}
static inline struct bpf_trampoline *bpf_trampoline_get(u64 key,
struct bpf_attach_target_info *tgt_info)
{@@ -980,6 +991,7 @@ struct bpf_prog_aux {
u64 load_time; /* ns since boottime */
u32 verified_insns;
struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
+ int cgroup_atype; /* enum cgroup_bpf_attach_type */
char name[BPF_OBJ_NAME_LEN];
#ifdef CONFIG_SECURITY
void *security;@@ -2383,6 +2395,7 @@ void *bpf_arch_text_copy(void *dst, void *src, size_t len);
struct btf_id_set;
bool btf_id_set_contains(const struct btf_id_set *set, u32 id);
+int btf_id_set_index(const struct btf_id_set *set, u32 id);
#define MAX_BPRINTF_VARARGS 12
diff --git a/include/linux/bpf_lsm.h b/include/linux/bpf_lsm.h
index 479c101546ad..7f0e59f5f9be 100644
--- a/include/linux/bpf_lsm.h
+++ b/include/linux/bpf_lsm.h
@@ -42,6 +42,9 @@ extern const struct bpf_func_proto bpf_inode_storage_get_proto;
extern const struct bpf_func_proto bpf_inode_storage_delete_proto;
void bpf_inode_storage_free(struct inode *inode);
+int bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog, bpf_func_t *bpf_func);
+int bpf_lsm_hook_idx(u32 btf_id);
+
#else /* !CONFIG_BPF_LSM */
static inline bool bpf_lsm_is_sleepable_hook(u32 btf_id)
@@ -65,6 +68,17 @@ static inline void bpf_inode_storage_free(struct inode *inode)
{
}
+static inline int bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog,
+ bpf_func_t *bpf_func)
+{
+ return -ENOENT;
+}
+
+static inline int bpf_lsm_hook_idx(u32 btf_id)
+{
+ return -EINVAL;
+}
+
#endif /* CONFIG_BPF_LSM */
#endif /* _LINUX_BPF_LSM_H */diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index d14b10b85e51..bbe48a2dd852 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -998,6 +998,7 @@ enum bpf_attach_type {
BPF_SK_REUSEPORT_SELECT_OR_MIGRATE,
BPF_PERF_EVENT,
BPF_TRACE_KPROBE_MULTI,
+ BPF_LSM_CGROUP,
__MAX_BPF_ATTACH_TYPE
};
diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c
index 064eccba641d..eca258ba71d8 100644
--- a/kernel/bpf/bpf_lsm.c
+++ b/kernel/bpf/bpf_lsm.c
@@ -35,6 +35,98 @@ BTF_SET_START(bpf_lsm_hooks)
#undef LSM_HOOK
BTF_SET_END(bpf_lsm_hooks)
+static unsigned int __cgroup_bpf_run_lsm_socket(const void *ctx,
+ const struct bpf_insn *insn)
+{
+ const struct bpf_prog *prog;
+ struct socket *sock;
+ struct cgroup *cgrp;
+ struct sock *sk;
+ int ret = 0;
+ u64 *regs;
+
+ regs = (u64 *)ctx;
+ sock = (void *)(unsigned long)regs[BPF_REG_0];
+ /*prog = container_of(insn, struct bpf_prog, insnsi);*/
+ prog = (const struct bpf_prog *)((void *)insn - offsetof(struct bpf_prog, insnsi));
+
+ if (unlikely(!sock))
+ return 0;
+
+ sk = sock->sk;
+ if (unlikely(!sk))
+ return 0;
+
+ cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
+ if (likely(cgrp))
+ ret = BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[prog->aux->cgroup_atype],
+ ctx, bpf_prog_run, 0);
+ return ret;
+}
+
+static unsigned int __cgroup_bpf_run_lsm_current(const void *ctx,
+ const struct bpf_insn *insn)
+{
+ const struct bpf_prog *prog;
+ struct cgroup *cgrp;
+ int ret = 0;
+
+ if (unlikely(!current))
+ return 0;
+
+ /*prog = container_of(insn, struct bpf_prog, insnsi);*/
+ prog = (const struct bpf_prog *)((void *)insn - offsetof(struct bpf_prog, insnsi));
+
+ rcu_read_lock();
+ cgrp = task_dfl_cgroup(current);
+ if (likely(cgrp))
+ ret = BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[prog->aux->cgroup_atype],
+ ctx, bpf_prog_run, 0);
+ rcu_read_unlock();
+ return ret;
+}
+
+int bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog,
+ bpf_func_t *bpf_func)
+{
+ const struct btf_type *first_arg_type;
+ const struct btf_type *sock_type;
+ const struct btf *btf_vmlinux;
+ const struct btf_param *args;
+ s32 type_id;
+
+ if (!prog->aux->attach_func_proto ||
+ !btf_type_is_func_proto(prog->aux->attach_func_proto))
+ return -EINVAL;
+
+ if (btf_type_vlen(prog->aux->attach_func_proto) < 1)
+ return -EINVAL;
+
+ args = (const struct btf_param *)(prog->aux->attach_func_proto + 1);
+
+ btf_vmlinux = bpf_get_btf_vmlinux();
+ if (!btf_vmlinux)
+ return -EINVAL;
+
+ type_id = btf_find_by_name_kind(btf_vmlinux, "socket", BTF_KIND_STRUCT);
+ if (type_id < 0)
+ return -EINVAL;
+ sock_type = btf_type_by_id(btf_vmlinux, type_id);
+
+ first_arg_type = btf_type_resolve_ptr(btf_vmlinux, args[0].type, NULL);
+ if (first_arg_type == sock_type)
+ *bpf_func = __cgroup_bpf_run_lsm_socket;
+ else
+ *bpf_func = __cgroup_bpf_run_lsm_current;
+
+ return 0;
+}
+
+int bpf_lsm_hook_idx(u32 btf_id)
+{
+ return btf_id_set_index(&bpf_lsm_hooks, btf_id);
+}
+
int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog,
const struct bpf_prog *prog)
{diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index 0918a39279f6..4199de31f49c 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -4971,6 +4971,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
if (arg == nr_args) {
switch (prog->expected_attach_type) {
+ case BPF_LSM_CGROUP:
case BPF_LSM_MAC:
case BPF_TRACE_FEXIT:
/* When LSM programs are attached to void LSM hooks@@ -6396,6 +6397,16 @@ static int btf_id_cmp_func(const void *a, const void *b)
return *pa - *pb;
}
+int btf_id_set_index(const struct btf_id_set *set, u32 id)
+{
+ const u32 *p;
+
+ p = bsearch(&id, set->ids, set->cnt, sizeof(u32), btf_id_cmp_func);
+ if (!p)
+ return -1;
+ return p - set->ids;
+}
+
bool btf_id_set_contains(const struct btf_id_set *set, u32 id)
{
return bsearch(&id, set->ids, set->cnt, sizeof(u32), btf_id_cmp_func) != NULL;diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
index 128028efda64..8c77703954f7 100644
--- a/kernel/bpf/cgroup.c
+++ b/kernel/bpf/cgroup.c
@@ -14,6 +14,9 @@
#include <linux/string.h>
#include <linux/bpf.h>
#include <linux/bpf-cgroup.h>
+#include <linux/btf_ids.h>
+#include <linux/bpf_lsm.h>
+#include <linux/bpf_verifier.h>
#include <net/sock.h>
#include <net/bpf_sk_storage.h>
@@ -22,6 +25,18 @@
DEFINE_STATIC_KEY_ARRAY_FALSE(cgroup_bpf_enabled_key, MAX_CGROUP_BPF_ATTACH_TYPE);
EXPORT_SYMBOL(cgroup_bpf_enabled_key);
+#ifdef CONFIG_BPF_LSM
+static enum cgroup_bpf_attach_type bpf_lsm_attach_type_get(u32 attach_btf_id)
+{
+ return CGROUP_LSM_START + bpf_lsm_hook_idx(attach_btf_id);
+}
+#else
+static enum cgroup_bpf_attach_type bpf_lsm_attach_type_get(u32 attach_btf_id)
+{
+ return -EOPNOTSUPP;
+}
+#endif
+
void cgroup_bpf_offline(struct cgroup *cgrp)
{
cgroup_get(cgrp);@@ -89,6 +104,14 @@ static void bpf_cgroup_storages_link(struct bpf_cgroup_storage *storages[],
bpf_cgroup_storage_link(storages[stype], cgrp, attach_type);
}
+static void bpf_cgroup_storages_unlink(struct bpf_cgroup_storage *storages[])
+{
+ enum bpf_cgroup_storage_type stype;
+
+ for_each_cgroup_storage_type(stype)
+ bpf_cgroup_storage_unlink(storages[stype]);
+}
+
/* Called when bpf_cgroup_link is auto-detached from dying cgroup.
* It drops cgroup and bpf_prog refcounts, and marks bpf_link as defunct. It
* doesn't free link memory, which will eventually be done by bpf_link's@@ -100,6 +123,15 @@ static void bpf_cgroup_link_auto_detach(struct bpf_cgroup_link *link)
link->cgroup = NULL;
}
+static void bpf_cgroup_lsm_shim_release(struct bpf_prog *prog,
+ enum cgroup_bpf_attach_type atype)
+{
+ if (!prog || atype != prog->aux->cgroup_atype)
+ return;
+
+ bpf_trampoline_unlink_cgroup_shim(prog);
+}
+
/**
* cgroup_bpf_release() - put references of all bpf programs and
* release all cgroup bpf data@@ -123,10 +155,16 @@ static void cgroup_bpf_release(struct work_struct *work)
list_for_each_entry_safe(pl, pltmp, progs, node) {
list_del(&pl->node);
- if (pl->prog)
+ if (pl->prog) {
+ bpf_cgroup_lsm_shim_release(pl->prog,
+ atype);
bpf_prog_put(pl->prog);
- if (pl->link)
+ }
+ if (pl->link) {
+ bpf_cgroup_lsm_shim_release(pl->link->link.prog,
+ atype);
bpf_cgroup_link_auto_detach(pl->link);
+ }
kfree(pl);
static_branch_dec(&cgroup_bpf_enabled_key[atype]);
}@@ -439,6 +477,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp,
struct bpf_prog *old_prog = NULL;
struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {};
struct bpf_cgroup_storage *new_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {};
+ struct bpf_attach_target_info tgt_info = {};
enum cgroup_bpf_attach_type atype;
struct bpf_prog_list *pl;
struct list_head *progs;@@ -455,9 +494,31 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp,
/* replace_prog implies BPF_F_REPLACE, and vice versa */
return -EINVAL;
- atype = to_cgroup_bpf_attach_type(type);
- if (atype < 0)
- return -EINVAL;
+ if (type == BPF_LSM_CGROUP) {
+ struct bpf_prog *p = prog ? : link->link.prog;
+
+ if (replace_prog) {
+ /* Reusing shim from the original program.
+ */
+ atype = replace_prog->aux->cgroup_atype;
+ } else {
+ err = bpf_check_attach_target(NULL, p, NULL,
+ p->aux->attach_btf_id,
+ &tgt_info);
+ if (err)
+ return -EINVAL;
+
+ atype = bpf_lsm_attach_type_get(p->aux->attach_btf_id);
+ if (atype < 0)
+ return atype;
+ }
+
+ p->aux->cgroup_atype = atype;
+ } else {
+ atype = to_cgroup_bpf_attach_type(type);
+ if (atype < 0)
+ return -EINVAL;
+ }
progs = &cgrp->bpf.progs[atype];
@@ -503,13 +564,27 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp,
if (err)
goto cleanup;
+ bpf_cgroup_storages_link(new_storage, cgrp, type);
+
+ if (type == BPF_LSM_CGROUP && !old_prog) {
+ struct bpf_prog *p = prog ? : link->link.prog;
+ int err;
+
+ err = bpf_trampoline_link_cgroup_shim(p, &tgt_info);
+ if (err)
+ goto cleanup_trampoline;
+ }
+
if (old_prog)
bpf_prog_put(old_prog);
else
static_branch_inc(&cgroup_bpf_enabled_key[atype]);
- bpf_cgroup_storages_link(new_storage, cgrp, type);
+
return 0;
+cleanup_trampoline:
+ bpf_cgroup_storages_unlink(new_storage);
+
cleanup:
if (old_prog) {
pl->prog = old_prog;@@ -601,9 +676,13 @@ static int __cgroup_bpf_replace(struct cgroup *cgrp,
struct list_head *progs;
bool found = false;
- atype = to_cgroup_bpf_attach_type(link->type);
- if (atype < 0)
- return -EINVAL;
+ if (link->type == BPF_LSM_CGROUP) {
+ atype = link->link.prog->aux->cgroup_atype;
+ } else {
+ atype = to_cgroup_bpf_attach_type(link->type);
+ if (atype < 0)
+ return -EINVAL;
+ }
progs = &cgrp->bpf.progs[atype];
@@ -619,6 +698,9 @@ static int __cgroup_bpf_replace(struct cgroup *cgrp,
if (!found)
return -ENOENT;
+ if (link->type == BPF_LSM_CGROUP)
+ new_prog->aux->cgroup_atype = atype;
+
old_prog = xchg(&link->link.prog, new_prog);
replace_effective_prog(cgrp, atype, link);
bpf_prog_put(old_prog);
@@ -702,9 +784,15 @@ static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
u32 flags;
int err;
- atype = to_cgroup_bpf_attach_type(type);
- if (atype < 0)
- return -EINVAL;
+ if (type == BPF_LSM_CGROUP) {
+ struct bpf_prog *p = prog ? : link->link.prog;
+
+ atype = p->aux->cgroup_atype;
+ } else {
+ atype = to_cgroup_bpf_attach_type(type);
+ if (atype < 0)
+ return -EINVAL;
+ }
progs = &cgrp->bpf.progs[atype];
flags = cgrp->bpf.flags[atype];@@ -726,6 +814,10 @@ static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
if (err)
goto cleanup;
+ if (type == BPF_LSM_CGROUP)
+ bpf_cgroup_lsm_shim_release(prog ? : link->link.prog,
+ atype);
+
/* now can actually delete it from this cgroup list */
list_del(&pl->node);
kfree(pl);
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index cdaa1152436a..351166cea25c 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -3139,6 +3139,11 @@ static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog,
return prog->enforce_expected_attach_type &&
prog->expected_attach_type != attach_type ?
-EINVAL : 0;
+ case BPF_PROG_TYPE_LSM:
+ if (prog->expected_attach_type != BPF_LSM_CGROUP)
+ return -EINVAL;
+ return 0;
+
default:
return 0;
}
@@ -3194,6 +3199,8 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type)
return BPF_PROG_TYPE_SK_LOOKUP;
case BPF_XDP:
return BPF_PROG_TYPE_XDP;
+ case BPF_LSM_CGROUP:
+ return BPF_PROG_TYPE_LSM;
default:
return BPF_PROG_TYPE_UNSPEC;
}
@@ -3247,6 +3254,7 @@ static int bpf_prog_attach(const union bpf_attr *attr)
case BPF_PROG_TYPE_CGROUP_SOCKOPT:
case BPF_PROG_TYPE_CGROUP_SYSCTL:
case BPF_PROG_TYPE_SOCK_OPS:
+ case BPF_PROG_TYPE_LSM:
ret = cgroup_bpf_prog_attach(attr, ptype, prog);
break;
default:
@@ -3284,6 +3292,7 @@ static int bpf_prog_detach(const union bpf_attr *attr)
case BPF_PROG_TYPE_CGROUP_SOCKOPT:
case BPF_PROG_TYPE_CGROUP_SYSCTL:
case BPF_PROG_TYPE_SOCK_OPS:
+ case BPF_PROG_TYPE_LSM:
return cgroup_bpf_prog_detach(attr, ptype);
default:
return -EINVAL;
@@ -4317,6 +4326,7 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr)
case BPF_PROG_TYPE_CGROUP_DEVICE:
case BPF_PROG_TYPE_CGROUP_SYSCTL:
case BPF_PROG_TYPE_CGROUP_SOCKOPT:
+ case BPF_PROG_TYPE_LSM:
ret = cgroup_bpf_link_attach(attr, prog);
break;
case BPF_PROG_TYPE_TRACING:
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index 0c4fd194e801..fca1dea786c7 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -11,6 +11,8 @@
#include <linux/rcupdate_wait.h>
#include <linux/module.h>
#include <linux/static_call.h>
+#include <linux/bpf_verifier.h>
+#include <linux/bpf_lsm.h>
/* dummy _ops. The verifier will operate on target program's ops. */
const struct bpf_verifier_ops bpf_extension_verifier_ops = {@@ -394,6 +396,7 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog)
return BPF_TRAMP_MODIFY_RETURN;
case BPF_TRACE_FEXIT:
return BPF_TRAMP_FEXIT;
+ case BPF_LSM_CGROUP:
case BPF_LSM_MAC:
if (!prog->aux->attach_func_proto->type)
/* The function returns void, we cannot modify its
@@ -485,6 +488,147 @@ int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
return err;
}
+static struct bpf_prog *cgroup_shim_alloc(const struct bpf_prog *prog,
+ bpf_func_t bpf_func)
+{
+ struct bpf_prog *p;
+
+ p = bpf_prog_alloc(1, 0);
+ if (!p)
+ return NULL;
+
+ p->jited = false;
+ p->bpf_func = bpf_func;
+
+ p->aux->cgroup_atype = prog->aux->cgroup_atype;
+ p->aux->attach_func_proto = prog->aux->attach_func_proto;
+ p->aux->attach_btf_id = prog->aux->attach_btf_id;
+ p->aux->attach_btf = prog->aux->attach_btf;
+ btf_get(p->aux->attach_btf);
+ p->type = BPF_PROG_TYPE_LSM;
+ p->expected_attach_type = BPF_LSM_MAC;
+ bpf_prog_inc(p);
+
+ return p;
+}
+
+static struct bpf_prog *cgroup_shim_find(struct bpf_trampoline *tr,
+ bpf_func_t bpf_func)
+{
+ const struct bpf_prog_aux *aux;
+ int kind;
+
+ for (kind = 0; kind < BPF_TRAMP_MAX; kind++) {
+ hlist_for_each_entry(aux, &tr->progs_hlist[kind], tramp_hlist) {
+ struct bpf_prog *p = aux->prog;
+
+ if (!p->jited && p->bpf_func == bpf_func)
+ return p;
+ }
+ }
+
+ return NULL;
+}
+
+int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
+ struct bpf_attach_target_info *tgt_info)
+{
+ struct bpf_prog *shim_prog = NULL;
+ struct bpf_trampoline *tr;
+ bpf_func_t bpf_func;
+ u64 key;
+ int err;
+
+ key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf,
+ prog->aux->attach_btf_id);
+
+ err = bpf_lsm_find_cgroup_shim(prog, &bpf_func);
+ if (err)
+ return err;
+
+ tr = bpf_trampoline_get(key, tgt_info);
+ if (!tr)
+ return -ENOMEM;
+
+ mutex_lock(&tr->mutex);
+
+ shim_prog = cgroup_shim_find(tr, bpf_func);
+ if (shim_prog) {
+ /* Reusing existing shim attached by the other program.
+ */
+ bpf_prog_inc(shim_prog);
+ mutex_unlock(&tr->mutex);
+ return 0;
+ }
+
+ /* Allocate and install new shim.
+ */
+
+ shim_prog = cgroup_shim_alloc(prog, bpf_func);
+ if (!shim_prog) {
+ err = -ENOMEM;
+ goto out;
+ }
+
+ err = __bpf_trampoline_link_prog(shim_prog, tr);
+ if (err)
+ goto out;
+
+ mutex_unlock(&tr->mutex);
+
+ return 0;
+out:
+ if (shim_prog)
+ bpf_prog_put(shim_prog);
+
+ mutex_unlock(&tr->mutex);
+ return err;
+}
+
+void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog)
+{
+ struct bpf_prog *shim_prog;
+ struct bpf_trampoline *tr;
+ bpf_func_t bpf_func;
+ u64 key;
+ int err;
+
+ key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf,
+ prog->aux->attach_btf_id);
+
+ err = bpf_lsm_find_cgroup_shim(prog, &bpf_func);
+ if (err)
+ return;
+
+ tr = bpf_trampoline_lookup(key);
+ if (!tr)
+ return;
+
+ mutex_lock(&tr->mutex);
+
+ shim_prog = cgroup_shim_find(tr, bpf_func);
+ if (shim_prog) {
+ /* We use shim_prog refcnt for tracking whether to
+ * remove the shim program from the trampoline.
+ * Trampoline's mutex is held while refcnt is
+ * added/subtracted so we don't need to care about
+ * potential races.
+ */
+
+ if (atomic64_read(&shim_prog->aux->refcnt) == 1)
+ WARN_ON_ONCE(__bpf_trampoline_unlink_prog(shim_prog, tr));
+
+ bpf_prog_put(shim_prog);
+ }
+
+ mutex_unlock(&tr->mutex);
+
+ bpf_trampoline_put(tr); /* bpf_trampoline_lookup */
+
+ if (shim_prog)
+ bpf_trampoline_put(tr);
+}
+
struct bpf_trampoline *bpf_trampoline_get(u64 key,
struct bpf_attach_target_info *tgt_info)
{diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 9c1a02b82ecd..cc84954846d7 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -14197,6 +14197,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
fallthrough;
case BPF_MODIFY_RETURN:
case BPF_LSM_MAC:
+ case BPF_LSM_CGROUP:
case BPF_TRACE_FENTRY:
case BPF_TRACE_FEXIT:
if (!btf_type_is_func(t)) {diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index d14b10b85e51..bbe48a2dd852 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -998,6 +998,7 @@ enum bpf_attach_type {
BPF_SK_REUSEPORT_SELECT_OR_MIGRATE,
BPF_PERF_EVENT,
BPF_TRACE_KPROBE_MULTI,
+ BPF_LSM_CGROUP,
__MAX_BPF_ATTACH_TYPE
};
--
2.35.1.1178.g4f1659d476-goog