diff --git a/include/linux/bpf-cgroup-defs.h b/include/linux/bpf-cgroup-defs.h
index 6c661b4df9fa..d42516e86b3a 100644
--- a/include/linux/bpf-cgroup-defs.h
+++ b/include/linux/bpf-cgroup-defs.h
@@ -10,7 +10,9 @@
struct bpf_prog_array;
-#define CGROUP_LSM_NUM 211 /* will be addressed in the next patch */
+/* Maximum number of concurrently attachable per-cgroup LSM hooks.
+ */
+#define CGROUP_LSM_NUM 10
enum cgroup_bpf_attach_type {
CGROUP_BPF_ATTACH_TYPE_INVALID = -1,diff --git a/include/linux/bpf_lsm.h b/include/linux/bpf_lsm.h
index 7f0e59f5f9be..613de44aa429 100644
--- a/include/linux/bpf_lsm.h
+++ b/include/linux/bpf_lsm.h
@@ -43,7 +43,6 @@ extern const struct bpf_func_proto bpf_inode_storage_delete_proto;
void bpf_inode_storage_free(struct inode *inode);
int bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog, bpf_func_t *bpf_func);
-int bpf_lsm_hook_idx(u32 btf_id);
#else /* !CONFIG_BPF_LSM */
@@ -74,11 +73,6 @@ static inline int bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog,
return -ENOENT;
}
-static inline int bpf_lsm_hook_idx(u32 btf_id)
-{
- return -EINVAL;
-}
-
#endif /* CONFIG_BPF_LSM */
#endif /* _LINUX_BPF_LSM_H */diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c
index eca258ba71d8..8b948ec9ab73 100644
--- a/kernel/bpf/bpf_lsm.c
+++ b/kernel/bpf/bpf_lsm.c
@@ -57,10 +57,12 @@ static unsigned int __cgroup_bpf_run_lsm_socket(const void *ctx,
if (unlikely(!sk))
return 0;
+ rcu_read_lock(); /* See bpf_lsm_attach_type_get(). */
cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
if (likely(cgrp))
ret = BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[prog->aux->cgroup_atype],
ctx, bpf_prog_run, 0);
+ rcu_read_unlock();
return ret;
}
@@ -77,7 +79,7 @@ static unsigned int __cgroup_bpf_run_lsm_current(const void *ctx,
/*prog = container_of(insn, struct bpf_prog, insnsi);*/
prog = (const struct bpf_prog *)((void *)insn - offsetof(struct bpf_prog, insnsi));
- rcu_read_lock();
+ rcu_read_lock(); /* See bpf_lsm_attach_type_get(). */
cgrp = task_dfl_cgroup(current);
if (likely(cgrp))
ret = BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[prog->aux->cgroup_atype],
@@ -122,11 +124,6 @@ int bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog,
return 0;
}
-int bpf_lsm_hook_idx(u32 btf_id)
-{
- return btf_id_set_index(&bpf_lsm_hooks, btf_id);
-}
-
int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog,
const struct bpf_prog *prog)
{diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
index 8c77703954f7..fca95e021e7e 100644
--- a/kernel/bpf/cgroup.c
+++ b/kernel/bpf/cgroup.c
@@ -26,15 +26,68 @@ DEFINE_STATIC_KEY_ARRAY_FALSE(cgroup_bpf_enabled_key, MAX_CGROUP_BPF_ATTACH_TYPE
EXPORT_SYMBOL(cgroup_bpf_enabled_key);
#ifdef CONFIG_BPF_LSM
+/* Readers are protected by rcu+synchronize_rcu.
+ * Writers are protected by cgroup_mutex.
+ */
+int cgroup_lsm_atype_usecnt[CGROUP_LSM_NUM];
+u32 cgroup_lsm_atype_btf_id[CGROUP_LSM_NUM];
+
static enum cgroup_bpf_attach_type bpf_lsm_attach_type_get(u32 attach_btf_id)
{
- return CGROUP_LSM_START + bpf_lsm_hook_idx(attach_btf_id);
+ int i;
+
+ WARN_ON_ONCE(!mutex_is_locked(&cgroup_mutex));
+
+ for (i = 0; i < ARRAY_SIZE(cgroup_lsm_atype_btf_id); i++) {
+ if (cgroup_lsm_atype_btf_id[i] != attach_btf_id)
+ continue;
+
+ cgroup_lsm_atype_usecnt[i]++;
+ return CGROUP_LSM_START + i;
+ }
+
+ for (i = 0; i < ARRAY_SIZE(cgroup_lsm_atype_usecnt); i++) {
+ if (cgroup_lsm_atype_usecnt[i] != 0)
+ continue;
+
+ cgroup_lsm_atype_btf_id[i] = attach_btf_id;
+ cgroup_lsm_atype_usecnt[i] = 1;
+ return CGROUP_LSM_START + i;
+ }
+
+ return -E2BIG;
+}
+
+static void bpf_lsm_attach_type_put(u32 attach_btf_id)
+{
+ int i;
+
+ WARN_ON_ONCE(!mutex_is_locked(&cgroup_mutex));
+
+ for (i = 0; i < ARRAY_SIZE(cgroup_lsm_atype_btf_id); i++) {
+ if (cgroup_lsm_atype_btf_id[i] != attach_btf_id)
+ continue;
+
+ if (--cgroup_lsm_atype_usecnt[i] <= 0) {
+ /* Wait for any existing users to finish.
+ */
+ synchronize_rcu();
+ WARN_ON_ONCE(cgroup_lsm_atype_usecnt[i] < 0);
+ }
+ return;
+ }
+
+ WARN_ON_ONCE(1);
}
#else
static enum cgroup_bpf_attach_type bpf_lsm_attach_type_get(u32 attach_btf_id)
{
return -EOPNOTSUPP;
}
+
+static void bpf_lsm_attach_type_put(u32 attach_btf_id)
+{
+}
#endif
void cgroup_bpf_offline(struct cgroup *cgrp)@@ -130,6 +183,7 @@ static void bpf_cgroup_lsm_shim_release(struct bpf_prog *prog,
return;
bpf_trampoline_unlink_cgroup_shim(prog);
+ bpf_lsm_attach_type_put(prog->aux->attach_btf_id);
}
/**
@@ -522,27 +576,37 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp,
progs = &cgrp->bpf.progs[atype];
- if (!hierarchy_allows_attach(cgrp, atype))
- return -EPERM;
+ if (!hierarchy_allows_attach(cgrp, atype)) {
+ err = -EPERM;
+ goto cleanup_attach_type;
+ }
- if (!list_empty(progs) && cgrp->bpf.flags[atype] != saved_flags)
+ if (!list_empty(progs) && cgrp->bpf.flags[atype] != saved_flags) {
/* Disallow attaching non-overridable on top
* of existing overridable in this cgroup.
* Disallow attaching multi-prog if overridable or none
*/
- return -EPERM;
+ err = -EPERM;
+ goto cleanup_attach_type;
+ }
- if (prog_list_length(progs) >= BPF_CGROUP_MAX_PROGS)
- return -E2BIG;
+ if (prog_list_length(progs) >= BPF_CGROUP_MAX_PROGS) {
+ err = -E2BIG;
+ goto cleanup_attach_type;
+ }
pl = find_attach_entry(progs, prog, link, replace_prog,
flags & BPF_F_ALLOW_MULTI);
- if (IS_ERR(pl))
- return PTR_ERR(pl);
+ if (IS_ERR(pl)) {
+ err = PTR_ERR(pl);
+ goto cleanup_attach_type;
+ }
if (bpf_cgroup_storages_alloc(storage, new_storage, type,
- prog ? : link->link.prog, cgrp))
- return -ENOMEM;
+ prog ? : link->link.prog, cgrp)) {
+ err = -ENOMEM;
+ goto cleanup_attach_type;
+ }
if (pl) {
old_prog = pl->prog;@@ -550,7 +614,8 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp,
pl = kmalloc(sizeof(*pl), GFP_KERNEL);
if (!pl) {
bpf_cgroup_storages_free(new_storage);
- return -ENOMEM;
+ err = -ENOMEM;
+ goto cleanup_attach_type;
}
list_add_tail(&pl->node, progs);
}@@ -595,6 +660,13 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp,
list_del(&pl->node);
kfree(pl);
}
+
+cleanup_attach_type:
+ if (type == BPF_LSM_CGROUP) {
+ struct bpf_prog *p = prog ? : link->link.prog;
+
+ bpf_lsm_attach_type_put(p->aux->attach_btf_id);
+ }
return err;
}
--
2.35.1.1178.g4f1659d476-goog