Skip to content

bpf: Nested rcu critical sections #10315

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: bpf-next_base
Choose a base branch
from
+66 −27
Open

bpf: Nested rcu critical sections #10315

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a961781
bpf: support nested rcu critical sections
puranjaymohan Nov 17, 2025
744e635
selftests: bpf: Add tests for unbalanced rcu_read_lock
puranjaymohan Nov 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion include/linux/bpf_verifier.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -416,7 +416,7 @@ struct bpf_verifier_state {
u32 active_irq_id;
u32 active_lock_id;
void *active_lock_ptr;
bool active_rcu_lock;
u32 active_rcu_locks;

bool speculative;
bool in_sleepable;
Expand Down
47 changes: 22 additions & 25 deletions kernel/bpf/verifier.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1437,7 +1437,7 @@ static int copy_reference_state(struct bpf_verifier_state *dst, const struct bpf
dst->acquired_refs = src->acquired_refs;
dst->active_locks = src->active_locks;
dst->active_preempt_locks = src->active_preempt_locks;
dst->active_rcu_lock = src->active_rcu_lock;
dst->active_rcu_locks = src->active_rcu_locks;
dst->active_irq_id = src->active_irq_id;
dst->active_lock_id = src->active_lock_id;
dst->active_lock_ptr = src->active_lock_ptr;
Expand Down Expand Up @@ -5880,7 +5880,7 @@ static bool in_sleepable(struct bpf_verifier_env *env)
*/
static bool in_rcu_cs(struct bpf_verifier_env *env)
{
return env->cur_state->active_rcu_lock ||
return env->cur_state->active_rcu_locks ||
env->cur_state->active_locks ||
!in_sleepable(env);
}
Expand Down Expand Up @@ -10735,7 +10735,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
}

if (env->subprog_info[subprog].might_sleep &&
(env->cur_state->active_rcu_lock || env->cur_state->active_preempt_locks ||
(env->cur_state->active_rcu_locks || env->cur_state->active_preempt_locks ||
env->cur_state->active_irq_id || !in_sleepable(env))) {
verbose(env, "global functions that may sleep are not allowed in non-sleepable context,\n"
"i.e., in a RCU/IRQ/preempt-disabled section, or in\n"
Expand Down Expand Up @@ -11314,7 +11314,7 @@ static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exit
return -EINVAL;
}

if (check_lock && env->cur_state->active_rcu_lock) {
if (check_lock && env->cur_state->active_rcu_locks) {
verbose(env, "%s cannot be used inside bpf_rcu_read_lock-ed region\n", prefix);
return -EINVAL;
}
Expand Down Expand Up @@ -11452,7 +11452,7 @@ static int get_helper_proto(struct bpf_verifier_env *env, int func_id,
/* Check if we're in a sleepable context. */
static inline bool in_sleepable_context(struct bpf_verifier_env *env)
{
return !env->cur_state->active_rcu_lock &&
return !env->cur_state->active_rcu_locks &&
!env->cur_state->active_preempt_locks &&
!env->cur_state->active_irq_id &&
in_sleepable(env);
Expand Down Expand Up @@ -11518,7 +11518,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
return err;
}

if (env->cur_state->active_rcu_lock) {
if (env->cur_state->active_rcu_locks) {
if (fn->might_sleep) {
verbose(env, "sleepable helper %s#%d in rcu_read_lock region\n",
func_id_name(func_id), func_id);
Expand Down Expand Up @@ -14006,36 +14006,33 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
preempt_disable = is_kfunc_bpf_preempt_disable(&meta);
preempt_enable = is_kfunc_bpf_preempt_enable(&meta);

if (env->cur_state->active_rcu_lock) {
if (rcu_lock) {
env->cur_state->active_rcu_locks++;
} else if (rcu_unlock) {
struct bpf_func_state *state;
struct bpf_reg_state *reg;
u32 clear_mask = (1 << STACK_SPILL) | (1 << STACK_ITER);

if (in_rbtree_lock_required_cb(env) && (rcu_lock || rcu_unlock)) {
verbose(env, "Calling bpf_rcu_read_{lock,unlock} in unnecessary rbtree callback\n");
return -EACCES;
}

if (rcu_lock) {
verbose(env, "nested rcu read lock (kernel function %s)\n", func_name);
if (env->cur_state->active_rcu_locks == 0) {
verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name);
return -EINVAL;
} else if (rcu_unlock) {
}
if (--env->cur_state->active_rcu_locks == 0) {
bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, clear_mask, ({
if (reg->type & MEM_RCU) {
reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL);
reg->type |= PTR_UNTRUSTED;
}
}));
env->cur_state->active_rcu_lock = false;
} else if (sleepable) {
verbose(env, "kernel func %s is sleepable within rcu_read_lock region\n", func_name);
return -EACCES;
}
} else if (rcu_lock) {
env->cur_state->active_rcu_lock = true;
} else if (rcu_unlock) {
verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name);
return -EINVAL;
} else if (sleepable && env->cur_state->active_rcu_locks) {
verbose(env, "kernel func %s is sleepable within rcu_read_lock region\n", func_name);
return -EACCES;
}

if (in_rbtree_lock_required_cb(env) && (rcu_lock || rcu_unlock)) {
verbose(env, "Calling bpf_rcu_read_{lock,unlock} in unnecessary rbtree callback\n");
return -EACCES;
}

if (env->cur_state->active_preempt_locks) {
Expand Down Expand Up @@ -19328,7 +19325,7 @@ static bool refsafe(struct bpf_verifier_state *old, struct bpf_verifier_state *c
if (old->active_preempt_locks != cur->active_preempt_locks)
return false;

if (old->active_rcu_lock != cur->active_rcu_lock)
if (old->active_rcu_locks != cur->active_rcu_locks)
return false;

if (!check_ids(old->active_irq_id, cur->active_irq_id, idmap))
Expand Down
4 changes: 3 additions & 1 deletion tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ static void test_success(void)
bpf_program__set_autoload(skel->progs.two_regions, true);
bpf_program__set_autoload(skel->progs.non_sleepable_1, true);
bpf_program__set_autoload(skel->progs.non_sleepable_2, true);
bpf_program__set_autoload(skel->progs.nested_rcu_region, true);
bpf_program__set_autoload(skel->progs.task_trusted_non_rcuptr, true);
bpf_program__set_autoload(skel->progs.rcu_read_lock_subprog, true);
bpf_program__set_autoload(skel->progs.rcu_read_lock_global_subprog, true);
Expand Down Expand Up @@ -78,7 +79,8 @@ static const char * const inproper_region_tests[] = {
"non_sleepable_rcu_mismatch",
"inproper_sleepable_helper",
"inproper_sleepable_kfunc",
"nested_rcu_region",
"nested_rcu_region_unbalanced_1",
"nested_rcu_region_unbalanced_2",
"rcu_read_lock_global_subprog_lock",
"rcu_read_lock_global_subprog_unlock",
"rcu_read_lock_sleepable_helper_global_subprog",
Expand Down
40 changes: 40 additions & 0 deletions tools/testing/selftests/bpf/progs/rcu_read_lock.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,46 @@ int nested_rcu_region(void *ctx)
return 0;
}

SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
int nested_rcu_region_unbalanced_1(void *ctx)
{
struct task_struct *task, *real_parent;

/* nested rcu read lock regions */
task = bpf_get_current_task_btf();
bpf_rcu_read_lock();
bpf_rcu_read_lock();
real_parent = task->real_parent;
if (!real_parent)
goto out;
(void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
out:
bpf_rcu_read_unlock();
bpf_rcu_read_unlock();
bpf_rcu_read_unlock();
return 0;
}

SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
int nested_rcu_region_unbalanced_2(void *ctx)
{
struct task_struct *task, *real_parent;

/* nested rcu read lock regions */
task = bpf_get_current_task_btf();
bpf_rcu_read_lock();
bpf_rcu_read_lock();
bpf_rcu_read_lock();
real_parent = task->real_parent;
if (!real_parent)
goto out;
(void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
out:
bpf_rcu_read_unlock();
bpf_rcu_read_unlock();
return 0;
}

SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
int task_trusted_non_rcuptr(void *ctx)
{
Expand Down
Loading