1 /* SPDX-License-Identifier: GPL-2.0 */
11  *    dispatching from the global queue to each CPU's local dsq from dispatch().
13  *    through per-CPU BPF queues. The current design is chosen to maximally
30  *    Kthreads are unconditionally queued to the head of a matching local dsq
104 	s32 pid = p->pid;  in BPF_STRUCT_OPS()  local
109 	 * Push per-cpu kthreads at the head of local dsq's and preempt the  in BPF_STRUCT_OPS()
114 	if ((p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) {  in BPF_STRUCT_OPS()
121 	if (bpf_map_push_elem(&central_q, &pid, 0)) {  in BPF_STRUCT_OPS()
136 	s32 pid;  in dispatch_to_cpu()  local
139 		if (bpf_map_pop_elem(&central_q, &pid))  in dispatch_to_cpu()
144 		p = bpf_task_from_pid(pid);  in dispatch_to_cpu()
154 		if (!bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) {  in dispatch_to_cpu()
160 			 * to the fallback DSQ, without dispatching to the local DSQ of the  in dispatch_to_cpu()
169 		/* dispatch to local and mark that @cpu doesn't need more */  in dispatch_to_cpu()
206 		 * core automatically retries if the local dsq is empty but we  in BPF_STRUCT_OPS()
285 			nr_to_kick--;  in central_timerfn()
303 	ret = scx_bpf_create_dsq(FALLBACK_DSQ_ID, -1);  in BPF_STRUCT_OPS_SLEEPABLE()
309 		return -ESRCH;  in BPF_STRUCT_OPS_SLEEPABLE()
312 		scx_bpf_error("init from non-central CPU");  in BPF_STRUCT_OPS_SLEEPABLE()
313 		return -EINVAL;  in BPF_STRUCT_OPS_SLEEPABLE()
322 	 * kernel which doesn't have it, bpf_timer_start() will return -EINVAL.  in BPF_STRUCT_OPS_SLEEPABLE()
327 	if (ret == -EINVAL) {  in BPF_STRUCT_OPS_SLEEPABLE()