Lines Matching full:cgc
165 struct fcg_cgrp_ctx *cgc; in find_cgrp_ctx() local
167 cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); in find_cgrp_ctx()
168 if (!cgc) { in find_cgrp_ctx()
172 return cgc; in find_cgrp_ctx()
177 struct fcg_cgrp_ctx *cgc; in find_ancestor_cgrp_ctx() local
185 cgc = find_cgrp_ctx(cgrp); in find_ancestor_cgrp_ctx()
186 if (!cgc) in find_ancestor_cgrp_ctx()
189 return cgc; in find_ancestor_cgrp_ctx()
192 static void cgrp_refresh_hweight(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) in cgrp_refresh_hweight() argument
196 if (!cgc->nr_active) { in cgrp_refresh_hweight()
201 if (cgc->hweight_gen == hweight_gen) { in cgrp_refresh_hweight()
208 struct fcg_cgrp_ctx *cgc; in cgrp_refresh_hweight() local
211 cgc = find_ancestor_cgrp_ctx(cgrp, level); in cgrp_refresh_hweight()
212 if (!cgc) in cgrp_refresh_hweight()
216 cgc->hweight = FCG_HWEIGHT_ONE; in cgrp_refresh_hweight()
217 cgc->hweight_gen = hweight_gen; in cgrp_refresh_hweight()
233 is_active = cgc->nr_active; in cgrp_refresh_hweight()
235 cgc->hweight_gen = pcgc->hweight_gen; in cgrp_refresh_hweight()
236 cgc->hweight = in cgrp_refresh_hweight()
237 div_round_up(pcgc->hweight * cgc->weight, in cgrp_refresh_hweight()
250 static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc) in cgrp_cap_budget() argument
259 delta = __sync_fetch_and_sub(&cgc->cvtime_delta, cgc->cvtime_delta); in cgrp_cap_budget()
267 max_budget = (cgrp_slice_ns * nr_cpus * cgc->hweight) / in cgrp_cap_budget()
275 static void cgrp_enqueued(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) in cgrp_enqueued() argument
282 if (__sync_val_compare_and_swap(&cgc->queued, 0, 1)) { in cgrp_enqueued()
301 cgrp_cap_budget(cgv_node, cgc); in cgrp_enqueued()
349 struct fcg_cgrp_ctx *cgc; in BPF_STRUCT_OPS() local
386 cgc = find_cgrp_ctx(cgrp); in BPF_STRUCT_OPS()
387 if (!cgc) in BPF_STRUCT_OPS()
399 if (time_before(tvtime, cgc->tvtime_now - SCX_SLICE_DFL)) in BPF_STRUCT_OPS()
400 tvtime = cgc->tvtime_now - SCX_SLICE_DFL; in BPF_STRUCT_OPS()
406 cgrp_enqueued(cgrp, cgc); in BPF_STRUCT_OPS()
418 struct fcg_cgrp_ctx *cgc; in update_active_weight_sums() local
422 cgc = find_cgrp_ctx(cgrp); in update_active_weight_sums()
423 if (!cgc) in update_active_weight_sums()
434 if (__sync_fetch_and_add(&cgc->nr_runnable, 1)) in update_active_weight_sums()
438 if (__sync_sub_and_fetch(&cgc->nr_runnable, 1)) in update_active_weight_sums()
451 cgrp_refresh_hweight(cgrp, cgc); in update_active_weight_sums()
456 struct fcg_cgrp_ctx *cgc, *pcgc = NULL; in update_active_weight_sums() local
459 cgc = find_ancestor_cgrp_ctx(cgrp, level); in update_active_weight_sums()
460 if (!cgc) in update_active_weight_sums()
477 if (!cgc->nr_active++) { in update_active_weight_sums()
481 pcgc->child_weight_sum += cgc->weight; in update_active_weight_sums()
485 if (!--cgc->nr_active) { in update_active_weight_sums()
489 pcgc->child_weight_sum -= cgc->weight; in update_active_weight_sums()
504 cgrp_refresh_hweight(cgrp, cgc); in update_active_weight_sums()
519 struct fcg_cgrp_ctx *cgc; in BPF_STRUCT_OPS() local
525 cgc = find_cgrp_ctx(cgrp); in BPF_STRUCT_OPS()
526 if (cgc) { in BPF_STRUCT_OPS()
528 * @cgc->tvtime_now always progresses forward as tasks start in BPF_STRUCT_OPS()
533 if (time_before(cgc->tvtime_now, p->scx.dsq_vtime)) in BPF_STRUCT_OPS()
534 cgc->tvtime_now = p->scx.dsq_vtime; in BPF_STRUCT_OPS()
543 struct fcg_cgrp_ctx *cgc; in BPF_STRUCT_OPS() local
568 cgc = find_cgrp_ctx(cgrp); in BPF_STRUCT_OPS()
569 if (cgc) { in BPF_STRUCT_OPS()
570 __sync_fetch_and_add(&cgc->cvtime_delta, in BPF_STRUCT_OPS()
588 struct fcg_cgrp_ctx *cgc, *pcgc = NULL; in BPF_STRUCT_OPS() local
590 cgc = find_cgrp_ctx(cgrp); in BPF_STRUCT_OPS()
591 if (!cgc) in BPF_STRUCT_OPS()
601 if (pcgc && cgc->nr_active) in BPF_STRUCT_OPS()
602 pcgc->child_weight_sum += (s64)weight - cgc->weight; in BPF_STRUCT_OPS()
603 cgc->weight = weight; in BPF_STRUCT_OPS()
612 struct fcg_cgrp_ctx *cgc; in try_pick_next_cgroup() local
656 cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); in try_pick_next_cgroup()
657 if (!cgc) { in try_pick_next_cgroup()
673 cgrp_refresh_hweight(cgrp, cgc); in try_pick_next_cgroup()
684 cgv_node->cvtime += cgrp_slice_ns * FCG_HWEIGHT_ONE / (cgc->hweight ?: 1); in try_pick_next_cgroup()
685 cgrp_cap_budget(cgv_node, cgc); in try_pick_next_cgroup()
705 __sync_val_compare_and_swap(&cgc->queued, 1, 0); in try_pick_next_cgroup()
730 struct fcg_cgrp_ctx *cgc; in BPF_STRUCT_OPS() local
762 cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); in BPF_STRUCT_OPS()
763 if (cgc) { in BPF_STRUCT_OPS()
770 __sync_fetch_and_add(&cgc->cvtime_delta, in BPF_STRUCT_OPS()
772 FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); in BPF_STRUCT_OPS()
809 struct fcg_cgrp_ctx *cgc; in BPF_STRUCT_OPS() local
822 if (!(cgc = find_cgrp_ctx(args->cgroup))) in BPF_STRUCT_OPS()
825 p->scx.dsq_vtime = cgc->tvtime_now; in BPF_STRUCT_OPS()
833 struct fcg_cgrp_ctx *cgc; in BPF_STRUCT_OPS_SLEEPABLE() local
848 cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, in BPF_STRUCT_OPS_SLEEPABLE()
850 if (!cgc) { in BPF_STRUCT_OPS_SLEEPABLE()
855 cgc->weight = args->weight; in BPF_STRUCT_OPS_SLEEPABLE()
856 cgc->hweight = FCG_HWEIGHT_ONE; in BPF_STRUCT_OPS_SLEEPABLE()