Lines Matching full:cs
145 struct cpuset *cs = task_cs(p); in inc_dl_tasks_cs() local
147 cs->nr_deadline_tasks++; in inc_dl_tasks_cs()
152 struct cpuset *cs = task_cs(p); in dec_dl_tasks_cs() local
154 cs->nr_deadline_tasks--; in dec_dl_tasks_cs()
157 static inline int is_partition_valid(const struct cpuset *cs) in is_partition_valid() argument
159 return cs->partition_root_state > 0; in is_partition_valid()
162 static inline int is_partition_invalid(const struct cpuset *cs) in is_partition_invalid() argument
164 return cs->partition_root_state < 0; in is_partition_invalid()
170 static inline void make_partition_invalid(struct cpuset *cs) in make_partition_invalid() argument
172 if (cs->partition_root_state > 0) in make_partition_invalid()
173 cs->partition_root_state = -cs->partition_root_state; in make_partition_invalid()
179 static inline void notify_partition_change(struct cpuset *cs, int old_prs) in notify_partition_change() argument
181 if (old_prs == cs->partition_root_state) in notify_partition_change()
183 cgroup_file_notify(&cs->partition_file); in notify_partition_change()
186 if (is_partition_valid(cs)) in notify_partition_change()
187 WRITE_ONCE(cs->prs_err, PERR_NONE); in notify_partition_change()
272 * decrease cs->attach_in_progress.
273 * wake_up cpuset_attach_wq if cs->attach_in_progress==0.
275 static inline void dec_attach_in_progress_locked(struct cpuset *cs) in dec_attach_in_progress_locked() argument
279 cs->attach_in_progress--; in dec_attach_in_progress_locked()
280 if (!cs->attach_in_progress) in dec_attach_in_progress_locked()
284 static inline void dec_attach_in_progress(struct cpuset *cs) in dec_attach_in_progress() argument
287 dec_attach_in_progress_locked(cs); in dec_attach_in_progress()
313 * @cs: partition root to be checked
317 * It is assumed that @cs is a valid partition root. @excluded_child should
320 static inline bool partition_is_populated(struct cpuset *cs, in partition_is_populated() argument
326 if (cs->css.cgroup->nr_populated_csets) in partition_is_populated()
328 if (!excluded_child && !cs->nr_subparts) in partition_is_populated()
329 return cgroup_is_populated(cs->css.cgroup); in partition_is_populated()
332 cpuset_for_each_child(child, css, cs) { in partition_is_populated()
361 struct cpuset *cs; in guarantee_online_cpus() local
367 cs = task_cs(tsk); in guarantee_online_cpus()
369 while (!cpumask_intersects(cs->effective_cpus, pmask)) in guarantee_online_cpus()
370 cs = parent_cs(cs); in guarantee_online_cpus()
372 cpumask_and(pmask, pmask, cs->effective_cpus); in guarantee_online_cpus()
387 static void guarantee_online_mems(struct cpuset *cs, nodemask_t *pmask) in guarantee_online_mems() argument
389 while (!nodes_intersects(cs->effective_mems, node_states[N_MEMORY])) in guarantee_online_mems()
390 cs = parent_cs(cs); in guarantee_online_mems()
391 nodes_and(*pmask, cs->effective_mems, node_states[N_MEMORY]); in guarantee_online_mems()
396 * @cs: the cpuset that have cpumasks to be allocated.
402 static inline int alloc_cpumasks(struct cpuset *cs, struct tmpmasks *tmp) in alloc_cpumasks() argument
406 if (cs) { in alloc_cpumasks()
407 pmask1 = &cs->cpus_allowed; in alloc_cpumasks()
408 pmask2 = &cs->effective_cpus; in alloc_cpumasks()
409 pmask3 = &cs->effective_xcpus; in alloc_cpumasks()
410 pmask4 = &cs->exclusive_cpus; in alloc_cpumasks()
444 * @cs: the cpuset that have cpumasks to be free.
447 static inline void free_cpumasks(struct cpuset *cs, struct tmpmasks *tmp) in free_cpumasks() argument
449 if (cs) { in free_cpumasks()
450 free_cpumask_var(cs->cpus_allowed); in free_cpumasks()
451 free_cpumask_var(cs->effective_cpus); in free_cpumasks()
452 free_cpumask_var(cs->effective_xcpus); in free_cpumasks()
453 free_cpumask_var(cs->exclusive_cpus); in free_cpumasks()
464 * @cs: the cpuset that the trial cpuset duplicates
466 static struct cpuset *alloc_trial_cpuset(struct cpuset *cs) in alloc_trial_cpuset() argument
470 trial = kmemdup(cs, sizeof(*cs), GFP_KERNEL); in alloc_trial_cpuset()
479 cpumask_copy(trial->cpus_allowed, cs->cpus_allowed); in alloc_trial_cpuset()
480 cpumask_copy(trial->effective_cpus, cs->effective_cpus); in alloc_trial_cpuset()
481 cpumask_copy(trial->effective_xcpus, cs->effective_xcpus); in alloc_trial_cpuset()
482 cpumask_copy(trial->exclusive_cpus, cs->exclusive_cpus); in alloc_trial_cpuset()
488 * @cs: the cpuset to be freed
490 static inline void free_cpuset(struct cpuset *cs) in free_cpuset() argument
492 free_cpumasks(cs, NULL); in free_cpuset()
493 kfree(cs); in free_cpuset()
497 static inline struct cpumask *user_xcpus(struct cpuset *cs) in user_xcpus() argument
499 return cpumask_empty(cs->exclusive_cpus) ? cs->cpus_allowed in user_xcpus()
500 : cs->exclusive_cpus; in user_xcpus()
503 static inline bool xcpus_empty(struct cpuset *cs) in xcpus_empty() argument
505 return cpumask_empty(cs->cpus_allowed) && in xcpus_empty()
506 cpumask_empty(cs->exclusive_cpus); in xcpus_empty()
941 static void dl_update_tasks_root_domain(struct cpuset *cs) in dl_update_tasks_root_domain() argument
946 if (cs->nr_deadline_tasks == 0) in dl_update_tasks_root_domain()
949 css_task_iter_start(&cs->css, 0, &it); in dl_update_tasks_root_domain()
959 struct cpuset *cs = NULL; in dl_rebuild_rd_accounting() local
977 cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) { in dl_rebuild_rd_accounting()
979 if (cpumask_empty(cs->effective_cpus)) { in dl_rebuild_rd_accounting()
984 css_get(&cs->css); in dl_rebuild_rd_accounting()
988 dl_update_tasks_root_domain(cs); in dl_rebuild_rd_accounting()
991 css_put(&cs->css); in dl_rebuild_rd_accounting()
1021 struct cpuset *cs; in rebuild_sched_domains_locked() local
1048 cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) { in rebuild_sched_domains_locked()
1049 if (!is_partition_valid(cs)) { in rebuild_sched_domains_locked()
1053 if (!cpumask_subset(cs->effective_cpus, in rebuild_sched_domains_locked()
1097 * @cs: the cpuset in which each task's cpus_allowed mask needs to be changed
1100 * Iterate through each task of @cs updating its cpus_allowed to the
1106 void cpuset_update_tasks_cpumask(struct cpuset *cs, struct cpumask *new_cpus) in cpuset_update_tasks_cpumask() argument
1110 bool top_cs = cs == &top_cpuset; in cpuset_update_tasks_cpumask()
1112 css_task_iter_start(&cs->css, 0, &it); in cpuset_update_tasks_cpumask()
1124 cpumask_and(new_cpus, possible_mask, cs->effective_cpus); in cpuset_update_tasks_cpumask()
1134 * @cs: the cpuset the need to recompute the new effective_cpus mask
1140 struct cpuset *cs, struct cpuset *parent) in compute_effective_cpumask() argument
1142 cpumask_and(new_cpus, cs->cpus_allowed, parent->effective_cpus); in compute_effective_cpumask()
1156 static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs,
1164 static int update_partition_exclusive(struct cpuset *cs, int new_prs) in update_partition_exclusive() argument
1168 if (exclusive && !is_cpu_exclusive(cs)) { in update_partition_exclusive()
1169 if (cpuset_update_flag(CS_CPU_EXCLUSIVE, cs, 1)) in update_partition_exclusive()
1171 } else if (!exclusive && is_cpu_exclusive(cs)) { in update_partition_exclusive()
1173 cpuset_update_flag(CS_CPU_EXCLUSIVE, cs, 0); in update_partition_exclusive()
1185 static void update_partition_sd_lb(struct cpuset *cs, int old_prs) in update_partition_sd_lb() argument
1187 int new_prs = cs->partition_root_state; in update_partition_sd_lb()
1192 * If cs is not a valid partition root, the load balance state in update_partition_sd_lb()
1198 new_lb = is_sched_load_balance(parent_cs(cs)); in update_partition_sd_lb()
1200 if (new_lb != !!is_sched_load_balance(cs)) { in update_partition_sd_lb()
1203 set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); in update_partition_sd_lb()
1205 clear_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); in update_partition_sd_lb()
1215 static bool tasks_nocpu_error(struct cpuset *parent, struct cpuset *cs, in tasks_nocpu_error() argument
1219 * A populated partition (cs or parent) can't have empty effective_cpus in tasks_nocpu_error()
1222 partition_is_populated(parent, cs)) || in tasks_nocpu_error()
1224 partition_is_populated(cs, NULL)); in tasks_nocpu_error()
1227 static void reset_partition_data(struct cpuset *cs) in reset_partition_data() argument
1229 struct cpuset *parent = parent_cs(cs); in reset_partition_data()
1236 cs->nr_subparts = 0; in reset_partition_data()
1237 if (cpumask_empty(cs->exclusive_cpus)) { in reset_partition_data()
1238 cpumask_clear(cs->effective_xcpus); in reset_partition_data()
1239 if (is_cpu_exclusive(cs)) in reset_partition_data()
1240 clear_bit(CS_CPU_EXCLUSIVE, &cs->flags); in reset_partition_data()
1242 if (!cpumask_and(cs->effective_cpus, parent->effective_cpus, cs->cpus_allowed)) in reset_partition_data()
1243 cpumask_copy(cs->effective_cpus, parent->effective_cpus); in reset_partition_data()
1351 * @cs: cpuset
1358 static bool compute_effective_exclusive_cpumask(struct cpuset *cs, in compute_effective_exclusive_cpumask() argument
1361 struct cpuset *parent = parent_cs(cs); in compute_effective_exclusive_cpumask()
1364 xcpus = cs->effective_xcpus; in compute_effective_exclusive_cpumask()
1366 return cpumask_and(xcpus, user_xcpus(cs), parent->effective_xcpus); in compute_effective_exclusive_cpumask()
1369 static inline bool is_remote_partition(struct cpuset *cs) in is_remote_partition() argument
1371 return !list_empty(&cs->remote_sibling); in is_remote_partition()
1374 static inline bool is_local_partition(struct cpuset *cs) in is_local_partition() argument
1376 return is_partition_valid(cs) && !is_remote_partition(cs); in is_local_partition()
1381 * @cs: the cpuset to update
1389 static int remote_partition_enable(struct cpuset *cs, int new_prs, in remote_partition_enable() argument
1408 compute_effective_exclusive_cpumask(cs, tmp->new_cpus); in remote_partition_enable()
1416 list_add(&cs->remote_sibling, &remote_children); in remote_partition_enable()
1419 cs->prs_err = 0; in remote_partition_enable()
1431 * @cs: the cpuset to update
1438 static void remote_partition_disable(struct cpuset *cs, struct tmpmasks *tmp) in remote_partition_disable() argument
1442 compute_effective_exclusive_cpumask(cs, tmp->new_cpus); in remote_partition_disable()
1443 WARN_ON_ONCE(!is_remote_partition(cs)); in remote_partition_disable()
1447 list_del_init(&cs->remote_sibling); in remote_partition_disable()
1448 isolcpus_updated = partition_xcpus_del(cs->partition_root_state, in remote_partition_disable()
1450 if (cs->prs_err) in remote_partition_disable()
1451 cs->partition_root_state = -cs->partition_root_state; in remote_partition_disable()
1453 cs->partition_root_state = PRS_MEMBER; in remote_partition_disable()
1455 reset_partition_data(cs); in remote_partition_disable()
1468 * @cs: the cpuset to be updated
1475 static void remote_cpus_update(struct cpuset *cs, struct cpumask *newmask, in remote_cpus_update() argument
1479 int prs = cs->partition_root_state; in remote_cpus_update()
1482 if (WARN_ON_ONCE(!is_remote_partition(cs))) in remote_cpus_update()
1485 WARN_ON_ONCE(!cpumask_subset(cs->effective_xcpus, subpartitions_cpus)); in remote_cpus_update()
1488 cs->prs_err = PERR_CPUSEMPTY; in remote_cpus_update()
1492 adding = cpumask_andnot(tmp->addmask, newmask, cs->effective_xcpus); in remote_cpus_update()
1493 deleting = cpumask_andnot(tmp->delmask, cs->effective_xcpus, newmask); in remote_cpus_update()
1502 cs->prs_err = PERR_ACCESS; in remote_cpus_update()
1505 cs->prs_err = PERR_NOCPUS; in remote_cpus_update()
1506 if (cs->prs_err) in remote_cpus_update()
1526 remote_partition_disable(cs, tmp); in remote_cpus_update()
1531 * @cs: the cpuset to be updated
1536 * This should be called before the given cs has updated its cpus_allowed
1539 static void remote_partition_check(struct cpuset *cs, struct cpumask *newmask, in remote_partition_check() argument
1548 if (!cpumask_andnot(delmask, cs->effective_xcpus, newmask) || in remote_partition_check()
1592 * @cs: The cpuset that requests change in partition root state
1627 static int update_parent_effective_cpumask(struct cpuset *cs, int cmd, in update_parent_effective_cpumask() argument
1631 struct cpuset *parent = parent_cs(cs); in update_parent_effective_cpumask()
1637 struct cpumask *xcpus; /* cs effective_xcpus */ in update_parent_effective_cpumask()
1648 old_prs = new_prs = cs->partition_root_state; in update_parent_effective_cpumask()
1649 xcpus = user_xcpus(cs); in update_parent_effective_cpumask()
1677 if (!newmask && xcpus_empty(cs)) in update_parent_effective_cpumask()
1680 nocpu = tasks_nocpu_error(parent, cs, xcpus); in update_parent_effective_cpumask()
1724 /* Check newmask again, whether cpus are available for parent/cs */ in update_parent_effective_cpumask()
1725 nocpu |= tasks_nocpu_error(parent, cs, newmask); in update_parent_effective_cpumask()
1784 if (is_partition_valid(cs)) in update_parent_effective_cpumask()
1787 } else if (is_partition_invalid(cs) && in update_parent_effective_cpumask()
1799 if (child == cs) in update_parent_effective_cpumask()
1801 if (!cpusets_are_exclusive(cs, child)) { in update_parent_effective_cpumask()
1817 WRITE_ONCE(cs->prs_err, part_error); in update_parent_effective_cpumask()
1824 switch (cs->partition_root_state) { in update_parent_effective_cpumask()
1849 * CPU lists in cs haven't been updated yet. So defer it to later. in update_parent_effective_cpumask()
1852 int err = update_partition_exclusive(cs, new_prs); in update_parent_effective_cpumask()
1867 cs->partition_root_state = new_prs; in update_parent_effective_cpumask()
1869 cs->nr_subparts = 0; in update_parent_effective_cpumask()
1872 * Adding to parent's effective_cpus means deletion CPUs from cs in update_parent_effective_cpumask()
1890 update_partition_exclusive(cs, new_prs); in update_parent_effective_cpumask()
1894 update_sibling_cpumasks(parent, cs, tmp); in update_parent_effective_cpumask()
1903 update_partition_sd_lb(cs, old_prs); in update_parent_effective_cpumask()
1905 notify_partition_change(cs, old_prs); in update_parent_effective_cpumask()
1911 * @cs: partition root cpuset
1925 static void compute_partition_effective_cpumask(struct cpuset *cs, in compute_partition_effective_cpumask() argument
1930 bool populated = partition_is_populated(cs, NULL); in compute_partition_effective_cpumask()
1940 compute_effective_exclusive_cpumask(cs, new_ecpus); in compute_partition_effective_cpumask()
1944 cpuset_for_each_child(child, css, cs) { in compute_partition_effective_cpumask()
1950 cs->effective_xcpus)) in compute_partition_effective_cpumask()
1964 cs->nr_subparts--; in compute_partition_effective_cpumask()
1978 * @cs: the cpuset to consider
1989 static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp, in update_cpumasks_hier() argument
1998 cpuset_for_each_descendant_pre(cp, pos_css, cs) { in update_cpumasks_hier()
2005 * directly from top cpuset unless it is cs. in update_cpumasks_hier()
2007 if (remote && (cp != cs)) { in update_cpumasks_hier()
2016 if (!cpumask_empty(cp->exclusive_cpus) && (cp != cs)) { in update_cpumasks_hier()
2069 * for cs already in update_cpumask(). We should also call in update_cpumasks_hier()
2073 if ((cp != cs) && old_prs) { in update_cpumasks_hier()
2165 * @cs: Current cpuset
2168 static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs, in update_sibling_cpumasks() argument
2190 if (sibling == cs) in update_sibling_cpumasks()
2211 * @cs: the cpuset to consider
2215 static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs, in update_cpumask() argument
2220 struct cpuset *parent = parent_cs(cs); in update_cpumask()
2223 int old_prs = cs->partition_root_state; in update_cpumask()
2226 if (cs == &top_cpuset) in update_cpumask()
2254 if (!cpumask_empty(trialcs->exclusive_cpus) || is_partition_valid(cs)) in update_cpumask()
2259 if (cpumask_equal(cs->cpus_allowed, trialcs->cpus_allowed)) in update_cpumask()
2266 if (is_partition_valid(cs) && in update_cpumask()
2269 cs->prs_err = PERR_INVCPUS; in update_cpumask()
2272 cs->prs_err = PERR_HKEEPING; in update_cpumask()
2273 } else if (tasks_nocpu_error(parent, cs, trialcs->effective_xcpus)) { in update_cpumask()
2275 cs->prs_err = PERR_NOCPUS; in update_cpumask()
2283 force = !cpumask_equal(cs->effective_xcpus, trialcs->effective_xcpus); in update_cpumask()
2285 retval = validate_change(cs, trialcs); in update_cpumask()
2317 if (is_partition_valid(cs) || in update_cpumask()
2318 (is_partition_invalid(cs) && !invalidate)) { in update_cpumask()
2321 if (cpumask_empty(xcpus) && is_partition_invalid(cs)) in update_cpumask()
2327 if (is_remote_partition(cs)) in update_cpumask()
2328 remote_cpus_update(cs, xcpus, &tmp); in update_cpumask()
2330 update_parent_effective_cpumask(cs, partcmd_invalidate, in update_cpumask()
2333 update_parent_effective_cpumask(cs, partcmd_update, in update_cpumask()
2335 } else if (!cpumask_empty(cs->exclusive_cpus)) { in update_cpumask()
2339 remote_partition_check(cs, trialcs->effective_xcpus, in update_cpumask()
2344 cpumask_copy(cs->cpus_allowed, trialcs->cpus_allowed); in update_cpumask()
2345 cpumask_copy(cs->effective_xcpus, trialcs->effective_xcpus); in update_cpumask()
2346 if ((old_prs > 0) && !is_partition_valid(cs)) in update_cpumask()
2347 reset_partition_data(cs); in update_cpumask()
2351 update_cpumasks_hier(cs, &tmp, force); in update_cpumask()
2354 if (cs->partition_root_state) in update_cpumask()
2355 update_partition_sd_lb(cs, old_prs); in update_cpumask()
2363 * @cs: the cpuset to consider
2367 * The tasks' cpumask will be updated if cs is a valid partition root.
2369 static int update_exclusive_cpumask(struct cpuset *cs, struct cpuset *trialcs, in update_exclusive_cpumask() argument
2374 struct cpuset *parent = parent_cs(cs); in update_exclusive_cpumask()
2377 int old_prs = cs->partition_root_state; in update_exclusive_cpumask()
2389 if (cpumask_equal(cs->exclusive_cpus, trialcs->exclusive_cpus)) in update_exclusive_cpumask()
2399 force = !cpumask_equal(cs->effective_xcpus, trialcs->effective_xcpus); in update_exclusive_cpumask()
2401 retval = validate_change(cs, trialcs); in update_exclusive_cpumask()
2411 cs->prs_err = PERR_INVCPUS; in update_exclusive_cpumask()
2414 cs->prs_err = PERR_HKEEPING; in update_exclusive_cpumask()
2415 } else if (tasks_nocpu_error(parent, cs, trialcs->effective_xcpus)) { in update_exclusive_cpumask()
2417 cs->prs_err = PERR_NOCPUS; in update_exclusive_cpumask()
2420 if (is_remote_partition(cs)) { in update_exclusive_cpumask()
2422 remote_partition_disable(cs, &tmp); in update_exclusive_cpumask()
2424 remote_cpus_update(cs, trialcs->effective_xcpus, in update_exclusive_cpumask()
2427 update_parent_effective_cpumask(cs, partcmd_invalidate, in update_exclusive_cpumask()
2430 update_parent_effective_cpumask(cs, partcmd_update, in update_exclusive_cpumask()
2437 remote_partition_check(cs, trialcs->effective_xcpus, in update_exclusive_cpumask()
2441 cpumask_copy(cs->exclusive_cpus, trialcs->exclusive_cpus); in update_exclusive_cpumask()
2442 cpumask_copy(cs->effective_xcpus, trialcs->effective_xcpus); in update_exclusive_cpumask()
2443 if ((old_prs > 0) && !is_partition_valid(cs)) in update_exclusive_cpumask()
2444 reset_partition_data(cs); in update_exclusive_cpumask()
2452 if (is_partition_valid(cs) || force) in update_exclusive_cpumask()
2453 update_cpumasks_hier(cs, &tmp, force); in update_exclusive_cpumask()
2456 if (cs->partition_root_state) in update_exclusive_cpumask()
2457 update_partition_sd_lb(cs, old_prs); in update_exclusive_cpumask()
2548 * @cs: the cpuset in which each task's mems_allowed mask needs to be changed
2550 * Iterate through each task of @cs updating its mems_allowed to the
2554 void cpuset_update_tasks_nodemask(struct cpuset *cs) in cpuset_update_tasks_nodemask() argument
2560 cpuset_being_rebound = cs; /* causes mpol_dup() rebind */ in cpuset_update_tasks_nodemask()
2562 guarantee_online_mems(cs, &newmems); in cpuset_update_tasks_nodemask()
2574 css_task_iter_start(&cs->css, 0, &it); in cpuset_update_tasks_nodemask()
2585 migrate = is_memory_migrate(cs); in cpuset_update_tasks_nodemask()
2587 mpol_rebind_mm(mm, &cs->mems_allowed); in cpuset_update_tasks_nodemask()
2589 cpuset_migrate_mm(mm, &cs->old_mems_allowed, &newmems); in cpuset_update_tasks_nodemask()
2597 * cs->old_mems_allowed. in cpuset_update_tasks_nodemask()
2599 cs->old_mems_allowed = newmems; in cpuset_update_tasks_nodemask()
2607 * @cs: the cpuset to consider
2617 static void update_nodemasks_hier(struct cpuset *cs, nodemask_t *new_mems) in update_nodemasks_hier() argument
2623 cpuset_for_each_descendant_pre(cp, pos_css, cs) { in update_nodemasks_hier()
2669 * Will take tasklist_lock, scan tasklist for tasks in cpuset cs,
2673 static int update_nodemask(struct cpuset *cs, struct cpuset *trialcs, in update_nodemask() argument
2682 if (cs == &top_cpuset) { in update_nodemask()
2707 if (nodes_equal(cs->mems_allowed, trialcs->mems_allowed)) { in update_nodemask()
2711 retval = validate_change(cs, trialcs); in update_nodemask()
2718 cs->mems_allowed = trialcs->mems_allowed; in update_nodemask()
2722 update_nodemasks_hier(cs, &trialcs->mems_allowed); in update_nodemask()
2741 * cs: the cpuset to update
2747 int cpuset_update_flag(cpuset_flagbits_t bit, struct cpuset *cs, in cpuset_update_flag() argument
2755 trialcs = alloc_trial_cpuset(cs); in cpuset_update_flag()
2764 err = validate_change(cs, trialcs); in cpuset_update_flag()
2768 balance_flag_changed = (is_sched_load_balance(cs) != in cpuset_update_flag()
2771 spread_flag_changed = ((is_spread_slab(cs) != is_spread_slab(trialcs)) in cpuset_update_flag()
2772 || (is_spread_page(cs) != is_spread_page(trialcs))); in cpuset_update_flag()
2775 cs->flags = trialcs->flags; in cpuset_update_flag()
2786 cpuset1_update_tasks_flags(cs); in cpuset_update_flag()
2794 * @cs: the cpuset to update
2800 static int update_prstate(struct cpuset *cs, int new_prs) in update_prstate() argument
2802 int err = PERR_NONE, old_prs = cs->partition_root_state; in update_prstate()
2803 struct cpuset *parent = parent_cs(cs); in update_prstate()
2823 if ((new_prs > 0) && cpumask_empty(cs->exclusive_cpus)) { in update_prstate()
2825 cpumask_and(cs->effective_xcpus, in update_prstate()
2826 cs->cpus_allowed, parent->effective_xcpus); in update_prstate()
2830 err = update_partition_exclusive(cs, new_prs); in update_prstate()
2838 if (xcpus_empty(cs)) { in update_prstate()
2851 err = update_parent_effective_cpumask(cs, cmd, NULL, &tmpmask); in update_prstate()
2853 err = remote_partition_enable(cs, new_prs, &tmpmask); in update_prstate()
2865 if (is_remote_partition(cs)) in update_prstate()
2866 remote_partition_disable(cs, &tmpmask); in update_prstate()
2868 update_parent_effective_cpumask(cs, partcmd_disable, in update_prstate()
2883 update_partition_exclusive(cs, new_prs); in update_prstate()
2887 cs->partition_root_state = new_prs; in update_prstate()
2888 WRITE_ONCE(cs->prs_err, err); in update_prstate()
2889 if (!is_partition_valid(cs)) in update_prstate()
2890 reset_partition_data(cs); in update_prstate()
2892 partition_xcpus_newstate(old_prs, new_prs, cs->effective_xcpus); in update_prstate()
2897 update_cpumasks_hier(cs, &tmpmask, !new_prs); in update_prstate()
2900 update_partition_sd_lb(cs, old_prs); in update_prstate()
2902 notify_partition_change(cs, old_prs); in update_prstate()
2917 static int cpuset_can_attach_check(struct cpuset *cs) in cpuset_can_attach_check() argument
2919 if (cpumask_empty(cs->effective_cpus) || in cpuset_can_attach_check()
2920 (!is_in_v2_mode() && nodes_empty(cs->mems_allowed))) in cpuset_can_attach_check()
2925 static void reset_migrate_dl_data(struct cpuset *cs) in reset_migrate_dl_data() argument
2927 cs->nr_migrate_dl_tasks = 0; in reset_migrate_dl_data()
2928 cs->sum_migrate_dl_bw = 0; in reset_migrate_dl_data()
2935 struct cpuset *cs, *oldcs; in cpuset_can_attach() local
2943 cs = css_cs(css); in cpuset_can_attach()
2948 ret = cpuset_can_attach_check(cs); in cpuset_can_attach()
2952 cpus_updated = !cpumask_equal(cs->effective_cpus, oldcs->effective_cpus); in cpuset_can_attach()
2953 mems_updated = !nodes_equal(cs->effective_mems, oldcs->effective_mems); in cpuset_can_attach()
2972 cs->nr_migrate_dl_tasks++; in cpuset_can_attach()
2973 cs->sum_migrate_dl_bw += task->dl.dl_bw; in cpuset_can_attach()
2977 if (!cs->nr_migrate_dl_tasks) in cpuset_can_attach()
2980 if (!cpumask_intersects(oldcs->effective_cpus, cs->effective_cpus)) { in cpuset_can_attach()
2981 int cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus); in cpuset_can_attach()
2984 reset_migrate_dl_data(cs); in cpuset_can_attach()
2989 ret = dl_bw_alloc(cpu, cs->sum_migrate_dl_bw); in cpuset_can_attach()
2991 reset_migrate_dl_data(cs); in cpuset_can_attach()
3001 cs->attach_in_progress++; in cpuset_can_attach()
3010 struct cpuset *cs; in cpuset_cancel_attach() local
3013 cs = css_cs(css); in cpuset_cancel_attach()
3016 dec_attach_in_progress_locked(cs); in cpuset_cancel_attach()
3018 if (cs->nr_migrate_dl_tasks) { in cpuset_cancel_attach()
3019 int cpu = cpumask_any(cs->effective_cpus); in cpuset_cancel_attach()
3021 dl_bw_free(cpu, cs->sum_migrate_dl_bw); in cpuset_cancel_attach()
3022 reset_migrate_dl_data(cs); in cpuset_cancel_attach()
3036 static void cpuset_attach_task(struct cpuset *cs, struct task_struct *task) in cpuset_attach_task() argument
3040 if (cs != &top_cpuset) in cpuset_attach_task()
3052 cpuset1_update_task_spread_flags(cs, task); in cpuset_attach_task()
3060 struct cpuset *cs; in cpuset_attach() local
3065 cs = css_cs(css); in cpuset_attach()
3069 cpus_updated = !cpumask_equal(cs->effective_cpus, in cpuset_attach()
3071 mems_updated = !nodes_equal(cs->effective_mems, oldcs->effective_mems); in cpuset_attach()
3080 cpuset_attach_nodemask_to = cs->effective_mems; in cpuset_attach()
3084 guarantee_online_mems(cs, &cpuset_attach_nodemask_to); in cpuset_attach()
3087 cpuset_attach_task(cs, task); in cpuset_attach()
3095 cpuset_attach_nodemask_to = cs->effective_mems; in cpuset_attach()
3096 if (!is_memory_migrate(cs) && !mems_updated) in cpuset_attach()
3113 if (is_memory_migrate(cs)) in cpuset_attach()
3122 cs->old_mems_allowed = cpuset_attach_nodemask_to; in cpuset_attach()
3124 if (cs->nr_migrate_dl_tasks) { in cpuset_attach()
3125 cs->nr_deadline_tasks += cs->nr_migrate_dl_tasks; in cpuset_attach()
3126 oldcs->nr_deadline_tasks -= cs->nr_migrate_dl_tasks; in cpuset_attach()
3127 reset_migrate_dl_data(cs); in cpuset_attach()
3130 dec_attach_in_progress_locked(cs); in cpuset_attach()
3141 struct cpuset *cs = css_cs(of_css(of)); in cpuset_write_resmask() local
3148 if (!is_cpuset_online(cs)) in cpuset_write_resmask()
3151 trialcs = alloc_trial_cpuset(cs); in cpuset_write_resmask()
3159 retval = update_cpumask(cs, trialcs, buf); in cpuset_write_resmask()
3162 retval = update_exclusive_cpumask(cs, trialcs, buf); in cpuset_write_resmask()
3165 retval = update_nodemask(cs, trialcs, buf); in cpuset_write_resmask()
3192 struct cpuset *cs = css_cs(seq_css(sf)); in cpuset_common_seq_show() local
3200 seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->cpus_allowed)); in cpuset_common_seq_show()
3203 seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->mems_allowed)); in cpuset_common_seq_show()
3206 seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->effective_cpus)); in cpuset_common_seq_show()
3209 seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->effective_mems)); in cpuset_common_seq_show()
3212 seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->exclusive_cpus)); in cpuset_common_seq_show()
3215 seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->effective_xcpus)); in cpuset_common_seq_show()
3233 struct cpuset *cs = css_cs(seq_css(seq)); in sched_partition_show() local
3236 switch (cs->partition_root_state) { in sched_partition_show()
3252 err = perr_strings[READ_ONCE(cs->prs_err)]; in sched_partition_show()
3265 struct cpuset *cs = css_cs(of_css(of)); in sched_partition_write() local
3280 css_get(&cs->css); in sched_partition_write()
3283 if (!is_cpuset_online(cs)) in sched_partition_write()
3286 retval = update_prstate(cs, val); in sched_partition_write()
3290 css_put(&cs->css); in sched_partition_write()
3384 struct cpuset *cs; in cpuset_css_alloc() local
3389 cs = kzalloc(sizeof(*cs), GFP_KERNEL); in cpuset_css_alloc()
3390 if (!cs) in cpuset_css_alloc()
3393 if (alloc_cpumasks(cs, NULL)) { in cpuset_css_alloc()
3394 kfree(cs); in cpuset_css_alloc()
3398 __set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); in cpuset_css_alloc()
3399 fmeter_init(&cs->fmeter); in cpuset_css_alloc()
3400 cs->relax_domain_level = -1; in cpuset_css_alloc()
3401 INIT_LIST_HEAD(&cs->remote_sibling); in cpuset_css_alloc()
3405 __set_bit(CS_MEMORY_MIGRATE, &cs->flags); in cpuset_css_alloc()
3407 return &cs->css; in cpuset_css_alloc()
3412 struct cpuset *cs = css_cs(css); in cpuset_css_online() local
3413 struct cpuset *parent = parent_cs(cs); in cpuset_css_online()
3423 set_bit(CS_ONLINE, &cs->flags); in cpuset_css_online()
3425 set_bit(CS_SPREAD_PAGE, &cs->flags); in cpuset_css_online()
3427 set_bit(CS_SPREAD_SLAB, &cs->flags); in cpuset_css_online()
3432 clear_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); in cpuset_css_online()
3438 cpumask_copy(cs->effective_cpus, parent->effective_cpus); in cpuset_css_online()
3439 cs->effective_mems = parent->effective_mems; in cpuset_css_online()
3469 cs->mems_allowed = parent->mems_allowed; in cpuset_css_online()
3470 cs->effective_mems = parent->mems_allowed; in cpuset_css_online()
3471 cpumask_copy(cs->cpus_allowed, parent->cpus_allowed); in cpuset_css_online()
3472 cpumask_copy(cs->effective_cpus, parent->cpus_allowed); in cpuset_css_online()
3493 struct cpuset *cs = css_cs(css); in cpuset_css_offline() local
3498 if (!cpuset_v2() && is_sched_load_balance(cs)) in cpuset_css_offline()
3499 cpuset_update_flag(CS_SCHED_LOAD_BALANCE, cs, 0); in cpuset_css_offline()
3502 clear_bit(CS_ONLINE, &cs->flags); in cpuset_css_offline()
3510 struct cpuset *cs = css_cs(css); in cpuset_css_killed() local
3516 if (is_partition_valid(cs)) in cpuset_css_killed()
3517 update_prstate(cs, PRS_MEMBER); in cpuset_css_killed()
3526 struct cpuset *cs = css_cs(css); in cpuset_css_free() local
3528 free_cpuset(cs); in cpuset_css_free()
3556 struct cpuset *cs = css_cs(cset->subsys[cpuset_cgrp_id]); in cpuset_can_fork() local
3561 same_cs = (cs == task_cs(current)); in cpuset_can_fork()
3571 ret = cpuset_can_attach_check(cs); in cpuset_can_fork()
3587 cs->attach_in_progress++; in cpuset_can_fork()
3595 struct cpuset *cs = css_cs(cset->subsys[cpuset_cgrp_id]); in cpuset_cancel_fork() local
3599 same_cs = (cs == task_cs(current)); in cpuset_cancel_fork()
3605 dec_attach_in_progress(cs); in cpuset_cancel_fork()
3615 struct cpuset *cs; in cpuset_fork() local
3619 cs = task_cs(task); in cpuset_fork()
3620 same_cs = (cs == task_cs(current)); in cpuset_fork()
3624 if (cs == &top_cpuset) in cpuset_fork()
3634 guarantee_online_mems(cs, &cpuset_attach_nodemask_to); in cpuset_fork()
3635 cpuset_attach_task(cs, task); in cpuset_fork()
3637 dec_attach_in_progress_locked(cs); in cpuset_fork()
3701 hotplug_update_tasks(struct cpuset *cs, in hotplug_update_tasks() argument
3706 if (cpumask_empty(new_cpus) && !is_partition_valid(cs)) in hotplug_update_tasks()
3707 cpumask_copy(new_cpus, parent_cs(cs)->effective_cpus); in hotplug_update_tasks()
3709 *new_mems = parent_cs(cs)->effective_mems; in hotplug_update_tasks()
3712 cpumask_copy(cs->effective_cpus, new_cpus); in hotplug_update_tasks()
3713 cs->effective_mems = *new_mems; in hotplug_update_tasks()
3717 cpuset_update_tasks_cpumask(cs, new_cpus); in hotplug_update_tasks()
3719 cpuset_update_tasks_nodemask(cs); in hotplug_update_tasks()
3729 * @cs: cpuset in interest
3732 * Compare @cs's cpu and mem masks against top_cpuset and if some have gone
3733 * offline, update @cs accordingly. If @cs ends up with no CPU or memory,
3736 static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp) in cpuset_hotplug_update_tasks() argument
3746 wait_event(cpuset_attach_wq, cs->attach_in_progress == 0); in cpuset_hotplug_update_tasks()
3754 if (cs->attach_in_progress) { in cpuset_hotplug_update_tasks()
3759 parent = parent_cs(cs); in cpuset_hotplug_update_tasks()
3760 compute_effective_cpumask(&new_cpus, cs, parent); in cpuset_hotplug_update_tasks()
3761 nodes_and(new_mems, cs->mems_allowed, parent->effective_mems); in cpuset_hotplug_update_tasks()
3763 if (!tmp || !cs->partition_root_state) in cpuset_hotplug_update_tasks()
3770 remote = is_remote_partition(cs); in cpuset_hotplug_update_tasks()
3771 if (remote || (is_partition_valid(cs) && is_partition_valid(parent))) in cpuset_hotplug_update_tasks()
3772 compute_partition_effective_cpumask(cs, &new_cpus); in cpuset_hotplug_update_tasks()
3775 partition_is_populated(cs, NULL)) { in cpuset_hotplug_update_tasks()
3776 cs->prs_err = PERR_HOTPLUG; in cpuset_hotplug_update_tasks()
3777 remote_partition_disable(cs, tmp); in cpuset_hotplug_update_tasks()
3778 compute_effective_cpumask(&new_cpus, cs, parent); in cpuset_hotplug_update_tasks()
3790 if (is_local_partition(cs) && (!is_partition_valid(parent) || in cpuset_hotplug_update_tasks()
3791 tasks_nocpu_error(parent, cs, &new_cpus))) in cpuset_hotplug_update_tasks()
3797 else if (is_partition_valid(parent) && is_partition_invalid(cs)) in cpuset_hotplug_update_tasks()
3801 update_parent_effective_cpumask(cs, partcmd, NULL, tmp); in cpuset_hotplug_update_tasks()
3802 if ((partcmd == partcmd_invalidate) || is_partition_valid(cs)) { in cpuset_hotplug_update_tasks()
3803 compute_partition_effective_cpumask(cs, &new_cpus); in cpuset_hotplug_update_tasks()
3809 cpus_updated = !cpumask_equal(&new_cpus, cs->effective_cpus); in cpuset_hotplug_update_tasks()
3810 mems_updated = !nodes_equal(new_mems, cs->effective_mems); in cpuset_hotplug_update_tasks()
3818 hotplug_update_tasks(cs, &new_cpus, &new_mems, in cpuset_hotplug_update_tasks()
3821 cpuset1_hotplug_update_tasks(cs, &new_cpus, &new_mems, in cpuset_hotplug_update_tasks()
3913 struct cpuset *cs; in cpuset_handle_hotplug() local
3917 cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) { in cpuset_handle_hotplug()
3918 if (cs == &top_cpuset || !css_tryget_online(&cs->css)) in cpuset_handle_hotplug()
3922 cpuset_hotplug_update_tasks(cs, ptmp); in cpuset_handle_hotplug()
3925 css_put(&cs->css); in cpuset_handle_hotplug()
3996 struct cpuset *cs; in cpuset_cpus_allowed() local
4001 cs = task_cs(tsk); in cpuset_cpus_allowed()
4002 if (cs != &top_cpuset) in cpuset_cpus_allowed()
4009 if ((cs == &top_cpuset) || cpumask_empty(pmask)) { in cpuset_cpus_allowed()
4056 * But we used cs && cs->cpus_allowed lockless and thus can in cpuset_cpus_allowed_fallback()
4119 static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs) in nearest_hardwall_ancestor() argument
4121 while (!(is_mem_exclusive(cs) || is_mem_hardwall(cs)) && parent_cs(cs)) in nearest_hardwall_ancestor()
4122 cs = parent_cs(cs); in nearest_hardwall_ancestor()
4123 return cs; in nearest_hardwall_ancestor()
4168 struct cpuset *cs; /* current cpuset ancestors */ in cpuset_node_allowed() local
4192 cs = nearest_hardwall_ancestor(task_cs(current)); in cpuset_node_allowed()
4193 allowed = node_isset(node, cs->mems_allowed); in cpuset_node_allowed()