Lines Matching +full:cpu +full:- +full:1
1 /* SPDX-License-Identifier: GPL-2.0 */
7 * set of CPUs in a system, one bit position per CPU number. In general,
20 * cpumask_pr_args - printf args to output a cpumask
27 #if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS)
35 #if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS) in set_nr_cpu_ids()
47 * optimized routines that work for the single-word case, but only when
57 * they set bits or just don't have any faster fixed-sized versions. We
63 * optimization comes from being able to potentially use a compile-time
64 * constant instead of a run-time generated exact number of CPUs.
82 * cpu_possible_mask- has bit 'cpu' set iff cpu is populatable
83 * cpu_present_mask - has bit 'cpu' set iff cpu is populated
84 * cpu_enabled_mask - has bit 'cpu' set iff cpu can be brought online
85 * cpu_online_mask - has bit 'cpu' set iff cpu available to scheduler
86 * cpu_active_mask - has bit 'cpu' set iff cpu available to migration
90 * The cpu_possible_mask is fixed at boot time, as the set of CPU IDs
105 * 1) UP ARCHes (NR_CPUS == 1, CONFIG_SMP not defined) hardcode
106 * assumption that their single CPU is online. The UP
110 * optimization - don't waste any instructions or memory references
112 * only one CPU.
132 static __always_inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits) in cpu_max_bits_warn() argument
135 WARN_ON_ONCE(cpu >= bits); in cpu_max_bits_warn()
139 /* verify cpu argument to cpumask_* operators */
140 static __always_inline unsigned int cpumask_check(unsigned int cpu) in cpumask_check() argument
142 cpu_max_bits_warn(cpu, small_cpumask_bits); in cpumask_check()
143 return cpu; in cpumask_check()
147 * cpumask_first - get the first cpu in a cpumask
158 * cpumask_first_zero - get the first unset cpu in a cpumask
169 * cpumask_first_and - return the first cpu from *srcp1 & *srcp2
182 * cpumask_first_and_and - return the first cpu from *srcp1 & *srcp2 & *srcp3
199 * cpumask_last - get the last CPU in a cpumask
200 * @srcp: - the cpumask pointer
210 * cpumask_next - get the next cpu in a cpumask
211 * @n: the cpu prior to the place to search (i.e. return will be > @n)
219 /* -1 is a legal arg here. */ in cpumask_next()
220 if (n != -1) in cpumask_next()
222 return find_next_bit(cpumask_bits(srcp), small_cpumask_bits, n + 1); in cpumask_next()
226 * cpumask_next_zero - get the next unset cpu in a cpumask
227 * @n: the cpu prior to the place to search (i.e. return will be > @n)
235 /* -1 is a legal arg here. */ in cpumask_next_zero()
236 if (n != -1) in cpumask_next_zero()
238 return find_next_zero_bit(cpumask_bits(srcp), small_cpumask_bits, n+1); in cpumask_next_zero()
241 #if NR_CPUS == 1
242 /* Uniprocessor: there is only one valid CPU */
269 * cpumask_next_and - get the next cpu in *src1p & *src2p
270 * @n: the cpu prior to the place to search (i.e. return will be > @n)
280 /* -1 is a legal arg here. */ in cpumask_next_and()
281 if (n != -1) in cpumask_next_and()
284 small_cpumask_bits, n + 1); in cpumask_next_and()
288 * for_each_cpu - iterate over every cpu in a mask
289 * @cpu: the (optionally unsigned) integer iterator
292 * After the loop, cpu is >= nr_cpu_ids.
294 #define for_each_cpu(cpu, mask) \ argument
295 for_each_set_bit(cpu, cpumask_bits(mask), small_cpumask_bits)
297 #if NR_CPUS == 1
302 if (n != -1) in cpumask_next_wrap()
306 * Return the first available CPU when wrapping, or when starting before cpu0, in cpumask_next_wrap()
319 * for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location
320 * @cpu: the (optionally unsigned) integer iterator
326 * After the loop, cpu is >= nr_cpu_ids.
328 #define for_each_cpu_wrap(cpu, mask, start) \ argument
329 for_each_set_bit_wrap(cpu, cpumask_bits(mask), small_cpumask_bits, start)
332 * for_each_cpu_and - iterate over every cpu in both masks
333 * @cpu: the (optionally unsigned) integer iterator
337 * This saves a temporary CPU mask in many places. It is equivalent to:
340 * for_each_cpu(cpu, &tmp)
343 * After the loop, cpu is >= nr_cpu_ids.
345 #define for_each_cpu_and(cpu, mask1, mask2) \ argument
346 for_each_and_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
349 * for_each_cpu_andnot - iterate over every cpu present in one mask, excluding
351 * @cpu: the (optionally unsigned) integer iterator
355 * This saves a temporary CPU mask in many places. It is equivalent to:
358 * for_each_cpu(cpu, &tmp)
361 * After the loop, cpu is >= nr_cpu_ids.
363 #define for_each_cpu_andnot(cpu, mask1, mask2) \ argument
364 for_each_andnot_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
367 * for_each_cpu_or - iterate over every cpu present in either mask
368 * @cpu: the (optionally unsigned) integer iterator
372 * This saves a temporary CPU mask in many places. It is equivalent to:
375 * for_each_cpu(cpu, &tmp)
378 * After the loop, cpu is >= nr_cpu_ids.
380 #define for_each_cpu_or(cpu, mask1, mask2) \ argument
381 for_each_or_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
384 * for_each_cpu_from - iterate over CPUs present in @mask, from @cpu to the end of @mask.
385 * @cpu: the (optionally unsigned) integer iterator
388 * After the loop, cpu is >= nr_cpu_ids.
390 #define for_each_cpu_from(cpu, mask) \ argument
391 for_each_set_bit_from(cpu, cpumask_bits(mask), small_cpumask_bits)
394 * cpumask_any_but - return an arbitrary cpu in a cpumask, but not this one.
396 * @cpu: the cpu to ignore.
398 * Often used to find any cpu but smp_processor_id() in a mask.
402 unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu) in cpumask_any_but() argument
406 cpumask_check(cpu); in cpumask_any_but()
408 if (i != cpu) in cpumask_any_but()
414 * cpumask_any_and_but - pick an arbitrary cpu from *mask1 & *mask2, but not this one.
417 * @cpu: the cpu to ignore
424 unsigned int cpu) in cpumask_any_and_but()
428 cpumask_check(cpu); in cpumask_any_and_but()
430 if (i != cpu) in cpumask_any_and_but()
433 return cpumask_next_and(cpu, mask1, mask2); in cpumask_any_and_but()
437 * cpumask_nth - get the Nth cpu in a cpumask
439 * @cpu: the Nth cpu to find, starting from 0
441 * Return: >= nr_cpu_ids if such cpu doesn't exist.
444 unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp) in cpumask_nth() argument
446 return find_nth_bit(cpumask_bits(srcp), small_cpumask_bits, cpumask_check(cpu)); in cpumask_nth()
450 * cpumask_nth_and - get the Nth cpu in 2 cpumasks
453 * @cpu: the Nth cpu to find, starting from 0
455 * Return: >= nr_cpu_ids if such cpu doesn't exist.
458 unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1, in cpumask_nth_and() argument
462 small_cpumask_bits, cpumask_check(cpu)); in cpumask_nth_and()
466 * cpumask_nth_andnot - get the Nth cpu set in 1st cpumask, and clear in 2nd.
469 * @cpu: the Nth cpu to find, starting from 0
471 * Return: >= nr_cpu_ids if such cpu doesn't exist.
474 unsigned int cpumask_nth_andnot(unsigned int cpu, const struct cpumask *srcp1, in cpumask_nth_andnot() argument
478 small_cpumask_bits, cpumask_check(cpu)); in cpumask_nth_andnot()
482 * cpumask_nth_and_andnot - get the Nth cpu set in 1st and 2nd cpumask, and clear in 3rd.
486 * @cpu: the Nth cpu to find, starting from 0
488 * Return: >= nr_cpu_ids if such cpu doesn't exist.
491 unsigned int cpumask_nth_and_andnot(unsigned int cpu, const struct cpumask *srcp1, in cpumask_nth_and_andnot() argument
498 small_cpumask_bits, cpumask_check(cpu)); in cpumask_nth_and_andnot()
503 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
508 [0] = 1UL \
512 * cpumask_set_cpu - set a cpu in a cpumask
513 * @cpu: cpu number (< nr_cpu_ids)
517 void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) in cpumask_set_cpu() argument
519 set_bit(cpumask_check(cpu), cpumask_bits(dstp)); in cpumask_set_cpu()
523 void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) in __cpumask_set_cpu() argument
525 __set_bit(cpumask_check(cpu), cpumask_bits(dstp)); in __cpumask_set_cpu()
530 * cpumask_clear_cpu - clear a cpu in a cpumask
531 * @cpu: cpu number (< nr_cpu_ids)
534 static __always_inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp) in cpumask_clear_cpu() argument
536 clear_bit(cpumask_check(cpu), cpumask_bits(dstp)); in cpumask_clear_cpu()
539 static __always_inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp) in __cpumask_clear_cpu() argument
541 __clear_bit(cpumask_check(cpu), cpumask_bits(dstp)); in __cpumask_clear_cpu()
545 * cpumask_assign_cpu - assign a cpu in a cpumask
546 * @cpu: cpu number (< nr_cpu_ids)
550 static __always_inline void cpumask_assign_cpu(int cpu, struct cpumask *dstp, bool value) in cpumask_assign_cpu() argument
552 assign_bit(cpumask_check(cpu), cpumask_bits(dstp), value); in cpumask_assign_cpu()
555 static __always_inline void __cpumask_assign_cpu(int cpu, struct cpumask *dstp, bool value) in __cpumask_assign_cpu() argument
557 __assign_bit(cpumask_check(cpu), cpumask_bits(dstp), value); in __cpumask_assign_cpu()
561 * cpumask_test_cpu - test for a cpu in a cpumask
562 * @cpu: cpu number (< nr_cpu_ids)
565 * Return: true if @cpu is set in @cpumask, else returns false
568 bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask) in cpumask_test_cpu() argument
570 return test_bit(cpumask_check(cpu), cpumask_bits((cpumask))); in cpumask_test_cpu()
574 * cpumask_test_and_set_cpu - atomically test and set a cpu in a cpumask
575 * @cpu: cpu number (< nr_cpu_ids)
580 * Return: true if @cpu is set in old bitmap of @cpumask, else returns false
583 bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask) in cpumask_test_and_set_cpu() argument
585 return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask)); in cpumask_test_and_set_cpu()
589 * cpumask_test_and_clear_cpu - atomically test and clear a cpu in a cpumask
590 * @cpu: cpu number (< nr_cpu_ids)
595 * Return: true if @cpu is set in old bitmap of @cpumask, else returns false
598 bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask) in cpumask_test_and_clear_cpu() argument
600 return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask)); in cpumask_test_and_clear_cpu()
604 * cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask
617 * cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask
626 * cpumask_and - *dstp = *src1p & *src2p
642 * cpumask_or - *dstp = *src1p | *src2p
656 * cpumask_xor - *dstp = *src1p ^ *src2p
670 * cpumask_andnot - *dstp = *src1p & ~*src2p
686 * cpumask_equal - *src1p == *src2p
700 * cpumask_or_equal - *src1p | *src2p == *src3p
717 * cpumask_intersects - (*src1p & *src2p) != 0
721 * Return: true if first cpumask ANDed with second cpumask is non-empty,
732 * cpumask_subset - (*src1p & ~*src2p) == 0
746 * cpumask_empty - *srcp == 0
757 * cpumask_full - *srcp == 0xFFFFFFFF...
768 * cpumask_weight - Count of bits in *srcp
779 * cpumask_weight_and - Count of bits in (*srcp1 & *srcp2)
792 * cpumask_weight_andnot - Count of bits in (*srcp1 & ~*srcp2)
806 * cpumask_shift_right - *dstp = *srcp >> n
819 * cpumask_shift_left - *dstp = *srcp << n
832 * cpumask_copy - *dstp = *srcp
843 * cpumask_any - pick an arbitrary cpu from *srcp
851 * cpumask_any_and - pick an arbitrary cpu from *mask1 & *mask2
860 * cpumask_of - the cpumask containing just a given cpu
861 * @cpu: the cpu (<= nr_cpu_ids)
863 #define cpumask_of(cpu) (get_cpu_mask(cpu)) argument
866 * cpumask_parse_user - extract a cpumask from a user string
871 * Return: -errno, or 0 for success.
880 * cpumask_parselist_user - extract a cpumask from a user string
885 * Return: -errno, or 0 for success.
895 * cpumask_parse - extract a cpumask from a string
899 * Return: -errno, or 0 for success.
907 * cpulist_parse - extract a cpumask from a user string of ranges
911 * Return: -errno, or 0 for success.
919 * cpumask_size - calculate size to allocate for a 'struct cpumask' in bytes
942 * alloc_cpumask_var - allocate a struct cpumask
947 * a nop returning a constant 1 (in <linux/cpumask.h>).
1031 #if NR_CPUS == 1
1032 /* Uniprocessor: the possible/online/present masks are always "1" */
1033 #define for_each_possible_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++) argument
1034 #define for_each_online_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++) argument
1035 #define for_each_present_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++) argument
1037 #define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask) argument
1038 #define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask) argument
1039 #define for_each_enabled_cpu(cpu) for_each_cpu((cpu), cpu_enabled_mask) argument
1040 #define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask) argument
1043 /* Wrappers for arch boot code to manipulate normally-constant masks */
1047 #define assign_cpu(cpu, mask, val) \ argument
1048 assign_bit(cpumask_check(cpu), cpumask_bits(mask), (val))
1050 #define set_cpu_possible(cpu, possible) assign_cpu((cpu), &__cpu_possible_mask, (possible)) argument
1051 #define set_cpu_enabled(cpu, enabled) assign_cpu((cpu), &__cpu_enabled_mask, (enabled)) argument
1052 #define set_cpu_present(cpu, present) assign_cpu((cpu), &__cpu_present_mask, (present)) argument
1053 #define set_cpu_active(cpu, active) assign_cpu((cpu), &__cpu_active_mask, (active)) argument
1054 #define set_cpu_dying(cpu, dying) assign_cpu((cpu), &__cpu_dying_mask, (dying)) argument
1056 void set_cpu_online(unsigned int cpu, bool online);
1059 * to_cpumask - convert a NR_CPUS bitmap to a struct cpumask *
1069 ((struct cpumask *)(1 ? (bitmap) \
1074 return 1; in __check_is_bitmap()
1078 * Special-case data structure for "single bit set only" constant CPU masks.
1080 * We pre-generate all the 64 (or 32) possible bit positions, with enough
1085 cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
1087 static __always_inline const struct cpumask *get_cpu_mask(unsigned int cpu) in get_cpu_mask() argument
1089 const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG]; in get_cpu_mask()
1090 p -= cpu / BITS_PER_LONG; in get_cpu_mask()
1094 #if NR_CPUS > 1
1096 * num_online_cpus() - Read the number of online CPUs
1100 * concurrent CPU hotplug operations unless invoked from a cpuhp_lock held
1114 static __always_inline bool cpu_online(unsigned int cpu) in cpu_online() argument
1116 return cpumask_test_cpu(cpu, cpu_online_mask); in cpu_online()
1119 static __always_inline bool cpu_enabled(unsigned int cpu) in cpu_enabled() argument
1121 return cpumask_test_cpu(cpu, cpu_enabled_mask); in cpu_enabled()
1124 static __always_inline bool cpu_possible(unsigned int cpu) in cpu_possible() argument
1126 return cpumask_test_cpu(cpu, cpu_possible_mask); in cpu_possible()
1129 static __always_inline bool cpu_present(unsigned int cpu) in cpu_present() argument
1131 return cpumask_test_cpu(cpu, cpu_present_mask); in cpu_present()
1134 static __always_inline bool cpu_active(unsigned int cpu) in cpu_active() argument
1136 return cpumask_test_cpu(cpu, cpu_active_mask); in cpu_active()
1139 static __always_inline bool cpu_dying(unsigned int cpu) in cpu_dying() argument
1141 return cpumask_test_cpu(cpu, cpu_dying_mask); in cpu_dying()
1146 #define num_online_cpus() 1U
1147 #define num_possible_cpus() 1U
1148 #define num_enabled_cpus() 1U
1149 #define num_present_cpus() 1U
1150 #define num_active_cpus() 1U
1152 static __always_inline bool cpu_online(unsigned int cpu) in cpu_online() argument
1154 return cpu == 0; in cpu_online()
1157 static __always_inline bool cpu_possible(unsigned int cpu) in cpu_possible() argument
1159 return cpu == 0; in cpu_possible()
1162 static __always_inline bool cpu_enabled(unsigned int cpu) in cpu_enabled() argument
1164 return cpu == 0; in cpu_enabled()
1167 static __always_inline bool cpu_present(unsigned int cpu) in cpu_present() argument
1169 return cpu == 0; in cpu_present()
1172 static __always_inline bool cpu_active(unsigned int cpu) in cpu_active() argument
1174 return cpu == 0; in cpu_active()
1177 static __always_inline bool cpu_dying(unsigned int cpu) in cpu_dying() argument
1182 #endif /* NR_CPUS > 1 */
1184 #define cpu_is_offline(cpu) unlikely(!cpu_online(cpu)) argument
1189 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
1196 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
1197 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
1202 * cpumap_print_to_pagebuf - copies the cpumask into the buffer either
1203 * as comma-separated list of cpus or hex values of cpumask
1208 * Return: the length of the (null-terminated) @buf string, zero if
1219 * cpumap_print_bitmask_to_buf - copies the cpumask into the buffer as
1239 nr_cpu_ids, off, count) - 1; in cpumap_print_bitmask_to_buf()
1243 * cpumap_print_list_to_buf - copies the cpumask into the buffer as
1244 * comma-separated list of cpus
1261 nr_cpu_ids, off, count) - 1; in cpumap_print_list_to_buf()
1267 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
1272 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
1273 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
1279 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
1284 [0] = 1UL \
1292 * for cpumap NR_CPUS * 9/32 - 1 should be an exact length.
1294 * For cpulist 7 is (ceil(log10(NR_CPUS)) + 1) allowing for NR_CPUS to be up
1296 * cover a worst-case of every other cpu being on one of two nodes for a
1300 * unsigned comparison to -1.
1303 ? (NR_CPUS * 9)/32 - 1 : PAGE_SIZE)