Lines Matching +full:multi +full:- +full:tt
1 /* SPDX-License-Identifier: GPL-2.0+ */
3 * Read-Copy Update mechanism for mutual exclusion
15 * For detailed explanation of Read-Copy Update mechanism see -
34 #define ULONG_CMP_GE(a, b) (ULONG_MAX / 2 >= (a) - (b))
35 #define ULONG_CMP_LT(a, b) (ULONG_MAX / 2 < (a) - (b))
38 #define RCU_SEQ_STATE_MASK ((1 << RCU_SEQ_CTR_SHIFT) - 1)
50 // not-yet-completed RCU grace periods.
54 * same_state_synchronize_rcu - Are two old-state values identical?
55 * @oldstate1: First old-state value.
56 * @oldstate2: Second old-state value.
58 * The two old-state values must have been obtained from either
62 * are tracked by old-state values to push these values to a list header,
78 * nesting depth, but makes sense only if CONFIG_PREEMPT_RCU -- in other
81 #define rcu_preempt_depth() READ_ONCE(current->rcu_read_lock_nesting)
155 static inline int rcu_nocb_cpu_offload(int cpu) { return -EINVAL; } in rcu_nocb_cpu_offload()
164 * Note a quasi-voluntary context switch for RCU-tasks's benefit.
172 if (!(preempt) && READ_ONCE((t)->rcu_tasks_holdout)) \
173 WRITE_ONCE((t)->rcu_tasks_holdout, false); \
177 void rcu_tasks_torture_stats_print(char *tt, char *tf);
185 // Bits for ->trc_reader_special.b.need_qs field.
194 int ___rttq_nesting = READ_ONCE((t)->trc_reader_nesting); \
196 if (unlikely(READ_ONCE((t)->trc_reader_special.b.need_qs) == TRC_NEED_QS) && \
200 !READ_ONCE((t)->trc_reader_special.b.blocked)) { \
204 void rcu_tasks_trace_torture_stats_print(char *tt, char *tf);
217 void rcu_tasks_rude_torture_stats_print(char *tt, char *tf);
234 * rcu_trace_implies_rcu_gp - does an RCU Tasks Trace grace period imply an RCU grace period?
246 * cond_resched_tasks_rcu_qs - Report potential quiescent states to RCU
249 * report potential quiescent states to RCU-tasks even if the cond_resched()
259 * rcu_softirq_qs_periodic - Report RCU and RCU-Tasks quiescent states
262 * This helper is for long-running softirq handlers, such as NAPI threads in
266 * provide both RCU and RCU-Tasks quiescent states. Note that this macro
269 * Because regions of code that have disabled softirq act as RCU read-side
276 * effect because cond_resched() does not provide RCU-Tasks quiescent states.
392 * RCU_LOCKDEP_WARN - emit lockdep splat if specified condition is met
398 * and rechecks it after checking (c) to prevent false-positive splats
416 "Illegal context switch in RCU read-side critical section"); in rcu_preempt_sleep_check()
427 "Illegal context switch in RCU-bh read-side critical section"); \
429 "Illegal context switch in RCU-sched read-side critical section"); \
442 * lockdep_assert_in_rcu_read_lock - WARN if not protected by rcu_read_lock()
450 * lockdep_assert_in_rcu_read_lock_bh - WARN if not protected by rcu_read_lock_bh()
460 * lockdep_assert_in_rcu_read_lock_sched - WARN if not protected by rcu_read_lock_sched()
470 * lockdep_assert_in_rcu_reader - WARN if not within some type of RCU reader
521 * unrcu_pointer - mark a pointer as not being RCU protected
558 * RCU_INITIALIZER() - statically initialize an RCU-protected global variable
564 * rcu_assign_pointer() - assign to RCU-protected pointer
568 * Assigns the specified value to the specified RCU-protected
576 * will be dereferenced by RCU read-side code.
583 * impossible-to-diagnose memory corruption. So please be careful.
590 * macros, this execute-arguments-only-once property is important, so
606 * rcu_replace_pointer() - replace an RCU pointer, returning its old value
611 * Perform a replacement, where @rcu_ptr is an RCU-annotated
624 * rcu_access_pointer() - fetch RCU pointer with no dereferencing
627 * Return the value of the specified RCU-protected pointer, but omit the
628 * lockdep checks for being in an RCU read-side critical section. This is
630 * not dereferenced, for example, when testing an RCU-protected pointer
632 * where update-side locks prevent the value of the pointer from changing,
634 * Within an RCU read-side critical section, there is little reason to
643 * It is also permissible to use rcu_access_pointer() when read-side
647 * down multi-linked structures after a grace period has elapsed. However,
653 * rcu_dereference_check() - rcu_dereference with debug checking
661 * An implicit check for being in an RCU read-side critical section
666 * bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock));
668 * could be used to indicate to lockdep that foo->bar may only be dereferenced
670 * the bar struct at foo->bar is held.
676 * bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock) ||
677 * atomic_read(&foo->usage) == 0);
690 * rcu_dereference_bh_check() - rcu_dereference_bh with debug checking
694 * This is the RCU-bh counterpart to rcu_dereference_check(). However,
706 * rcu_dereference_sched_check() - rcu_dereference_sched with debug checking
710 * This is the RCU-sched counterpart to rcu_dereference_check().
726 * The no-tracing version of rcu_dereference_raw() must not call
733 * rcu_dereference_protected() - fetch RCU pointer when updates prevented
737 * Return the value of the specified RCU-protected pointer, but omit
738 * the READ_ONCE(). This is useful in cases where update-side locks
744 * This function is only for update-side use. Using this function
753 * rcu_dereference() - fetch RCU-protected pointer for dereferencing
761 * rcu_dereference_bh() - fetch an RCU-bh-protected pointer for dereferencing
769 * rcu_dereference_sched() - fetch RCU-sched-protected pointer for dereferencing
777 * rcu_pointer_handoff() - Hand off a pointer from RCU to other mechanism
789 * if (!atomic_inc_not_zero(p->refcnt))
799 * rcu_read_lock() - mark the beginning of an RCU read-side critical section
802 * are within RCU read-side critical sections, then the
805 * on one CPU while other CPUs are within RCU read-side critical
811 * code with interrupts or softirqs disabled. In pre-v5.0 kernels, which
816 * with new RCU read-side critical sections. One way that this can happen
818 * read-side critical section, (2) CPU 1 invokes call_rcu() to register
819 * an RCU callback, (3) CPU 0 exits the RCU read-side critical section,
820 * (4) CPU 2 enters a RCU read-side critical section, (5) the RCU
821 * callback is invoked. This is legal, because the RCU read-side critical
827 * RCU read-side critical sections may be nested. Any deferred actions
828 * will be deferred until the outermost RCU read-side critical section
833 * read-side critical section that would block in a !PREEMPTION kernel.
836 * In non-preemptible RCU implementations (pure TREE_RCU and TINY_RCU),
837 * it is illegal to block while in an RCU read-side critical section.
839 * kernel builds, RCU read-side critical sections may be preempted,
841 * implementations in real-time (with -rt patchset) kernel builds, RCU
842 * read-side critical sections may be preempted and they may also block, but
857 * a bug -- this property is what provides RCU's performance benefits.
865 * rcu_read_unlock() - marks the end of an RCU read-side critical section.
870 * also extends to the scheduler's runqueue and priority-inheritance
871 * spinlocks, courtesy of the quiescent-state deferral that is carried
886 * rcu_read_lock_bh() - mark the beginning of an RCU-bh critical section
890 * read-side critical section. However, please note that this equivalence
909 * rcu_read_unlock_bh() - marks the end of a softirq-only RCU critical section
923 * rcu_read_lock_sched() - mark the beginning of a RCU-sched critical section
926 * Read-side critical sections can also be introduced by anything else that
954 * rcu_read_unlock_sched() - marks the end of a RCU-classic critical section
975 * RCU_INIT_POINTER() - initialize an RCU protected pointer
979 * Initialize an RCU-protected pointer in special cases where readers
989 * a. You have not made *any* reader-visible changes to
997 * result in impossible-to-diagnose memory corruption. As in the structures
999 * see pre-initialized values of the referenced data structure. So
1002 * If you are creating an RCU-protected linked structure that is accessed
1003 * by a single external-to-structure RCU-protected pointer, then you may
1004 * use RCU_INIT_POINTER() to initialize the internal RCU-protected
1006 * external-to-structure pointer *after* you have completely initialized
1007 * the reader-accessible portions of the linked structure.
1019 * RCU_POINTER_INITIALIZER() - statically initialize an RCU protected pointer
1023 * GCC-style initialization for an RCU-protected pointer in a structure field.
1035 * kfree_rcu() - kfree an object after a grace period.
1036 * @ptr: pointer to kfree for double-argument invocations.
1042 * high-latency rcu_barrier() function at module-unload time.
1047 * Because the functions are not allowed in the low-order 4096 bytes of
1049 * If the offset is larger than 4095 bytes, a compile-time error will
1066 * kfree_rcu_mightsleep() - kfree an object after a grace period.
1067 * @ptr: pointer to kfree for single-argument invocations.
1069 * When it comes to head-less variant, only one argument
1077 * Please note, head-less way of freeing is permitted to
1091 kvfree_call_rcu(&((___p)->rhf), (void *) (___p)); \
1104 * Place this after a lock-acquisition primitive to guarantee that
1119 * rcu_head_init - Initialize rcu_head for rcu_head_after_call_rcu()
1130 rhp->func = (rcu_callback_t)~0L; in rcu_head_init()
1134 * rcu_head_after_call_rcu() - Has this rcu_head been passed to call_rcu()?
1143 * in an RCU read-side critical section that includes a read-side fetch
1149 rcu_callback_t func = READ_ONCE(rhp->func); in rcu_head_after_call_rcu()