125 	struct srcu_data *sdp;  in init_srcu_struct_data()  local
131 	BUILD_BUG_ON(ARRAY_SIZE(sdp->srcu_lock_count) !=  in init_srcu_struct_data()
132 		     ARRAY_SIZE(sdp->srcu_unlock_count));  in init_srcu_struct_data()
134 		sdp = per_cpu_ptr(ssp->sda, cpu);  in init_srcu_struct_data()
135 		spin_lock_init(&ACCESS_PRIVATE(sdp, lock));  in init_srcu_struct_data()
136 		rcu_segcblist_init(&sdp->srcu_cblist);  in init_srcu_struct_data()
137 		sdp->srcu_cblist_invoking = false;  in init_srcu_struct_data()
138 		sdp->srcu_gp_seq_needed = ssp->srcu_sup->srcu_gp_seq;  in init_srcu_struct_data()
139 		sdp->srcu_gp_seq_needed_exp = ssp->srcu_sup->srcu_gp_seq;  in init_srcu_struct_data()
140 		sdp->srcu_barrier_head.next = &sdp->srcu_barrier_head;  in init_srcu_struct_data()
141 		sdp->mynode = NULL;  in init_srcu_struct_data()
142 		sdp->cpu = cpu;  in init_srcu_struct_data()
143 		INIT_WORK(&sdp->work, srcu_invoke_callbacks);  in init_srcu_struct_data()
144 		timer_setup(&sdp->delay_work, srcu_delay_timer, 0);  in init_srcu_struct_data()
145 		sdp->ssp = ssp;  in init_srcu_struct_data()
171 	struct srcu_data *sdp;  in init_srcu_struct_nodes()  local
220 		sdp = per_cpu_ptr(ssp->sda, cpu);  in init_srcu_struct_nodes()
221 		sdp->mynode = &snp_first[cpu / levelspread[level]];  in init_srcu_struct_nodes()
222 		for (snp = sdp->mynode; snp != NULL; snp = snp->srcu_parent) {  in init_srcu_struct_nodes()
227 		sdp->grpmask = 1UL << (cpu - sdp->mynode->grplo);  in init_srcu_struct_nodes()
371 static void spin_lock_irqsave_sdp_contention(struct srcu_data *sdp, unsigned long *flags)  in spin_lock_irqsave_sdp_contention()  argument
373 	struct srcu_struct *ssp = sdp->ssp;  in spin_lock_irqsave_sdp_contention()
375 	if (spin_trylock_irqsave_rcu_node(sdp, *flags))  in spin_lock_irqsave_sdp_contention()
380 	spin_lock_irqsave_rcu_node(sdp, *flags);  in spin_lock_irqsave_sdp_contention()
444 		struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_readers_lock_idx()  local
446 		sum += atomic_long_read(&sdp->srcu_lock_count[idx]);  in srcu_readers_lock_idx()
448 			mask = mask | READ_ONCE(sdp->srcu_reader_flavor);  in srcu_readers_lock_idx()
468 		struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_readers_unlock_idx()  local
470 		sum += atomic_long_read(&sdp->srcu_unlock_count[idx]);  in srcu_readers_unlock_idx()
471 		mask = mask | READ_ONCE(sdp->srcu_reader_flavor);  in srcu_readers_unlock_idx()
582 		struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_readers_active()  local
584 		sum += atomic_long_read(&sdp->srcu_lock_count[0]);  in srcu_readers_active()
585 		sum += atomic_long_read(&sdp->srcu_lock_count[1]);  in srcu_readers_active()
586 		sum -= atomic_long_read(&sdp->srcu_unlock_count[0]);  in srcu_readers_active()
587 		sum -= atomic_long_read(&sdp->srcu_unlock_count[1]);  in srcu_readers_active()
690 		struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);  in cleanup_srcu_struct()  local
692 		del_timer_sync(&sdp->delay_work);  in cleanup_srcu_struct()
693 		flush_work(&sdp->work);  in cleanup_srcu_struct()
694 		if (WARN_ON(rcu_segcblist_n_cbs(&sdp->srcu_cblist)))  in cleanup_srcu_struct()
726 	struct srcu_data *sdp;  in __srcu_check_read_flavor()  local
732 	sdp = raw_cpu_ptr(ssp->sda);  in __srcu_check_read_flavor()
733 	old_read_flavor = READ_ONCE(sdp->srcu_reader_flavor);  in __srcu_check_read_flavor()
735 		old_read_flavor = cmpxchg(&sdp->srcu_reader_flavor, 0, read_flavor);  in __srcu_check_read_flavor()
739 …WARN_ONCE(old_read_flavor != read_flavor, "CPU %d old state %d new state %d\n", sdp->cpu, old_read…  in __srcu_check_read_flavor()
782 	struct srcu_data *sdp = raw_cpu_ptr(ssp->sda);  in __srcu_read_lock_nmisafe()  local
785 	atomic_long_inc(&sdp->srcu_lock_count[idx]);  in __srcu_read_lock_nmisafe()
798 	struct srcu_data *sdp = raw_cpu_ptr(ssp->sda);  in __srcu_read_unlock_nmisafe()  local
801 	atomic_long_inc(&sdp->srcu_unlock_count[idx]);  in __srcu_read_unlock_nmisafe()
827 	struct srcu_data *sdp = container_of(t, struct srcu_data, delay_work);  in srcu_delay_timer()  local
829 	queue_work_on(sdp->cpu, rcu_gp_wq, &sdp->work);  in srcu_delay_timer()
832 static void srcu_queue_delayed_work_on(struct srcu_data *sdp,  in srcu_queue_delayed_work_on()  argument
836 		queue_work_on(sdp->cpu, rcu_gp_wq, &sdp->work);  in srcu_queue_delayed_work_on()
840 	timer_reduce(&sdp->delay_work, jiffies + delay);  in srcu_queue_delayed_work_on()
847 static void srcu_schedule_cbs_sdp(struct srcu_data *sdp, unsigned long delay)  in srcu_schedule_cbs_sdp()  argument
849 	srcu_queue_delayed_work_on(sdp, delay);  in srcu_schedule_cbs_sdp()
888 	struct srcu_data *sdp;  in srcu_gp_end()  local
945 			sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_gp_end()
946 			spin_lock_irq_rcu_node(sdp);  in srcu_gp_end()
947 			if (ULONG_CMP_GE(gpseq, sdp->srcu_gp_seq_needed + 100))  in srcu_gp_end()
948 				sdp->srcu_gp_seq_needed = gpseq;  in srcu_gp_end()
949 			if (ULONG_CMP_GE(gpseq, sdp->srcu_gp_seq_needed_exp + 100))  in srcu_gp_end()
950 				sdp->srcu_gp_seq_needed_exp = gpseq;  in srcu_gp_end()
951 			spin_unlock_irq_rcu_node(sdp);  in srcu_gp_end()
1025 static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp,  in srcu_funnel_gp_start()  argument
1029 	int idx = rcu_seq_ctr(s) % ARRAY_SIZE(sdp->mynode->srcu_have_cbs);  in srcu_funnel_gp_start()
1040 		snp_leaf = sdp->mynode;  in srcu_funnel_gp_start()
1051 					snp->srcu_data_have_cbs[idx] |= sdp->grpmask;  in srcu_funnel_gp_start()
1054 					srcu_schedule_cbs_sdp(sdp, do_norm ? SRCU_INTERVAL : 0);  in srcu_funnel_gp_start()
1063 				snp->srcu_data_have_cbs[idx] |= sdp->grpmask;  in srcu_funnel_gp_start()
1202 	struct srcu_data *sdp;  in srcu_should_expedite()  local
1211 	sdp = raw_cpu_ptr(ssp->sda);  in srcu_should_expedite()
1212 	spin_lock_irqsave_rcu_node(sdp, flags);  in srcu_should_expedite()
1213 	if (rcu_segcblist_pend_cbs(&sdp->srcu_cblist)) {  in srcu_should_expedite()
1214 		spin_unlock_irqrestore_rcu_node(sdp, flags);  in srcu_should_expedite()
1217 	spin_unlock_irqrestore_rcu_node(sdp, flags);  in srcu_should_expedite()
1261 	struct srcu_data *sdp;  in srcu_gp_start_if_needed()  local
1274 		sdp = per_cpu_ptr(ssp->sda, get_boot_cpu_id());  in srcu_gp_start_if_needed()
1276 		sdp = raw_cpu_ptr(ssp->sda);  in srcu_gp_start_if_needed()
1277 	spin_lock_irqsave_sdp_contention(sdp, &flags);  in srcu_gp_start_if_needed()
1279 		rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp);  in srcu_gp_start_if_needed()
1318 		rcu_segcblist_advance(&sdp->srcu_cblist,  in srcu_gp_start_if_needed()
1327 		WARN_ON_ONCE(!rcu_segcblist_accelerate(&sdp->srcu_cblist, s));  in srcu_gp_start_if_needed()
1329 	if (ULONG_CMP_LT(sdp->srcu_gp_seq_needed, s)) {  in srcu_gp_start_if_needed()
1330 		sdp->srcu_gp_seq_needed = s;  in srcu_gp_start_if_needed()
1333 	if (!do_norm && ULONG_CMP_LT(sdp->srcu_gp_seq_needed_exp, s)) {  in srcu_gp_start_if_needed()
1334 		sdp->srcu_gp_seq_needed_exp = s;  in srcu_gp_start_if_needed()
1337 	spin_unlock_irqrestore_rcu_node(sdp, flags);  in srcu_gp_start_if_needed()
1343 		sdp_mynode = sdp->mynode;  in srcu_gp_start_if_needed()
1346 		srcu_funnel_gp_start(ssp, sdp, s, do_norm);  in srcu_gp_start_if_needed()
1603 	struct srcu_data *sdp;  in srcu_barrier_cb()  local
1607 	sdp = container_of(rhp, struct srcu_data, srcu_barrier_head);  in srcu_barrier_cb()
1608 	ssp = sdp->ssp;  in srcu_barrier_cb()
1621 static void srcu_barrier_one_cpu(struct srcu_struct *ssp, struct srcu_data *sdp)  in srcu_barrier_one_cpu()  argument
1623 	spin_lock_irq_rcu_node(sdp);  in srcu_barrier_one_cpu()
1625 	sdp->srcu_barrier_head.func = srcu_barrier_cb;  in srcu_barrier_one_cpu()
1626 	debug_rcu_head_queue(&sdp->srcu_barrier_head);  in srcu_barrier_one_cpu()
1627 	if (!rcu_segcblist_entrain(&sdp->srcu_cblist,  in srcu_barrier_one_cpu()
1628 				   &sdp->srcu_barrier_head)) {  in srcu_barrier_one_cpu()
1629 		debug_rcu_head_unqueue(&sdp->srcu_barrier_head);  in srcu_barrier_one_cpu()
1632 	spin_unlock_irq_rcu_node(sdp);  in srcu_barrier_one_cpu()
1770 	struct srcu_data *sdp;  in srcu_invoke_callbacks()  local
1773 	sdp = container_of(work, struct srcu_data, work);  in srcu_invoke_callbacks()
1775 	ssp = sdp->ssp;  in srcu_invoke_callbacks()
1777 	spin_lock_irq_rcu_node(sdp);  in srcu_invoke_callbacks()
1778 	WARN_ON_ONCE(!rcu_segcblist_segempty(&sdp->srcu_cblist, RCU_NEXT_TAIL));  in srcu_invoke_callbacks()
1779 	rcu_segcblist_advance(&sdp->srcu_cblist,  in srcu_invoke_callbacks()
1786 	if (sdp->srcu_cblist_invoking ||  in srcu_invoke_callbacks()
1787 	    !rcu_segcblist_ready_cbs(&sdp->srcu_cblist)) {  in srcu_invoke_callbacks()
1788 		spin_unlock_irq_rcu_node(sdp);  in srcu_invoke_callbacks()
1793 	sdp->srcu_cblist_invoking = true;  in srcu_invoke_callbacks()
1794 	rcu_segcblist_extract_done_cbs(&sdp->srcu_cblist, &ready_cbs);  in srcu_invoke_callbacks()
1796 	spin_unlock_irq_rcu_node(sdp);  in srcu_invoke_callbacks()
1811 	spin_lock_irq_rcu_node(sdp);  in srcu_invoke_callbacks()
1812 	rcu_segcblist_add_len(&sdp->srcu_cblist, -len);  in srcu_invoke_callbacks()
1813 	sdp->srcu_cblist_invoking = false;  in srcu_invoke_callbacks()
1814 	more = rcu_segcblist_ready_cbs(&sdp->srcu_cblist);  in srcu_invoke_callbacks()
1815 	spin_unlock_irq_rcu_node(sdp);  in srcu_invoke_callbacks()
1818 		srcu_schedule_cbs_sdp(sdp, 0);  in srcu_invoke_callbacks()
1923 			struct srcu_data *sdp;  in srcu_torture_stats_print()  local
1925 			sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_torture_stats_print()
1926 			u0 = data_race(atomic_long_read(&sdp->srcu_unlock_count[!idx]));  in srcu_torture_stats_print()
1927 			u1 = data_race(atomic_long_read(&sdp->srcu_unlock_count[idx]));  in srcu_torture_stats_print()
1935 			l0 = data_race(atomic_long_read(&sdp->srcu_lock_count[!idx]));  in srcu_torture_stats_print()
1936 			l1 = data_race(atomic_long_read(&sdp->srcu_lock_count[idx]));  in srcu_torture_stats_print()
1942 				"C."[rcu_segcblist_empty(&sdp->srcu_cblist)]);  in srcu_torture_stats_print()