1 // SPDX-License-Identifier: GPL-2.0
3  *    Virtual cpu timer based timer functions.
39 	u64 timer;  in set_vtimer()  local
42 		"	stpt	%0\n"	/* Store current cpu timer value */  in set_vtimer()
44 		: "=Q" (timer) : "Q" (expires));  in set_vtimer()
45 	lc->system_timer += lc->last_update_timer - timer;  in set_vtimer()
46 	lc->last_update_timer = expires;  in set_vtimer()
70 		delta = cycles_new[i] - cycles_old[i];  in update_mt_scaling()
91 	delta = new - *tsk_vtime;  in update_tsk_timer()
110 	p->stimescaled += cputime_to_nsecs(scale_vtime(cputime));  in account_system_index_scaled()
120 	u64 timer, clock, user, guest, system, hardirq, softirq;  in do_account_vtime()  local
123 	timer = lc->last_update_timer;  in do_account_vtime()
124 	clock = lc->last_update_clock;  in do_account_vtime()
126 		"	stpt	%0\n"	/* Store current cpu timer value */  in do_account_vtime()
128 		: "=Q" (lc->last_update_timer),  in do_account_vtime()
129 		  "=Q" (lc->last_update_clock)  in do_account_vtime()
131 	clock = lc->last_update_clock - clock;  in do_account_vtime()
132 	timer -= lc->last_update_timer;  in do_account_vtime()
135 		lc->hardirq_timer += timer;  in do_account_vtime()
137 		lc->system_timer += timer;  in do_account_vtime()
145 	user = update_tsk_timer(&tsk->thread.user_timer,  in do_account_vtime()
146 				READ_ONCE(lc->user_timer));  in do_account_vtime()
147 	guest = update_tsk_timer(&tsk->thread.guest_timer,  in do_account_vtime()
148 				 READ_ONCE(lc->guest_timer));  in do_account_vtime()
149 	system = update_tsk_timer(&tsk->thread.system_timer,  in do_account_vtime()
150 				  READ_ONCE(lc->system_timer));  in do_account_vtime()
151 	hardirq = update_tsk_timer(&tsk->thread.hardirq_timer,  in do_account_vtime()
152 				   READ_ONCE(lc->hardirq_timer));  in do_account_vtime()
153 	softirq = update_tsk_timer(&tsk->thread.softirq_timer,  in do_account_vtime()
154 				   READ_ONCE(lc->softirq_timer));  in do_account_vtime()
155 	lc->steal_timer +=  in do_account_vtime()
156 		clock - user - guest - system - hardirq - softirq;  in do_account_vtime()
161 		tsk->utimescaled += cputime_to_nsecs(scale_vtime(user));  in do_account_vtime()
166 		tsk->utimescaled += cputime_to_nsecs(scale_vtime(guest));  in do_account_vtime()
184 	prev->thread.user_timer = lc->user_timer;  in vtime_task_switch()
185 	prev->thread.guest_timer = lc->guest_timer;  in vtime_task_switch()
186 	prev->thread.system_timer = lc->system_timer;  in vtime_task_switch()
187 	prev->thread.hardirq_timer = lc->hardirq_timer;  in vtime_task_switch()
188 	prev->thread.softirq_timer = lc->softirq_timer;  in vtime_task_switch()
189 	lc->user_timer = current->thread.user_timer;  in vtime_task_switch()
190 	lc->guest_timer = current->thread.guest_timer;  in vtime_task_switch()
191 	lc->system_timer = current->thread.system_timer;  in vtime_task_switch()
192 	lc->hardirq_timer = current->thread.hardirq_timer;  in vtime_task_switch()
193 	lc->softirq_timer = current->thread.softirq_timer;  in vtime_task_switch()
209 	steal = lc->steal_timer;  in vtime_flush()
210 	avg_steal = lc->avg_steal_timer;  in vtime_flush()
212 		lc->steal_timer = 0;  in vtime_flush()
216 	lc->avg_steal_timer = avg_steal / 2;  in vtime_flush()
222 	u64 timer = lc->last_update_timer;  in vtime_delta()  local
224 	lc->last_update_timer = get_cpu_timer();  in vtime_delta()
225 	return timer - lc->last_update_timer;  in vtime_delta()
237 	if (tsk->flags & PF_VCPU)  in vtime_account_kernel()
238 		lc->guest_timer += delta;  in vtime_account_kernel()
240 		lc->system_timer += delta;  in vtime_account_kernel()
250 	get_lowcore()->softirq_timer += delta;  in vtime_account_softirq()
259 	get_lowcore()->hardirq_timer += delta;  in vtime_account_hardirq()
268 static void list_add_sorted(struct vtimer_list *timer, struct list_head *head)  in list_add_sorted()  argument
273 		if (tmp->expires > timer->expires) {  in list_add_sorted()
274 			list_add_tail(&timer->entry, &tmp->entry);  in list_add_sorted()
278 	list_add_tail(&timer->entry, head);  in list_add_sorted()
282  * Handler for expired virtual CPU timer.
286 	struct vtimer_list *timer, *tmp;  in virt_timer_expire()  local
290 	/* walk timer list, fire all expired timers */  in virt_timer_expire()
293 	list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) {  in virt_timer_expire()
294 		if (timer->expires < elapsed)  in virt_timer_expire()
295 			/* move expired timer to the callback queue */  in virt_timer_expire()
296 			list_move_tail(&timer->entry, &cb_list);  in virt_timer_expire()
298 			timer->expires -= elapsed;  in virt_timer_expire()
301 		timer = list_first_entry(&virt_timer_list,  in virt_timer_expire()
303 		atomic64_set(&virt_timer_current, timer->expires);  in virt_timer_expire()
309 	list_for_each_entry_safe(timer, tmp, &cb_list, entry) {  in virt_timer_expire()
310 		list_del_init(&timer->entry);  in virt_timer_expire()
311 		timer->function(timer->data);  in virt_timer_expire()
312 		if (timer->interval) {  in virt_timer_expire()
313 			/* Recharge interval timer */  in virt_timer_expire()
314 			timer->expires = timer->interval +  in virt_timer_expire()
317 			list_add_sorted(timer, &virt_timer_list);  in virt_timer_expire()
323 void init_virt_timer(struct vtimer_list *timer)  in init_virt_timer()  argument
325 	timer->function = NULL;  in init_virt_timer()
326 	INIT_LIST_HEAD(&timer->entry);  in init_virt_timer()
330 static inline int vtimer_pending(struct vtimer_list *timer)  in vtimer_pending()  argument
332 	return !list_empty(&timer->entry);  in vtimer_pending()
335 static void internal_add_vtimer(struct vtimer_list *timer)  in internal_add_vtimer()  argument
338 		/* First timer, just program it. */  in internal_add_vtimer()
339 		atomic64_set(&virt_timer_current, timer->expires);  in internal_add_vtimer()
341 		list_add(&timer->entry, &virt_timer_list);  in internal_add_vtimer()
343 		/* Update timer against current base. */  in internal_add_vtimer()
344 		timer->expires += atomic64_read(&virt_timer_elapsed);  in internal_add_vtimer()
345 		if (likely((s64) timer->expires <  in internal_add_vtimer()
347 			/* The new timer expires before the current timer. */  in internal_add_vtimer()
348 			atomic64_set(&virt_timer_current, timer->expires);  in internal_add_vtimer()
349 		/* Insert new timer into the list. */  in internal_add_vtimer()
350 		list_add_sorted(timer, &virt_timer_list);  in internal_add_vtimer()
354 static void __add_vtimer(struct vtimer_list *timer, int periodic)  in __add_vtimer()  argument
358 	timer->interval = periodic ? timer->expires : 0;  in __add_vtimer()
360 	internal_add_vtimer(timer);  in __add_vtimer()
365  * add_virt_timer - add a oneshot virtual CPU timer
367 void add_virt_timer(struct vtimer_list *timer)  in add_virt_timer()  argument
369 	__add_vtimer(timer, 0);  in add_virt_timer()
374  * add_virt_timer_int - add an interval virtual CPU timer
376 void add_virt_timer_periodic(struct vtimer_list *timer)  in add_virt_timer_periodic()  argument
378 	__add_vtimer(timer, 1);  in add_virt_timer_periodic()
382 static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic)  in __mod_vtimer()  argument
387 	BUG_ON(!timer->function);  in __mod_vtimer()
389 	if (timer->expires == expires && vtimer_pending(timer))  in __mod_vtimer()
392 	rc = vtimer_pending(timer);  in __mod_vtimer()
394 		list_del_init(&timer->entry);  in __mod_vtimer()
395 	timer->interval = periodic ? expires : 0;  in __mod_vtimer()
396 	timer->expires = expires;  in __mod_vtimer()
397 	internal_add_vtimer(timer);  in __mod_vtimer()
403  * returns whether it has modified a pending timer (1) or not (0)
405 int mod_virt_timer(struct vtimer_list *timer, u64 expires)  in mod_virt_timer()  argument
407 	return __mod_vtimer(timer, expires, 0);  in mod_virt_timer()
412  * returns whether it has modified a pending timer (1) or not (0)
414 int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires)  in mod_virt_timer_periodic()  argument
416 	return __mod_vtimer(timer, expires, 1);  in mod_virt_timer_periodic()
421  * Delete a virtual timer.
423  * returns whether the deleted timer was pending (1) or not (0)
425 int del_virt_timer(struct vtimer_list *timer)  in del_virt_timer()  argument
429 	if (!vtimer_pending(timer))  in del_virt_timer()
432 	list_del_init(&timer->entry);  in del_virt_timer()
439  * Start the virtual CPU timer on the current CPU.
443 	/* set initial cpu timer */  in vtime_init()