1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /* delayacct.h - per-task delay accounting
3 *
4 * Copyright (C) Shailabh Nagar, IBM Corp. 2006
5 */
6
7 #ifndef _LINUX_DELAYACCT_H
8 #define _LINUX_DELAYACCT_H
9
10 #include <uapi/linux/taskstats.h>
11
12 #ifdef CONFIG_TASK_DELAY_ACCT
13 struct task_delay_info {
14 raw_spinlock_t lock;
15
16 /* For each stat XXX, add following, aligned appropriately
17 *
18 * struct timespec XXX_start, XXX_end;
19 * u64 XXX_delay;
20 * u32 XXX_count;
21 *
22 * Atomicity of updates to XXX_delay, XXX_count protected by
23 * single lock above (split into XXX_lock if contention is an issue).
24 */
25
26 /*
27 * XXX_count is incremented on every XXX operation, the delay
28 * associated with the operation is added to XXX_delay.
29 * XXX_delay contains the accumulated delay time in nanoseconds.
30 */
31 u64 blkio_start;
32 u64 blkio_delay_max;
33 u64 blkio_delay_min;
34 u64 blkio_delay; /* wait for sync block io completion */
35 u64 swapin_start;
36 u64 swapin_delay_max;
37 u64 swapin_delay_min;
38 u64 swapin_delay; /* wait for swapin */
39 u32 blkio_count; /* total count of the number of sync block */
40 /* io operations performed */
41 u32 swapin_count; /* total count of swapin */
42
43 u64 freepages_start;
44 u64 freepages_delay_max;
45 u64 freepages_delay_min;
46 u64 freepages_delay; /* wait for memory reclaim */
47
48 u64 thrashing_start;
49 u64 thrashing_delay_max;
50 u64 thrashing_delay_min;
51 u64 thrashing_delay; /* wait for thrashing page */
52
53 u64 compact_start;
54 u64 compact_delay_max;
55 u64 compact_delay_min;
56 u64 compact_delay; /* wait for memory compact */
57
58 u64 wpcopy_start;
59 u64 wpcopy_delay_max;
60 u64 wpcopy_delay_min;
61 u64 wpcopy_delay; /* wait for write-protect copy */
62
63 u64 irq_delay_max;
64 u64 irq_delay_min;
65 u64 irq_delay; /* wait for IRQ/SOFTIRQ */
66
67 u32 freepages_count; /* total count of memory reclaim */
68 u32 thrashing_count; /* total count of thrash waits */
69 u32 compact_count; /* total count of memory compact */
70 u32 wpcopy_count; /* total count of write-protect copy */
71 u32 irq_count; /* total count of IRQ/SOFTIRQ */
72 };
73 #endif
74
75 #include <linux/sched.h>
76 #include <linux/slab.h>
77 #include <linux/jump_label.h>
78
79 #ifdef CONFIG_TASK_DELAY_ACCT
80 DECLARE_STATIC_KEY_FALSE(delayacct_key);
81 extern int delayacct_on; /* Delay accounting turned on/off */
82 extern struct kmem_cache *delayacct_cache;
83 extern void delayacct_init(void);
84
85 extern void __delayacct_tsk_init(struct task_struct *);
86 extern void __delayacct_tsk_exit(struct task_struct *);
87 extern void __delayacct_blkio_start(void);
88 extern void __delayacct_blkio_end(struct task_struct *);
89 extern int delayacct_add_tsk(struct taskstats *, struct task_struct *);
90 extern __u64 __delayacct_blkio_ticks(struct task_struct *);
91 extern void __delayacct_freepages_start(void);
92 extern void __delayacct_freepages_end(void);
93 extern void __delayacct_thrashing_start(bool *in_thrashing);
94 extern void __delayacct_thrashing_end(bool *in_thrashing);
95 extern void __delayacct_swapin_start(void);
96 extern void __delayacct_swapin_end(void);
97 extern void __delayacct_compact_start(void);
98 extern void __delayacct_compact_end(void);
99 extern void __delayacct_wpcopy_start(void);
100 extern void __delayacct_wpcopy_end(void);
101 extern void __delayacct_irq(struct task_struct *task, u32 delta);
102
delayacct_tsk_init(struct task_struct * tsk)103 static inline void delayacct_tsk_init(struct task_struct *tsk)
104 {
105 /* reinitialize in case parent's non-null pointer was dup'ed*/
106 tsk->delays = NULL;
107 if (delayacct_on)
108 __delayacct_tsk_init(tsk);
109 }
110
111 /* Free tsk->delays. Called from bad fork and __put_task_struct
112 * where there's no risk of tsk->delays being accessed elsewhere
113 */
delayacct_tsk_free(struct task_struct * tsk)114 static inline void delayacct_tsk_free(struct task_struct *tsk)
115 {
116 if (tsk->delays)
117 kmem_cache_free(delayacct_cache, tsk->delays);
118 tsk->delays = NULL;
119 }
120
delayacct_blkio_start(void)121 static inline void delayacct_blkio_start(void)
122 {
123 if (!static_branch_unlikely(&delayacct_key))
124 return;
125
126 if (current->delays)
127 __delayacct_blkio_start();
128 }
129
delayacct_blkio_end(struct task_struct * p)130 static inline void delayacct_blkio_end(struct task_struct *p)
131 {
132 if (!static_branch_unlikely(&delayacct_key))
133 return;
134
135 if (p->delays)
136 __delayacct_blkio_end(p);
137 }
138
delayacct_blkio_ticks(struct task_struct * tsk)139 static inline __u64 delayacct_blkio_ticks(struct task_struct *tsk)
140 {
141 if (tsk->delays)
142 return __delayacct_blkio_ticks(tsk);
143 return 0;
144 }
145
delayacct_freepages_start(void)146 static inline void delayacct_freepages_start(void)
147 {
148 if (!static_branch_unlikely(&delayacct_key))
149 return;
150
151 if (current->delays)
152 __delayacct_freepages_start();
153 }
154
delayacct_freepages_end(void)155 static inline void delayacct_freepages_end(void)
156 {
157 if (!static_branch_unlikely(&delayacct_key))
158 return;
159
160 if (current->delays)
161 __delayacct_freepages_end();
162 }
163
delayacct_thrashing_start(bool * in_thrashing)164 static inline void delayacct_thrashing_start(bool *in_thrashing)
165 {
166 if (!static_branch_unlikely(&delayacct_key))
167 return;
168
169 if (current->delays)
170 __delayacct_thrashing_start(in_thrashing);
171 }
172
delayacct_thrashing_end(bool * in_thrashing)173 static inline void delayacct_thrashing_end(bool *in_thrashing)
174 {
175 if (!static_branch_unlikely(&delayacct_key))
176 return;
177
178 if (current->delays)
179 __delayacct_thrashing_end(in_thrashing);
180 }
181
delayacct_swapin_start(void)182 static inline void delayacct_swapin_start(void)
183 {
184 if (!static_branch_unlikely(&delayacct_key))
185 return;
186
187 if (current->delays)
188 __delayacct_swapin_start();
189 }
190
delayacct_swapin_end(void)191 static inline void delayacct_swapin_end(void)
192 {
193 if (!static_branch_unlikely(&delayacct_key))
194 return;
195
196 if (current->delays)
197 __delayacct_swapin_end();
198 }
199
delayacct_compact_start(void)200 static inline void delayacct_compact_start(void)
201 {
202 if (!static_branch_unlikely(&delayacct_key))
203 return;
204
205 if (current->delays)
206 __delayacct_compact_start();
207 }
208
delayacct_compact_end(void)209 static inline void delayacct_compact_end(void)
210 {
211 if (!static_branch_unlikely(&delayacct_key))
212 return;
213
214 if (current->delays)
215 __delayacct_compact_end();
216 }
217
delayacct_wpcopy_start(void)218 static inline void delayacct_wpcopy_start(void)
219 {
220 if (!static_branch_unlikely(&delayacct_key))
221 return;
222
223 if (current->delays)
224 __delayacct_wpcopy_start();
225 }
226
delayacct_wpcopy_end(void)227 static inline void delayacct_wpcopy_end(void)
228 {
229 if (!static_branch_unlikely(&delayacct_key))
230 return;
231
232 if (current->delays)
233 __delayacct_wpcopy_end();
234 }
235
delayacct_irq(struct task_struct * task,u32 delta)236 static inline void delayacct_irq(struct task_struct *task, u32 delta)
237 {
238 if (!static_branch_unlikely(&delayacct_key))
239 return;
240
241 if (task->delays)
242 __delayacct_irq(task, delta);
243 }
244
245 #else
delayacct_init(void)246 static inline void delayacct_init(void)
247 {}
delayacct_tsk_init(struct task_struct * tsk)248 static inline void delayacct_tsk_init(struct task_struct *tsk)
249 {}
delayacct_tsk_free(struct task_struct * tsk)250 static inline void delayacct_tsk_free(struct task_struct *tsk)
251 {}
delayacct_blkio_start(void)252 static inline void delayacct_blkio_start(void)
253 {}
delayacct_blkio_end(struct task_struct * p)254 static inline void delayacct_blkio_end(struct task_struct *p)
255 {}
delayacct_add_tsk(struct taskstats * d,struct task_struct * tsk)256 static inline int delayacct_add_tsk(struct taskstats *d,
257 struct task_struct *tsk)
258 { return 0; }
delayacct_blkio_ticks(struct task_struct * tsk)259 static inline __u64 delayacct_blkio_ticks(struct task_struct *tsk)
260 { return 0; }
delayacct_is_task_waiting_on_io(struct task_struct * p)261 static inline int delayacct_is_task_waiting_on_io(struct task_struct *p)
262 { return 0; }
delayacct_freepages_start(void)263 static inline void delayacct_freepages_start(void)
264 {}
delayacct_freepages_end(void)265 static inline void delayacct_freepages_end(void)
266 {}
delayacct_thrashing_start(bool * in_thrashing)267 static inline void delayacct_thrashing_start(bool *in_thrashing)
268 {}
delayacct_thrashing_end(bool * in_thrashing)269 static inline void delayacct_thrashing_end(bool *in_thrashing)
270 {}
delayacct_swapin_start(void)271 static inline void delayacct_swapin_start(void)
272 {}
delayacct_swapin_end(void)273 static inline void delayacct_swapin_end(void)
274 {}
delayacct_compact_start(void)275 static inline void delayacct_compact_start(void)
276 {}
delayacct_compact_end(void)277 static inline void delayacct_compact_end(void)
278 {}
delayacct_wpcopy_start(void)279 static inline void delayacct_wpcopy_start(void)
280 {}
delayacct_wpcopy_end(void)281 static inline void delayacct_wpcopy_end(void)
282 {}
delayacct_irq(struct task_struct * task,u32 delta)283 static inline void delayacct_irq(struct task_struct *task, u32 delta)
284 {}
285
286 #endif /* CONFIG_TASK_DELAY_ACCT */
287
288 #endif
289