1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
19 			return -EINVAL;  in __counter_set_mode()
20 		if (port_counter->num_counters)  in __counter_set_mode()
21 			return -EBUSY;  in __counter_set_mode()
24 	port_counter->mode.mode = new_mode;  in __counter_set_mode()
25 	port_counter->mode.mask = new_mask;  in __counter_set_mode()
26 	return 0;  in __counter_set_mode()
30  * rdma_counter_set_auto_mode() - Turn on/off per-port auto mode
37  * Return 0 on success. If counter mode wasn't changed then it is considered
39  * Return -EBUSY when changing to auto mode while there are bounded counters.
50 	port_counter = &dev->port_data[port].port_counter;  in rdma_counter_set_auto_mode()
51 	if (!port_counter->hstats)  in rdma_counter_set_auto_mode()
52 		return -EOPNOTSUPP;  in rdma_counter_set_auto_mode()
54 	mutex_lock(&port_counter->lock);  in rdma_counter_set_auto_mode()
58 		mode = (port_counter->num_counters) ? RDMA_COUNTER_MODE_MANUAL :  in rdma_counter_set_auto_mode()
61 	if (port_counter->mode.mode == mode &&  in rdma_counter_set_auto_mode()
62 	    port_counter->mode.mask == mask) {  in rdma_counter_set_auto_mode()
63 		ret = 0;  in rdma_counter_set_auto_mode()
70 	mutex_unlock(&port_counter->lock);  in rdma_counter_set_auto_mode()
71 	if (ret == -EBUSY)  in rdma_counter_set_auto_mode()
78 static void auto_mode_init_counter(struct rdma_counter *counter,  in auto_mode_init_counter()  argument
82 	struct auto_mode_param *param = &counter->mode.param;  in auto_mode_init_counter()
84 	counter->mode.mode = RDMA_COUNTER_MODE_AUTO;  in auto_mode_init_counter()
85 	counter->mode.mask = new_mask;  in auto_mode_init_counter()
88 		param->qp_type = qp->qp_type;  in auto_mode_init_counter()
91 static int __rdma_counter_bind_qp(struct rdma_counter *counter,  in __rdma_counter_bind_qp()  argument
96 	if (qp->counter)  in __rdma_counter_bind_qp()
97 		return -EINVAL;  in __rdma_counter_bind_qp()
99 	if (!qp->device->ops.counter_bind_qp)  in __rdma_counter_bind_qp()
100 		return -EOPNOTSUPP;  in __rdma_counter_bind_qp()
102 	mutex_lock(&counter->lock);  in __rdma_counter_bind_qp()
103 	ret = qp->device->ops.counter_bind_qp(counter, qp);  in __rdma_counter_bind_qp()
104 	mutex_unlock(&counter->lock);  in __rdma_counter_bind_qp()
113 	int ret = 0;  in rdma_counter_modify()
115 	if (!dev->ops.modify_hw_stat)  in rdma_counter_modify()
116 		return -EOPNOTSUPP;  in rdma_counter_modify()
119 	if (!stats || index >= stats->num_counters ||  in rdma_counter_modify()
120 	    !(stats->descs[index].flags & IB_STAT_FLAG_OPTIONAL))  in rdma_counter_modify()
121 		return -EINVAL;  in rdma_counter_modify()
123 	mutex_lock(&stats->lock);  in rdma_counter_modify()
125 	if (enable != test_bit(index, stats->is_disabled))  in rdma_counter_modify()
128 	ret = dev->ops.modify_hw_stat(dev, port, index, enable);  in rdma_counter_modify()
133 		clear_bit(index, stats->is_disabled);  in rdma_counter_modify()
135 		set_bit(index, stats->is_disabled);  in rdma_counter_modify()
137 	mutex_unlock(&stats->lock);  in rdma_counter_modify()
146 	struct rdma_counter *counter;  in alloc_and_bind()  local
149 	if (!dev->ops.counter_dealloc || !dev->ops.counter_alloc_stats)  in alloc_and_bind()
152 	counter = kzalloc(sizeof(*counter), GFP_KERNEL);  in alloc_and_bind()
153 	if (!counter)  in alloc_and_bind()
156 	counter->device    = dev;  in alloc_and_bind()
157 	counter->port      = port;  in alloc_and_bind()
159 	rdma_restrack_new(&counter->res, RDMA_RESTRACK_COUNTER);  in alloc_and_bind()
160 	counter->stats = dev->ops.counter_alloc_stats(counter);  in alloc_and_bind()
161 	if (!counter->stats)  in alloc_and_bind()
164 	port_counter = &dev->port_data[port].port_counter;  in alloc_and_bind()
165 	mutex_lock(&port_counter->lock);  in alloc_and_bind()
169 					 0);  in alloc_and_bind()
171 			mutex_unlock(&port_counter->lock);  in alloc_and_bind()
176 		auto_mode_init_counter(counter, qp, port_counter->mode.mask);  in alloc_and_bind()
179 		ret = -EOPNOTSUPP;  in alloc_and_bind()
180 		mutex_unlock(&port_counter->lock);  in alloc_and_bind()
184 	port_counter->num_counters++;  in alloc_and_bind()
185 	mutex_unlock(&port_counter->lock);  in alloc_and_bind()
187 	counter->mode.mode = mode;  in alloc_and_bind()
188 	kref_init(&counter->kref);  in alloc_and_bind()
189 	mutex_init(&counter->lock);  in alloc_and_bind()
191 	ret = __rdma_counter_bind_qp(counter, qp);  in alloc_and_bind()
195 	rdma_restrack_parent_name(&counter->res, &qp->res);  in alloc_and_bind()
196 	rdma_restrack_add(&counter->res);  in alloc_and_bind()
197 	return counter;  in alloc_and_bind()
200 	rdma_free_hw_stats_struct(counter->stats);  in alloc_and_bind()
202 	rdma_restrack_put(&counter->res);  in alloc_and_bind()
203 	kfree(counter);  in alloc_and_bind()
207 static void rdma_counter_free(struct rdma_counter *counter)  in rdma_counter_free()  argument
211 	port_counter = &counter->device->port_data[counter->port].port_counter;  in rdma_counter_free()
212 	mutex_lock(&port_counter->lock);  in rdma_counter_free()
213 	port_counter->num_counters--;  in rdma_counter_free()
214 	if (!port_counter->num_counters &&  in rdma_counter_free()
215 	    (port_counter->mode.mode == RDMA_COUNTER_MODE_MANUAL))  in rdma_counter_free()
216 		__counter_set_mode(port_counter, RDMA_COUNTER_MODE_NONE, 0);  in rdma_counter_free()
218 	mutex_unlock(&port_counter->lock);  in rdma_counter_free()
220 	rdma_restrack_del(&counter->res);  in rdma_counter_free()
221 	rdma_free_hw_stats_struct(counter->stats);  in rdma_counter_free()
222 	kfree(counter);  in rdma_counter_free()
225 static bool auto_mode_match(struct ib_qp *qp, struct rdma_counter *counter,  in auto_mode_match()  argument
228 	struct auto_mode_param *param = &counter->mode.param;  in auto_mode_match()
232 		match &= (param->qp_type == qp->qp_type);  in auto_mode_match()
235 		match &= (task_pid_nr(counter->res.task) ==  in auto_mode_match()
236 			  task_pid_nr(qp->res.task));  in auto_mode_match()
243 	struct rdma_counter *counter = qp->counter;  in __rdma_counter_unbind_qp()  local
246 	if (!qp->device->ops.counter_unbind_qp)  in __rdma_counter_unbind_qp()
247 		return -EOPNOTSUPP;  in __rdma_counter_unbind_qp()
249 	mutex_lock(&counter->lock);  in __rdma_counter_unbind_qp()
250 	ret = qp->device->ops.counter_unbind_qp(qp);  in __rdma_counter_unbind_qp()
251 	mutex_unlock(&counter->lock);  in __rdma_counter_unbind_qp()
256 static void counter_history_stat_update(struct rdma_counter *counter)  in counter_history_stat_update()  argument
258 	struct ib_device *dev = counter->device;  in counter_history_stat_update()
262 	port_counter = &dev->port_data[counter->port].port_counter;  in counter_history_stat_update()
263 	if (!port_counter->hstats)  in counter_history_stat_update()
266 	rdma_counter_query_stats(counter);  in counter_history_stat_update()
268 	for (i = 0; i < counter->stats->num_counters; i++)  in counter_history_stat_update()
269 		port_counter->hstats->value[i] += counter->stats->value[i];  in counter_history_stat_update()
273  * rdma_get_counter_auto_mode - Find the counter that @qp should be bound
276  * Return: The counter (with ref-count increased) if found
282 	struct rdma_counter *counter = NULL;  in rdma_get_counter_auto_mode()  local
283 	struct ib_device *dev = qp->device;  in rdma_get_counter_auto_mode()
286 	unsigned long id = 0;  in rdma_get_counter_auto_mode()
288 	port_counter = &dev->port_data[port].port_counter;  in rdma_get_counter_auto_mode()
289 	rt = &dev->res[RDMA_RESTRACK_COUNTER];  in rdma_get_counter_auto_mode()
290 	xa_lock(&rt->xa);  in rdma_get_counter_auto_mode()
291 	xa_for_each(&rt->xa, id, res) {  in rdma_get_counter_auto_mode()
292 		counter = container_of(res, struct rdma_counter, res);  in rdma_get_counter_auto_mode()
293 		if ((counter->device != qp->device) || (counter->port != port))  in rdma_get_counter_auto_mode()
296 		if (auto_mode_match(qp, counter, port_counter->mode.mask))  in rdma_get_counter_auto_mode()
299 		counter = NULL;  in rdma_get_counter_auto_mode()
302 	if (counter && !kref_get_unless_zero(&counter->kref))  in rdma_get_counter_auto_mode()
303 		counter = NULL;  in rdma_get_counter_auto_mode()
305 	xa_unlock(&rt->xa);  in rdma_get_counter_auto_mode()
306 	return counter;  in rdma_get_counter_auto_mode()
311 	struct rdma_counter *counter;  in counter_release()  local
313 	counter = container_of(kref, struct rdma_counter, kref);  in counter_release()
314 	counter_history_stat_update(counter);  in counter_release()
315 	counter->device->ops.counter_dealloc(counter);  in counter_release()
316 	rdma_counter_free(counter);  in counter_release()
320  * rdma_counter_bind_qp_auto - Check and bind the QP to a counter base on
321  *   the auto-mode rule
326 	struct ib_device *dev = qp->device;  in rdma_counter_bind_qp_auto()
327 	struct rdma_counter *counter;  in rdma_counter_bind_qp_auto()  local
330 	if (!rdma_restrack_is_tracked(&qp->res) || rdma_is_kernel_res(&qp->res))  in rdma_counter_bind_qp_auto()
331 		return 0;  in rdma_counter_bind_qp_auto()
334 		return -EINVAL;  in rdma_counter_bind_qp_auto()
336 	port_counter = &dev->port_data[port].port_counter;  in rdma_counter_bind_qp_auto()
337 	if (port_counter->mode.mode != RDMA_COUNTER_MODE_AUTO)  in rdma_counter_bind_qp_auto()
338 		return 0;  in rdma_counter_bind_qp_auto()
340 	counter = rdma_get_counter_auto_mode(qp, port);  in rdma_counter_bind_qp_auto()
341 	if (counter) {  in rdma_counter_bind_qp_auto()
342 		ret = __rdma_counter_bind_qp(counter, qp);  in rdma_counter_bind_qp_auto()
344 			kref_put(&counter->kref, counter_release);  in rdma_counter_bind_qp_auto()
348 		counter = alloc_and_bind(dev, port, qp, RDMA_COUNTER_MODE_AUTO);  in rdma_counter_bind_qp_auto()
349 		if (!counter)  in rdma_counter_bind_qp_auto()
350 			return -ENOMEM;  in rdma_counter_bind_qp_auto()
353 	return 0;  in rdma_counter_bind_qp_auto()
357  * rdma_counter_unbind_qp - Unbind a qp from a counter
359  *   true - Decrease the counter ref-count anyway (e.g., qp destroy)
363 	struct rdma_counter *counter = qp->counter;  in rdma_counter_unbind_qp()  local
366 	if (!counter)  in rdma_counter_unbind_qp()
367 		return -EINVAL;  in rdma_counter_unbind_qp()
373 	kref_put(&counter->kref, counter_release);  in rdma_counter_unbind_qp()
374 	return 0;  in rdma_counter_unbind_qp()
377 int rdma_counter_query_stats(struct rdma_counter *counter)  in rdma_counter_query_stats()  argument
379 	struct ib_device *dev = counter->device;  in rdma_counter_query_stats()
382 	if (!dev->ops.counter_update_stats)  in rdma_counter_query_stats()
383 		return -EINVAL;  in rdma_counter_query_stats()
385 	mutex_lock(&counter->lock);  in rdma_counter_query_stats()
386 	ret = dev->ops.counter_update_stats(counter);  in rdma_counter_query_stats()
387 	mutex_unlock(&counter->lock);  in rdma_counter_query_stats()
397 	struct rdma_counter *counter;  in get_running_counters_hwstat_sum()  local
398 	unsigned long id = 0;  in get_running_counters_hwstat_sum()
399 	u64 sum = 0;  in get_running_counters_hwstat_sum()
401 	rt = &dev->res[RDMA_RESTRACK_COUNTER];  in get_running_counters_hwstat_sum()
402 	xa_lock(&rt->xa);  in get_running_counters_hwstat_sum()
403 	xa_for_each(&rt->xa, id, res) {  in get_running_counters_hwstat_sum()
407 		xa_unlock(&rt->xa);  in get_running_counters_hwstat_sum()
409 		counter = container_of(res, struct rdma_counter, res);  in get_running_counters_hwstat_sum()
410 		if ((counter->device != dev) || (counter->port != port) ||  in get_running_counters_hwstat_sum()
411 		    rdma_counter_query_stats(counter))  in get_running_counters_hwstat_sum()
414 		sum += counter->stats->value[index];  in get_running_counters_hwstat_sum()
417 		xa_lock(&rt->xa);  in get_running_counters_hwstat_sum()
421 	xa_unlock(&rt->xa);  in get_running_counters_hwstat_sum()
426  * rdma_counter_get_hwstat_value() - Get the sum value of all counters on a
434 	port_counter = &dev->port_data[port].port_counter;  in rdma_counter_get_hwstat_value()
435 	if (!port_counter->hstats)  in rdma_counter_get_hwstat_value()
436 		return 0;  in rdma_counter_get_hwstat_value()
439 	sum += port_counter->hstats->value[index];  in rdma_counter_get_hwstat_value()
454 	if (qp->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW))  in rdma_counter_get_qp()
468 	struct rdma_counter *counter;  in rdma_get_counter_by_id()  local
474 	counter = container_of(res, struct rdma_counter, res);  in rdma_get_counter_by_id()
475 	kref_get(&counter->kref);  in rdma_get_counter_by_id()
478 	return counter;  in rdma_get_counter_by_id()
482  * rdma_counter_bind_qpn() - Bind QP @qp_num to counter @counter_id
488 	struct rdma_counter *counter;  in rdma_counter_bind_qpn()  local
492 	port_counter = &dev->port_data[port].port_counter;  in rdma_counter_bind_qpn()
493 	if (port_counter->mode.mode == RDMA_COUNTER_MODE_AUTO)  in rdma_counter_bind_qpn()
494 		return -EINVAL;  in rdma_counter_bind_qpn()
498 		return -ENOENT;  in rdma_counter_bind_qpn()
500 	counter = rdma_get_counter_by_id(dev, counter_id);  in rdma_counter_bind_qpn()
501 	if (!counter) {  in rdma_counter_bind_qpn()
502 		ret = -ENOENT;  in rdma_counter_bind_qpn()
506 	if (rdma_is_kernel_res(&counter->res) != rdma_is_kernel_res(&qp->res)) {  in rdma_counter_bind_qpn()
507 		ret = -EINVAL;  in rdma_counter_bind_qpn()
511 	if ((counter->device != qp->device) || (counter->port != qp->port)) {  in rdma_counter_bind_qpn()
512 		ret = -EINVAL;  in rdma_counter_bind_qpn()
516 	ret = __rdma_counter_bind_qp(counter, qp);  in rdma_counter_bind_qpn()
520 	rdma_restrack_put(&qp->res);  in rdma_counter_bind_qpn()
521 	return 0;  in rdma_counter_bind_qpn()
524 	kref_put(&counter->kref, counter_release);  in rdma_counter_bind_qpn()
526 	rdma_restrack_put(&qp->res);  in rdma_counter_bind_qpn()
531  * rdma_counter_bind_qpn_alloc() - Alloc a counter and bind QP @qp_num to it
532  *   The id of new counter is returned in @counter_id
538 	struct rdma_counter *counter;  in rdma_counter_bind_qpn_alloc()  local
543 		return -EINVAL;  in rdma_counter_bind_qpn_alloc()
545 	port_counter = &dev->port_data[port].port_counter;  in rdma_counter_bind_qpn_alloc()
546 	if (!port_counter->hstats)  in rdma_counter_bind_qpn_alloc()
547 		return -EOPNOTSUPP;  in rdma_counter_bind_qpn_alloc()
549 	if (port_counter->mode.mode == RDMA_COUNTER_MODE_AUTO)  in rdma_counter_bind_qpn_alloc()
550 		return -EINVAL;  in rdma_counter_bind_qpn_alloc()
554 		return -ENOENT;  in rdma_counter_bind_qpn_alloc()
556 	if (rdma_is_port_valid(dev, qp->port) && (qp->port != port)) {  in rdma_counter_bind_qpn_alloc()
557 		ret = -EINVAL;  in rdma_counter_bind_qpn_alloc()
561 	counter = alloc_and_bind(dev, port, qp, RDMA_COUNTER_MODE_MANUAL);  in rdma_counter_bind_qpn_alloc()
562 	if (!counter) {  in rdma_counter_bind_qpn_alloc()
563 		ret = -ENOMEM;  in rdma_counter_bind_qpn_alloc()
568 		*counter_id = counter->id;  in rdma_counter_bind_qpn_alloc()
570 	rdma_restrack_put(&qp->res);  in rdma_counter_bind_qpn_alloc()
571 	return 0;  in rdma_counter_bind_qpn_alloc()
574 	rdma_restrack_put(&qp->res);  in rdma_counter_bind_qpn_alloc()
579  * rdma_counter_unbind_qpn() - Unbind QP @qp_num from a counter
589 		return -EINVAL;  in rdma_counter_unbind_qpn()
593 		return -ENOENT;  in rdma_counter_unbind_qpn()
595 	if (rdma_is_port_valid(dev, qp->port) && (qp->port != port)) {  in rdma_counter_unbind_qpn()
596 		ret = -EINVAL;  in rdma_counter_unbind_qpn()
600 	port_counter = &dev->port_data[port].port_counter;  in rdma_counter_unbind_qpn()
601 	if (!qp->counter || qp->counter->id != counter_id ||  in rdma_counter_unbind_qpn()
602 	    port_counter->mode.mode != RDMA_COUNTER_MODE_MANUAL) {  in rdma_counter_unbind_qpn()
603 		ret = -EINVAL;  in rdma_counter_unbind_qpn()
610 	rdma_restrack_put(&qp->res);  in rdma_counter_unbind_qpn()
620 	port_counter = &dev->port_data[port].port_counter;  in rdma_counter_get_mode()
621 	*mode = port_counter->mode.mode;  in rdma_counter_get_mode()
622 	*mask = port_counter->mode.mask;  in rdma_counter_get_mode()
624 	return 0;  in rdma_counter_get_mode()
632 	if (!dev->port_data)  in rdma_counter_init()
636 		port_counter = &dev->port_data[port].port_counter;  in rdma_counter_init()
637 		port_counter->mode.mode = RDMA_COUNTER_MODE_NONE;  in rdma_counter_init()
638 		mutex_init(&port_counter->lock);  in rdma_counter_init()
640 		if (!dev->ops.alloc_hw_port_stats)  in rdma_counter_init()
643 		port_counter->hstats = dev->ops.alloc_hw_port_stats(dev, port);  in rdma_counter_init()
644 		if (!port_counter->hstats)  in rdma_counter_init()
651 	for (i = port; i >= rdma_start_port(dev); i--) {  in rdma_counter_init()
652 		port_counter = &dev->port_data[port].port_counter;  in rdma_counter_init()
653 		rdma_free_hw_stats_struct(port_counter->hstats);  in rdma_counter_init()
654 		port_counter->hstats = NULL;  in rdma_counter_init()
655 		mutex_destroy(&port_counter->lock);  in rdma_counter_init()
665 		port_counter = &dev->port_data[port].port_counter;  in rdma_counter_release()
666 		rdma_free_hw_stats_struct(port_counter->hstats);  in rdma_counter_release()
667 		mutex_destroy(&port_counter->lock);  in rdma_counter_release()