3  * Adjunct processor matrix VFIO device driver callbacks.
157 		if (test_bit_inv(AP_QID_CARD(apqn), matrix_mdev->matrix.apm) &&  in get_update_locks_by_apqn()
158 		    test_bit_inv(AP_QID_QUEUE(apqn), matrix_mdev->matrix.aqm)) {  in get_update_locks_by_apqn()
175  *			       KVM guest to which the matrix mdev linked to a
199  *			    hash table of queues assigned to a matrix mdev
200  * @matrix_mdev: the matrix mdev
646 				struct ap_matrix *matrix)  in vfio_ap_matrix_init()  argument
648 	matrix->apm_max = info->apxa ? info->na : 63;  in vfio_ap_matrix_init()
649 	matrix->aqm_max = info->apxa ? info->nd : 15;  in vfio_ap_matrix_init()
650 	matrix->adm_max = info->apxa ? info->nd : 15;  in vfio_ap_matrix_init()
667 	bitmap_and(matrix_mdev->shadow_apcb.adm, matrix_mdev->matrix.adm,  in vfio_ap_mdev_filter_cdoms()
690  * vfio_ap_mdev_filter_matrix - filter the APQNs assigned to the matrix mdev
695  * @matrix_mdev: the matrix mdev whose matrix is to be filtered.
701  *	 driver, its APID will be filtered from the guest's APCB. The matrix
725 	 * from the matrix mdev, but only those that are assigned to the host's  in vfio_ap_mdev_filter_matrix()
728 	bitmap_and(matrix_mdev->shadow_apcb.apm, matrix_mdev->matrix.apm,  in vfio_ap_mdev_filter_matrix()
730 	bitmap_and(matrix_mdev->shadow_apcb.aqm, matrix_mdev->matrix.aqm,  in vfio_ap_mdev_filter_matrix()
773 	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix);  in vfio_ap_mdev_init_dev()
839 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) {  in vfio_ap_mdev_unlink_fr_queues()
840 		for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm,  in vfio_ap_mdev_unlink_fr_queues()
883  * vfio_ap_mdev_verify_no_sharing - verify APQNs are not shared by matrix mdevs
889  * AP adapter IDs and AP queue indexes is not configured for any matrix
906 		if (mdev_apm == matrix_mdev->matrix.apm &&  in vfio_ap_mdev_verify_no_sharing()
907 		    mdev_aqm == matrix_mdev->matrix.aqm)  in vfio_ap_mdev_verify_no_sharing()
917 		if (!bitmap_and(apm, mdev_apm, matrix_mdev->matrix.apm,  in vfio_ap_mdev_verify_no_sharing()
921 		if (!bitmap_and(aqm, mdev_aqm, matrix_mdev->matrix.aqm,  in vfio_ap_mdev_verify_no_sharing()
950 	if (ap_apqn_in_matrix_owned_by_def_drv(matrix_mdev->matrix.apm,  in vfio_ap_mdev_validate_masks()
951 					       matrix_mdev->matrix.aqm))  in vfio_ap_mdev_validate_masks()
954 	return vfio_ap_mdev_verify_no_sharing(matrix_mdev->matrix.apm,  in vfio_ap_mdev_validate_masks()
955 					      matrix_mdev->matrix.aqm);  in vfio_ap_mdev_validate_masks()
963 	for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS)  in vfio_ap_mdev_link_adapter()
1011  * corresponding bit in the mediated matrix device's APM
1013  * @dev:	the matrix device
1014  * @attr:	the mediated matrix device's assign_adapter attribute
1037  *	   matrix device
1059 	if (apid > matrix_mdev->matrix.apm_max) {  in assign_adapter_store()
1064 	if (test_bit_inv(apid, matrix_mdev->matrix.apm)) {  in assign_adapter_store()
1069 	set_bit_inv(apid, matrix_mdev->matrix.apm);  in assign_adapter_store()
1073 		clear_bit_inv(apid, matrix_mdev->matrix.apm);  in assign_adapter_store()
1100 	/* If the queue is assigned to the matrix mdev, unlink it. */  in vfio_ap_unlink_apqn_fr_mdev()
1109  *				 adapter from the matrix mdev to which the
1111  * @matrix_mdev: the matrix mediated device to which the adapter was assigned.
1123 	for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS) {  in vfio_ap_mdev_unlink_adapter()
1177  * corresponding bit in the mediated matrix device's APM
1179  * @dev:	the matrix device
1180  * @attr:	the mediated matrix device's unassign_adapter attribute
1204 	if (apid > matrix_mdev->matrix.apm_max) {  in unassign_adapter_store()
1209 	if (!test_bit_inv(apid, matrix_mdev->matrix.apm)) {  in unassign_adapter_store()
1214 	clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm);  in unassign_adapter_store()
1228 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES)  in vfio_ap_mdev_link_domain()
1235  * corresponding bit in the mediated matrix device's AQM
1237  * @dev:	the matrix device
1238  * @attr:	the mediated matrix device's assign_domain attribute
1261  *	   matrix device
1283 	if (apqi > matrix_mdev->matrix.aqm_max) {  in assign_domain_store()
1288 	if (test_bit_inv(apqi, matrix_mdev->matrix.aqm)) {  in assign_domain_store()
1293 	set_bit_inv(apqi, matrix_mdev->matrix.aqm);  in assign_domain_store()
1297 		clear_bit_inv(apqi, matrix_mdev->matrix.aqm);  in assign_domain_store()
1324 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) {  in vfio_ap_mdev_unlink_domain()
1378  * corresponding bit in the mediated matrix device's AQM
1380  * @dev:	the matrix device
1381  * @attr:	the mediated matrix device's unassign_domain attribute
1405 	if (apqi > matrix_mdev->matrix.aqm_max) {  in unassign_domain_store()
1410 	if (!test_bit_inv(apqi, matrix_mdev->matrix.aqm)) {  in unassign_domain_store()
1415 	clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm);  in unassign_domain_store()
1427  * the corresponding bit in the mediated matrix device's ADM
1429  * @dev:	the matrix device
1430  * @attr:	the mediated matrix device's assign_control_domain attribute
1453 	if (id > matrix_mdev->matrix.adm_max) {  in assign_control_domain_store()
1458 	if (test_bit_inv(id, matrix_mdev->matrix.adm)) {  in assign_control_domain_store()
1468 	set_bit_inv(id, matrix_mdev->matrix.adm);  in assign_control_domain_store()
1481  * clears the corresponding bit in the mediated matrix device's ADM
1483  * @dev:	the matrix device
1484  * @attr:	the mediated matrix device's unassign_control_domain attribute
1507 	if (domid > matrix_mdev->matrix.adm_max) {  in unassign_control_domain_store()
1512 	if (!test_bit_inv(domid, matrix_mdev->matrix.adm)) {  in unassign_control_domain_store()
1517 	clear_bit_inv(domid, matrix_mdev->matrix.adm);  in unassign_control_domain_store()
1537 	unsigned long max_domid = matrix_mdev->matrix.adm_max;  in control_domains_show()
1541 	for_each_set_bit_inv(id, matrix_mdev->matrix.adm, max_domid + 1)  in control_domains_show()
1549 static ssize_t vfio_ap_mdev_matrix_show(struct ap_matrix *matrix, char *buf)  in vfio_ap_mdev_matrix_show()  argument
1555 	unsigned long napm_bits = matrix->apm_max + 1;  in vfio_ap_mdev_matrix_show()
1556 	unsigned long naqm_bits = matrix->aqm_max + 1;  in vfio_ap_mdev_matrix_show()
1559 	apid1 = find_first_bit_inv(matrix->apm, napm_bits);  in vfio_ap_mdev_matrix_show()
1560 	apqi1 = find_first_bit_inv(matrix->aqm, naqm_bits);  in vfio_ap_mdev_matrix_show()
1563 		for_each_set_bit_inv(apid, matrix->apm, napm_bits) {  in vfio_ap_mdev_matrix_show()
1564 			for_each_set_bit_inv(apqi, matrix->aqm, naqm_bits)  in vfio_ap_mdev_matrix_show()
1568 		for_each_set_bit_inv(apid, matrix->apm, napm_bits)  in vfio_ap_mdev_matrix_show()
1571 		for_each_set_bit_inv(apqi, matrix->aqm, naqm_bits)  in vfio_ap_mdev_matrix_show()
1585 	nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->matrix, buf);  in matrix_show()
1590 static DEVICE_ATTR_RO(matrix);
1618 	idx += write_ap_bitmap(matrix_mdev->matrix.apm, buf, idx, ',');  in ap_config_show()
1619 	idx += write_ap_bitmap(matrix_mdev->matrix.aqm, buf, idx, ',');  in ap_config_show()
1620 	idx += write_ap_bitmap(matrix_mdev->matrix.adm, buf, idx, '\n');  in ap_config_show()
1646 	for_each_set_bit_inv(bit, matrix_mdev->matrix.apm, AP_DEVICES) {  in ap_matrix_overflow_check()
1647 		if (bit > matrix_mdev->matrix.apm_max)  in ap_matrix_overflow_check()
1651 	for_each_set_bit_inv(bit, matrix_mdev->matrix.aqm, AP_DOMAINS) {  in ap_matrix_overflow_check()
1652 		if (bit > matrix_mdev->matrix.aqm_max)  in ap_matrix_overflow_check()
1656 	for_each_set_bit_inv(bit, matrix_mdev->matrix.adm, AP_DOMAINS) {  in ap_matrix_overflow_check()
1657 		if (bit > matrix_mdev->matrix.adm_max)  in ap_matrix_overflow_check()
1695 	ap_matrix_copy(&m_old, &matrix_mdev->matrix);  in ap_config_store()
1709 	ap_matrix_copy(&matrix_mdev->matrix, &m_new);  in ap_config_store()
1714 		ap_matrix_copy(&matrix_mdev->matrix, &m_old);  in ap_config_store()
1719 		ap_matrix_copy(&matrix_mdev->matrix, &m_old);  in ap_config_store()
1725 	ap_matrix_copy(&matrix_mdev->matrix, &m_old);  in ap_config_store()
1732 	ap_matrix_copy(&matrix_mdev->matrix, &m_new);  in ap_config_store()
1785  * @matrix_mdev: a mediated matrix device
1788  * Return: 0 if no other mediated matrix device has a reference to @kvm;
1849  * @matrix_mdev: a matrix mediated device
2223 		if (test_bit_inv(apid, matrix_mdev->matrix.apm) &&  in vfio_ap_mdev_for_queue()
2224 		    test_bit_inv(apqi, matrix_mdev->matrix.aqm))  in vfio_ap_mdev_for_queue()
2245 	/* If the queue is assigned to the matrix mediated device, then  in status_show()
2255 		 * merely assigned to a matrix mediated device.  in status_show()
2368 		 * matrix and update the guest's AP configuration after all of  in vfio_ap_mdev_probe_queue()
2410 			 * Since the queues are defined via a matrix of adapters  in vfio_ap_mdev_remove_queue()
2535 					  matrix_mdev->matrix.apm,  in vfio_ap_mdev_cfg_remove()
2538 					  matrix_mdev->matrix.aqm,  in vfio_ap_mdev_cfg_remove()
2541 					     matrix_mdev->matrix.adm,  in vfio_ap_mdev_cfg_remove()
2653  *			  AP configuration for each matrix mdev to which they
2675 			   matrix_mdev->matrix.apm, apm_add, AP_DEVICES);  in vfio_ap_mdev_cfg_add()
2677 			   matrix_mdev->matrix.aqm, aqm_add, AP_DOMAINS);  in vfio_ap_mdev_cfg_add()
2679 			   matrix_mdev->matrix.adm, adm_add, AP_DEVICES);  in vfio_ap_mdev_cfg_add()
2749 	filter_adapters = bitmap_intersects(matrix_mdev->matrix.apm,  in vfio_ap_mdev_hot_plug_cfg()
2751 	filter_domains = bitmap_intersects(matrix_mdev->matrix.aqm,  in vfio_ap_mdev_hot_plug_cfg()
2753 	filter_cdoms = bitmap_intersects(matrix_mdev->matrix.adm,  in vfio_ap_mdev_hot_plug_cfg()