1 // SPDX-License-Identifier: GPL-2.0-only
147 	 * 64 KB ring buffer + 4 KB header should be sufficient size for any Hyper-V device apart
179 	channel->rescind = true;  in vmbus_rescind_cleanup()
183 		if (msginfo->waiting_channel == channel) {  in vmbus_rescind_cleanup()
184 			complete(&msginfo->waitevent);  in vmbus_rescind_cleanup()
203 	const guid_t *guid = &channel->offermsg.offer.if_type;  in hv_get_dev_type()
218  * vmbus_prep_negotiate_resp() - Create default response for Negotiate message
231  * Set up and fill in default negotiate response message.
232  * Mainly used by Hyper-V drivers.
253 	icmsghdrp->icmsgsize = 0x10;  in vmbus_prep_negotiate_resp()
256 	icframe_major = negop->icframe_vercnt;  in vmbus_prep_negotiate_resp()
259 	icmsg_major = negop->icmsg_vercnt;  in vmbus_prep_negotiate_resp()
266 		pr_err_ratelimited("Invalid icmsg negotiate - icframe_major: %u, icmsg_major: %u\n",  in vmbus_prep_negotiate_resp()
280 		for (j = 0; j < negop->icframe_vercnt; j++) {  in vmbus_prep_negotiate_resp()
281 			if ((negop->icversion_data[j].major == fw_major) &&  in vmbus_prep_negotiate_resp()
282 			    (negop->icversion_data[j].minor == fw_minor)) {  in vmbus_prep_negotiate_resp()
283 				icframe_major = negop->icversion_data[j].major;  in vmbus_prep_negotiate_resp()
284 				icframe_minor = negop->icversion_data[j].minor;  in vmbus_prep_negotiate_resp()
303 		for (j = negop->icframe_vercnt;  in vmbus_prep_negotiate_resp()
304 			(j < negop->icframe_vercnt + negop->icmsg_vercnt);  in vmbus_prep_negotiate_resp()
307 			if ((negop->icversion_data[j].major == srv_major) &&  in vmbus_prep_negotiate_resp()
308 				(negop->icversion_data[j].minor == srv_minor)) {  in vmbus_prep_negotiate_resp()
310 				icmsg_major = negop->icversion_data[j].major;  in vmbus_prep_negotiate_resp()
311 				icmsg_minor = negop->icversion_data[j].minor;  in vmbus_prep_negotiate_resp()
328 		negop->icframe_vercnt = 0;  in vmbus_prep_negotiate_resp()
329 		negop->icmsg_vercnt = 0;  in vmbus_prep_negotiate_resp()
331 		negop->icframe_vercnt = 1;  in vmbus_prep_negotiate_resp()
332 		negop->icmsg_vercnt = 1;  in vmbus_prep_negotiate_resp()
341 	negop->icversion_data[0].major = icframe_major;  in vmbus_prep_negotiate_resp()
342 	negop->icversion_data[0].minor = icframe_minor;  in vmbus_prep_negotiate_resp()
343 	negop->icversion_data[1].major = icmsg_major;  in vmbus_prep_negotiate_resp()
344 	negop->icversion_data[1].minor = icmsg_minor;  in vmbus_prep_negotiate_resp()
350  * alloc_channel - Allocate and initialize a vmbus channel object
360 	spin_lock_init(&channel->sched_lock);  in alloc_channel()
361 	init_completion(&channel->rescind_event);  in alloc_channel()
363 	INIT_LIST_HEAD(&channel->sc_list);  in alloc_channel()
365 	tasklet_init(&channel->callback_event,  in alloc_channel()
374  * free_channel - Release the resources used by the vmbus channel object
378 	tasklet_kill(&channel->callback_event);  in free_channel()
381 	kobject_put(&channel->kobj);  in free_channel()
386 	if (WARN_ON(channel->offermsg.child_relid >= MAX_CHANNEL_RELIDS))  in vmbus_channel_map_relid()
398 	 *      OPENCHANNEL message for the channel is sent in vmbus_open().  in vmbus_channel_map_relid()
399 	 *      Hyper-V won't start sending the interrupts for the channel  in vmbus_channel_map_relid()
400 	 *      before the OPENCHANNEL message is acked.  The memory barrier  in vmbus_channel_map_relid()
401 	 *      in vmbus_chan_sched() -> sync_test_and_clear_bit() ensures  in vmbus_channel_map_relid()
414 		vmbus_connection.channels[channel->offermsg.child_relid],  in vmbus_channel_map_relid()
420 	if (WARN_ON(channel->offermsg.child_relid >= MAX_CHANNEL_RELIDS))  in vmbus_channel_unmap_relid()
423 		vmbus_connection.channels[channel->offermsg.child_relid],  in vmbus_channel_unmap_relid()
444 	BUG_ON(!channel->rescind);  in hv_process_channel_removal()
450 	WARN_ON(channel->offermsg.child_relid == INVALID_RELID &&  in hv_process_channel_removal()
454 	 * Upon suspend, an in-use hv_sock channel is removed from the array of  in hv_process_channel_removal()
456 	 * user-space application destroys the channel, it's unnecessary and  in hv_process_channel_removal()
460 	if (channel->offermsg.child_relid != INVALID_RELID)  in hv_process_channel_removal()
463 	if (channel->primary_channel == NULL)  in hv_process_channel_removal()
464 		list_del(&channel->listentry);  in hv_process_channel_removal()
466 		list_del(&channel->sc_list);  in hv_process_channel_removal()
470 	 * init_vp_index() can (re-)use the CPU.  in hv_process_channel_removal()
473 		hv_clear_allocated_cpu(channel->target_cpu);  in hv_process_channel_removal()
476 	 * Upon suspend, an in-use hv_sock channel is marked as "rescinded" and  in hv_process_channel_removal()
477 	 * the relid is invalidated; after hibernation, when the user-space app  in hv_process_channel_removal()
482 	if (channel->offermsg.child_relid != INVALID_RELID)  in hv_process_channel_removal()
483 		vmbus_release_relid(channel->offermsg.child_relid);  in hv_process_channel_removal()
495 		channel->rescind = true;  in vmbus_free_channels()
497 		vmbus_device_unregister(channel->device_obj);  in vmbus_free_channels()
501 /* Note: the function can run concurrently for primary/sub channels. */
506 	struct vmbus_channel *primary_channel = newchannel->primary_channel;  in vmbus_add_channel_work()
514 	newchannel->state = CHANNEL_OPEN_STATE;  in vmbus_add_channel_work()
517 		/* newchannel is a sub-channel. */  in vmbus_add_channel_work()
518 		struct hv_device *dev = primary_channel->device_obj;  in vmbus_add_channel_work()
523 		if (primary_channel->sc_creation_callback != NULL)  in vmbus_add_channel_work()
524 			primary_channel->sc_creation_callback(newchannel);  in vmbus_add_channel_work()
526 		newchannel->probe_done = true;  in vmbus_add_channel_work()
533 	newchannel->device_obj = vmbus_device_create(  in vmbus_add_channel_work()
534 		&newchannel->offermsg.offer.if_type,  in vmbus_add_channel_work()
535 		&newchannel->offermsg.offer.if_instance,  in vmbus_add_channel_work()
537 	if (!newchannel->device_obj)  in vmbus_add_channel_work()
540 	newchannel->device_obj->device_id = newchannel->device_id;  in vmbus_add_channel_work()
542 	 * Add the new device to the bus. This will kick off device-driver  in vmbus_add_channel_work()
551 	ret = vmbus_device_register(newchannel->device_obj);  in vmbus_add_channel_work()
555 			newchannel->offermsg.child_relid);  in vmbus_add_channel_work()
559 	newchannel->probe_done = true;  in vmbus_add_channel_work()
569 	newchannel->probe_done = true;  in vmbus_add_channel_work()
572 		list_del(&newchannel->listentry);  in vmbus_add_channel_work()
574 		list_del(&newchannel->sc_list);  in vmbus_add_channel_work()
581 	vmbus_release_relid(newchannel->offermsg.child_relid);  in vmbus_add_channel_work()
587  * vmbus_process_offer - Process the offer by creating a channel/device
624 		if (guid_equal(&channel->offermsg.offer.if_type,  in vmbus_process_offer()
625 			       &newchannel->offermsg.offer.if_type) &&  in vmbus_process_offer()
626 		    guid_equal(&channel->offermsg.offer.if_instance,  in vmbus_process_offer()
627 			       &newchannel->offermsg.offer.if_instance)) {  in vmbus_process_offer()
629 			newchannel->primary_channel = channel;  in vmbus_process_offer()
647 		list_add_tail(&newchannel->listentry,  in vmbus_process_offer()
651 		 * Check to see if this is a valid sub-channel.  in vmbus_process_offer()
653 		if (newchannel->offermsg.offer.sub_channel_index == 0) {  in vmbus_process_offer()
657 			 * Don't call free_channel(), because newchannel->kobj  in vmbus_process_offer()
665 		 * Process the sub-channel.  in vmbus_process_offer()
667 		list_add_tail(&newchannel->sc_list, &channel->sc_list);  in vmbus_process_offer()
676 	 * vmbus_process_offer() mustn't call channel->sc_creation_callback()  in vmbus_process_offer()
677 	 * directly for sub-channels, because sc_creation_callback() ->  in vmbus_process_offer()
679 	 * OPEN_CHANNEL message (the host may rescind a channel at any time,  in vmbus_process_offer()
681 	 * may not wake up the vmbus_open() as it's blocked due to a non-zero  in vmbus_process_offer()
687 	 * And, usually the handling of primary channels and sub-channels can  in vmbus_process_offer()
689 	 * workqueues to avoid possible deadlock, e.g. in sync-probing mode,  in vmbus_process_offer()
690 	 * NIC1's netvsc_subchan_work() can race with NIC2's netvsc_probe() ->  in vmbus_process_offer()
692 	 * and waits for all the sub-channels to appear, but the latter  in vmbus_process_offer()
694 	 * sub-channels.  in vmbus_process_offer()
696 	INIT_WORK(&newchannel->add_channel_work, vmbus_add_channel_work);  in vmbus_process_offer()
699 	queue_work(wq, &newchannel->add_channel_work);  in vmbus_process_offer()
708 	struct vmbus_channel *primary = chn->primary_channel;  in hv_cpuself_used()
716 	if (primary->target_cpu == cpu)  in hv_cpuself_used()
719 	list_for_each_entry(sc, &primary->sc_list, sc_list)  in hv_cpuself_used()
720 		if (sc != chn && sc->target_cpu == cpu)  in hv_cpuself_used()
735  * For non-performance critical channels we assign the VMBUS_CONNECT_CPU.
761 		channel->target_cpu = VMBUS_CONNECT_CPU;  in init_vp_index()
796 		if (channel->offermsg.offer.sub_channel_index >= ncpu ||  in init_vp_index()
801 	channel->target_cpu = target_cpu;  in init_vp_index()
827 	 * read message pages for all CPUs directly.  in vmbus_wait_for_unload()
835 	 * message isn't seen.  in vmbus_wait_for_unload()
850 			 * not all present CPUs are online, the message page  in vmbus_wait_for_unload()
853 			page_addr = hv_cpu->synic_message_page;  in vmbus_wait_for_unload()
860 			message_type = READ_ONCE(msg->header.message_type);  in vmbus_wait_for_unload()
865 				msg->u.payload;  in vmbus_wait_for_unload()
867 			if (hdr->msgtype == CHANNELMSG_UNLOAD_RESPONSE)  in vmbus_wait_for_unload()
887 	 * maybe-pending messages on all CPUs to be able to receive new  in vmbus_wait_for_unload()
894 		page_addr = hv_cpu->synic_message_page;  in vmbus_wait_for_unload()
899 		msg->header.message_type = HVMSG_NONE;  in vmbus_wait_for_unload()
904  * vmbus_unload_response - Handler for the unload response.
912 	 * NB.  A malicious or compromised Hyper-V could send a spurious  in vmbus_unload_response()
913 	 * message of type CHANNELMSG_UNLOAD_RESPONSE, and trigger a call  in vmbus_unload_response()
927 	/* Pre-Win2012R2 hosts don't support reconnect */  in vmbus_initiate_unload()
953 	channel->sig_event = VMBUS_EVENT_CONNECTION_ID;  in vmbus_setup_channel_state()
955 	channel->is_dedicated_interrupt =  in vmbus_setup_channel_state()
956 			(offer->is_dedicated_interrupt != 0);  in vmbus_setup_channel_state()
957 	channel->sig_event = offer->connection_id;  in vmbus_setup_channel_state()
959 	memcpy(&channel->offermsg, offer,  in vmbus_setup_channel_state()
961 	channel->monitor_grp = (u8)offer->monitorid / 32;  in vmbus_setup_channel_state()
962 	channel->monitor_bit = (u8)offer->monitorid % 32;  in vmbus_setup_channel_state()
963 	channel->device_id = hv_get_dev_type(channel);  in vmbus_setup_channel_state()
967  * find_primary_channel_by_offer - Get the channel object given the new offer.
976 	/* Ignore sub-channel offers. */  in find_primary_channel_by_offer()
977 	if (offer->offer.sub_channel_index != 0)  in find_primary_channel_by_offer()
983 		inst1 = &iter->offermsg.offer.if_instance;  in find_primary_channel_by_offer()
984 		inst2 = &offer->offer.if_instance;  in find_primary_channel_by_offer()
999 	const guid_t *guid = &offer->offer.if_type;  in vmbus_is_valid_offer()
1016  * vmbus_onoffer - Handler for channel offers from vmbus in parent partition.
1031 				   offer->child_relid);  in vmbus_onoffer()
1040 		 * We're resuming from hibernation: all the sub-channel and  in vmbus_onoffer()
1042 		 * been cleaned up, and now we must be seeing a re-offered  in vmbus_onoffer()
1065 		 * suspend are re-offered upon the resume.  See the WARN_ON()  in vmbus_onoffer()
1072 		WARN_ON(oldchannel->offermsg.child_relid != INVALID_RELID);  in vmbus_onoffer()
1074 		oldchannel->offermsg.child_relid = offer->child_relid;  in vmbus_onoffer()
1077 		if (memcmp(offer, &oldchannel->offermsg, offer_sz) != 0) {  in vmbus_onoffer()
1081 			 * (Build 17763), the offer->connection_id of the  in vmbus_onoffer()
1086 				 offer->child_relid);  in vmbus_onoffer()
1090 					     &oldchannel->offermsg, offer_sz,  in vmbus_onoffer()
1109 		vmbus_release_relid(offer->child_relid);  in vmbus_onoffer()
1123 	 * If all the sub-channels or hv_sock channels have been cleaned up,  in check_ready_for_suspend_event()
1131  * vmbus_onoffer_rescind - Rescind offer handler.
1148 	 * from the host are guranteed to be ordered -  in vmbus_onoffer_rescind()
1179 	channel = relid2channel(rescind->child_relid);  in vmbus_onoffer_rescind()
1186 		if (channel->rescind_ref) {  in vmbus_onoffer_rescind()
1190 		channel->rescind_ref = true;  in vmbus_onoffer_rescind()
1196 		 * We failed in processing the offer message;  in vmbus_onoffer_rescind()
1206 	 * Before setting channel->rescind in vmbus_rescind_cleanup(), we  in vmbus_onoffer_rescind()
1215 	while (READ_ONCE(channel->probe_done) == false) {  in vmbus_onoffer_rescind()
1227 	if (channel->device_obj) {  in vmbus_onoffer_rescind()
1228 		if (channel->chn_rescind_callback) {  in vmbus_onoffer_rescind()
1229 			channel->chn_rescind_callback(channel);  in vmbus_onoffer_rescind()
1240 		dev = get_device(&channel->device_obj->device);  in vmbus_onoffer_rescind()
1242 			vmbus_device_unregister(channel->device_obj);  in vmbus_onoffer_rescind()
1245 	} else if (channel->primary_channel != NULL) {  in vmbus_onoffer_rescind()
1247 		 * Sub-channel is being rescinded. Following is the channel  in vmbus_onoffer_rescind()
1250 		 * 1. Close all sub-channels first  in vmbus_onoffer_rescind()
1254 		if (channel->state == CHANNEL_OPEN_STATE) {  in vmbus_onoffer_rescind()
1261 			complete(&channel->rescind_event);  in vmbus_onoffer_rescind()
1277 	while (!READ_ONCE(channel->probe_done) || !READ_ONCE(channel->rescind))  in vmbus_hvsock_device_unregister()
1280 	vmbus_device_unregister(channel->device_obj);  in vmbus_hvsock_device_unregister()
1286  * vmbus_onoffers_delivered -
1287  * The CHANNELMSG_ALLOFFERS_DELIVERED message arrives after all
1288  * boot-time offers are delivered. A boot-time offer is for the primary
1290  * Boot-time offers include offers for physical devices assigned to the VM
1291  * via Hyper-V's Discrete Device Assignment (DDA) functionality that are
1293  * Boot-time offers do not include offers for VMBus sub-channels. Because
1294  * devices can be hot-added to the VM after it is booted, additional channel
1295  * offers that aren't boot-time offers can be received at any time after the
1296  * all-offers-delivered message.
1298  * SR-IOV NIC Virtual Functions (VFs) assigned to a VM are not considered
1299  * to be assigned to the VM at boot-time, and offers for VFs may occur after
1300  * the all-offers-delivered message. VFs are optional accelerators to the
1301  * synthetic VMBus NIC and are effectively hot-added only after the VMBus
1312  * vmbus_onopen_result - Open result handler.
1338 			(struct vmbus_channel_message_header *)msginfo->msg;  in vmbus_onopen_result()
1340 		if (requestheader->msgtype == CHANNELMSG_OPENCHANNEL) {  in vmbus_onopen_result()
1342 			(struct vmbus_channel_open_channel *)msginfo->msg;  in vmbus_onopen_result()
1343 			if (openmsg->child_relid == result->child_relid &&  in vmbus_onopen_result()
1344 			    openmsg->openid == result->openid) {  in vmbus_onopen_result()
1345 				memcpy(&msginfo->response.open_result,  in vmbus_onopen_result()
1349 				complete(&msginfo->waitevent);  in vmbus_onopen_result()
1358  * vmbus_ongpadl_created - GPADL created handler.
1385 			(struct vmbus_channel_message_header *)msginfo->msg;  in vmbus_ongpadl_created()
1387 		if (requestheader->msgtype == CHANNELMSG_GPADL_HEADER) {  in vmbus_ongpadl_created()
1391 			if ((gpadlcreated->child_relid ==  in vmbus_ongpadl_created()
1392 			     gpadlheader->child_relid) &&  in vmbus_ongpadl_created()
1393 			    (gpadlcreated->gpadl == gpadlheader->gpadl)) {  in vmbus_ongpadl_created()
1394 				memcpy(&msginfo->response.gpadl_created,  in vmbus_ongpadl_created()
1398 				complete(&msginfo->waitevent);  in vmbus_ongpadl_created()
1407  * vmbus_onmodifychannel_response - Modify Channel response handler.
1429 				(struct vmbus_channel_message_header *)msginfo->msg;  in vmbus_onmodifychannel_response()
1431 		if (responseheader->msgtype == CHANNELMSG_MODIFYCHANNEL) {  in vmbus_onmodifychannel_response()
1434 			modifymsg = (struct vmbus_channel_modifychannel *)msginfo->msg;  in vmbus_onmodifychannel_response()
1435 			if (modifymsg->child_relid == response->child_relid) {  in vmbus_onmodifychannel_response()
1436 				memcpy(&msginfo->response.modify_response, response,  in vmbus_onmodifychannel_response()
1438 				complete(&msginfo->waitevent);  in vmbus_onmodifychannel_response()
1447  * vmbus_ongpadl_torndown - GPADL torndown handler.
1474 			(struct vmbus_channel_message_header *)msginfo->msg;  in vmbus_ongpadl_torndown()
1476 		if (requestheader->msgtype == CHANNELMSG_GPADL_TEARDOWN) {  in vmbus_ongpadl_torndown()
1480 			if (gpadl_torndown->gpadl == gpadl_teardown->gpadl) {  in vmbus_ongpadl_torndown()
1481 				memcpy(&msginfo->response.gpadl_torndown,  in vmbus_ongpadl_torndown()
1485 				complete(&msginfo->waitevent);  in vmbus_ongpadl_torndown()
1494  * vmbus_onversion_response - Version response handler
1517 			(struct vmbus_channel_message_header *)msginfo->msg;  in vmbus_onversion_response()
1519 		if (requestheader->msgtype ==  in vmbus_onversion_response()
1521 			memcpy(&msginfo->response.version_response,  in vmbus_onversion_response()
1524 			complete(&msginfo->waitevent);  in vmbus_onversion_response()
1530 /* Channel message dispatch table */
1568  * vmbus_onmessage - Handler for channel protocol messages.
1577 	 * vmbus_on_msg_dpc() makes sure the hdr->msgtype here can not go  in vmbus_onmessage()
1580 	channel_message_table[hdr->msgtype].message_handler(hdr);  in vmbus_onmessage()
1584  * vmbus_request_offers - Send a request to get all our pending offers
1585  * and wait for all boot-time offers to arrive.
1597 		return -ENOMEM;  in vmbus_request_offers()
1599 	msg = (struct vmbus_channel_message_header *)msginfo->msg;  in vmbus_request_offers()
1601 	msg->msgtype = CHANNELMSG_REQUESTOFFERS;  in vmbus_request_offers()
1604 	 * This REQUESTOFFERS message will result in the host sending an all  in vmbus_request_offers()
1605 	 * offers delivered message after all the boot-time offers are sent.  in vmbus_request_offers()
1613 		pr_err("Unable to request offers - %d\n", ret);  in vmbus_request_offers()
1619 	 * Wait for the host to send all boot-time offers.  in vmbus_request_offers()
1620 	 * Keeping it as a best-effort mechanism, where a warning is  in vmbus_request_offers()
1625 		pr_warn("timed out waiting for all boot-time offers to be delivered.\n");  in vmbus_request_offers()
1650 	primary_channel->sc_creation_callback = sc_cr_cb;  in vmbus_set_sc_create_callback()
1657 	channel->chn_rescind_callback = chn_rescind_cb;  in vmbus_set_chn_rescind_callback()