99 	struct vring vring;  member
120 	 * vring.
133 	} vring;  member
291 static bool vring_need_unmap_buffer(const struct vring_virtqueue *vring,  in vring_need_unmap_buffer()  argument
294 	return vring->use_dma_api && (extra->addr != DMA_MAPPING_ERROR);  in vring_need_unmap_buffer()
354  * making all of the arch DMA ops work on the vring device itself
567 		WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect);  in virtqueue_add_split()
579 		desc = vq->split.vring.desc;  in virtqueue_add_split()
637 		vq->split.desc_extra[prev & (vq->split.vring.num - 1)].flags &=  in virtqueue_add_split()
648 		virtqueue_add_desc_split(_vq, vq->split.vring.desc,  in virtqueue_add_split()
673 	avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1);  in virtqueue_add_split()
674 	vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);  in virtqueue_add_split()
680 	vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,  in virtqueue_add_split()
736 					vring_avail_event(&vq->split.vring)),  in virtqueue_kick_prepare_split()
739 		needs_kick = !(vq->split.vring.used->flags &  in virtqueue_kick_prepare_split()
762 	while (vq->split.vring.desc[i].flags & nextflag) {  in detach_buf_split()
808 			vq->split.vring.used->idx);  in more_used_split()
836 	last_used = (vq->last_used_idx & (vq->split.vring.num - 1));  in virtqueue_get_buf_ctx_split()
838 			vq->split.vring.used->ring[last_used].id);  in virtqueue_get_buf_ctx_split()
840 			vq->split.vring.used->ring[last_used].len);  in virtqueue_get_buf_ctx_split()
842 	if (unlikely(i >= vq->split.vring.num)) {  in virtqueue_get_buf_ctx_split()
860 				&vring_used_event(&vq->split.vring),  in virtqueue_get_buf_ctx_split()
885 			vring_used_event(&vq->split.vring) = 0x0;  in virtqueue_disable_cb_split()
887 			vq->split.vring.avail->flags =  in virtqueue_disable_cb_split()
908 			vq->split.vring.avail->flags =  in virtqueue_enable_cb_prepare_split()
912 	vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev,  in virtqueue_enable_cb_prepare_split()
923 			vq->split.vring.used->idx);  in virtqueue_poll_split()
941 			vq->split.vring.avail->flags =  in virtqueue_enable_cb_delayed_split()
949 			&vring_used_event(&vq->split.vring),  in virtqueue_enable_cb_delayed_split()
952 	if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx)  in virtqueue_enable_cb_delayed_split()
970 	for (i = 0; i < vq->split.vring.num; i++) {  in virtqueue_detach_unused_buf_split()
977 		vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,  in virtqueue_detach_unused_buf_split()
983 	BUG_ON(vq->vq.num_free != vq->split.vring.num);  in virtqueue_detach_unused_buf_split()
1003 			vring_split->vring.avail->flags = cpu_to_virtio16(vdev,  in virtqueue_vring_init_split()
1012 	num = vq->split.vring.num;  in virtqueue_reinit_split()
1014 	vq->split.vring.avail->flags = 0;  in virtqueue_reinit_split()
1015 	vq->split.vring.avail->idx = 0;  in virtqueue_reinit_split()
1018 	vq->split.vring.avail->ring[num] = 0;  in virtqueue_reinit_split()
1020 	vq->split.vring.used->flags = 0;  in virtqueue_reinit_split()
1021 	vq->split.vring.used->idx = 0;  in virtqueue_reinit_split()
1024 	*(__virtio16 *)&(vq->split.vring.used->ring[num]) = 0;  in virtqueue_reinit_split()
1044 	u32 num = vring_split->vring.num;  in vring_alloc_state_extra_split()
1070 			 vring_split->vring.desc,  in vring_free_split()
1118 	vring_init(&vring_split->vring, num, queue, vring_align);  in vring_alloc_queue_split()
1178 	virtqueue_init(vq, vring_split->vring.num);  in __vring_new_virtqueue_split()
1243 	virtqueue_init(vq, vring_split.vring.num);  in virtqueue_resize_split()
1355 	BUG_ON(id == vq->packed.vring.num);  in virtqueue_add_indirect_packed()
1386 	vq->packed.vring.desc[head].addr = cpu_to_le64(addr);  in virtqueue_add_indirect_packed()
1387 	vq->packed.vring.desc[head].len = cpu_to_le32(total_sg *  in virtqueue_add_indirect_packed()
1389 	vq->packed.vring.desc[head].id = cpu_to_le16(id);  in virtqueue_add_indirect_packed()
1405 	vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT |  in virtqueue_add_indirect_packed()
1413 	if (n >= vq->packed.vring.num) {  in virtqueue_add_indirect_packed()
1494 	WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);  in virtqueue_add_packed()
1496 	desc = vq->packed.vring.desc;  in virtqueue_add_packed()
1508 	BUG_ON(id == vq->packed.vring.num);  in virtqueue_add_packed()
1543 			if ((unlikely(++i >= vq->packed.vring.num))) {  in virtqueue_add_packed()
1574 	vq->packed.vring.desc[head].flags = head_flags;  in virtqueue_add_packed()
1595 		if (i >= vq->packed.vring.num)  in virtqueue_add_packed()
1628 	snapshot.u32 = *(u32 *)vq->packed.vring.device;  in virtqueue_kick_prepare_packed()
1644 		event_idx -= vq->packed.vring.num;  in virtqueue_kick_prepare_packed()
1708 	flags = le16_to_cpu(vq->packed.vring.desc[idx].flags);  in is_used_desc_packed()
1755 	id = le16_to_cpu(vq->packed.vring.desc[last_used].id);  in virtqueue_get_buf_ctx_packed()
1756 	*len = le32_to_cpu(vq->packed.vring.desc[last_used].len);  in virtqueue_get_buf_ctx_packed()
1758 	if (unlikely(id >= vq->packed.vring.num)) {  in virtqueue_get_buf_ctx_packed()
1772 	if (unlikely(last_used >= vq->packed.vring.num)) {  in virtqueue_get_buf_ctx_packed()
1773 		last_used -= vq->packed.vring.num;  in virtqueue_get_buf_ctx_packed()
1787 				&vq->packed.vring.driver->off_wrap,  in virtqueue_get_buf_ctx_packed()
1810 		vq->packed.vring.driver->flags =  in virtqueue_disable_cb_packed()
1827 		vq->packed.vring.driver->off_wrap =  in virtqueue_enable_cb_prepare_packed()
1840 		vq->packed.vring.driver->flags =  in virtqueue_enable_cb_prepare_packed()
1875 		bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4;  in virtqueue_enable_cb_delayed_packed()
1880 		if (used_idx >= vq->packed.vring.num) {  in virtqueue_enable_cb_delayed_packed()
1881 			used_idx -= vq->packed.vring.num;  in virtqueue_enable_cb_delayed_packed()
1885 		vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx |  in virtqueue_enable_cb_delayed_packed()
1899 		vq->packed.vring.driver->flags =  in virtqueue_enable_cb_delayed_packed()
1929 	for (i = 0; i < vq->packed.vring.num; i++) {  in virtqueue_detach_unused_buf_packed()
1939 	BUG_ON(vq->vq.num_free != vq->packed.vring.num);  in virtqueue_detach_unused_buf_packed()
1967 	if (vring_packed->vring.desc)  in vring_free_packed()
1969 				 vring_packed->vring.desc,  in vring_free_packed()
1973 	if (vring_packed->vring.driver)  in vring_free_packed()
1975 				 vring_packed->vring.driver,  in vring_free_packed()
1979 	if (vring_packed->vring.device)  in vring_free_packed()
1981 				 vring_packed->vring.device,  in vring_free_packed()
2007 	vring_packed->vring.desc         = ring;  in vring_alloc_queue_packed()
2020 	vring_packed->vring.driver          = driver;  in vring_alloc_queue_packed()
2031 	vring_packed->vring.device          = device;  in vring_alloc_queue_packed()
2034 	vring_packed->vring.num = num;  in vring_alloc_queue_packed()
2047 	u32 num = vring_packed->vring.num;  in vring_alloc_state_extra_packed()
2081 		vring_packed->vring.driver->flags =  in virtqueue_vring_init_packed()
2097 	memset(vq->packed.vring.device, 0, vq->packed.event_size_in_bytes);  in virtqueue_reinit_packed()
2098 	memset(vq->packed.vring.driver, 0, vq->packed.event_size_in_bytes);  in virtqueue_reinit_packed()
2101 	memset(vq->packed.vring.desc, 0, vq->packed.ring_size_in_bytes);  in virtqueue_reinit_packed()
2103 	virtqueue_init(vq, vq->packed.vring.num);  in virtqueue_reinit_packed()
2156 	virtqueue_init(vq, vring_packed->vring.num);  in __vring_new_virtqueue_packed()
2214 	virtqueue_init(vq, vring_packed.vring.num);  in virtqueue_resize_packed()
2702 			      "virtio vring IRQ raised before DRIVER_OK");  in vring_interrupt()
2771  * virtqueue_resize - resize the vring of vq
2777  * When it is really necessary to create a new vring, it will set the current vq
2779  * that is no longer used. Only after the new vring is successfully created, the
2780  * old vring will be released.
2808 	if ((vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num) == num)  in virtqueue_resize()
2879 		vring_packed.vring.num = num;  in vring_new_virtqueue()
2880 		vring_packed.vring.desc = pages;  in vring_new_virtqueue()
2887 	vring_init(&vring_split.vring, num, pages, vring_align);  in vring_new_virtqueue()
2902 					 vq->packed.vring.desc,  in vring_free()
2908 					 vq->packed.vring.driver,  in vring_free()
2914 					 vq->packed.vring.device,  in vring_free()
2923 					 vq->split.vring.desc,  in vring_free()
2995  * virtqueue_get_vring_size - return the size of the virtqueue's vring
2996  * @_vq: the struct virtqueue containing the vring of interest.
2998  * Returns the size of the vring.  This is mainly used for boasting to
3006 	return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;  in virtqueue_get_vring_size()
3106 		((char *)vq->split.vring.avail - (char *)vq->split.vring.desc);  in virtqueue_get_avail_addr()
3120 		((char *)vq->split.vring.used - (char *)vq->split.vring.desc);  in virtqueue_get_used_addr()
3125 const struct vring *virtqueue_get_vring(const struct virtqueue *vq)  in virtqueue_get_vring()
3127 	return &to_vvq(vq)->split.vring;  in virtqueue_get_vring()