384 static void fbnic_clean_twq0(struct fbnic_napi_vector *nv, int napi_budget,  in fbnic_clean_twq0()  argument
420 		fbnic_unmap_single_twd(nv->dev, &ring->desc[head]);  in fbnic_clean_twq0()
426 			fbnic_unmap_page_twd(nv->dev, &ring->desc[head]);  in fbnic_clean_twq0()
442 	txq = txring_txq(nv->napi.dev, ring);  in fbnic_clean_twq0()
464 static void fbnic_clean_tsq(struct fbnic_napi_vector *nv,  in fbnic_clean_tsq()  argument
481 				netdev_err(nv->napi.dev,  in fbnic_clean_tsq()
493 	fbn = netdev_priv(nv->napi.dev);  in fbnic_clean_tsq()
535 				  struct fbnic_napi_vector *nv, int budget)  in fbnic_page_pool_drain()  argument
541 		page_pool_put_unrefed_page(nv->page_pool, page, -1, !!budget);  in fbnic_page_pool_drain()
546 static void fbnic_clean_twq(struct fbnic_napi_vector *nv, int napi_budget,  in fbnic_clean_twq()  argument
550 		fbnic_clean_twq0(nv, napi_budget, &qt->sub0, false, head0);  in fbnic_clean_twq()
552 		fbnic_clean_twq0(nv, napi_budget, &qt->sub0, false, ts_head);  in fbnic_clean_twq()
556 fbnic_clean_tcq(struct fbnic_napi_vector *nv, struct fbnic_q_triad *qt,  in fbnic_clean_tcq()  argument
589 			fbnic_clean_tsq(nv, &qt->sub0, tcd, &ts_head, &head0);  in fbnic_clean_tcq()
610 	fbnic_clean_twq(nv, napi_budget, qt, ts_head, head0);  in fbnic_clean_tcq()
613 static void fbnic_clean_bdq(struct fbnic_napi_vector *nv, int napi_budget,  in fbnic_clean_bdq()  argument
622 		fbnic_page_pool_drain(ring, head, nv, napi_budget);  in fbnic_clean_bdq()
652 static void fbnic_fill_bdq(struct fbnic_napi_vector *nv, struct fbnic_ring *bdq)  in fbnic_fill_bdq()  argument
663 		page = page_pool_dev_alloc_pages(nv->page_pool);  in fbnic_fill_bdq()
709 static void fbnic_pkt_prepare(struct fbnic_napi_vector *nv, u64 rcd,  in fbnic_pkt_prepare()  argument
735 	dma_sync_single_range_for_cpu(nv->dev, page_pool_get_dma_addr(page),  in fbnic_pkt_prepare()
750 static void fbnic_add_rx_frag(struct fbnic_napi_vector *nv, u64 rcd,  in fbnic_add_rx_frag()  argument
768 	dma_sync_single_range_for_cpu(nv->dev, page_pool_get_dma_addr(page),  in fbnic_add_rx_frag()
783 static void fbnic_put_pkt_buff(struct fbnic_napi_vector *nv,  in fbnic_put_pkt_buff()  argument
798 		page_pool_put_full_page(nv->page_pool, page, !!budget);  in fbnic_put_pkt_buff()
802 	page_pool_put_full_page(nv->page_pool, page, !!budget);  in fbnic_put_pkt_buff()
805 static struct sk_buff *fbnic_build_skb(struct fbnic_napi_vector *nv,  in fbnic_build_skb()  argument
843 	skb->protocol = eth_type_trans(skb, nv->napi.dev);  in fbnic_build_skb()
859 static void fbnic_rx_tstamp(struct fbnic_napi_vector *nv, u64 rcd,  in fbnic_rx_tstamp()  argument
868 	fbn = netdev_priv(nv->napi.dev);  in fbnic_rx_tstamp()
876 static void fbnic_populate_skb_fields(struct fbnic_napi_vector *nv,  in fbnic_populate_skb_fields()  argument
880 	struct net_device *netdev = nv->napi.dev;  in fbnic_populate_skb_fields()
898 static int fbnic_clean_rcq(struct fbnic_napi_vector *nv,  in fbnic_clean_rcq()  argument
927 			fbnic_pkt_prepare(nv, rcd, pkt, qt);  in fbnic_clean_rcq()
932 			fbnic_add_rx_frag(nv, rcd, pkt, qt);  in fbnic_clean_rcq()
940 			fbnic_rx_tstamp(nv, rcd, pkt);  in fbnic_clean_rcq()
946 				skb = fbnic_build_skb(nv, pkt);  in fbnic_clean_rcq()
950 				fbnic_populate_skb_fields(nv, rcd, skb, qt);  in fbnic_clean_rcq()
955 				napi_gro_receive(&nv->napi, skb);  in fbnic_clean_rcq()
958 				fbnic_put_pkt_buff(nv, pkt, 1);  in fbnic_clean_rcq()
984 		fbnic_clean_bdq(nv, budget, &qt->sub0, head0);  in fbnic_clean_rcq()
985 	fbnic_fill_bdq(nv, &qt->sub0);  in fbnic_clean_rcq()
988 		fbnic_clean_bdq(nv, budget, &qt->sub1, head1);  in fbnic_clean_rcq()
989 	fbnic_fill_bdq(nv, &qt->sub1);  in fbnic_clean_rcq()
1000 static void fbnic_nv_irq_disable(struct fbnic_napi_vector *nv)  in fbnic_nv_irq_disable()  argument
1002 	struct fbnic_dev *fbd = nv->fbd;  in fbnic_nv_irq_disable()
1003 	u32 v_idx = nv->v_idx;  in fbnic_nv_irq_disable()
1008 static void fbnic_nv_irq_rearm(struct fbnic_napi_vector *nv)  in fbnic_nv_irq_rearm()  argument
1010 	struct fbnic_dev *fbd = nv->fbd;  in fbnic_nv_irq_rearm()
1011 	u32 v_idx = nv->v_idx;  in fbnic_nv_irq_rearm()
1019 	struct fbnic_napi_vector *nv = container_of(napi,  in fbnic_poll()  local
1024 	for (i = 0; i < nv->txt_count; i++)  in fbnic_poll()
1025 		fbnic_clean_tcq(nv, &nv->qt[i], budget);  in fbnic_poll()
1027 	for (j = 0; j < nv->rxt_count; j++, i++)  in fbnic_poll()
1028 		work_done += fbnic_clean_rcq(nv, &nv->qt[i], budget);  in fbnic_poll()
1034 		fbnic_nv_irq_rearm(nv);  in fbnic_poll()
1041 	struct fbnic_napi_vector *nv = *(void **)data;  in fbnic_msix_clean_rings()  local
1043 	napi_schedule_irqoff(&nv->napi);  in fbnic_msix_clean_rings()
1099 				   struct fbnic_napi_vector *nv)  in fbnic_free_napi_vector()  argument
1101 	struct fbnic_dev *fbd = nv->fbd;  in fbnic_free_napi_vector()
1104 	for (i = 0; i < nv->txt_count; i++) {  in fbnic_free_napi_vector()
1105 		fbnic_remove_tx_ring(fbn, &nv->qt[i].sub0);  in fbnic_free_napi_vector()
1106 		fbnic_remove_tx_ring(fbn, &nv->qt[i].cmpl);  in fbnic_free_napi_vector()
1109 	for (j = 0; j < nv->rxt_count; j++, i++) {  in fbnic_free_napi_vector()
1110 		fbnic_remove_rx_ring(fbn, &nv->qt[i].sub0);  in fbnic_free_napi_vector()
1111 		fbnic_remove_rx_ring(fbn, &nv->qt[i].sub1);  in fbnic_free_napi_vector()
1112 		fbnic_remove_rx_ring(fbn, &nv->qt[i].cmpl);  in fbnic_free_napi_vector()
1115 	fbnic_napi_free_irq(fbd, nv);  in fbnic_free_napi_vector()
1116 	page_pool_destroy(nv->page_pool);  in fbnic_free_napi_vector()
1117 	netif_napi_del(&nv->napi);  in fbnic_free_napi_vector()
1118 	fbn->napi[fbnic_napi_idx(nv)] = NULL;  in fbnic_free_napi_vector()
1119 	kfree(nv);  in fbnic_free_napi_vector()
1135 				    struct fbnic_napi_vector *nv)  in fbnic_alloc_nv_page_pool()  argument
1140 		.pool_size = (fbn->hpq_size + fbn->ppq_size) * nv->rxt_count,  in fbnic_alloc_nv_page_pool()
1142 		.dev = nv->dev,  in fbnic_alloc_nv_page_pool()
1165 	nv->page_pool = pp;  in fbnic_alloc_nv_page_pool()
1186 	struct fbnic_napi_vector *nv;  in fbnic_alloc_napi_vector()  local
1200 	nv = kzalloc(struct_size(nv, qt, qt_count), GFP_KERNEL);  in fbnic_alloc_napi_vector()
1201 	if (!nv)  in fbnic_alloc_napi_vector()
1205 	nv->txt_count = txt_count;  in fbnic_alloc_napi_vector()
1206 	nv->rxt_count = rxt_count;  in fbnic_alloc_napi_vector()
1209 	nv->fbd = fbd;  in fbnic_alloc_napi_vector()
1210 	nv->v_idx = v_idx;  in fbnic_alloc_napi_vector()
1213 	fbn->napi[fbnic_napi_idx(nv)] = nv;  in fbnic_alloc_napi_vector()
1214 	netif_napi_add(fbn->netdev, &nv->napi, fbnic_poll);  in fbnic_alloc_napi_vector()
1217 	netif_napi_set_irq(&nv->napi,  in fbnic_alloc_napi_vector()
1218 			   pci_irq_vector(to_pci_dev(fbd->dev), nv->v_idx));  in fbnic_alloc_napi_vector()
1220 	/* Tie nv back to PCIe dev */  in fbnic_alloc_napi_vector()
1221 	nv->dev = fbd->dev;  in fbnic_alloc_napi_vector()
1225 		err = fbnic_alloc_nv_page_pool(fbn, nv);  in fbnic_alloc_napi_vector()
1231 	err = fbnic_napi_request_irq(fbd, nv);  in fbnic_alloc_napi_vector()
1236 	qt = nv->qt;  in fbnic_alloc_napi_vector()
1291 	page_pool_destroy(nv->page_pool);  in fbnic_alloc_napi_vector()
1293 	netif_napi_del(&nv->napi);  in fbnic_alloc_napi_vector()
1294 	fbn->napi[fbnic_napi_idx(nv)] = NULL;  in fbnic_alloc_napi_vector()
1295 	kfree(nv);  in fbnic_alloc_napi_vector()
1558 				    struct fbnic_napi_vector *nv)  in fbnic_free_nv_resources()  argument
1563 	for (i = 0; i < nv->txt_count; i++)  in fbnic_free_nv_resources()
1564 		fbnic_free_qt_resources(fbn, &nv->qt[i]);  in fbnic_free_nv_resources()
1566 	for (j = 0; j < nv->rxt_count; j++, i++)  in fbnic_free_nv_resources()
1567 		fbnic_free_qt_resources(fbn, &nv->qt[i]);  in fbnic_free_nv_resources()
1571 				    struct fbnic_napi_vector *nv)  in fbnic_alloc_nv_resources()  argument
1576 	for (i = 0; i < nv->txt_count; i++) {  in fbnic_alloc_nv_resources()
1577 		err = fbnic_alloc_tx_qt_resources(fbn, &nv->qt[i]);  in fbnic_alloc_nv_resources()
1583 	for (j = 0; j < nv->rxt_count; j++, i++) {  in fbnic_alloc_nv_resources()
1584 		err = fbnic_alloc_rx_qt_resources(fbn, &nv->qt[i]);  in fbnic_alloc_nv_resources()
1593 		fbnic_free_qt_resources(fbn, &nv->qt[i]);  in fbnic_alloc_nv_resources()
1624 static void fbnic_set_netif_napi(struct fbnic_napi_vector *nv)  in fbnic_set_netif_napi()  argument
1629 	for (i = 0; i < nv->txt_count; i++) {  in fbnic_set_netif_napi()
1630 		struct fbnic_q_triad *qt = &nv->qt[i];  in fbnic_set_netif_napi()
1632 		netif_queue_set_napi(nv->napi.dev, qt->sub0.q_idx,  in fbnic_set_netif_napi()
1633 				     NETDEV_QUEUE_TYPE_TX, &nv->napi);  in fbnic_set_netif_napi()
1637 	for (j = 0; j < nv->rxt_count; j++, i++) {  in fbnic_set_netif_napi()
1638 		struct fbnic_q_triad *qt = &nv->qt[i];  in fbnic_set_netif_napi()
1640 		netif_queue_set_napi(nv->napi.dev, qt->cmpl.q_idx,  in fbnic_set_netif_napi()
1641 				     NETDEV_QUEUE_TYPE_RX, &nv->napi);  in fbnic_set_netif_napi()
1645 static void fbnic_reset_netif_napi(struct fbnic_napi_vector *nv)  in fbnic_reset_netif_napi()  argument
1650 	for (i = 0; i < nv->txt_count; i++) {  in fbnic_reset_netif_napi()
1651 		struct fbnic_q_triad *qt = &nv->qt[i];  in fbnic_reset_netif_napi()
1653 		netif_queue_set_napi(nv->napi.dev, qt->sub0.q_idx,  in fbnic_reset_netif_napi()
1658 	for (j = 0; j < nv->rxt_count; j++, i++) {  in fbnic_reset_netif_napi()
1659 		struct fbnic_q_triad *qt = &nv->qt[i];  in fbnic_reset_netif_napi()
1661 		netif_queue_set_napi(nv->napi.dev, qt->cmpl.q_idx,  in fbnic_reset_netif_napi()
1736 		struct fbnic_napi_vector *nv = fbn->napi[i];  in fbnic_disable()  local
1739 		for (t = 0; t < nv->txt_count; t++) {  in fbnic_disable()
1740 			struct fbnic_q_triad *qt = &nv->qt[t];  in fbnic_disable()
1747 		for (j = 0; j < nv->rxt_count; j++, t++) {  in fbnic_disable()
1748 			struct fbnic_q_triad *qt = &nv->qt[t];  in fbnic_disable()
1847 		struct fbnic_napi_vector *nv = fbn->napi[i];  in fbnic_flush()  local
1851 		for (t = 0; t < nv->txt_count; t++) {  in fbnic_flush()
1852 			struct fbnic_q_triad *qt = &nv->qt[t];  in fbnic_flush()
1856 			fbnic_clean_twq0(nv, 0, &qt->sub0, true, qt->sub0.tail);  in fbnic_flush()
1866 			tx_queue = netdev_get_tx_queue(nv->napi.dev,  in fbnic_flush()
1872 		for (j = 0; j < nv->rxt_count; j++, t++) {  in fbnic_flush()
1873 			struct fbnic_q_triad *qt = &nv->qt[t];  in fbnic_flush()
1876 			fbnic_clean_bdq(nv, 0, &qt->sub0, qt->sub0.tail);  in fbnic_flush()
1877 			fbnic_clean_bdq(nv, 0, &qt->sub1, qt->sub1.tail);  in fbnic_flush()
1882 			fbnic_put_pkt_buff(nv, qt->cmpl.pkt, 0);  in fbnic_flush()
1893 		struct fbnic_napi_vector *nv = fbn->napi[i];  in fbnic_fill()  local
1899 		for (j = 0, t = nv->txt_count; j < nv->rxt_count; j++, t++) {  in fbnic_fill()
1900 			struct fbnic_q_triad *qt = &nv->qt[t];  in fbnic_fill()
1903 			fbnic_fill_bdq(nv, &qt->sub0);  in fbnic_fill()
1904 			fbnic_fill_bdq(nv, &qt->sub1);  in fbnic_fill()
1931 static void fbnic_enable_tcq(struct fbnic_napi_vector *nv,  in fbnic_enable_tcq()  argument
1952 	fbnic_ring_wr32(tcq, FBNIC_QUEUE_TIM_CTL, nv->v_idx);  in fbnic_enable_tcq()
1998 static void fbnic_config_drop_mode_rcq(struct fbnic_napi_vector *nv,  in fbnic_config_drop_mode_rcq()  argument
2013 static void fbnic_enable_rcq(struct fbnic_napi_vector *nv,  in fbnic_enable_rcq()  argument
2019 	fbnic_config_drop_mode_rcq(nv, rcq);  in fbnic_enable_rcq()
2043 	fbnic_ring_wr32(rcq, FBNIC_QUEUE_RIM_CTL, nv->v_idx);  in fbnic_enable_rcq()
2057 		struct fbnic_napi_vector *nv = fbn->napi[i];  in fbnic_enable()  local
2061 		for (t = 0; t < nv->txt_count; t++) {  in fbnic_enable()
2062 			struct fbnic_q_triad *qt = &nv->qt[t];  in fbnic_enable()
2065 			fbnic_enable_tcq(nv, &qt->cmpl);  in fbnic_enable()
2069 		for (j = 0; j < nv->rxt_count; j++, t++) {  in fbnic_enable()
2070 			struct fbnic_q_triad *qt = &nv->qt[t];  in fbnic_enable()
2073 			fbnic_config_drop_mode_rcq(nv, &qt->cmpl);  in fbnic_enable()
2074 			fbnic_enable_rcq(nv, &qt->cmpl);  in fbnic_enable()
2081 static void fbnic_nv_irq_enable(struct fbnic_napi_vector *nv)  in fbnic_nv_irq_enable()  argument
2083 	struct fbnic_dev *fbd = nv->fbd;  in fbnic_nv_irq_enable()
2088 	fbnic_wr32(fbd, FBNIC_INTR_CQ_REARM(nv->v_idx), val);  in fbnic_nv_irq_enable()
2098 		struct fbnic_napi_vector *nv = fbn->napi[i];  in fbnic_napi_enable()  local
2100 		napi_enable(&nv->napi);  in fbnic_napi_enable()
2102 		fbnic_nv_irq_enable(nv);  in fbnic_napi_enable()
2107 		irqs[nv->v_idx / 32] |= BIT(nv->v_idx % 32);  in fbnic_napi_enable()
2131 		struct fbnic_napi_vector *nv = fbn->napi[i];  in fbnic_napi_depletion_check()  local
2134 		for (t = nv->txt_count, j = 0; j < nv->rxt_count; j++, t++) {  in fbnic_napi_depletion_check()
2138 			if (fbnic_desc_used(&nv->qt[t].sub0) < 4 ||  in fbnic_napi_depletion_check()
2139 			    fbnic_desc_used(&nv->qt[t].sub1) < 4)  in fbnic_napi_depletion_check()
2140 				irqs[nv->v_idx / 32] |= BIT(nv->v_idx % 32);  in fbnic_napi_depletion_check()