412 	struct am65_cpsw_tx_chn *tx_chn;  in am65_cpsw_nuss_ndo_host_tx_timeout()  local
417 	tx_chn = &common->tx_chns[txqueue];  in am65_cpsw_nuss_ndo_host_tx_timeout()
425 		   k3_cppi_desc_pool_avail(tx_chn->desc_pool));  in am65_cpsw_nuss_ndo_host_tx_timeout()
662 	struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[id];  in am65_cpsw_destroy_txq()  local
664 	napi_disable(&tx_chn->napi_tx);  in am65_cpsw_destroy_txq()
665 	hrtimer_cancel(&tx_chn->tx_hrtimer);  in am65_cpsw_destroy_txq()
666 	k3_udma_glue_reset_tx_chn(tx_chn->tx_chn, tx_chn,  in am65_cpsw_destroy_txq()
668 	k3_udma_glue_disable_tx_chn(tx_chn->tx_chn);  in am65_cpsw_destroy_txq()
673 	struct am65_cpsw_tx_chn *tx_chn = common->tx_chns;  in am65_cpsw_destroy_txqs()  local
683 		k3_udma_glue_tdown_tx_chn(tx_chn[id].tx_chn, false);  in am65_cpsw_destroy_txqs()
696 	struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[id];  in am65_cpsw_create_txq()  local
699 	ret = k3_udma_glue_enable_tx_chn(tx_chn->tx_chn);  in am65_cpsw_create_txq()
703 	napi_enable(&tx_chn->napi_tx);  in am65_cpsw_create_txq()
739 static void am65_cpsw_nuss_set_buf_type(struct am65_cpsw_tx_chn *tx_chn,  in am65_cpsw_nuss_set_buf_type()  argument
745 	desc_idx = am65_cpsw_nuss_desc_idx(tx_chn->desc_pool, desc,  in am65_cpsw_nuss_set_buf_type()
746 					   tx_chn->dsize_log2);  in am65_cpsw_nuss_set_buf_type()
747 	k3_cppi_desc_pool_desc_info_set(tx_chn->desc_pool, desc_idx,  in am65_cpsw_nuss_set_buf_type()
751 static enum am65_cpsw_tx_buf_type am65_cpsw_nuss_buf_type(struct am65_cpsw_tx_chn *tx_chn,  in am65_cpsw_nuss_buf_type()  argument
757 	desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma);  in am65_cpsw_nuss_buf_type()
758 	desc_idx = am65_cpsw_nuss_desc_idx(tx_chn->desc_pool, desc_tx,  in am65_cpsw_nuss_buf_type()
759 					   tx_chn->dsize_log2);  in am65_cpsw_nuss_buf_type()
761 	return (enum am65_cpsw_tx_buf_type)k3_cppi_desc_pool_desc_info(tx_chn->desc_pool,  in am65_cpsw_nuss_buf_type()
793 static void am65_cpsw_nuss_xmit_free(struct am65_cpsw_tx_chn *tx_chn,  in am65_cpsw_nuss_xmit_free()  argument
804 	k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);  in am65_cpsw_nuss_xmit_free()
806 	dma_unmap_single(tx_chn->dma_dev, buf_dma, buf_dma_len, DMA_TO_DEVICE);  in am65_cpsw_nuss_xmit_free()
809 	k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma);  in am65_cpsw_nuss_xmit_free()
811 		next_desc = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool,  in am65_cpsw_nuss_xmit_free()
814 		k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);  in am65_cpsw_nuss_xmit_free()
816 		dma_unmap_page(tx_chn->dma_dev, buf_dma, buf_dma_len,  in am65_cpsw_nuss_xmit_free()
820 		k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma);  in am65_cpsw_nuss_xmit_free()
822 		k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);  in am65_cpsw_nuss_xmit_free()
825 	k3_cppi_desc_pool_free(tx_chn->desc_pool, first_desc);  in am65_cpsw_nuss_xmit_free()
830 	struct am65_cpsw_tx_chn *tx_chn = data;  in am65_cpsw_nuss_tx_cleanup()  local
837 	desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma);  in am65_cpsw_nuss_tx_cleanup()
839 	buf_type = am65_cpsw_nuss_buf_type(tx_chn, desc_dma);  in am65_cpsw_nuss_tx_cleanup()
848 	am65_cpsw_nuss_xmit_free(tx_chn, desc_tx);  in am65_cpsw_nuss_tx_cleanup()
1094 				  struct am65_cpsw_tx_chn *tx_chn,  in am65_cpsw_xdp_tx_frame()  argument
1107 	host_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);  in am65_cpsw_xdp_tx_frame()
1113 	am65_cpsw_nuss_set_buf_type(tx_chn, host_desc, buf_type);  in am65_cpsw_xdp_tx_frame()
1115 	dma_buf = dma_map_single(tx_chn->dma_dev, xdpf->data,  in am65_cpsw_xdp_tx_frame()
1117 	if (unlikely(dma_mapping_error(tx_chn->dma_dev, dma_buf))) {  in am65_cpsw_xdp_tx_frame()
1130 	k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &dma_buf);  in am65_cpsw_xdp_tx_frame()
1137 	netif_txq = netdev_get_tx_queue(ndev, tx_chn->id);  in am65_cpsw_xdp_tx_frame()
1140 	dma_desc = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, host_desc);  in am65_cpsw_xdp_tx_frame()
1142 		ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, host_desc,  in am65_cpsw_xdp_tx_frame()
1145 		spin_lock_bh(&tx_chn->lock);  in am65_cpsw_xdp_tx_frame()
1146 		ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, host_desc,  in am65_cpsw_xdp_tx_frame()
1148 		spin_unlock_bh(&tx_chn->lock);  in am65_cpsw_xdp_tx_frame()
1161 	k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &dma_buf);  in am65_cpsw_xdp_tx_frame()
1162 	dma_unmap_single(tx_chn->dma_dev, dma_buf, pkt_len, DMA_TO_DEVICE);  in am65_cpsw_xdp_tx_frame()
1164 	k3_cppi_desc_pool_free(tx_chn->desc_pool, host_desc);  in am65_cpsw_xdp_tx_frame()
1176 	struct am65_cpsw_tx_chn *tx_chn;  in am65_cpsw_run_xdp()  local
1199 		tx_chn = &common->tx_chns[cpu % AM65_CPSW_MAX_QUEUES];  in am65_cpsw_run_xdp()
1200 		netif_txq = netdev_get_tx_queue(ndev, tx_chn->id);  in am65_cpsw_run_xdp()
1209 		err = am65_cpsw_xdp_tx_frame(ndev, tx_chn, xdpf,  in am65_cpsw_run_xdp()
1442 am65_cpsw_nuss_tx_compl_packet_skb(struct am65_cpsw_tx_chn *tx_chn,  in am65_cpsw_nuss_tx_compl_packet_skb()  argument
1449 	desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool,  in am65_cpsw_nuss_tx_compl_packet_skb()
1453 	am65_cpsw_nuss_xmit_free(tx_chn, desc_tx);  in am65_cpsw_nuss_tx_compl_packet_skb()
1455 	am65_cpts_tx_timestamp(tx_chn->common->cpts, skb);  in am65_cpsw_nuss_tx_compl_packet_skb()
1464 				   struct am65_cpsw_tx_chn *tx_chn,  in am65_cpsw_nuss_tx_compl_packet_xdp()  argument
1474 	desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma);  in am65_cpsw_nuss_tx_compl_packet_xdp()
1478 	am65_cpsw_nuss_xmit_free(tx_chn, desc_tx);  in am65_cpsw_nuss_tx_compl_packet_xdp()
1487 static void am65_cpsw_nuss_tx_wake(struct am65_cpsw_tx_chn *tx_chn, struct net_device *ndev,  in am65_cpsw_nuss_tx_wake()  argument
1497 		    (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >= MAX_SKB_FRAGS))  in am65_cpsw_nuss_tx_wake()
1509 	struct am65_cpsw_tx_chn *tx_chn;  in am65_cpsw_nuss_tx_compl_packets()  local
1518 	tx_chn = &common->tx_chns[chn];  in am65_cpsw_nuss_tx_compl_packets()
1521 		spin_lock(&tx_chn->lock);  in am65_cpsw_nuss_tx_compl_packets()
1522 		res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma);  in am65_cpsw_nuss_tx_compl_packets()
1523 		spin_unlock(&tx_chn->lock);  in am65_cpsw_nuss_tx_compl_packets()
1534 		buf_type = am65_cpsw_nuss_buf_type(tx_chn, desc_dma);  in am65_cpsw_nuss_tx_compl_packets()
1536 			skb = am65_cpsw_nuss_tx_compl_packet_skb(tx_chn, desc_dma);  in am65_cpsw_nuss_tx_compl_packets()
1541 			xdpf = am65_cpsw_nuss_tx_compl_packet_xdp(common, tx_chn,  in am65_cpsw_nuss_tx_compl_packets()
1555 		am65_cpsw_nuss_tx_wake(tx_chn, ndev, netif_txq);  in am65_cpsw_nuss_tx_compl_packets()
1568 	struct am65_cpsw_tx_chn *tx_chn;  in am65_cpsw_nuss_tx_compl_packets_2g()  local
1577 	tx_chn = &common->tx_chns[chn];  in am65_cpsw_nuss_tx_compl_packets_2g()
1580 		res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma);  in am65_cpsw_nuss_tx_compl_packets_2g()
1591 		buf_type = am65_cpsw_nuss_buf_type(tx_chn, desc_dma);  in am65_cpsw_nuss_tx_compl_packets_2g()
1593 			skb = am65_cpsw_nuss_tx_compl_packet_skb(tx_chn, desc_dma);  in am65_cpsw_nuss_tx_compl_packets_2g()
1598 			xdpf = am65_cpsw_nuss_tx_compl_packet_xdp(common, tx_chn,  in am65_cpsw_nuss_tx_compl_packets_2g()
1616 	am65_cpsw_nuss_tx_wake(tx_chn, ndev, netif_txq);  in am65_cpsw_nuss_tx_compl_packets_2g()
1634 	struct am65_cpsw_tx_chn *tx_chn = am65_cpsw_napi_to_tx_chn(napi_tx);  in am65_cpsw_nuss_tx_poll()  local
1638 	if (AM65_CPSW_IS_CPSW2G(tx_chn->common))  in am65_cpsw_nuss_tx_poll()
1639 		num_tx = am65_cpsw_nuss_tx_compl_packets_2g(tx_chn->common, tx_chn->id,  in am65_cpsw_nuss_tx_poll()
1642 		num_tx = am65_cpsw_nuss_tx_compl_packets(tx_chn->common,  in am65_cpsw_nuss_tx_poll()
1643 							 tx_chn->id, budget, &tdown);  in am65_cpsw_nuss_tx_poll()
1649 		if (unlikely(tx_chn->tx_pace_timeout && !tdown)) {  in am65_cpsw_nuss_tx_poll()
1650 			hrtimer_start(&tx_chn->tx_hrtimer,  in am65_cpsw_nuss_tx_poll()
1651 				      ns_to_ktime(tx_chn->tx_pace_timeout),  in am65_cpsw_nuss_tx_poll()
1654 			enable_irq(tx_chn->irq);  in am65_cpsw_nuss_tx_poll()
1674 	struct am65_cpsw_tx_chn *tx_chn = dev_id;  in am65_cpsw_nuss_tx_irq()  local
1677 	napi_schedule(&tx_chn->napi_tx);  in am65_cpsw_nuss_tx_irq()
1689 	struct am65_cpsw_tx_chn *tx_chn;  in am65_cpsw_nuss_ndo_slave_xmit()  local
1707 	tx_chn = &common->tx_chns[q_idx];  in am65_cpsw_nuss_ndo_slave_xmit()
1711 	buf_dma = dma_map_single(tx_chn->dma_dev, skb->data, pkt_len,  in am65_cpsw_nuss_ndo_slave_xmit()
1713 	if (unlikely(dma_mapping_error(tx_chn->dma_dev, buf_dma))) {  in am65_cpsw_nuss_ndo_slave_xmit()
1719 	first_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);  in am65_cpsw_nuss_ndo_slave_xmit()
1722 		dma_unmap_single(tx_chn->dma_dev, buf_dma, pkt_len,  in am65_cpsw_nuss_ndo_slave_xmit()
1727 	am65_cpsw_nuss_set_buf_type(tx_chn, first_desc,  in am65_cpsw_nuss_ndo_slave_xmit()
1736 	k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma);  in am65_cpsw_nuss_ndo_slave_xmit()
1766 		next_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);  in am65_cpsw_nuss_ndo_slave_xmit()
1772 		am65_cpsw_nuss_set_buf_type(tx_chn, next_desc,  in am65_cpsw_nuss_ndo_slave_xmit()
1775 		buf_dma = skb_frag_dma_map(tx_chn->dma_dev, frag, 0, frag_size,  in am65_cpsw_nuss_ndo_slave_xmit()
1777 		if (unlikely(dma_mapping_error(tx_chn->dma_dev, buf_dma))) {  in am65_cpsw_nuss_ndo_slave_xmit()
1779 			k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);  in am65_cpsw_nuss_ndo_slave_xmit()
1785 		k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma);  in am65_cpsw_nuss_ndo_slave_xmit()
1789 		desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool,  in am65_cpsw_nuss_ndo_slave_xmit()
1791 		k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &desc_dma);  in am65_cpsw_nuss_ndo_slave_xmit()
1806 	desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, first_desc);  in am65_cpsw_nuss_ndo_slave_xmit()
1808 		ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma);  in am65_cpsw_nuss_ndo_slave_xmit()
1810 		spin_lock_bh(&tx_chn->lock);  in am65_cpsw_nuss_ndo_slave_xmit()
1811 		ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma);  in am65_cpsw_nuss_ndo_slave_xmit()
1812 		spin_unlock_bh(&tx_chn->lock);  in am65_cpsw_nuss_ndo_slave_xmit()
1822 	if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) < MAX_SKB_FRAGS) {  in am65_cpsw_nuss_ndo_slave_xmit()
1829 		if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >=  in am65_cpsw_nuss_ndo_slave_xmit()
1839 	am65_cpsw_nuss_xmit_free(tx_chn, first_desc);  in am65_cpsw_nuss_ndo_slave_xmit()
1846 	am65_cpsw_nuss_xmit_free(tx_chn, first_desc);  in am65_cpsw_nuss_ndo_slave_xmit()
2046 	struct am65_cpsw_tx_chn *tx_chn;  in am65_cpsw_ndo_xdp_xmit()  local
2051 	tx_chn = &common->tx_chns[cpu % common->tx_ch_num];  in am65_cpsw_ndo_xdp_xmit()
2052 	netif_txq = netdev_get_tx_queue(ndev, tx_chn->id);  in am65_cpsw_ndo_xdp_xmit()
2056 		if (am65_cpsw_xdp_tx_frame(ndev, tx_chn, frames[i],  in am65_cpsw_ndo_xdp_xmit()
2276 		struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];  in am65_cpsw_nuss_free_tx_chns()  local
2278 		if (!IS_ERR_OR_NULL(tx_chn->desc_pool))  in am65_cpsw_nuss_free_tx_chns()
2279 			k3_cppi_desc_pool_destroy(tx_chn->desc_pool);  in am65_cpsw_nuss_free_tx_chns()
2281 		if (!IS_ERR_OR_NULL(tx_chn->tx_chn))  in am65_cpsw_nuss_free_tx_chns()
2282 			k3_udma_glue_release_tx_chn(tx_chn->tx_chn);  in am65_cpsw_nuss_free_tx_chns()
2284 		memset(tx_chn, 0, sizeof(*tx_chn));  in am65_cpsw_nuss_free_tx_chns()
2295 		struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];  in am65_cpsw_nuss_remove_tx_chns()  local
2297 		if (tx_chn->irq > 0)  in am65_cpsw_nuss_remove_tx_chns()
2298 			devm_free_irq(dev, tx_chn->irq, tx_chn);  in am65_cpsw_nuss_remove_tx_chns()
2300 		netif_napi_del(&tx_chn->napi_tx);  in am65_cpsw_nuss_remove_tx_chns()
2309 	struct am65_cpsw_tx_chn *tx_chn;  in am65_cpsw_nuss_ndev_add_tx_napi()  local
2313 		tx_chn = &common->tx_chns[i];  in am65_cpsw_nuss_ndev_add_tx_napi()
2315 		hrtimer_init(&tx_chn->tx_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);  in am65_cpsw_nuss_ndev_add_tx_napi()
2316 		tx_chn->tx_hrtimer.function = &am65_cpsw_nuss_tx_timer_callback;  in am65_cpsw_nuss_ndev_add_tx_napi()
2318 		netif_napi_add_tx(common->dma_ndev, &tx_chn->napi_tx,  in am65_cpsw_nuss_ndev_add_tx_napi()
2321 		ret = devm_request_irq(dev, tx_chn->irq,  in am65_cpsw_nuss_ndev_add_tx_napi()
2324 				       tx_chn->tx_chn_name, tx_chn);  in am65_cpsw_nuss_ndev_add_tx_napi()
2327 				tx_chn->id, tx_chn->irq, ret);  in am65_cpsw_nuss_ndev_add_tx_napi()
2335 	netif_napi_del(&tx_chn->napi_tx);  in am65_cpsw_nuss_ndev_add_tx_napi()
2337 		tx_chn = &common->tx_chns[i];  in am65_cpsw_nuss_ndev_add_tx_napi()
2338 		devm_free_irq(dev, tx_chn->irq, tx_chn);  in am65_cpsw_nuss_ndev_add_tx_napi()
2339 		netif_napi_del(&tx_chn->napi_tx);  in am65_cpsw_nuss_ndev_add_tx_napi()
2368 		struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];  in am65_cpsw_nuss_init_tx_chns()  local
2370 		snprintf(tx_chn->tx_chn_name,  in am65_cpsw_nuss_init_tx_chns()
2371 			 sizeof(tx_chn->tx_chn_name), "tx%d", i);  in am65_cpsw_nuss_init_tx_chns()
2373 		spin_lock_init(&tx_chn->lock);  in am65_cpsw_nuss_init_tx_chns()
2374 		tx_chn->common = common;  in am65_cpsw_nuss_init_tx_chns()
2375 		tx_chn->id = i;  in am65_cpsw_nuss_init_tx_chns()
2376 		tx_chn->descs_num = max_desc_num;  in am65_cpsw_nuss_init_tx_chns()
2378 		tx_chn->tx_chn =  in am65_cpsw_nuss_init_tx_chns()
2380 						    tx_chn->tx_chn_name,  in am65_cpsw_nuss_init_tx_chns()
2382 		if (IS_ERR(tx_chn->tx_chn)) {  in am65_cpsw_nuss_init_tx_chns()
2383 			ret = dev_err_probe(dev, PTR_ERR(tx_chn->tx_chn),  in am65_cpsw_nuss_init_tx_chns()
2387 		tx_chn->dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn->tx_chn);  in am65_cpsw_nuss_init_tx_chns()
2389 		tx_chn->desc_pool = k3_cppi_desc_pool_create_name(tx_chn->dma_dev,  in am65_cpsw_nuss_init_tx_chns()
2390 								  tx_chn->descs_num,  in am65_cpsw_nuss_init_tx_chns()
2392 								  tx_chn->tx_chn_name);  in am65_cpsw_nuss_init_tx_chns()
2393 		if (IS_ERR(tx_chn->desc_pool)) {  in am65_cpsw_nuss_init_tx_chns()
2394 			ret = PTR_ERR(tx_chn->desc_pool);  in am65_cpsw_nuss_init_tx_chns()
2399 		hdesc_size_out = k3_cppi_desc_pool_desc_size(tx_chn->desc_pool);  in am65_cpsw_nuss_init_tx_chns()
2400 		tx_chn->dsize_log2 = __fls(hdesc_size_out);  in am65_cpsw_nuss_init_tx_chns()
2401 		WARN_ON(hdesc_size_out != (1 << tx_chn->dsize_log2));  in am65_cpsw_nuss_init_tx_chns()
2403 		tx_chn->irq = k3_udma_glue_tx_get_irq(tx_chn->tx_chn);  in am65_cpsw_nuss_init_tx_chns()
2404 		if (tx_chn->irq < 0) {  in am65_cpsw_nuss_init_tx_chns()
2406 				tx_chn->irq);  in am65_cpsw_nuss_init_tx_chns()
2407 			ret = tx_chn->irq;  in am65_cpsw_nuss_init_tx_chns()
2411 		snprintf(tx_chn->tx_chn_name,  in am65_cpsw_nuss_init_tx_chns()
2412 			 sizeof(tx_chn->tx_chn_name), "%s-tx%d",  in am65_cpsw_nuss_init_tx_chns()
2413 			 dev_name(dev), tx_chn->id);  in am65_cpsw_nuss_init_tx_chns()
3428 		k3_udma_glue_reset_tx_chn(tx_chan[i].tx_chn, &tx_chan[i],  in am65_cpsw_nuss_register_ndevs()
3430 		k3_udma_glue_disable_tx_chn(tx_chan[i].tx_chn);  in am65_cpsw_nuss_register_ndevs()