437 	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;  in am65_cpsw_nuss_rx_push()  local
444 	desc_rx = k3_cppi_desc_pool_alloc(rx_chn->desc_pool);  in am65_cpsw_nuss_rx_push()
449 	desc_dma = k3_cppi_desc_pool_virt2dma(rx_chn->desc_pool, desc_rx);  in am65_cpsw_nuss_rx_push()
451 	buf_dma = dma_map_single(rx_chn->dma_dev,  in am65_cpsw_nuss_rx_push()
454 	if (unlikely(dma_mapping_error(rx_chn->dma_dev, buf_dma))) {  in am65_cpsw_nuss_rx_push()
455 		k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);  in am65_cpsw_nuss_rx_push()
462 	k3_udma_glue_rx_dma_to_cppi5_addr(rx_chn->rx_chn, &buf_dma);  in am65_cpsw_nuss_rx_push()
469 	return k3_udma_glue_push_rx_chn(rx_chn->rx_chn, flow_idx,  in am65_cpsw_nuss_rx_push()
509 	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;  in am65_cpsw_destroy_rxq()  local
514 	flow = &rx_chn->flows[id];  in am65_cpsw_destroy_rxq()
517 	k3_udma_glue_reset_rx_chn(rx_chn->rx_chn, id, rx_chn,  in am65_cpsw_destroy_rxq()
538 	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;  in am65_cpsw_destroy_rxqs()  local
542 	k3_udma_glue_tdown_rx_chn(rx_chn->rx_chn, true);  in am65_cpsw_destroy_rxqs()
553 	k3_udma_glue_disable_rx_chn(common->rx_chns.rx_chn);  in am65_cpsw_destroy_rxqs()
558 	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;  in am65_cpsw_create_rxq()  local
574 	flow = &rx_chn->flows[id];  in am65_cpsw_create_rxq()
645 	ret = k3_udma_glue_enable_rx_chn(common->rx_chns.rx_chn);  in am65_cpsw_create_rxqs()
774 	struct am65_cpsw_rx_chn *rx_chn = data;  in am65_cpsw_nuss_rx_cleanup()  local
782 	desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);  in am65_cpsw_nuss_rx_cleanup()
787 	k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma);  in am65_cpsw_nuss_rx_cleanup()
788 	dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);  in am65_cpsw_nuss_rx_cleanup()
789 	k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);  in am65_cpsw_nuss_rx_cleanup()
790 	am65_cpsw_put_page(&rx_chn->flows[flow_id], page, false);  in am65_cpsw_nuss_rx_cleanup()
1279 	struct am65_cpsw_rx_chn *rx_chn = &flow->common->rx_chns;  in am65_cpsw_nuss_rx_packets()  local
1298 	ret = k3_udma_glue_pop_rx_chn(rx_chn->rx_chn, flow_idx, &desc_dma);  in am65_cpsw_nuss_rx_packets()
1312 	desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);  in am65_cpsw_nuss_rx_packets()
1320 	k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma);  in am65_cpsw_nuss_rx_packets()
1330 	dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);  in am65_cpsw_nuss_rx_packets()
1331 	k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);  in am65_cpsw_nuss_rx_packets()
2433 	struct am65_cpsw_rx_chn *rx_chn;  in am65_cpsw_nuss_free_rx_chns()  local
2435 	rx_chn = &common->rx_chns;  in am65_cpsw_nuss_free_rx_chns()
2437 	if (!IS_ERR_OR_NULL(rx_chn->desc_pool))  in am65_cpsw_nuss_free_rx_chns()
2438 		k3_cppi_desc_pool_destroy(rx_chn->desc_pool);  in am65_cpsw_nuss_free_rx_chns()
2440 	if (!IS_ERR_OR_NULL(rx_chn->rx_chn))  in am65_cpsw_nuss_free_rx_chns()
2441 		k3_udma_glue_release_rx_chn(rx_chn->rx_chn);  in am65_cpsw_nuss_free_rx_chns()
2447 	struct am65_cpsw_rx_chn *rx_chn;  in am65_cpsw_nuss_remove_rx_chns()  local
2451 	rx_chn = &common->rx_chns;  in am65_cpsw_nuss_remove_rx_chns()
2452 	flows = rx_chn->flows;  in am65_cpsw_nuss_remove_rx_chns()
2467 	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;  in am65_cpsw_nuss_init_rx_chns()  local
2484 	rx_chn->dev = dev;  in am65_cpsw_nuss_init_rx_chns()
2485 	rx_chn->descs_num = max_desc_num * rx_cfg.flow_id_num;  in am65_cpsw_nuss_init_rx_chns()
2488 		flow = &rx_chn->flows[i];  in am65_cpsw_nuss_init_rx_chns()
2492 	rx_chn->rx_chn = k3_udma_glue_request_rx_chn(dev, "rx", &rx_cfg);  in am65_cpsw_nuss_init_rx_chns()
2493 	if (IS_ERR(rx_chn->rx_chn)) {  in am65_cpsw_nuss_init_rx_chns()
2494 		ret = dev_err_probe(dev, PTR_ERR(rx_chn->rx_chn),  in am65_cpsw_nuss_init_rx_chns()
2498 	rx_chn->dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn->rx_chn);  in am65_cpsw_nuss_init_rx_chns()
2500 	rx_chn->desc_pool = k3_cppi_desc_pool_create_name(rx_chn->dma_dev,  in am65_cpsw_nuss_init_rx_chns()
2501 							  rx_chn->descs_num,  in am65_cpsw_nuss_init_rx_chns()
2503 	if (IS_ERR(rx_chn->desc_pool)) {  in am65_cpsw_nuss_init_rx_chns()
2504 		ret = PTR_ERR(rx_chn->desc_pool);  in am65_cpsw_nuss_init_rx_chns()
2509 	hdesc_size_out = k3_cppi_desc_pool_desc_size(rx_chn->desc_pool);  in am65_cpsw_nuss_init_rx_chns()
2510 	rx_chn->dsize_log2 = __fls(hdesc_size_out);  in am65_cpsw_nuss_init_rx_chns()
2511 	WARN_ON(hdesc_size_out != (1 << rx_chn->dsize_log2));  in am65_cpsw_nuss_init_rx_chns()
2514 			k3_udma_glue_rx_get_flow_id_base(rx_chn->rx_chn);  in am65_cpsw_nuss_init_rx_chns()
2536 		flow = &rx_chn->flows[i];  in am65_cpsw_nuss_init_rx_chns()
2547 		ret = k3_udma_glue_rx_flow_init(rx_chn->rx_chn,  in am65_cpsw_nuss_init_rx_chns()
2555 				k3_udma_glue_rx_flow_get_fdq_id(rx_chn->rx_chn,  in am65_cpsw_nuss_init_rx_chns()
2558 		flow->irq = k3_udma_glue_rx_get_irq(rx_chn->rx_chn, i);  in am65_cpsw_nuss_init_rx_chns()
2598 		flow = &rx_chn->flows[i];  in am65_cpsw_nuss_init_rx_chns()
3434 		k3_udma_glue_reset_rx_chn(rx_chan->rx_chn, i,  in am65_cpsw_nuss_register_ndevs()
3438 	k3_udma_glue_disable_rx_chn(rx_chan->rx_chn);  in am65_cpsw_nuss_register_ndevs()