1 // SPDX-License-Identifier: GPL-2.0-only
18 #include "virt-dma.h"
72  * struct tegra_adma_chip_data - Tegra chip specific data
110  * struct tegra_adma_chan_regs - Tegra ADMA channel registers
123  * struct tegra_adma_desc - Tegra ADMA descriptor to manage transfer requests.
134  * struct tegra_adma_chan - Tegra ADMA channel information
156  * struct tegra_adma - Tegra ADMA controller information
181 	writel(val, tdma->base_addr + tdma->cdata->global_reg_offset + reg);  in tdma_write()
186 	return readl(tdma->base_addr + tdma->cdata->global_reg_offset + reg);  in tdma_read()
191 	writel(val, tdma->ch_base_addr + tdma->cdata->global_reg_offset + reg);  in tdma_ch_global_write()
196 	writel(val, tdc->chan_addr + reg);  in tdma_ch_write()
201 	return readl(tdc->chan_addr + reg);  in tdma_ch_read()
204 static inline struct tegra_adma_chan *to_tegra_adma_chan(struct dma_chan *dc)  in to_tegra_adma_chan()  argument
206 	return container_of(dc, struct tegra_adma_chan, vc.chan);  in to_tegra_adma_chan()
217 	return tdc->tdma->dev;  in tdc2dev()
225 static int tegra_adma_slave_config(struct dma_chan *dc,  in tegra_adma_slave_config()  argument
228 	struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);  in tegra_adma_slave_config()
230 	memcpy(&tdc->sconfig, sconfig, sizeof(*sconfig));  in tegra_adma_slave_config()
244 	tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_CHGRP + (tdma->ch_page_no * 0x4), 0xff);  in tegra186_adma_global_page_config()
245 	tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_RX_REQ + (tdma->ch_page_no * 0x4), 0x1ffffff);  in tegra186_adma_global_page_config()
246 	tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_TX_REQ + (tdma->ch_page_no * 0x4), 0xffffff);  in tegra186_adma_global_page_config()
255 	tdma_ch_global_write(tdma, tdma->cdata->global_int_clear, 0x1);  in tegra_adma_init()
257 	if (!tdma->base_addr)  in tegra_adma_init()
265 				 tdma->base_addr +  in tegra_adma_init()
266 				 tdma->cdata->global_reg_offset +  in tegra_adma_init()
272 	if (tdma->cdata->set_global_pg_config)  in tegra_adma_init()
273 		tdma->cdata->set_global_pg_config(tdma);  in tegra_adma_init()
284 	struct tegra_adma *tdma = tdc->tdma;  in tegra_adma_request_alloc()
285 	unsigned int sreq_index = tdc->sreq_index;  in tegra_adma_request_alloc()
287 	if (tdc->sreq_reserved)  in tegra_adma_request_alloc()
288 		return tdc->sreq_dir == direction ? 0 : -EINVAL;  in tegra_adma_request_alloc()
290 	if (sreq_index > tdma->cdata->ch_req_max) {  in tegra_adma_request_alloc()
291 		dev_err(tdma->dev, "invalid DMA request\n");  in tegra_adma_request_alloc()
292 		return -EINVAL;  in tegra_adma_request_alloc()
297 		if (test_and_set_bit(sreq_index, &tdma->tx_requests_reserved)) {  in tegra_adma_request_alloc()
298 			dev_err(tdma->dev, "DMA request reserved\n");  in tegra_adma_request_alloc()
299 			return -EINVAL;  in tegra_adma_request_alloc()
304 		if (test_and_set_bit(sreq_index, &tdma->rx_requests_reserved)) {  in tegra_adma_request_alloc()
305 			dev_err(tdma->dev, "DMA request reserved\n");  in tegra_adma_request_alloc()
306 			return -EINVAL;  in tegra_adma_request_alloc()
311 		dev_WARN(tdma->dev, "channel %s has invalid transfer type\n",  in tegra_adma_request_alloc()
312 			 dma_chan_name(&tdc->vc.chan));  in tegra_adma_request_alloc()
313 		return -EINVAL;  in tegra_adma_request_alloc()
316 	tdc->sreq_dir = direction;  in tegra_adma_request_alloc()
317 	tdc->sreq_reserved = true;  in tegra_adma_request_alloc()
324 	struct tegra_adma *tdma = tdc->tdma;  in tegra_adma_request_free()
326 	if (!tdc->sreq_reserved)  in tegra_adma_request_free()
329 	switch (tdc->sreq_dir) {  in tegra_adma_request_free()
331 		clear_bit(tdc->sreq_index, &tdma->tx_requests_reserved);  in tegra_adma_request_free()
335 		clear_bit(tdc->sreq_index, &tdma->rx_requests_reserved);  in tegra_adma_request_free()
339 		dev_WARN(tdma->dev, "channel %s has invalid transfer type\n",  in tegra_adma_request_free()
340 			 dma_chan_name(&tdc->vc.chan));  in tegra_adma_request_free()
344 	tdc->sreq_reserved = false;  in tegra_adma_request_free()
374 	if (readx_poll_timeout_atomic(readl, tdc->chan_addr + ADMA_CH_STATUS,  in tegra_adma_stop()
381 	kfree(tdc->desc);  in tegra_adma_stop()
382 	tdc->desc = NULL;  in tegra_adma_stop()
387 	struct virt_dma_desc *vd = vchan_next_desc(&tdc->vc);  in tegra_adma_start()
394 	list_del(&vd->node);  in tegra_adma_start()
396 	desc = to_tegra_adma_desc(&vd->tx);  in tegra_adma_start()
403 	ch_regs = &desc->ch_regs;  in tegra_adma_start()
405 	tdc->tx_buf_pos = 0;  in tegra_adma_start()
406 	tdc->tx_buf_count = 0;  in tegra_adma_start()
407 	tdma_ch_write(tdc, ADMA_CH_TC, ch_regs->tc);  in tegra_adma_start()
408 	tdma_ch_write(tdc, ADMA_CH_CTRL, ch_regs->ctrl);  in tegra_adma_start()
409 	tdma_ch_write(tdc, ADMA_CH_LOWER_SRC_ADDR, ch_regs->src_addr);  in tegra_adma_start()
410 	tdma_ch_write(tdc, ADMA_CH_LOWER_TRG_ADDR, ch_regs->trg_addr);  in tegra_adma_start()
411 	tdma_ch_write(tdc, ADMA_CH_FIFO_CTRL, ch_regs->fifo_ctrl);  in tegra_adma_start()
412 	tdma_ch_write(tdc, ADMA_CH_CONFIG, ch_regs->config);  in tegra_adma_start()
417 	tdc->desc = desc;  in tegra_adma_start()
422 	struct tegra_adma_desc *desc = tdc->desc;  in tegra_adma_get_residue()
430 	if (pos < tdc->tx_buf_pos)  in tegra_adma_get_residue()
431 		tdc->tx_buf_count += pos + (max - tdc->tx_buf_pos);  in tegra_adma_get_residue()
433 		tdc->tx_buf_count += pos - tdc->tx_buf_pos;  in tegra_adma_get_residue()
435 	periods_remaining = tdc->tx_buf_count % desc->num_periods;  in tegra_adma_get_residue()
436 	tdc->tx_buf_pos = pos;  in tegra_adma_get_residue()
438 	return desc->buf_len - (periods_remaining * desc->period_len);  in tegra_adma_get_residue()
446 	spin_lock(&tdc->vc.lock);  in tegra_adma_isr()
449 	if (status == 0 || !tdc->desc) {  in tegra_adma_isr()
450 		spin_unlock(&tdc->vc.lock);  in tegra_adma_isr()
454 	vchan_cyclic_callback(&tdc->desc->vd);  in tegra_adma_isr()
456 	spin_unlock(&tdc->vc.lock);  in tegra_adma_isr()
461 static void tegra_adma_issue_pending(struct dma_chan *dc)  in tegra_adma_issue_pending()  argument
463 	struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);  in tegra_adma_issue_pending()
466 	spin_lock_irqsave(&tdc->vc.lock, flags);  in tegra_adma_issue_pending()
468 	if (vchan_issue_pending(&tdc->vc)) {  in tegra_adma_issue_pending()
469 		if (!tdc->desc)  in tegra_adma_issue_pending()
473 	spin_unlock_irqrestore(&tdc->vc.lock, flags);  in tegra_adma_issue_pending()
486 static int tegra_adma_pause(struct dma_chan *dc)  in tegra_adma_pause()  argument
488 	struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);  in tegra_adma_pause()
489 	struct tegra_adma_desc *desc = tdc->desc;  in tegra_adma_pause()
490 	struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs;  in tegra_adma_pause()
493 	ch_regs->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL);  in tegra_adma_pause()
494 	ch_regs->ctrl |= (1 << ADMA_CH_CTRL_XFER_PAUSE_SHIFT);  in tegra_adma_pause()
495 	tdma_ch_write(tdc, ADMA_CH_CTRL, ch_regs->ctrl);  in tegra_adma_pause()
497 	while (dcnt-- && !tegra_adma_is_paused(tdc))  in tegra_adma_pause()
502 		return -EBUSY;  in tegra_adma_pause()
508 static int tegra_adma_resume(struct dma_chan *dc)  in tegra_adma_resume()  argument
510 	struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);  in tegra_adma_resume()
511 	struct tegra_adma_desc *desc = tdc->desc;  in tegra_adma_resume()
512 	struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs;  in tegra_adma_resume()
514 	ch_regs->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL);  in tegra_adma_resume()
515 	ch_regs->ctrl &= ~(1 << ADMA_CH_CTRL_XFER_PAUSE_SHIFT);  in tegra_adma_resume()
516 	tdma_ch_write(tdc, ADMA_CH_CTRL, ch_regs->ctrl);  in tegra_adma_resume()
521 static int tegra_adma_terminate_all(struct dma_chan *dc)  in tegra_adma_terminate_all()  argument
523 	struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);  in tegra_adma_terminate_all()
527 	spin_lock_irqsave(&tdc->vc.lock, flags);  in tegra_adma_terminate_all()
529 	if (tdc->desc)  in tegra_adma_terminate_all()
533 	vchan_get_all_descriptors(&tdc->vc, &head);  in tegra_adma_terminate_all()
534 	spin_unlock_irqrestore(&tdc->vc.lock, flags);  in tegra_adma_terminate_all()
535 	vchan_dma_desc_free_list(&tdc->vc, &head);  in tegra_adma_terminate_all()
540 static enum dma_status tegra_adma_tx_status(struct dma_chan *dc,  in tegra_adma_tx_status()  argument
544 	struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);  in tegra_adma_tx_status()
551 	ret = dma_cookie_status(dc, cookie, txstate);  in tegra_adma_tx_status()
555 	spin_lock_irqsave(&tdc->vc.lock, flags);  in tegra_adma_tx_status()
557 	vd = vchan_find_desc(&tdc->vc, cookie);  in tegra_adma_tx_status()
559 		desc = to_tegra_adma_desc(&vd->tx);  in tegra_adma_tx_status()
560 		residual = desc->ch_regs.tc;  in tegra_adma_tx_status()
561 	} else if (tdc->desc && tdc->desc->vd.tx.cookie == cookie) {  in tegra_adma_tx_status()
567 	spin_unlock_irqrestore(&tdc->vc.lock, flags);  in tegra_adma_tx_status()
587 	return (burst_size - 1) << ADMA_CH_CONFIG_BURST_SIZE_SHIFT;  in tegra186_adma_get_burst_config()
595 	struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs;  in tegra_adma_set_xfer_params()
596 	const struct tegra_adma_chip_data *cdata = tdc->tdma->cdata;  in tegra_adma_set_xfer_params()
599 	if (desc->num_periods > ADMA_CH_CONFIG_MAX_BUFS)  in tegra_adma_set_xfer_params()
600 		return -EINVAL;  in tegra_adma_set_xfer_params()
606 		burst_size = tdc->sconfig.dst_maxburst;  in tegra_adma_set_xfer_params()
607 		ch_regs->config = ADMA_CH_CONFIG_SRC_BUF(desc->num_periods - 1);  in tegra_adma_set_xfer_params()
608 		ch_regs->ctrl = ADMA_CH_REG_FIELD_VAL(tdc->sreq_index,  in tegra_adma_set_xfer_params()
609 						      cdata->ch_req_mask,  in tegra_adma_set_xfer_params()
610 						      cdata->ch_req_tx_shift);  in tegra_adma_set_xfer_params()
611 		ch_regs->src_addr = buf_addr;  in tegra_adma_set_xfer_params()
617 		burst_size = tdc->sconfig.src_maxburst;  in tegra_adma_set_xfer_params()
618 		ch_regs->config = ADMA_CH_CONFIG_TRG_BUF(desc->num_periods - 1);  in tegra_adma_set_xfer_params()
619 		ch_regs->ctrl = ADMA_CH_REG_FIELD_VAL(tdc->sreq_index,  in tegra_adma_set_xfer_params()
620 						      cdata->ch_req_mask,  in tegra_adma_set_xfer_params()
621 						      cdata->ch_req_rx_shift);  in tegra_adma_set_xfer_params()
622 		ch_regs->trg_addr = buf_addr;  in tegra_adma_set_xfer_params()
627 		return -EINVAL;  in tegra_adma_set_xfer_params()
630 	ch_regs->ctrl |= ADMA_CH_CTRL_DIR(adma_dir) |  in tegra_adma_set_xfer_params()
633 	ch_regs->config |= cdata->adma_get_burst_config(burst_size);  in tegra_adma_set_xfer_params()
634 	ch_regs->config |= ADMA_CH_CONFIG_WEIGHT_FOR_WRR(1);  in tegra_adma_set_xfer_params()
635 	if (cdata->has_outstanding_reqs)  in tegra_adma_set_xfer_params()
636 		ch_regs->config |= TEGRA186_ADMA_CH_CONFIG_OUTSTANDING_REQS(8);  in tegra_adma_set_xfer_params()
649 	if (tdc->sreq_index > cdata->sreq_index_offset)  in tegra_adma_set_xfer_params()
650 		ch_regs->fifo_ctrl =  in tegra_adma_set_xfer_params()
651 			ADMA_CH_REG_FIELD_VAL(2, cdata->ch_fifo_size_mask,  in tegra_adma_set_xfer_params()
654 		ch_regs->fifo_ctrl =  in tegra_adma_set_xfer_params()
655 			ADMA_CH_REG_FIELD_VAL(3, cdata->ch_fifo_size_mask,  in tegra_adma_set_xfer_params()
658 	ch_regs->tc = desc->period_len & ADMA_CH_TC_COUNT_MASK;  in tegra_adma_set_xfer_params()
664 	struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_len,  in tegra_adma_prep_dma_cyclic()  argument
668 	struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);  in tegra_adma_prep_dma_cyclic()
690 	desc->buf_len = buf_len;  in tegra_adma_prep_dma_cyclic()
691 	desc->period_len = period_len;  in tegra_adma_prep_dma_cyclic()
692 	desc->num_periods = buf_len / period_len;  in tegra_adma_prep_dma_cyclic()
699 	return vchan_tx_prep(&tdc->vc, &desc->vd, flags);  in tegra_adma_prep_dma_cyclic()
702 static int tegra_adma_alloc_chan_resources(struct dma_chan *dc)  in tegra_adma_alloc_chan_resources()  argument
704 	struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);  in tegra_adma_alloc_chan_resources()
707 	ret = request_irq(tdc->irq, tegra_adma_isr, 0, dma_chan_name(dc), tdc);  in tegra_adma_alloc_chan_resources()
710 			dma_chan_name(dc));  in tegra_adma_alloc_chan_resources()
716 		free_irq(tdc->irq, tdc);  in tegra_adma_alloc_chan_resources()
720 	dma_cookie_init(&tdc->vc.chan);  in tegra_adma_alloc_chan_resources()
725 static void tegra_adma_free_chan_resources(struct dma_chan *dc)  in tegra_adma_free_chan_resources()  argument
727 	struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);  in tegra_adma_free_chan_resources()
729 	tegra_adma_terminate_all(dc);  in tegra_adma_free_chan_resources()
730 	vchan_free_chan_resources(&tdc->vc);  in tegra_adma_free_chan_resources()
731 	tasklet_kill(&tdc->vc.task);  in tegra_adma_free_chan_resources()
732 	free_irq(tdc->irq, tdc);  in tegra_adma_free_chan_resources()
735 	tdc->sreq_index = 0;  in tegra_adma_free_chan_resources()
736 	tdc->sreq_dir = DMA_TRANS_NONE;  in tegra_adma_free_chan_resources()
742 	struct tegra_adma *tdma = ofdma->of_dma_data;  in tegra_dma_of_xlate()
747 	if (dma_spec->args_count != 1)  in tegra_dma_of_xlate()
750 	sreq_index = dma_spec->args[0];  in tegra_dma_of_xlate()
753 		dev_err(tdma->dev, "DMA request must not be 0\n");  in tegra_dma_of_xlate()
757 	chan = dma_get_any_slave_channel(&tdma->dma_dev);  in tegra_dma_of_xlate()
762 	tdc->sreq_index = sreq_index;  in tegra_dma_of_xlate()
774 	if (tdma->base_addr)  in tegra_adma_runtime_suspend()
775 		tdma->global_cmd = tdma_read(tdma, ADMA_GLOBAL_CMD);  in tegra_adma_runtime_suspend()
777 	if (!tdma->global_cmd)  in tegra_adma_runtime_suspend()
780 	for (i = 0; i < tdma->nr_channels; i++) {  in tegra_adma_runtime_suspend()
781 		tdc = &tdma->channels[i];  in tegra_adma_runtime_suspend()
783 		if (!tdc->tdma)  in tegra_adma_runtime_suspend()
786 		ch_reg = &tdc->ch_regs;  in tegra_adma_runtime_suspend()
787 		ch_reg->cmd = tdma_ch_read(tdc, ADMA_CH_CMD);  in tegra_adma_runtime_suspend()
789 		if (!ch_reg->cmd)  in tegra_adma_runtime_suspend()
791 		ch_reg->tc = tdma_ch_read(tdc, ADMA_CH_TC);  in tegra_adma_runtime_suspend()
792 		ch_reg->src_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_SRC_ADDR);  in tegra_adma_runtime_suspend()
793 		ch_reg->trg_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_TRG_ADDR);  in tegra_adma_runtime_suspend()
794 		ch_reg->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL);  in tegra_adma_runtime_suspend()
795 		ch_reg->fifo_ctrl = tdma_ch_read(tdc, ADMA_CH_FIFO_CTRL);  in tegra_adma_runtime_suspend()
796 		ch_reg->config = tdma_ch_read(tdc, ADMA_CH_CONFIG);  in tegra_adma_runtime_suspend()
800 	clk_disable_unprepare(tdma->ahub_clk);  in tegra_adma_runtime_suspend()
812 	ret = clk_prepare_enable(tdma->ahub_clk);  in tegra_adma_runtime_resume()
817 	if (tdma->base_addr) {  in tegra_adma_runtime_resume()
818 		tdma_write(tdma, ADMA_GLOBAL_CMD, tdma->global_cmd);  in tegra_adma_runtime_resume()
819 		if (tdma->cdata->set_global_pg_config)  in tegra_adma_runtime_resume()
820 			tdma->cdata->set_global_pg_config(tdma);  in tegra_adma_runtime_resume()
823 	if (!tdma->global_cmd)  in tegra_adma_runtime_resume()
826 	for (i = 0; i < tdma->nr_channels; i++) {  in tegra_adma_runtime_resume()
827 		tdc = &tdma->channels[i];  in tegra_adma_runtime_resume()
829 		if (!tdc->tdma)  in tegra_adma_runtime_resume()
831 		ch_reg = &tdc->ch_regs;  in tegra_adma_runtime_resume()
833 		if (!ch_reg->cmd)  in tegra_adma_runtime_resume()
835 		tdma_ch_write(tdc, ADMA_CH_TC, ch_reg->tc);  in tegra_adma_runtime_resume()
836 		tdma_ch_write(tdc, ADMA_CH_LOWER_SRC_ADDR, ch_reg->src_addr);  in tegra_adma_runtime_resume()
837 		tdma_ch_write(tdc, ADMA_CH_LOWER_TRG_ADDR, ch_reg->trg_addr);  in tegra_adma_runtime_resume()
838 		tdma_ch_write(tdc, ADMA_CH_CTRL, ch_reg->ctrl);  in tegra_adma_runtime_resume()
839 		tdma_ch_write(tdc, ADMA_CH_FIFO_CTRL, ch_reg->fifo_ctrl);  in tegra_adma_runtime_resume()
840 		tdma_ch_write(tdc, ADMA_CH_CONFIG, ch_reg->config);  in tegra_adma_runtime_resume()
841 		tdma_ch_write(tdc, ADMA_CH_CMD, ch_reg->cmd);  in tegra_adma_runtime_resume()
884 	{ .compatible = "nvidia,tegra210-adma", .data = &tegra210_chip_data },
885 	{ .compatible = "nvidia,tegra186-adma", .data = &tegra186_chip_data },
897 	cdata = of_device_get_match_data(&pdev->dev);  in tegra_adma_probe()
899 		dev_err(&pdev->dev, "device match data not found\n");  in tegra_adma_probe()
900 		return -ENODEV;  in tegra_adma_probe()
903 	tdma = devm_kzalloc(&pdev->dev,  in tegra_adma_probe()
904 			    struct_size(tdma, channels, cdata->nr_channels),  in tegra_adma_probe()
907 		return -ENOMEM;  in tegra_adma_probe()
909 	tdma->dev = &pdev->dev;  in tegra_adma_probe()
910 	tdma->cdata = cdata;  in tegra_adma_probe()
911 	tdma->nr_channels = cdata->nr_channels;  in tegra_adma_probe()
916 		tdma->ch_base_addr = devm_ioremap_resource(&pdev->dev, res_page);  in tegra_adma_probe()
917 		if (IS_ERR(tdma->ch_base_addr))  in tegra_adma_probe()
918 			return PTR_ERR(tdma->ch_base_addr);  in tegra_adma_probe()
925 			if (res_page->start < res_base->start)  in tegra_adma_probe()
926 				return -EINVAL;  in tegra_adma_probe()
927 			page_offset = res_page->start - res_base->start;  in tegra_adma_probe()
928 			ch_base_offset = cdata->ch_base_offset;  in tegra_adma_probe()
930 				return -EINVAL;  in tegra_adma_probe()
934 				return -EINVAL;  in tegra_adma_probe()
936 			tdma->ch_page_no = page_no - 1;  in tegra_adma_probe()
937 			tdma->base_addr = devm_ioremap_resource(&pdev->dev, res_base);  in tegra_adma_probe()
938 			if (IS_ERR(tdma->base_addr))  in tegra_adma_probe()
939 				return PTR_ERR(tdma->base_addr);  in tegra_adma_probe()
945 			tdma->base_addr = devm_ioremap_resource(&pdev->dev, res_base);  in tegra_adma_probe()
946 			if (IS_ERR(tdma->base_addr))  in tegra_adma_probe()
947 				return PTR_ERR(tdma->base_addr);  in tegra_adma_probe()
949 			return -ENODEV;  in tegra_adma_probe()
952 		tdma->ch_base_addr = tdma->base_addr + cdata->ch_base_offset;  in tegra_adma_probe()
955 	tdma->ahub_clk = devm_clk_get(&pdev->dev, "d_audio");  in tegra_adma_probe()
956 	if (IS_ERR(tdma->ahub_clk)) {  in tegra_adma_probe()
957 		dev_err(&pdev->dev, "Error: Missing ahub controller clock\n");  in tegra_adma_probe()
958 		return PTR_ERR(tdma->ahub_clk);  in tegra_adma_probe()
961 	tdma->dma_chan_mask = devm_kzalloc(&pdev->dev,  in tegra_adma_probe()
962 					   BITS_TO_LONGS(tdma->nr_channels) * sizeof(unsigned long),  in tegra_adma_probe()
964 	if (!tdma->dma_chan_mask)  in tegra_adma_probe()
965 		return -ENOMEM;  in tegra_adma_probe()
968 	bitmap_fill(tdma->dma_chan_mask, tdma->nr_channels);  in tegra_adma_probe()
970 	ret = of_property_read_u32_array(pdev->dev.of_node, "dma-channel-mask",  in tegra_adma_probe()
971 					 (u32 *)tdma->dma_chan_mask,  in tegra_adma_probe()
972 					 BITS_TO_U32(tdma->nr_channels));  in tegra_adma_probe()
973 	if (ret < 0 && (ret != -EINVAL)) {  in tegra_adma_probe()
974 		dev_err(&pdev->dev, "dma-channel-mask is not complete.\n");  in tegra_adma_probe()
978 	INIT_LIST_HEAD(&tdma->dma_dev.channels);  in tegra_adma_probe()
979 	for (i = 0; i < tdma->nr_channels; i++) {  in tegra_adma_probe()
980 		struct tegra_adma_chan *tdc = &tdma->channels[i];  in tegra_adma_probe()
983 		if (!test_bit(i, tdma->dma_chan_mask))  in tegra_adma_probe()
986 		tdc->chan_addr = tdma->ch_base_addr + (cdata->ch_reg_size * i);  in tegra_adma_probe()
988 		tdc->irq = of_irq_get(pdev->dev.of_node, i);  in tegra_adma_probe()
989 		if (tdc->irq <= 0) {  in tegra_adma_probe()
990 			ret = tdc->irq ?: -ENXIO;  in tegra_adma_probe()
994 		vchan_init(&tdc->vc, &tdma->dma_dev);  in tegra_adma_probe()
995 		tdc->vc.desc_free = tegra_adma_desc_free;  in tegra_adma_probe()
996 		tdc->tdma = tdma;  in tegra_adma_probe()
999 	pm_runtime_enable(&pdev->dev);  in tegra_adma_probe()
1001 	ret = pm_runtime_resume_and_get(&pdev->dev);  in tegra_adma_probe()
1009 	dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask);  in tegra_adma_probe()
1010 	dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask);  in tegra_adma_probe()
1011 	dma_cap_set(DMA_CYCLIC, tdma->dma_dev.cap_mask);  in tegra_adma_probe()
1013 	tdma->dma_dev.dev = &pdev->dev;  in tegra_adma_probe()
1014 	tdma->dma_dev.device_alloc_chan_resources =  in tegra_adma_probe()
1016 	tdma->dma_dev.device_free_chan_resources =  in tegra_adma_probe()
1018 	tdma->dma_dev.device_issue_pending = tegra_adma_issue_pending;  in tegra_adma_probe()
1019 	tdma->dma_dev.device_prep_dma_cyclic = tegra_adma_prep_dma_cyclic;  in tegra_adma_probe()
1020 	tdma->dma_dev.device_config = tegra_adma_slave_config;  in tegra_adma_probe()
1021 	tdma->dma_dev.device_tx_status = tegra_adma_tx_status;  in tegra_adma_probe()
1022 	tdma->dma_dev.device_terminate_all = tegra_adma_terminate_all;  in tegra_adma_probe()
1023 	tdma->dma_dev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);  in tegra_adma_probe()
1024 	tdma->dma_dev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);  in tegra_adma_probe()
1025 	tdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);  in tegra_adma_probe()
1026 	tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;  in tegra_adma_probe()
1027 	tdma->dma_dev.device_pause = tegra_adma_pause;  in tegra_adma_probe()
1028 	tdma->dma_dev.device_resume = tegra_adma_resume;  in tegra_adma_probe()
1030 	ret = dma_async_device_register(&tdma->dma_dev);  in tegra_adma_probe()
1032 		dev_err(&pdev->dev, "ADMA registration failed: %d\n", ret);  in tegra_adma_probe()
1036 	ret = of_dma_controller_register(pdev->dev.of_node,  in tegra_adma_probe()
1039 		dev_err(&pdev->dev, "ADMA OF registration failed %d\n", ret);  in tegra_adma_probe()
1043 	pm_runtime_put(&pdev->dev);  in tegra_adma_probe()
1045 	dev_info(&pdev->dev, "Tegra210 ADMA driver registered %d channels\n",  in tegra_adma_probe()
1046 		 tdma->nr_channels);  in tegra_adma_probe()
1051 	dma_async_device_unregister(&tdma->dma_dev);  in tegra_adma_probe()
1053 	pm_runtime_put_sync(&pdev->dev);  in tegra_adma_probe()
1055 	pm_runtime_disable(&pdev->dev);  in tegra_adma_probe()
1057 	while (--i >= 0)  in tegra_adma_probe()
1058 		irq_dispose_mapping(tdma->channels[i].irq);  in tegra_adma_probe()
1068 	of_dma_controller_free(pdev->dev.of_node);  in tegra_adma_remove()
1069 	dma_async_device_unregister(&tdma->dma_dev);  in tegra_adma_remove()
1071 	for (i = 0; i < tdma->nr_channels; ++i) {  in tegra_adma_remove()
1072 		if (tdma->channels[i].irq)  in tegra_adma_remove()
1073 			irq_dispose_mapping(tdma->channels[i].irq);  in tegra_adma_remove()
1076 	pm_runtime_disable(&pdev->dev);  in tegra_adma_remove()
1088 		.name	= "tegra-adma",
1098 MODULE_ALIAS("platform:tegra210-adma");