1 // SPDX-License-Identifier: GPL-2.0
6  * Copyright (C) 2011-2016 Google, Inc.
44  * guest user addresses into host ones.
62 #include <linux/dma-mapping.h>
68  * Update this when something changes in the driver's behavior so the host
84 /* A per-pipe command structure, shared with the host */
86 	s32 cmd;	/* PipeCmdCode, guest -> host */
87 	s32 id;		/* pipe id, guest -> host */
88 	s32 status;	/* command execution status, host -> guest */
89 	s32 reserved;	/* to pad to 64-bit boundary */
93 			/* number of buffers, guest -> host */
95 			/* number of consumed bytes, host -> guest */
97 			/* buffer pointers, guest -> host */
99 			/* buffer sizes, guest -> host */
117 /* Device-level set of buffers shared with the host */
126 	/* pipe ID - index into goldfish_pipe_dev::pipes array */
129 	/* The wake flags pipe is waiting for
131 	 *  and barriers to make it thread-safe.
135 	/* wake flags host have signalled,
136 	 *  - protected by goldfish_pipe_dev::lock
151 	 *  - *command_buffer - makes sure a command can safely write its
152 	 *    parameters to the host and read the results back.
156 	/* A wake queue for sleeping until host signals an event */
167  * communicate with the emulator, and a wake queue for blocked tasks
176 	 *  - pipes, pipes_capacity
177 	 *  - [*pipes, *pipes + pipes_capacity) - array data
178 	 *  - first_signalled_pipe,
181 	 *      goldfish_pipe::signalled_flags - all singnalled-related fields,
183 	 *  - open_command_params - PIPE_CMD_OPEN-related buffers
200 	/* Pointers to the buffers host uses for interaction with this driver */
209 	/* Some device-specific data */
220 	pipe->command_buffer->cmd = cmd;  in goldfish_pipe_cmd_locked()
222 	pipe->command_buffer->status = PIPE_ERROR_INVAL;  in goldfish_pipe_cmd_locked()
223 	writel(pipe->id, pipe->dev->base + PIPE_REG_CMD);  in goldfish_pipe_cmd_locked()
224 	return pipe->command_buffer->status;  in goldfish_pipe_cmd_locked()
231 	if (mutex_lock_interruptible(&pipe->lock))  in goldfish_pipe_cmd()
234 	mutex_unlock(&pipe->lock);  in goldfish_pipe_cmd()
246 		return -EAGAIN;  in goldfish_pipe_error_convert()
248 		return -ENOMEM;  in goldfish_pipe_error_convert()
250 		return -EIO;  in goldfish_pipe_error_convert()
252 		return -EINVAL;  in goldfish_pipe_error_convert()
264 	int requested_pages = ((last_page - first_page) >> PAGE_SHIFT) + 1;  in goldfish_pin_pages()
277 		return -EFAULT;  in goldfish_pin_pages()
296 	 * Process the first page separately - it's the only page that  in populate_rw_params()
304 			? (int)(address_end - address)  in populate_rw_params()
305 			: (PAGE_SIZE - (address & ~PAGE_MASK));  in populate_rw_params()
306 	command->rw_params.ptrs[0] = (u64)(xaddr | (address & ~PAGE_MASK));  in populate_rw_params()
307 	command->rw_params.sizes[0] = size_on_page;  in populate_rw_params()
310 		size_on_page = (i == pages_count - 1) ?  in populate_rw_params()
313 			command->rw_params.sizes[buffer_idx] += size_on_page;  in populate_rw_params()
316 			command->rw_params.ptrs[buffer_idx] = (u64)xaddr;  in populate_rw_params()
317 			command->rw_params.sizes[buffer_idx] = size_on_page;  in populate_rw_params()
321 	command->rw_params.buffers_count = buffer_idx + 1;  in populate_rw_params()
338 	if (mutex_lock_interruptible(&pipe->lock))  in transfer_max_buffers()
339 		return -ERESTARTSYS;  in transfer_max_buffers()
343 					 pipe->pages, &iter_last_page_size);  in transfer_max_buffers()
345 		mutex_unlock(&pipe->lock);  in transfer_max_buffers()
349 	populate_rw_params(pipe->pages, pages_count, address, address_end,  in transfer_max_buffers()
351 			   pipe->command_buffer);  in transfer_max_buffers()
357 	*consumed_size = pipe->command_buffer->rw_params.consumed_size;  in transfer_max_buffers()
359 	unpin_user_pages_dirty_lock(pipe->pages, pages_count,  in transfer_max_buffers()
362 	mutex_unlock(&pipe->lock);  in transfer_max_buffers()
370 	set_bit(wake_bit, &pipe->flags);  in wait_for_host_signal()
372 	/* Tell the emulator we're going to wait for a wake event */  in wait_for_host_signal()
376 	while (test_bit(wake_bit, &pipe->flags)) {  in wait_for_host_signal()
377 		if (wait_event_interruptible(pipe->wake_queue,  in wait_for_host_signal()
378 					     !test_bit(wake_bit, &pipe->flags)))  in wait_for_host_signal()
379 			return -ERESTARTSYS;  in wait_for_host_signal()
381 		if (test_bit(BIT_CLOSED_ON_HOST, &pipe->flags))  in wait_for_host_signal()
382 			return -EIO;  in wait_for_host_signal()
393 	struct goldfish_pipe *pipe = filp->private_data;  in goldfish_pipe_read_write()
394 	int count = 0, ret = -EINVAL;  in goldfish_pipe_read_write()
399 	if (unlikely(test_bit(BIT_CLOSED_ON_HOST, &pipe->flags)))  in goldfish_pipe_read_write()
400 		return -EIO;  in goldfish_pipe_read_write()
406 		return -EFAULT;  in goldfish_pipe_read_write()
410 	last_page = (address_end - 1) & PAGE_MASK;  in goldfish_pipe_read_write()
411 	last_page_size = ((address_end - 1) & ~PAGE_MASK) + 1;  in goldfish_pipe_read_write()
445 				dev_err_ratelimited(pipe->dev->pdev_dev,  in goldfish_pipe_read_write()
453 		 * non-blocking mode, just return the error code.  in goldfish_pipe_read_write()
456 			(filp->f_flags & O_NONBLOCK) != 0) {  in goldfish_pipe_read_write()
491 	struct goldfish_pipe *pipe = filp->private_data;  in goldfish_pipe_poll()
495 	poll_wait(filp, &pipe->wake_queue, wait);  in goldfish_pipe_poll()
499 		return -ERESTARTSYS;  in goldfish_pipe_poll()
507 	if (test_bit(BIT_CLOSED_ON_HOST, &pipe->flags))  in goldfish_pipe_poll()
518 	if (WARN_ON(id >= dev->pipes_capacity))  in signalled_pipes_add_locked()
521 	pipe = dev->pipes[id];  in signalled_pipes_add_locked()
524 	pipe->signalled_flags |= flags;  in signalled_pipes_add_locked()
526 	if (pipe->prev_signalled || pipe->next_signalled ||  in signalled_pipes_add_locked()
527 		dev->first_signalled_pipe == pipe)  in signalled_pipes_add_locked()
529 	pipe->next_signalled = dev->first_signalled_pipe;  in signalled_pipes_add_locked()
530 	if (dev->first_signalled_pipe)  in signalled_pipes_add_locked()
531 		dev->first_signalled_pipe->prev_signalled = pipe;  in signalled_pipes_add_locked()
532 	dev->first_signalled_pipe = pipe;  in signalled_pipes_add_locked()
538 	if (pipe->prev_signalled)  in signalled_pipes_remove_locked()
539 		pipe->prev_signalled->next_signalled = pipe->next_signalled;  in signalled_pipes_remove_locked()
540 	if (pipe->next_signalled)  in signalled_pipes_remove_locked()
541 		pipe->next_signalled->prev_signalled = pipe->prev_signalled;  in signalled_pipes_remove_locked()
542 	if (pipe == dev->first_signalled_pipe)  in signalled_pipes_remove_locked()
543 		dev->first_signalled_pipe = pipe->next_signalled;  in signalled_pipes_remove_locked()
544 	pipe->prev_signalled = NULL;  in signalled_pipes_remove_locked()
545 	pipe->next_signalled = NULL;  in signalled_pipes_remove_locked()
554 	spin_lock_irqsave(&dev->lock, flags);  in signalled_pipes_pop_front()
556 	pipe = dev->first_signalled_pipe;  in signalled_pipes_pop_front()
558 		*wakes = pipe->signalled_flags;  in signalled_pipes_pop_front()
559 		pipe->signalled_flags = 0;  in signalled_pipes_pop_front()
563 		 * - We want to make it as fast as possible to  in signalled_pipes_pop_front()
564 		 * wake the sleeping pipe operations faster.  in signalled_pipes_pop_front()
566 		dev->first_signalled_pipe = pipe->next_signalled;  in signalled_pipes_pop_front()
567 		if (dev->first_signalled_pipe)  in signalled_pipes_pop_front()
568 			dev->first_signalled_pipe->prev_signalled = NULL;  in signalled_pipes_pop_front()
569 		pipe->next_signalled = NULL;  in signalled_pipes_pop_front()
572 	spin_unlock_irqrestore(&dev->lock, flags);  in signalled_pipes_pop_front()
578 	/* Iterate over the signalled pipes and wake them one by one */  in goldfish_interrupt_task()
585 			pipe->flags = 1 << BIT_CLOSED_ON_HOST;  in goldfish_interrupt_task()
588 				clear_bit(BIT_WAKE_ON_READ, &pipe->flags);  in goldfish_interrupt_task()
590 				clear_bit(BIT_WAKE_ON_WRITE, &pipe->flags);  in goldfish_interrupt_task()
596 		wake_up_interruptible(&pipe->wake_queue);  in goldfish_interrupt_task()
624 	if (dev->magic != &goldfish_pipe_device_deinit)  in goldfish_pipe_interrupt()
628 	spin_lock_irqsave(&dev->lock, flags);  in goldfish_pipe_interrupt()
630 	count = readl(dev->base + PIPE_REG_GET_SIGNALLED);  in goldfish_pipe_interrupt()
632 		spin_unlock_irqrestore(&dev->lock, flags);  in goldfish_pipe_interrupt()
640 			dev->buffers->signalled_pipe_buffers[i].id,  in goldfish_pipe_interrupt()
641 			dev->buffers->signalled_pipe_buffers[i].flags);  in goldfish_pipe_interrupt()
643 	spin_unlock_irqrestore(&dev->lock, flags);  in goldfish_pipe_interrupt()
652 	for (id = 0; id < dev->pipes_capacity; ++id)  in get_free_pipe_id_locked()
653 		if (!dev->pipes[id])  in get_free_pipe_id_locked()
661 		u32 new_capacity = 2 * dev->pipes_capacity;  in get_free_pipe_id_locked()
665 			return -ENOMEM;  in get_free_pipe_id_locked()
666 		memcpy(pipes, dev->pipes, sizeof(*pipes) * dev->pipes_capacity);  in get_free_pipe_id_locked()
667 		kfree(dev->pipes);  in get_free_pipe_id_locked()
668 		dev->pipes = pipes;  in get_free_pipe_id_locked()
669 		id = dev->pipes_capacity;  in get_free_pipe_id_locked()
670 		dev->pipes_capacity = new_capacity;  in get_free_pipe_id_locked()
678 	struct miscdevice *miscdev = file->private_data;  in to_goldfish_pipe_dev()
684  *	goldfish_pipe_open - open a channel to the AVD
705 		return -ENOMEM;  in goldfish_pipe_open()
707 	pipe->dev = dev;  in goldfish_pipe_open()
708 	mutex_init(&pipe->lock);  in goldfish_pipe_open()
709 	init_waitqueue_head(&pipe->wake_queue);  in goldfish_pipe_open()
713 	 * it is physically contiguous in host's address space.  in goldfish_pipe_open()
716 	pipe->command_buffer =  in goldfish_pipe_open()
718 	if (!pipe->command_buffer) {  in goldfish_pipe_open()
719 		status = -ENOMEM;  in goldfish_pipe_open()
723 	spin_lock_irqsave(&dev->lock, flags);  in goldfish_pipe_open()
731 	dev->pipes[id] = pipe;  in goldfish_pipe_open()
732 	pipe->id = id;  in goldfish_pipe_open()
733 	pipe->command_buffer->id = id;  in goldfish_pipe_open()
736 	dev->buffers->open_command_params.rw_params_max_count =  in goldfish_pipe_open()
738 	dev->buffers->open_command_params.command_buffer_ptr =  in goldfish_pipe_open()
739 			(u64)(unsigned long)__pa(pipe->command_buffer);  in goldfish_pipe_open()
741 	spin_unlock_irqrestore(&dev->lock, flags);  in goldfish_pipe_open()
745 	file->private_data = pipe;  in goldfish_pipe_open()
749 	spin_lock_irqsave(&dev->lock, flags);  in goldfish_pipe_open()
750 	dev->pipes[id] = NULL;  in goldfish_pipe_open()
752 	spin_unlock_irqrestore(&dev->lock, flags);  in goldfish_pipe_open()
753 	free_page((unsigned long)pipe->command_buffer);  in goldfish_pipe_open()
762 	struct goldfish_pipe *pipe = filp->private_data;  in goldfish_pipe_release()
763 	struct goldfish_pipe_dev *dev = pipe->dev;  in goldfish_pipe_release()
768 	spin_lock_irqsave(&dev->lock, flags);  in goldfish_pipe_release()
769 	dev->pipes[pipe->id] = NULL;  in goldfish_pipe_release()
771 	spin_unlock_irqrestore(&dev->lock, flags);  in goldfish_pipe_release()
773 	filp->private_data = NULL;  in goldfish_pipe_release()
774 	free_page((unsigned long)pipe->command_buffer);  in goldfish_pipe_release()
792 	miscdev->minor = MISC_DYNAMIC_MINOR;  in init_miscdevice()
793 	miscdev->name = "goldfish_pipe";  in init_miscdevice()
794 	miscdev->fops = &goldfish_pipe_fops;  in init_miscdevice()
810 	err = devm_request_threaded_irq(&pdev->dev, dev->irq,  in goldfish_pipe_device_init()
815 		dev_err(&pdev->dev, "unable to allocate IRQ for v2\n");  in goldfish_pipe_device_init()
819 	init_miscdevice(&dev->miscdev);  in goldfish_pipe_device_init()
820 	err = misc_register(&dev->miscdev);  in goldfish_pipe_device_init()
822 		dev_err(&pdev->dev, "unable to register v2 device\n");  in goldfish_pipe_device_init()
826 	dev->pdev_dev = &pdev->dev;  in goldfish_pipe_device_init()
827 	dev->first_signalled_pipe = NULL;  in goldfish_pipe_device_init()
828 	dev->pipes_capacity = INITIAL_PIPES_CAPACITY;  in goldfish_pipe_device_init()
829 	dev->pipes = kcalloc(dev->pipes_capacity, sizeof(*dev->pipes),  in goldfish_pipe_device_init()
831 	if (!dev->pipes) {  in goldfish_pipe_device_init()
832 		misc_deregister(&dev->miscdev);  in goldfish_pipe_device_init()
833 		return -ENOMEM;  in goldfish_pipe_device_init()
838 	 * signalled_pipe_buffers, to the host. This means each of those buffers  in goldfish_pipe_device_init()
843 	dev->buffers = (struct goldfish_pipe_dev_buffers *)  in goldfish_pipe_device_init()
845 	if (!dev->buffers) {  in goldfish_pipe_device_init()
846 		kfree(dev->pipes);  in goldfish_pipe_device_init()
847 		misc_deregister(&dev->miscdev);  in goldfish_pipe_device_init()
848 		return -ENOMEM;  in goldfish_pipe_device_init()
851 	/* Send the buffer addresses to the host */  in goldfish_pipe_device_init()
852 	write_pa_addr(&dev->buffers->signalled_pipe_buffers,  in goldfish_pipe_device_init()
853 		      dev->base + PIPE_REG_SIGNAL_BUFFER,  in goldfish_pipe_device_init()
854 		      dev->base + PIPE_REG_SIGNAL_BUFFER_HIGH);  in goldfish_pipe_device_init()
857 	       dev->base + PIPE_REG_SIGNAL_BUFFER_COUNT);  in goldfish_pipe_device_init()
859 	write_pa_addr(&dev->buffers->open_command_params,  in goldfish_pipe_device_init()
860 		      dev->base + PIPE_REG_OPEN_BUFFER,  in goldfish_pipe_device_init()
861 		      dev->base + PIPE_REG_OPEN_BUFFER_HIGH);  in goldfish_pipe_device_init()
870 	misc_deregister(&dev->miscdev);  in goldfish_pipe_device_deinit()
871 	kfree(dev->pipes);  in goldfish_pipe_device_deinit()
872 	free_page((unsigned long)dev->buffers);  in goldfish_pipe_device_deinit()
880 	dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL);  in goldfish_pipe_probe()
882 		return -ENOMEM;  in goldfish_pipe_probe()
884 	dev->magic = &goldfish_pipe_device_deinit;  in goldfish_pipe_probe()
885 	spin_lock_init(&dev->lock);  in goldfish_pipe_probe()
889 		dev_err(&pdev->dev, "can't allocate i/o page\n");  in goldfish_pipe_probe()
890 		return -EINVAL;  in goldfish_pipe_probe()
892 	dev->base = devm_ioremap(&pdev->dev, r->start, PAGE_SIZE);  in goldfish_pipe_probe()
893 	if (!dev->base) {  in goldfish_pipe_probe()
894 		dev_err(&pdev->dev, "ioremap failed\n");  in goldfish_pipe_probe()
895 		return -EINVAL;  in goldfish_pipe_probe()
898 	dev->irq = platform_get_irq(pdev, 0);  in goldfish_pipe_probe()
899 	if (dev->irq < 0)  in goldfish_pipe_probe()
900 		return dev->irq;  in goldfish_pipe_probe()
903 	 * Exchange the versions with the host device  in goldfish_pipe_probe()
906 	 *  reading device version back: this allows the host implementation to  in goldfish_pipe_probe()
909 	writel(PIPE_DRIVER_VERSION, dev->base + PIPE_REG_VERSION);  in goldfish_pipe_probe()
910 	dev->version = readl(dev->base + PIPE_REG_VERSION);  in goldfish_pipe_probe()
911 	if (WARN_ON(dev->version < PIPE_CURRENT_DEVICE_VERSION))  in goldfish_pipe_probe()
912 		return -EINVAL;  in goldfish_pipe_probe()
931 	{ .compatible = "google,android-pipe", },