1 // SPDX-License-Identifier: GPL-2.0-only
17 #include "dax-private.h"
20  * struct dax_device - anchor object for dax services
76  * fs_dax_get_by_bdev() - temporary lookup mechanism for filesystem-dax
89 	if (!blk_queue_dax(bdev->bd_disk->queue))  in fs_dax_get_by_bdev()
95 		pr_info("%pg: error: unaligned partition for dax\n", bdev);  in fs_dax_get_by_bdev()
100 	dax_dev = xa_load(&dax_hosts, (unsigned long)bdev->bd_disk);  in fs_dax_get_by_bdev()
101 	if (!dax_dev || !dax_alive(dax_dev) || !igrab(&dax_dev->inode))  in fs_dax_get_by_bdev()
104 		if (!cmpxchg(&dax_dev->holder_data, NULL, holder))  in fs_dax_get_by_bdev()
105 			dax_dev->holder_ops = ops;  in fs_dax_get_by_bdev()
118 	    cmpxchg(&dax_dev->holder_data, holder, NULL) == holder)  in fs_put_dax()
119 		dax_dev->holder_ops = NULL;  in fs_put_dax()
126 	/* !alive + rcu grace period == no new operations / mappings */
139  * dax_direct_access() - translate a device pgoff to an absolute pfn
143  * @mode: indicator on normal access or recovery write
156 		return -EOPNOTSUPP;  in dax_direct_access()
159 		return -ENXIO;  in dax_direct_access()
162 		return -EINVAL;  in dax_direct_access()
164 	avail = dax_dev->ops->direct_access(dax_dev, pgoff, nr_pages,  in dax_direct_access()
167 		return -ERANGE;  in dax_direct_access()
180 	 * via access_ok() in vfs_write, so use the 'no check' version to bypass  in dax_copy_from_iter()
183 	if (test_bit(DAXDEV_NOCACHE, &dax_dev->flags))  in dax_copy_from_iter()
196 	 * via access_ok() in vfs_red, so use the 'no check' version to bypass  in dax_copy_to_iter()
199 	if (test_bit(DAXDEV_NOMC, &dax_dev->flags))  in dax_copy_to_iter()
210 		return -ENXIO;  in dax_zero_page_range()
212 	 * There are no callers that want to zero more than one page as of now.  in dax_zero_page_range()
217 		return -EIO;  in dax_zero_page_range()
219 	ret = dax_dev->ops->zero_page_range(dax_dev, pgoff, nr_pages);  in dax_zero_page_range()
227 	if (!dax_dev->ops->recovery_write)  in dax_recovery_write()
229 	return dax_dev->ops->recovery_write(dax_dev, pgoff, addr, bytes, iter);  in dax_recovery_write()
240 		rc = -ENXIO;  in dax_holder_notify_failure()
244 	if (!dax_dev->holder_ops) {  in dax_holder_notify_failure()
245 		rc = -EOPNOTSUPP;  in dax_holder_notify_failure()
249 	rc = dax_dev->holder_ops->notify_failure(dax_dev, off, len, mf_flags);  in dax_holder_notify_failure()
275 		set_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);  in dax_write_cache()
277 		clear_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);  in dax_write_cache()
283 	return test_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);  in dax_write_cache_enabled()
289 	return test_bit(DAXDEV_SYNC, &dax_dev->flags);  in dax_synchronous()
295 	set_bit(DAXDEV_SYNC, &dax_dev->flags);  in set_dax_synchronous()
301 	set_bit(DAXDEV_NOCACHE, &dax_dev->flags);  in set_dax_nocache()
307 	set_bit(DAXDEV_NOMC, &dax_dev->flags);  in set_dax_nomc()
314 	return test_bit(DAXDEV_ALIVE, &dax_dev->flags);  in dax_alive()
334 	if (dax_dev->holder_data != NULL)  in kill_dax()
338 	clear_bit(DAXDEV_ALIVE, &dax_dev->flags);  in kill_dax()
342 	dax_dev->holder_ops = NULL;  in kill_dax()
343 	dax_dev->holder_data = NULL;  in kill_dax()
349 	set_bit(DAXDEV_ALIVE, &dax_dev->flags);  in run_dax()
362 	inode = &dax_dev->inode;  in dax_alloc_inode()
363 	inode->i_rdev = 0;  in dax_alloc_inode()
375 	if (inode->i_rdev)  in dax_free_inode()
383 	WARN_ONCE(test_bit(DAXDEV_ALIVE, &dax_dev->flags),  in dax_destroy_inode()
399 		return -ENOMEM;  in dax_init_fs_context()
400 	ctx->ops = &dax_sops;  in dax_init_fs_context()
414 	return inode->i_rdev == devt;  in dax_test()
421 	inode->i_rdev = devt;  in dax_set()
437 	if (inode->i_state & I_NEW) {  in dax_dev_get()
438 		set_bit(DAXDEV_ALIVE, &dax_dev->flags);  in dax_dev_get()
439 		inode->i_cdev = &dax_dev->cdev;  in dax_dev_get()
440 		inode->i_mode = S_IFCHR;  in dax_dev_get()
441 		inode->i_flags = S_DAX;  in dax_dev_get()
442 		mapping_set_gfp_mask(&inode->i_data, GFP_USER);  in dax_dev_get()
457 	 * except for device-dax (NULL operations pointer), which does  in alloc_dax()
461 		return ERR_PTR(-EOPNOTSUPP);  in alloc_dax()
463 	if (WARN_ON_ONCE(ops && !ops->zero_page_range))  in alloc_dax()
464 		return ERR_PTR(-EINVAL);  in alloc_dax()
468 		return ERR_PTR(-ENOMEM);  in alloc_dax()
475 	dax_dev->ops = ops;  in alloc_dax()
476 	dax_dev->private = private;  in alloc_dax()
481 	return ERR_PTR(-ENOMEM);  in alloc_dax()
489 	iput(&dax_dev->inode);  in put_dax()
494  * dax_holder() - obtain the holder of a dax device
502 	return dax_dev->holder_data;  in dax_holder()
515 	struct cdev *cdev = inode->i_cdev;  in inode_dax()
523 	return &dax_dev->inode;  in dax_inode()
529 	if (!test_bit(DAXDEV_ALIVE, &dax_dev->flags))  in dax_get_private()
531 	return dax_dev->private;  in dax_get_private()
538 	struct inode *inode = &dax_dev->inode;  in init_once()
552 		return -ENOMEM;  in dax_fs_init()
559 	dax_superblock = dax_mnt->mnt_sb;  in dax_fs_init()
609 MODULE_DESCRIPTION("DAX: direct access to differentiated memory");