1 // SPDX-License-Identifier: GPL-2.0-only
8  * DOC: GPU page-table tree walking.
9  * The utilities in this file are similar to the CPU page-table walk
11  * the various levels of a page-table tree with an unsigned integer rather
12  * than by name. 0 is the lowest level, and page-tables with level 0 can
14  * can. The user of the utilities determines the highest level.
17  * Each struct xe_ptw, regardless of level is referred to as a page table, and
18  * multiple page tables typically form a page table tree with page tables at
19  * intermediate levels being page directories pointing at page tables at lower
20  * levels. A shared page table for a given address range is a page-table which
24  * Please keep this code generic so that it can used as a drm-wide page-
27 static u64 xe_pt_addr_end(u64 addr, u64 end, unsigned int level,  in xe_pt_addr_end()  argument
30 	u64 size = 1ull << walk->shifts[level];  in xe_pt_addr_end()
37 		       unsigned int level, const struct xe_pt_walk *walk)  in xe_pt_next()  argument
42 	if (unlikely(walk->shared_pt_mode)) {  in xe_pt_next()
43 		unsigned int shift = walk->shifts[level];  in xe_pt_next()
47 			step += (skip_to - next) >> shift;  in xe_pt_next()
59  * xe_pt_walk_range() - Walk a range of a gpu page table tree with callbacks
60  * for each page-table entry in all levels.
61  * @parent: The root page table for walk start.
62  * @level: The root page table level.
67  * Similar to the CPU page-table walker, this is a helper to walk
68  * a gpu page table and call a provided callback function for each entry.
73 int xe_pt_walk_range(struct xe_ptw *parent, unsigned int level,  in xe_pt_walk_range()  argument
76 	pgoff_t offset = xe_pt_offset(addr, level, walk);  in xe_pt_walk_range()
77 	struct xe_ptw **entries = walk->staging ? (parent->staging ?: NULL) :  in xe_pt_walk_range()
78 		(parent->children ?: NULL);  in xe_pt_walk_range()
79 	const struct xe_pt_walk_ops *ops = walk->ops;  in xe_pt_walk_range()
86 		next = xe_pt_addr_end(addr, end, level, walk);  in xe_pt_walk_range()
87 		if (walk->shared_pt_mode && xe_pt_covers(addr, next, level,  in xe_pt_walk_range()
93 		err = ops->pt_entry(parent, offset, level, addr, next,  in xe_pt_walk_range()
102 		if (likely(!level || !child || action == ACTION_CONTINUE))  in xe_pt_walk_range()
105 		err = xe_pt_walk_range(child, level - 1, addr, next, walk);  in xe_pt_walk_range()
107 		if (!err && ops->pt_post_descend)  in xe_pt_walk_range()
108 			err = ops->pt_post_descend(parent, offset, level, addr,  in xe_pt_walk_range()
113 	} while (xe_pt_next(&offset, &addr, next, end, level, walk));  in xe_pt_walk_range()
119  * xe_pt_walk_shared() - Walk shared page tables of a page-table tree.
120  * @parent: Root page table directory.
121  * @level: Level of the root.
126  * This function is similar to xe_pt_walk_range() but it skips page tables
127  * that are private to the range. Since the root (or @parent) page table is
128  * typically also a shared page table this function is different in that it
132  * Walking only the shared page tables is common for unbind-type operations
133  * where the page-table entries for an address range are cleared or detached
134  * from the main page-table tree.
140 int xe_pt_walk_shared(struct xe_ptw *parent, unsigned int level,  in xe_pt_walk_shared()  argument
143 	const struct xe_pt_walk_ops *ops = walk->ops;  in xe_pt_walk_shared()
148 	walk->shared_pt_mode = true;  in xe_pt_walk_shared()
149 	err = walk->ops->pt_entry(parent, 0, level + 1, addr, end,  in xe_pt_walk_shared()
155 	err = xe_pt_walk_range(parent, level, addr, end, walk);  in xe_pt_walk_shared()
156 	if (!err && ops->pt_post_descend) {  in xe_pt_walk_shared()
157 		err = ops->pt_post_descend(parent, 0, level + 1, addr, end,  in xe_pt_walk_shared()