1 // SPDX-License-Identifier: GPL-2.0
76 /* Read-protected by RCU, write-protected by kmmio_lock. */
92 /* Accessed per-cpu */
101  * Space Efficient Dynamic Stabbing with Fast Queries - Mikkel Thorup
108 		if (addr >= p->addr && addr < (p->addr + p->len))  in get_kmmio_probe()
127 		if (f->addr == addr)  in get_kmmio_fault_page()
163 	pte_t *pte = lookup_address(f->addr, &level);  in clear_page_presence()
166 		pr_err("no pte for addr 0x%08lx\n", f->addr);  in clear_page_presence()
167 		return -1;  in clear_page_presence()
172 		clear_pmd_presence((pmd_t *)pte, clear, &f->old_presence);  in clear_page_presence()
175 		clear_pte_presence(pte, clear, &f->old_presence);  in clear_page_presence()
179 		return -1;  in clear_page_presence()
182 	flush_tlb_one_kernel(f->addr);  in clear_page_presence()
192  * that double arming the same virtual address (page) cannot occur.
194  * Double disarming on the other hand is allowed, and may occur when a fault
200 	WARN_ONCE(f->armed, KERN_ERR pr_fmt("kmmio page already armed.\n"));  in arm_kmmio_fault_page()
201 	if (f->armed) {  in arm_kmmio_fault_page()
202 		pr_warn("double-arm: addr 0x%08lx, ref %d, old %d\n",  in arm_kmmio_fault_page()
203 			f->addr, f->count, !!f->old_presence);  in arm_kmmio_fault_page()
207 		  f->addr);  in arm_kmmio_fault_page()
208 	f->armed = true;  in arm_kmmio_fault_page()
217 			KERN_ERR "kmmio disarming at 0x%08lx failed.\n", f->addr);  in disarm_kmmio_fault_page()
218 	f->armed = false;  in disarm_kmmio_fault_page()
245 		return -EINVAL;  in kmmio_handler()
269 	if (ctx->active) {  in kmmio_handler()
270 		if (page_base == ctx->addr) {  in kmmio_handler()
273 			 * condition needs handling by do_page_fault(), the  in kmmio_handler()
279 			if (!faultpage->old_presence)  in kmmio_handler()
284 			 * Prevent overwriting already in-flight context.  in kmmio_handler()
290 			pr_emerg("previous hit was at 0x%08lx.\n", ctx->addr);  in kmmio_handler()
295 	ctx->active++;  in kmmio_handler()
297 	ctx->fpage = faultpage;  in kmmio_handler()
298 	ctx->probe = get_kmmio_probe(page_base);  in kmmio_handler()
299 	ctx->saved_flags = (regs->flags & (X86_EFLAGS_TF | X86_EFLAGS_IF));  in kmmio_handler()
300 	ctx->addr = page_base;  in kmmio_handler()
302 	if (ctx->probe && ctx->probe->pre_handler)  in kmmio_handler()
303 		ctx->probe->pre_handler(ctx->probe, regs, addr);  in kmmio_handler()
306 	 * Enable single-stepping and disable interrupts for the faulting  in kmmio_handler()
309 	regs->flags |= X86_EFLAGS_TF;  in kmmio_handler()
310 	regs->flags &= ~X86_EFLAGS_IF;  in kmmio_handler()
313 	disarm_kmmio_fault_page(ctx->fpage);  in kmmio_handler()
339 	if (!ctx->active) {  in post_kmmio_handler()
349 	if (ctx->probe && ctx->probe->post_handler)  in post_kmmio_handler()
350 		ctx->probe->post_handler(ctx->probe, condition, regs);  in post_kmmio_handler()
354 	if (ctx->fpage->count)  in post_kmmio_handler()
355 		arm_kmmio_fault_page(ctx->fpage);  in post_kmmio_handler()
358 	regs->flags &= ~X86_EFLAGS_TF;  in post_kmmio_handler()
359 	regs->flags |= ctx->saved_flags;  in post_kmmio_handler()
362 	ctx->active--;  in post_kmmio_handler()
363 	BUG_ON(ctx->active);  in post_kmmio_handler()
371 	if (!(regs->flags & X86_EFLAGS_TF))  in post_kmmio_handler()
384 		if (!f->count)  in add_kmmio_fault_page()
386 		f->count++;  in add_kmmio_fault_page()
392 		return -1;  in add_kmmio_fault_page()
394 	f->count = 1;  in add_kmmio_fault_page()
395 	f->addr = addr;  in add_kmmio_fault_page()
399 		return -1;  in add_kmmio_fault_page()
402 	list_add_rcu(&f->list, kmmio_page_list(f->addr));  in add_kmmio_fault_page()
417 	f->count--;  in release_kmmio_fault_page()
418 	BUG_ON(f->count < 0);  in release_kmmio_fault_page()
419 	if (!f->count) {  in release_kmmio_fault_page()
421 		if (!f->scheduled_for_release) {  in release_kmmio_fault_page()
422 			f->release_next = *release_list;  in release_kmmio_fault_page()
424 			f->scheduled_for_release = true;  in release_kmmio_fault_page()
430  * With page-unaligned ioremaps, one or two armed pages may contain
441 	unsigned long addr = p->addr & PAGE_MASK;  in register_kmmio_probe()
442 	const unsigned long size_lim = p->len + (p->addr & ~PAGE_MASK);  in register_kmmio_probe()
449 		ret = -EEXIST;  in register_kmmio_probe()
455 		ret = -EINVAL;  in register_kmmio_probe()
460 	list_add_rcu(&p->list, &kmmio_probes);  in register_kmmio_probe()
485 	struct kmmio_fault_page *f = dr->release_list;  in rcu_free_kmmio_fault_pages()
487 		struct kmmio_fault_page *next = f->release_next;  in rcu_free_kmmio_fault_pages()
488 		BUG_ON(f->count);  in rcu_free_kmmio_fault_pages()
499 	struct kmmio_fault_page *f = dr->release_list;  in remove_kmmio_fault_pages()
500 	struct kmmio_fault_page **prevp = &dr->release_list;  in remove_kmmio_fault_pages()
506 		if (!f->count) {  in remove_kmmio_fault_pages()
507 			list_del_rcu(&f->list);  in remove_kmmio_fault_pages()
508 			prevp = &f->release_next;  in remove_kmmio_fault_pages()
510 			*prevp = f->release_next;  in remove_kmmio_fault_pages()
511 			f->release_next = NULL;  in remove_kmmio_fault_pages()
512 			f->scheduled_for_release = false;  in remove_kmmio_fault_pages()
520 	call_rcu(&dr->rcu, rcu_free_kmmio_fault_pages);  in remove_kmmio_fault_pages()
540 	unsigned long addr = p->addr & PAGE_MASK;  in unregister_kmmio_probe()
541 	const unsigned long size_lim = p->len + (p->addr & ~PAGE_MASK);  in unregister_kmmio_probe()
557 	list_del_rcu(&p->list);  in unregister_kmmio_probe()
558 	kmmio_count--;  in unregister_kmmio_probe()
570 	drelease->release_list = release_list;  in unregister_kmmio_probe()
586 	call_rcu(&drelease->rcu, remove_kmmio_fault_pages);  in unregister_kmmio_probe()
594 	unsigned long* dr6_p = (unsigned long *)ERR_PTR(arg->err);  in kmmio_die_notifier()
597 		if (post_kmmio_handler(*dr6_p, arg->regs) == 1) {  in kmmio_die_notifier()
599 			 * Reset the BS bit in dr6 (pointed by args->err) to  in kmmio_die_notifier()