27 static void kvmhv_update_ptbl_cache(struct kvm_nested_guest *gp);
554 static void kvmhv_set_nested_ptbl(struct kvm_nested_guest *gp)  in kvmhv_set_nested_ptbl()  argument
559 		__pa(gp->shadow_pgtable) | RADIX_PGD_INDEX_SIZE;  in kvmhv_set_nested_ptbl()
560 	kvmhv_set_ptbl_entry(gp->shadow_lpid, dw0, gp->process_table);  in kvmhv_set_nested_ptbl()
598 	struct kvm_nested_guest *gp;  in kvmhv_copy_tofrom_guest_nested()  local
619 	gp = kvmhv_get_nested(vcpu->kvm, l1_lpid, false);  in kvmhv_copy_tofrom_guest_nested()
620 	if (!gp) {  in kvmhv_copy_tofrom_guest_nested()
625 	mutex_lock(&gp->tlb_lock);  in kvmhv_copy_tofrom_guest_nested()
629 		rc = __kvmhv_copy_tofrom_guest_radix(gp->shadow_lpid, pid,  in kvmhv_copy_tofrom_guest_nested()
649 		rc = __kvmhv_copy_tofrom_guest_radix(gp->shadow_lpid, pid,  in kvmhv_copy_tofrom_guest_nested()
656 	mutex_unlock(&gp->tlb_lock);  in kvmhv_copy_tofrom_guest_nested()
657 	kvmhv_put_nested(gp);  in kvmhv_copy_tofrom_guest_nested()
668  * Caller must hold gp->tlb_lock.
670 static void kvmhv_update_ptbl_cache(struct kvm_nested_guest *gp)  in kvmhv_update_ptbl_cache()  argument
675 	struct kvm *kvm = gp->l1_host;  in kvmhv_update_ptbl_cache()
678 	ptbl_addr = (kvm->arch.l1_ptcr & PRTB_MASK) + (gp->l1_lpid << 4);  in kvmhv_update_ptbl_cache()
679 	if (gp->l1_lpid < (1ul << ((kvm->arch.l1_ptcr & PRTS_MASK) + 12 - 4))) {  in kvmhv_update_ptbl_cache()
686 		gp->l1_gr_to_hr = 0;  in kvmhv_update_ptbl_cache()
687 		gp->process_table = 0;  in kvmhv_update_ptbl_cache()
689 		gp->l1_gr_to_hr = be64_to_cpu(ptbl_entry.patb0);  in kvmhv_update_ptbl_cache()
690 		gp->process_table = be64_to_cpu(ptbl_entry.patb1);  in kvmhv_update_ptbl_cache()
692 	kvmhv_set_nested_ptbl(gp);  in kvmhv_update_ptbl_cache()
713 static void __add_nested(struct kvm *kvm, int lpid, struct kvm_nested_guest *gp)  in __add_nested()  argument
715 	if (idr_replace(&kvm->arch.kvm_nested_guest_idr, gp, lpid))  in __add_nested()
726 	struct kvm_nested_guest *gp;  in kvmhv_alloc_nested()  local
729 	gp = kzalloc(sizeof(*gp), GFP_KERNEL);  in kvmhv_alloc_nested()
730 	if (!gp)  in kvmhv_alloc_nested()
732 	gp->l1_host = kvm;  in kvmhv_alloc_nested()
733 	gp->l1_lpid = lpid;  in kvmhv_alloc_nested()
734 	mutex_init(&gp->tlb_lock);  in kvmhv_alloc_nested()
735 	gp->shadow_pgtable = pgd_alloc(kvm->mm);  in kvmhv_alloc_nested()
736 	if (!gp->shadow_pgtable)  in kvmhv_alloc_nested()
741 	gp->shadow_lpid = shadow_lpid;  in kvmhv_alloc_nested()
742 	gp->radix = 1;  in kvmhv_alloc_nested()
744 	memset(gp->prev_cpu, -1, sizeof(gp->prev_cpu));  in kvmhv_alloc_nested()
746 	return gp;  in kvmhv_alloc_nested()
749 	pgd_free(kvm->mm, gp->shadow_pgtable);  in kvmhv_alloc_nested()
751 	kfree(gp);  in kvmhv_alloc_nested()
758 static void kvmhv_release_nested(struct kvm_nested_guest *gp)  in kvmhv_release_nested()  argument
760 	struct kvm *kvm = gp->l1_host;  in kvmhv_release_nested()
762 	if (gp->shadow_pgtable) {  in kvmhv_release_nested()
768 		kvmppc_free_pgtable_radix(kvm, gp->shadow_pgtable,  in kvmhv_release_nested()
769 					  gp->shadow_lpid);  in kvmhv_release_nested()
770 		pgd_free(kvm->mm, gp->shadow_pgtable);  in kvmhv_release_nested()
772 	kvmhv_set_ptbl_entry(gp->shadow_lpid, 0, 0);  in kvmhv_release_nested()
773 	kvmppc_free_lpid(gp->shadow_lpid);  in kvmhv_release_nested()
774 	kfree(gp);  in kvmhv_release_nested()
777 static void kvmhv_remove_nested(struct kvm_nested_guest *gp)  in kvmhv_remove_nested()  argument
779 	struct kvm *kvm = gp->l1_host;  in kvmhv_remove_nested()
780 	int lpid = gp->l1_lpid;  in kvmhv_remove_nested()
784 	if (gp == __find_nested(kvm, lpid)) {  in kvmhv_remove_nested()
786 		--gp->refcnt;  in kvmhv_remove_nested()
788 	ref = gp->refcnt;  in kvmhv_remove_nested()
791 		kvmhv_release_nested(gp);  in kvmhv_remove_nested()
803 	struct kvm_nested_guest *gp;  in kvmhv_release_all_nested()  local
809 	idr_for_each_entry(&kvm->arch.kvm_nested_guest_idr, gp, lpid) {  in kvmhv_release_all_nested()
811 		if (--gp->refcnt == 0) {  in kvmhv_release_all_nested()
812 			gp->next = freelist;  in kvmhv_release_all_nested()
813 			freelist = gp;  in kvmhv_release_all_nested()
819 	while ((gp = freelist) != NULL) {  in kvmhv_release_all_nested()
820 		freelist = gp->next;  in kvmhv_release_all_nested()
821 		kvmhv_release_nested(gp);  in kvmhv_release_all_nested()
830 /* caller must hold gp->tlb_lock */
831 static void kvmhv_flush_nested(struct kvm_nested_guest *gp)  in kvmhv_flush_nested()  argument
833 	struct kvm *kvm = gp->l1_host;  in kvmhv_flush_nested()
836 	kvmppc_free_pgtable_radix(kvm, gp->shadow_pgtable, gp->shadow_lpid);  in kvmhv_flush_nested()
838 	kvmhv_flush_lpid(gp->shadow_lpid);  in kvmhv_flush_nested()
839 	kvmhv_update_ptbl_cache(gp);  in kvmhv_flush_nested()
840 	if (gp->l1_gr_to_hr == 0)  in kvmhv_flush_nested()
841 		kvmhv_remove_nested(gp);  in kvmhv_flush_nested()
847 	struct kvm_nested_guest *gp, *newgp;  in kvmhv_get_nested()  local
853 	gp = __find_nested(kvm, l1_lpid);  in kvmhv_get_nested()
854 	if (gp)  in kvmhv_get_nested()
855 		++gp->refcnt;  in kvmhv_get_nested()
858 	if (gp || !create)  in kvmhv_get_nested()
859 		return gp;  in kvmhv_get_nested()
871 	gp = __find_nested(kvm, l1_lpid);  in kvmhv_get_nested()
872 	if (!gp) {  in kvmhv_get_nested()
875 		gp = newgp;  in kvmhv_get_nested()
878 	++gp->refcnt;  in kvmhv_get_nested()
884 	return gp;  in kvmhv_get_nested()
887 void kvmhv_put_nested(struct kvm_nested_guest *gp)  in kvmhv_put_nested()  argument
889 	struct kvm *kvm = gp->l1_host;  in kvmhv_put_nested()
893 	ref = --gp->refcnt;  in kvmhv_put_nested()
896 		kvmhv_release_nested(gp);  in kvmhv_put_nested()
902 	struct kvm_nested_guest *gp;  in find_kvm_nested_guest_pte()  local
905 	gp = __find_nested(kvm, lpid);  in find_kvm_nested_guest_pte()
906 	if (!gp)  in find_kvm_nested_guest_pte()
911 	pte = __find_linux_pte(gp->shadow_pgtable, ea, NULL, hshift);  in find_kvm_nested_guest_pte()
1004 	struct kvm_nested_guest *gp;  in kvmhv_remove_nest_rmap()  local
1011 	gp = __find_nested(kvm, lpid);  in kvmhv_remove_nest_rmap()
1012 	if (!gp)  in kvmhv_remove_nest_rmap()
1019 		kvmppc_unmap_pte(kvm, ptep, gpa, shift, NULL, gp->shadow_lpid);  in kvmhv_remove_nest_rmap()
1074 					struct kvm_nested_guest *gp,  in kvmhv_invalidate_shadow_pte()  argument
1083 	ptep = find_kvm_nested_guest_pte(kvm, gp->l1_lpid, gpa, &shift);  in kvmhv_invalidate_shadow_pte()
1087 		kvmppc_unmap_pte(kvm, ptep, gpa, shift, NULL, gp->shadow_lpid);  in kvmhv_invalidate_shadow_pte()
1136 	struct kvm_nested_guest *gp;  in kvmhv_emulate_tlbie_tlb_addr()  local
1150 	gp = kvmhv_get_nested(kvm, lpid, false);  in kvmhv_emulate_tlbie_tlb_addr()
1151 	if (!gp) /* No such guest -> nothing to do */  in kvmhv_emulate_tlbie_tlb_addr()
1153 	mutex_lock(&gp->tlb_lock);  in kvmhv_emulate_tlbie_tlb_addr()
1157 		kvmhv_invalidate_shadow_pte(vcpu, gp, addr, &shadow_shift);  in kvmhv_emulate_tlbie_tlb_addr()
1163 	mutex_unlock(&gp->tlb_lock);  in kvmhv_emulate_tlbie_tlb_addr()
1164 	kvmhv_put_nested(gp);  in kvmhv_emulate_tlbie_tlb_addr()
1169 				     struct kvm_nested_guest *gp, int ric)  in kvmhv_emulate_tlbie_lpid()  argument
1173 	mutex_lock(&gp->tlb_lock);  in kvmhv_emulate_tlbie_lpid()
1178 		kvmppc_free_pgtable_radix(kvm, gp->shadow_pgtable,  in kvmhv_emulate_tlbie_lpid()
1179 					  gp->shadow_lpid);  in kvmhv_emulate_tlbie_lpid()
1180 		kvmhv_flush_lpid(gp->shadow_lpid);  in kvmhv_emulate_tlbie_lpid()
1191 		kvmhv_flush_nested(gp);  in kvmhv_emulate_tlbie_lpid()
1196 	mutex_unlock(&gp->tlb_lock);  in kvmhv_emulate_tlbie_lpid()
1202 	struct kvm_nested_guest *gp;  in kvmhv_emulate_tlbie_all_lpid()  local
1206 	idr_for_each_entry(&kvm->arch.kvm_nested_guest_idr, gp, lpid) {  in kvmhv_emulate_tlbie_all_lpid()
1208 		kvmhv_emulate_tlbie_lpid(vcpu, gp, ric);  in kvmhv_emulate_tlbie_all_lpid()
1218 	struct kvm_nested_guest *gp;  in kvmhv_emulate_priv_tlbie()  local
1254 		gp = kvmhv_get_nested(kvm, lpid, false);  in kvmhv_emulate_priv_tlbie()
1255 		if (gp) {  in kvmhv_emulate_priv_tlbie()
1256 			kvmhv_emulate_tlbie_lpid(vcpu, gp, ric);  in kvmhv_emulate_priv_tlbie()
1257 			kvmhv_put_nested(gp);  in kvmhv_emulate_priv_tlbie()
1292 	struct kvm_nested_guest *gp;  in do_tlb_invalidate_nested_all()  local
1294 	gp = kvmhv_get_nested(kvm, lpid, false);  in do_tlb_invalidate_nested_all()
1295 	if (gp) {  in do_tlb_invalidate_nested_all()
1296 		kvmhv_emulate_tlbie_lpid(vcpu, gp, ric);  in do_tlb_invalidate_nested_all()
1297 		kvmhv_put_nested(gp);  in do_tlb_invalidate_nested_all()
1397 				       struct kvm_nested_guest *gp,  in kvmhv_translate_addr_nested()  argument
1404 	ret = kvmppc_mmu_walk_radix_tree(vcpu, n_gpa, gpte_p, gp->l1_gr_to_hr,  in kvmhv_translate_addr_nested()
1459 				       struct kvm_nested_guest *gp,  in kvmhv_handle_nested_set_rc()  argument
1487 				      n_gpa, gp->l1_lpid);  in kvmhv_handle_nested_set_rc()
1522 /* called with gp->tlb_lock held */
1524 					  struct kvm_nested_guest *gp)  in __kvmhv_nested_page_fault()  argument
1540 	if (!gp->l1_gr_to_hr) {  in __kvmhv_nested_page_fault()
1541 		kvmhv_update_ptbl_cache(gp);  in __kvmhv_nested_page_fault()
1542 		if (!gp->l1_gr_to_hr)  in __kvmhv_nested_page_fault()
1551 	ret = kvmhv_translate_addr_nested(vcpu, gp, n_gpa, dsisr, &gpte);  in __kvmhv_nested_page_fault()
1567 		ret = kvmhv_handle_nested_set_rc(vcpu, gp, n_gpa, gpte, dsisr);  in __kvmhv_nested_page_fault()
1678 		(((unsigned long) gp->l1_lpid) << RMAP_NESTED_LPID_SHIFT);  in __kvmhv_nested_page_fault()
1680 	ret = kvmppc_create_pte(kvm, gp->shadow_pgtable, pte, n_gpa, level,  in __kvmhv_nested_page_fault()
1681 				mmu_seq, gp->shadow_lpid, rmapp, &n_rmap);  in __kvmhv_nested_page_fault()
1689 	kvmhv_invalidate_shadow_pte(vcpu, gp, n_gpa, NULL);  in __kvmhv_nested_page_fault()
1695 	struct kvm_nested_guest *gp = vcpu->arch.nested;  in kvmhv_nested_page_fault()  local
1698 	mutex_lock(&gp->tlb_lock);  in kvmhv_nested_page_fault()
1699 	ret = __kvmhv_nested_page_fault(vcpu, gp);  in kvmhv_nested_page_fault()
1700 	mutex_unlock(&gp->tlb_lock);  in kvmhv_nested_page_fault()