Lines Matching +full:scrubber +full:- +full:done
1 // SPDX-License-Identifier: GPL-2.0-only
7 * hardware as being corrupted usually due to a multi-bit ECC memory or cache
11 * not-yet-corrupted-by-suspicious pages without killing anything.
23 * - You know how to test it.
24 * - You have a test that can be added to mce-test
25 * https://git.kernel.org/cgit/utils/cpu/mce/mce-test.git/
26 * - The case actually shows up as a frequent (top 10) page state in
27 * tools/mm/page-types when running a real workload.
41 #include <linux/page-flags.h>
50 #include <linux/backing-dev.h>
88 if (pfn != -1UL) in num_poisoned_pages_sub()
93 * MF_ATTR_RO - Create sysfs entry for each memory failure statistics.
102 &NODE_DATA(dev->id)->mf_stats; \
103 return sysfs_emit(buf, "%lu\n", mf_stats->_name); \
193 * dissolve_free_hugetlb_folio() returns 0 for non-hugetlb folios as well. in page_handle_poison()
199 * acceptable because soft-offlined page is not broken in page_handle_poison()
239 if (mapping == NULL || mapping->host == NULL) in hwpoison_filter_dev()
240 return -EINVAL; in hwpoison_filter_dev()
242 dev = mapping->host->i_sb->s_dev; in hwpoison_filter_dev()
245 return -EINVAL; in hwpoison_filter_dev()
248 return -EINVAL; in hwpoison_filter_dev()
262 return -EINVAL; in hwpoison_filter_flags()
284 return -EINVAL; in hwpoison_filter_task()
298 return -EINVAL; in hwpoison_filter()
301 return -EINVAL; in hwpoison_filter()
304 return -EINVAL; in hwpoison_filter()
327 * from the VMAs. So do a brute-force search over all
352 struct task_struct *t = tk->tsk; in kill_proc()
353 short addr_lsb = tk->size_shift; in kill_proc()
357 pfn, t->comm, task_pid_nr(t)); in kill_proc()
361 (void __user *)tk->addr, addr_lsb); in kill_proc()
369 ret = send_sig_mceerr(BUS_MCEERR_AO, (void __user *)tk->addr, in kill_proc()
373 t->comm, task_pid_nr(t), ret); in kill_proc()
386 * TODO: Could shrink slab caches here if a lightweight range-based in shake_folio()
412 VM_BUG_ON_VMA(address == -EFAULT, vma); in dev_pagemap_mapping_shift()
413 pgd = pgd_offset(vma->vm_mm, address); in dev_pagemap_mapping_shift()
460 tk->addr = addr; in __add_to_kill()
462 tk->size_shift = dev_pagemap_mapping_shift(vma, tk->addr); in __add_to_kill()
464 tk->size_shift = folio_shift(page_folio(p)); in __add_to_kill()
467 * Send SIGKILL if "tk->addr == -EFAULT". Also, as in __add_to_kill()
468 * "tk->size_shift" is always non-zero for !is_zone_device_page(), in __add_to_kill()
469 * so "tk->size_shift == 0" effectively checks no mapping on in __add_to_kill()
476 if (tk->addr == -EFAULT) { in __add_to_kill()
478 page_to_pfn(p), tsk->comm); in __add_to_kill()
479 } else if (tk->size_shift == 0) { in __add_to_kill()
485 tk->tsk = tsk; in __add_to_kill()
486 list_add_tail(&tk->nd, to_kill); in __add_to_kill()
493 if (addr == -EFAULT) in add_to_kill_anon_file()
505 if (tk->tsk == tsk) in task_in_to_kill_list()
533 if (tk->addr == -EFAULT) { in kill_procs()
535 pfn, tk->tsk->comm, task_pid_nr(tk->tsk)); in kill_procs()
537 tk->tsk, PIDTYPE_PID); in kill_procs()
542 * something else on the address in-between. We could in kill_procs()
548 pfn, tk->tsk->comm, task_pid_nr(tk->tsk)); in kill_procs()
550 list_del(&tk->nd); in kill_procs()
551 put_task_struct(tk->tsk); in kill_procs()
569 if (t->flags & PF_MCE_PROCESS) { in find_early_kill_thread()
570 if (t->flags & PF_MCE_EARLY) in find_early_kill_thread()
594 if (!tsk->mm) in task_early_kill()
597 * Comparing ->mm here because current task might represent in task_early_kill()
600 if (force_early && tsk->mm == current->mm) in task_early_kill()
631 anon_vma_interval_tree_foreach(vmac, &av->rb_root, in collect_procs_anon()
633 vma = vmac->vma; in collect_procs_anon()
634 if (vma->vm_mm != t->mm) in collect_procs_anon()
653 struct address_space *mapping = folio->mapping; in collect_procs_file()
665 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, in collect_procs_file()
674 if (vma->vm_mm != t->mm) in collect_procs_file()
717 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { in collect_procs_fsdax()
718 if (vma->vm_mm == t->mm) in collect_procs_fsdax()
733 if (!folio->mapping) in collect_procs()
751 tk->addr = addr; in set_to_kill()
752 tk->size_shift = shift; in set_to_kill()
787 if (pfn <= hwp->pfn && hwp->pfn < pfn + HPAGE_PMD_NR) { in check_hwpoisoned_pmd_entry()
788 hwpoison_vaddr = addr + ((hwp->pfn - pfn) << PAGE_SHIFT); in check_hwpoisoned_pmd_entry()
789 set_to_kill(&hwp->tk, hwpoison_vaddr, PAGE_SHIFT); in check_hwpoisoned_pmd_entry()
805 struct hwpoison_walk *hwp = walk->private; in hwpoison_pte_range()
810 ptl = pmd_trans_huge_lock(pmdp, walk->vma); in hwpoison_pte_range()
817 mapped_pte = ptep = pte_offset_map_lock(walk->vma->vm_mm, pmdp, in hwpoison_pte_range()
824 hwp->pfn, &hwp->tk); in hwpoison_pte_range()
839 struct hwpoison_walk *hwp = walk->private; in hwpoison_hugetlb_range()
840 pte_t pte = huge_ptep_get(walk->mm, addr, ptep); in hwpoison_hugetlb_range()
841 struct hstate *h = hstate_vma(walk->vma); in hwpoison_hugetlb_range()
844 hwp->pfn, &hwp->tk); in hwpoison_hugetlb_range()
878 if (!p->mm) in kill_accessing_process()
879 return -EFAULT; in kill_accessing_process()
881 mmap_read_lock(p->mm); in kill_accessing_process()
882 ret = walk_page_range(p->mm, 0, TASK_SIZE, &hwpoison_walk_ops, in kill_accessing_process()
892 mmap_read_unlock(p->mm); in kill_accessing_process()
894 return ret > 0 ? -EHWPOISON : 0; in kill_accessing_process()
898 * MF_IGNORED - The m-f() handler marks the page as PG_hwpoisoned'ed.
904 * MF_FAILED - The m-f() handler marks the page as PG_hwpoisoned'ed.
908 * by the m-f() handler immediately.
910 * MF_DELAYED - The m-f() handler marks the page as PG_hwpoisoned'ed.
915 * MF_RECOVERED - The m-f() handler marks the page as PG_hwpoisoned'ed.
917 * the buddy system, or hole-punnched out of the file mapping.
928 [MF_MSG_KERNEL_HIGH_ORDER] = "high-order kernel page",
960 * complain when the folio is unpoison-and-freed. in delete_from_lru_cache()
977 return -EIO; in delete_from_lru_cache()
985 if (mapping->a_ops->error_remove_folio) { in truncate_error_folio()
986 int err = mapping->a_ops->error_remove_folio(mapping, folio); in truncate_error_folio()
1013 /* Callback ->action() has to unlock the relevant page inside it. */
1026 int count = page_count(p) - 1; in has_extra_refcount()
1029 count -= folio_nr_pages(page_folio(p)); in has_extra_refcount()
1033 page_to_pfn(p), action_page_types[ps->type], count); in has_extra_refcount()
1053 * This is a catch-all in case we fail to make sense of the page state.
1099 * so is expected to have an extra refcount after error-handling. in me_pagecache_clean()
1136 mapping_set_error(mapping, -EIO); in me_pagecache_dirty()
1151 * - clear dirty bit to prevent IO
1152 * - remove from LRU
1153 * - but keep in the swap cache, so that when we return to it on
1202 * - Error on hugepage is contained in hugepage unit (not in raw page unit.)
1243 * A page state is defined by its current page->flags bits.
1308 mf_stats = &NODE_DATA(nid)->mf_stats; in update_per_node_mf_stats()
1311 ++mf_stats->ignored; in update_per_node_mf_stats()
1314 ++mf_stats->failed; in update_per_node_mf_stats()
1317 ++mf_stats->delayed; in update_per_node_mf_stats()
1320 ++mf_stats->recovered; in update_per_node_mf_stats()
1326 ++mf_stats->total; in update_per_node_mf_stats()
1345 return (result == MF_RECOVERED || result == MF_DELAYED) ? 0 : -EBUSY; in action_result()
1353 /* page p should be unlocked after returning from ps->action(). */ in page_action()
1354 result = ps->action(ps, p); in page_action()
1361 return action_result(pfn, ps->type, result); in page_action()
1391 /* Soft offline could migrate non-LRU movable pages */ in HWPoisonHandlable()
1420 if (!HWPoisonHandlable(&folio->page, flags)) in __get_hwpoison_page()
1421 return -EBUSY; in __get_hwpoison_page()
1452 ret = -EBUSY; in get_any_page()
1457 ret = -EIO; in get_any_page()
1460 } else if (ret == -EBUSY) { in get_any_page()
1469 ret = -EIO; in get_any_page()
1488 ret = -EIO; in get_any_page()
1491 if (ret == -EIO) in get_any_page()
1518 return -EHWPOISON; in __get_unpoison_page()
1524 * get_hwpoison_page() - Get refcount for memory error handling
1529 * error on it, after checking that the error page is in a well-defined state
1530 * (defined as a page-type we can successfully handle the memory error on it,
1536 * extra care for the error page's state (as done in __get_hwpoison_page()),
1544 * 1 on success for in-use pages in a well-defined state,
1545 * -EIO for pages on which we can not handle memory errors,
1546 * -EBUSY when get_hwpoison_page() has raced with page lifecycle
1548 * -EHWPOISON when the page is hwpoisoned and taken off from buddy.
1604 return -EBUSY; in unmap_poisoned_folio()
1613 return folio_mapped(folio) ? -EBUSY : 0; in unmap_poisoned_folio()
1629 * Here we are interested only in user-mapped pages, so skip any in hwpoison_user_mappings()
1647 * mapped in dirty form. This has to be done before try_to_unmap, in hwpoison_user_mappings()
1666 * struct page and all unmaps done we can decide if in hwpoison_user_mappings()
1671 * use a more force-full uncatchable kill to prevent in hwpoison_user_mappings()
1692 if ((p->flags & ps->mask) == ps->res) in identify_page_state()
1695 page_flags |= (p->flags & (1UL << PG_dirty)); in identify_page_state()
1697 if (!ps->mask) in identify_page_state()
1699 if ((page_flags & ps->mask) == ps->res) in identify_page_state()
1730 if (tk->size_shift) in unmap_and_kill()
1731 size = max(size, 1UL << tk->size_shift); in unmap_and_kill()
1735 * Unmap the largest mapping to avoid breaking up device-dax in unmap_and_kill()
1740 loff_t start = ((loff_t)index << PAGE_SHIFT) & ~(size - 1); in unmap_and_kill()
1774 return -EBUSY; in mf_generic_kill_procs()
1776 if (hwpoison_filter(&folio->page)) { in mf_generic_kill_procs()
1777 rc = -EOPNOTSUPP; in mf_generic_kill_procs()
1781 switch (pgmap->type) { in mf_generic_kill_procs()
1786 * with device-side memory. in mf_generic_kill_procs()
1788 rc = -ENXIO; in mf_generic_kill_procs()
1798 SetPageHWPoison(&folio->page); in mf_generic_kill_procs()
1801 * Unlike System-RAM there is no possibility to swap in a in mf_generic_kill_procs()
1807 collect_procs(folio, &folio->page, &to_kill, true); in mf_generic_kill_procs()
1809 unmap_and_kill(&to_kill, pfn, folio->mapping, folio->index, flags); in mf_generic_kill_procs()
1817 * mf_dax_kill_procs - Collect and kill processes who are using this file range
1838 return -EBUSY; in mf_dax_kill_procs()
1864 * constructing singly linked list from ->_hugetlb_hwpoison field of folio.
1873 return (struct llist_head *)&folio->_hugetlb_hwpoison; in raw_hwp_list_head()
1899 llist_for_each_entry(p, raw_hwp_head->first, node) { in is_raw_hwpoison_page_in_hugepage()
1900 if (page == p->page) { in is_raw_hwpoison_page_in_hugepage()
1920 SetPageHWPoison(p->page); in __folio_free_raw_hwp()
1922 num_poisoned_pages_sub(page_to_pfn(p->page), 1); in __folio_free_raw_hwp()
1934 int ret = folio_test_set_hwpoison(folio) ? -EHWPOISON : 0; in folio_set_hugetlb_hwpoison()
1942 return -EHWPOISON; in folio_set_hugetlb_hwpoison()
1944 llist_for_each_entry(p, head->first, node) { in folio_set_hugetlb_hwpoison()
1945 if (p->page == page) in folio_set_hugetlb_hwpoison()
1946 return -EHWPOISON; in folio_set_hugetlb_hwpoison()
1951 raw_hwp->page = page; in folio_set_hugetlb_hwpoison()
1952 llist_add(&raw_hwp->node, head); in folio_set_hugetlb_hwpoison()
2005 * 0 - free hugepage
2006 * 1 - in-use hugepage
2007 * 2 - not a hugepage
2008 * -EBUSY - the hugepage is busy (try to retry)
2009 * -EHWPOISON - the hugepage is already hwpoisoned
2032 ret = -EBUSY; in __get_huge_page_for_hwpoison()
2038 ret = -EHWPOISON; in __get_huge_page_for_hwpoison()
2062 * PageHWPoison) should be done in single hugetlb_lock range.
2078 } else if (res == -EHWPOISON) { in try_memory_failure_hugetlb()
2086 } else if (res == -EBUSY) { in try_memory_failure_hugetlb()
2104 return -EOPNOTSUPP; in try_memory_failure_hugetlb()
2122 page_flags = folio->flags; in try_memory_failure_hugetlb()
2156 int rc = -ENXIO; in memory_failure_dev_pagemap()
2167 rc = pgmap->ops->memory_failure(pgmap, pfn, 1, flags); in memory_failure_dev_pagemap()
2172 if (rc != -EOPNOTSUPP) in memory_failure_dev_pagemap()
2180 if (rc != -EOPNOTSUPP) in memory_failure_dev_pagemap()
2187 * been RDMA pinned, not much can be done for recovery.
2202 * memory_failure - Handle memory failure of a page.
2213 * detected by a background scrubber)
2219 * -EOPNOTSUPP for hwpoison_filter() filtered the error event,
2220 * < 0(except -EOPNOTSUPP) on failure.
2256 res = -ENXIO; in memory_failure()
2267 res = -EHWPOISON; in memory_failure()
2280 * 2) it's part of a non-compound high order page. in memory_failure()
2322 res = -EOPNOTSUPP; in memory_failure()
2343 res = -EHWPOISON; in memory_failure()
2354 * We ignore non-LRU pages for good reasons. in memory_failure()
2355 * - PG_locked is only well defined for LRU pages and a few others in memory_failure()
2356 * - to avoid races with __SetPageLocked() in memory_failure()
2357 * - to avoid races with __SetPageSlab*() (and more non-atomic ops) in memory_failure()
2366 * We're only intended to deal with the non-Compound page here. in memory_failure()
2379 page_flags = folio->flags; in memory_failure()
2408 folio->mapping == NULL) { in memory_failure()
2443 * memory_failure_queue - Schedule handling memory failure of a page.
2454 * detected by a background scrubber)
2469 raw_spin_lock_irqsave(&mf_cpu->lock, proc_flags); in memory_failure_queue()
2470 buffer_overflow = !kfifo_put(&mf_cpu->fifo, entry); in memory_failure_queue()
2472 schedule_work_on(smp_processor_id(), &mf_cpu->work); in memory_failure_queue()
2473 raw_spin_unlock_irqrestore(&mf_cpu->lock, proc_flags); in memory_failure_queue()
2490 raw_spin_lock_irqsave(&mf_cpu->lock, proc_flags); in memory_failure_work_func()
2491 gotten = kfifo_get(&mf_cpu->fifo, &entry); in memory_failure_work_func()
2492 raw_spin_unlock_irqrestore(&mf_cpu->lock, proc_flags); in memory_failure_work_func()
2504 * Used to avoid return-to-userspace racing with the memory_failure workqueue.
2511 cancel_work_sync(&mf_cpu->work); in memory_failure_queue_kick()
2512 memory_failure_work_func(&mf_cpu->work); in memory_failure_queue_kick()
2522 raw_spin_lock_init(&mf_cpu->lock); in memory_failure_init()
2523 INIT_KFIFO(mf_cpu->fifo); in memory_failure_init()
2524 INIT_WORK(&mf_cpu->work, memory_failure_work_func); in memory_failure_init()
2542 * unpoison_memory - Unpoison a previously poisoned page
2545 * Software-unpoison a page that has been poisoned by
2548 * This is only done on the software-level, so it only works
2551 * Returns 0 for success, otherwise -errno.
2557 int ret = -EBUSY, ghp; in unpoison_memory()
2564 return -ENXIO; in unpoison_memory()
2574 ret = -EOPNOTSUPP; in unpoison_memory()
2581 ret = -EOPNOTSUPP; in unpoison_memory()
2608 unpoison_pr_info("%#lx: the hwpoison page has non-NULL mapping\n", in unpoison_memory()
2621 ret = folio_test_clear_hwpoison(folio) ? 0 : -EBUSY; in unpoison_memory()
2623 if (ghp == -EHWPOISON) { in unpoison_memory()
2624 ret = put_page_back_buddy(p) ? 0 : -EBUSY; in unpoison_memory()
2652 unpoison_pr_info("%#lx: software-unpoisoned page\n", in unpoison_memory()
2663 * soft_offline_in_use_page handles hugetlb-pages and non-hugetlb pages.
2664 * If the page is a non-dirty unmapped page-cache page, it simply invalidates.
2685 return -EBUSY; in soft_offline_in_use_page()
2732 ret = -EBUSY; in soft_offline_in_use_page()
2738 pfn, msg_page[huge], ret, &page->flags); in soft_offline_in_use_page()
2740 ret = -EBUSY; in soft_offline_in_use_page()
2744 pfn, msg_page[huge], page_count(page), &page->flags); in soft_offline_in_use_page()
2745 ret = -EBUSY; in soft_offline_in_use_page()
2751 * soft_offline_page - Soft offline a page.
2752 * @pfn: pfn to soft-offline
2756 * -EOPNOTSUPP for hwpoison_filter() filtered the error event, or
2783 return -ENXIO; in soft_offline_page()
2786 /* Only online pages can be soft-offlined (esp., not ZONE_DEVICE). */ in soft_offline_page()
2790 return -EIO; in soft_offline_page()
2796 return -EOPNOTSUPP; in soft_offline_page()
2818 return -EOPNOTSUPP; in soft_offline_page()
2830 ret = -EBUSY; in soft_offline_page()