1 /* SPDX-License-Identifier: GPL-2.0 */
19 #include <linux/pageblock-flags.h>
20 #include <linux/page-flags-layout.h>
23 #include <linux/page-flags.h>
28 /* Free memory management - zoned buddy allocator.  */
80 	get_pfnblock_flags_mask(&folio->page, pfn, MIGRATETYPE_MASK))
109 #define MIGRATETYPE_MASK ((1UL << PB_migratetype_bits) - 1)
115 	get_pfnblock_flags_mask(&folio->page, folio_pfn(folio),		\
139 	/* First 128 byte cacheline (assuming 64 bit words) */
200 	NR_KERNEL_MISC_RECLAIMABLE,	/* reclaimable non-slab kernel pages */
254 	 * Global and per-node slab counters track slab pages.  in vmstat_item_in_bytes()
258 	 * Per-memcg and per-lruvec counters track memory, consumed  in vmstat_item_in_bytes()
260 	 * byte-precise.  in vmstat_item_in_bytes()
317 	 * 1. LRUVEC_CGROUP_CONGESTED is set by cgroup-level reclaim.
319 	 * 2. LRUVEC_NODE_CONGESTED is set by kswapd node-level reclaim.
336  * oldest generation numbers, max_seq and min_seq, are monotonically increasing.
339  * corresponding generation. The gen counter in folio->flags stores gen+1 while
340  * a folio is on one of lrugen->folios[]. Otherwise it stores 0.
343  * least twice before handing this folio over to the eviction. The first check
351  * PG_active is always cleared while a folio is on one of lrugen->folios[] so
353  * a folio considered active is isolated for non-reclaiming purposes, e.g.,
356  * MAX_NR_GENS is set to 4 so that the multi-gen LRU can support twice the
359  * in folio->flags, masked by LRU_GEN_MASK.
365  * Each generation is divided into multiple tiers. A folio accessed N times
366  * through file descriptors is in tier order_base_2(N). A folio in the first
368  * tables or read ahead. A folio in the last tier (MAX_NR_TIERS-1) is marked by
369  * PG_workingset. A folio in any other tier (1<N<5) between the first and last
370  * is marked by additional bits of LRU_REFS_WIDTH in folio->flags.
373  * across tiers only involves atomic operations on folio->flags and therefore
375  * comparisons of refaulted/(evicted+protected) from the first tier and the rest
379  * MAX_NR_TIERS is set to 4 so that the multi-gen LRU can support twice the
381  * accesses through file descriptors. This uses MAX_NR_TIERS-2 spare bits in
382  * folio->flags, masked by LRU_REFS_MASK.
388 #define LRU_GEN_MASK		((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF)
389 #define LRU_REFS_MASK		((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF)
393  * lru_gen_inc_refs() sets additional bits of LRU_REFS_WIDTH in folio->flags
396  * promoted into the second oldest generation in the eviction path. And when
403  * PG_referenced after the accessed bit is cleared for the first time.
405  * youngest generation. Like folio_inc_gen(), folio_update_gen() also clears
442  * The youngest generation number is stored in max_seq for both anon and file
443  * types as they are aged on an equal footing. The oldest generation numbers are
448  * MAX_NR_GENS-MIN_NR_GENS-1.
450  * The number of pages in each generation is eventually consistent and therefore
454 	/* the aging increments the youngest generation number */
456 	/* the eviction increments the oldest generation numbers */
458 	/* the birth time of each generation in jiffies */
460 	/* the multi-gen LRU lists, lazily sorted on eviction */
462 	/* the multi-gen LRU sizes, eventually consistent */
473 	/* whether the multi-gen LRU is enabled */
475 	/* the memcg generation this lru_gen_folio belongs to */
479 	/* per-node lru_gen_folio list for global reclaim */
486 	MM_NONLEAF_FOUND,	/* non-leaf entries found in Bloom filters */
487 	MM_NONLEAF_ADDED,	/* non-leaf entries added to Bloom filters */
491 /* double-buffering Bloom filters */
526  * young. For each generation, memcgs are randomly sharded into multiple bins
530  * An onlining memcg is added to the tail of a random bin in the old generation.
531  * The eviction starts at the head of a random bin in the old generation. The
532  * per-node memcg generation counter, whose reminder (mod MEMCG_NR_GENS) indexes
533  * the old generation, is incremented when all its bins become empty.
537  *    current generation (old or young) and updates its "seg" to "head";
539  *    current generation (old or young) and updates its "seg" to "tail";
541  *    generation, updates its "gen" to "old" and resets its "seg" to "default";
543  *    young generation, updates its "gen" to "young" and resets its "seg" to
548  * 2. The first attempt to reclaim a memcg below low, which triggers
550  * 3. The first attempt to reclaim a memcg offlined or below reclaimable size
559  * 1. Memcg LRU only applies to global reclaim, and the round-robin incrementing
563  *    MEMCG_NR_GENS is set to three so that when reading the generation counter
564  *    locklessly, a stale value (seq-1) does not wraparound to young.
570 	/* the per-node memcg generation counter */
574 	/* per-node lru_gen_folio list for global reclaim */
637 	 * These track the cost of reclaiming one LRU - file or anon -
643 	/* Non-resident age, driven by LRU movement */
693  * Flags used in pcp->flags field.
695  * PCPF_PREV_FREE_HIGH_ORDER: a high-order page is freed in the
697  * high-order page freeing.
699  * PCPF_FREE_HIGH_BATCH: preserve "pcp->batch" pages in PCP before
700  * draining PCP for consecutive high-order pages freeing without
702  * zone lock contention and keep cache-hot pages reusing.
714 	u8 flags;		/* protected by pcp->lock */
721 	/* Lists of pages, one per migrate type stored on the pcp-lists */
754 	 * DMA mask is assumed when ZONE_DMA32 is defined. Some 64-bit
785 	 * likely to succeed, and to locally limit unmovable allocations - e.g.,
788 	 * 1. Pinned pages: (long-term) pinning of movable pages might
790 	 *    pinning long-term pages in ZONE_MOVABLE. When pages are pinned and
795 	 *    to a different zone. When migration fails - pinning fails.
807 	 *    buddy (e.g., via XEN-balloon, Hyper-V balloon, virtio-mem). The
809 	 *    some cases (virtio-mem), such pages can be skipped during
813 	 *    of memory unplug in virtio-mem).
818 	 * 7. Memory-hotplug: when using memmap_on_memory and onlining the
821 	 *    self-stored in the range, but they are treated as movable when
843 	/* Read-mostly fields */
879 	 * Flags for a pageblock_nr_pages block. See pageblock-flags.h.
891 	 * 	spanned_pages = zone_end_pfn - zone_start_pfn;
895 	 *	present_pages = spanned_pages - absent_pages(pages in holes);
904 	 *	managed_pages = present_pages - reserved_pages;
911 	 * (present_pages - managed_pages). And managed_pages should be used
918 	 * It is a seqlock because it has to be read outside of zone->lock,
922 	 * The span_seq lock is declared along with zone->lock because it is
923 	 * frequently read in proximity to zone->lock.  It's good to
946 	 * of pageblock. Protected by zone->lock.
958 	/* Write-intensive fields used from the page allocator */
975 	/* Write-intensive fields used by compaction and vmstats. */
980 	 * when reading the number of free pages to avoid per-cpu counter
1041 	return z->_watermark[w] + z->watermark_boost;  in wmark_pages()
1066 	return (unsigned long)atomic_long_read(&zone->managed_pages);  in zone_managed_pages()
1072 	return zone->cma_pages;  in zone_cma_pages()
1080 	return zone->zone_start_pfn + zone->spanned_pages;  in zone_end_pfn()
1085 	return zone->zone_start_pfn <= pfn && pfn < zone_end_pfn(zone);  in zone_spans_pfn()
1090 	return zone->initialized;  in zone_is_initialized()
1095 	return zone->spanned_pages == 0;  in zone_is_empty()
1105 #define SECTIONS_PGOFF		((sizeof(unsigned long)*8) - SECTIONS_WIDTH)
1106 #define NODES_PGOFF		(SECTIONS_PGOFF - NODES_WIDTH)
1107 #define ZONES_PGOFF		(NODES_PGOFF - ZONES_WIDTH)
1108 #define LAST_CPUPID_PGOFF	(ZONES_PGOFF - LAST_CPUPID_WIDTH)
1109 #define KASAN_TAG_PGOFF		(LAST_CPUPID_PGOFF - KASAN_TAG_WIDTH)
1110 #define LRU_GEN_PGOFF		(KASAN_TAG_PGOFF - LRU_GEN_WIDTH)
1111 #define LRU_REFS_PGOFF		(LRU_GEN_PGOFF - LRU_REFS_WIDTH)
1114  * Define the bit shifts to access each section.  For non-existent
1137 #define ZONES_MASK		((1UL << ZONES_WIDTH) - 1)
1138 #define NODES_MASK		((1UL << NODES_WIDTH) - 1)
1139 #define SECTIONS_MASK		((1UL << SECTIONS_WIDTH) - 1)
1140 #define LAST_CPUPID_MASK	((1UL << LAST_CPUPID_SHIFT) - 1)
1141 #define KASAN_TAG_MASK		((1UL << KASAN_TAG_WIDTH) - 1)
1142 #define ZONEID_MASK		((1UL << ZONEID_SHIFT) - 1)
1146 	ASSERT_EXCLUSIVE_BITS(page->flags, ZONES_MASK << ZONES_PGSHIFT);  in page_zonenum()
1147 	return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;  in page_zonenum()
1152 	return page_zonenum(&folio->page);  in folio_zonenum()
1176 	return a->pgmap == b->pgmap;  in zone_device_pages_have_same_pgmap()
1195 	return is_zone_device_page(&folio->page);  in folio_is_zone_device()
1210  * Return true if [start_pfn, start_pfn + nr_pages) range has a non-empty
1219 	    start_pfn + nr_pages <= zone->zone_start_pfn)  in zone_intersects()
1253 	int zone_idx;		/* zone_idx(zoneref->zone) */
1258  * is a list of zones, the first one is the 'goal' of the
1266  * zonelist_zone()	- Return the struct zone * for an entry in _zonerefs
1267  * zonelist_zone_idx()	- Return the index of the zone for an entry
1268  * zonelist_node_idx()	- Return the index of the node for an entry
1297 	 * arch-specific memory_failure (SGX), hwpoison_filter() filtered
1320  * per-zone basis.
1332 	 * Generally the first zones will be references to this node's
1348 	 * Also synchronizes pgdat->first_deferred_pfn during deferred page
1355 	 * Nests above zone->lock and zone->span_seqlock
1370 	atomic_t nr_writeback_throttled;/* nr of writeback-throttled tasks */
1390 	 * This is a per-node reserve of pages that are not available
1403 	/* Write-intensive fields used by page reclaim */
1409 	 * is the first PFN that needs to be initialised.
1453 	/* Per-node vmstats */
1464 #define node_present_pages(nid)	(NODE_DATA(nid)->node_present_pages)
1465 #define node_spanned_pages(nid)	(NODE_DATA(nid)->node_spanned_pages)
1467 #define node_start_pfn(nid)	(NODE_DATA(nid)->node_start_pfn)
1472 	return pgdat->node_start_pfn + pgdat->node_spanned_pages;  in pgdat_end_pfn()
1505 	return lruvec->pgdat;  in lruvec_pgdat()
1520 #define zone_idx(zone)		((zone) - (zone)->zone_pgdat->node_zones)
1548 	return zone->present_pages;  in populated_zone()
1554 	return zone->node;  in zone_to_nid()
1559 	zone->node = nid;  in zone_set_nid()
1583  * is_highmem - helper function to quickly check if a struct zone is a
1623  * for_each_online_pgdat - helper macro to iterate over all online nodes
1631  * for_each_zone - helper macro to iterate over all memory zones
1638 	for (zone = (first_online_pgdat())->node_zones; \
1643 	for (zone = (first_online_pgdat())->node_zones; \
1652 	return zoneref->zone;  in zonelist_zone()
1657 	return zoneref->zone_idx;  in zonelist_zone_idx()
1662 	return zone_to_nid(zoneref->zone);  in zonelist_node_idx()
1670 …* next_zones_zonelist - Returns the next zone at or below highest_zoneidx within the allowed nodem…
1694 …* first_zones_zonelist - Returns the first zone at or below highest_zoneidx within the allowed nod…
1699  * This function returns the first zone at or below a given zone index that is
1704  * When no eligible zone is found, zoneref->zone is NULL (zoneref itself is
1708  * Return: Zoneref pointer for the first suitable zone found
1714 	return next_zones_zonelist(zonelist->_zonerefs,  in first_zones_zonelist()
1719 …* for_each_zone_zonelist_nodemask - helper macro to iterate over valid zones in a zonelist at or b…
1721  * @z: The current pointer within zonelist->_zonerefs being iterated
1743 …* for_each_zone_zonelist - helper macro to iterate over valid zones in a zonelist at or below a gi…
1745  * @z: The current pointer within zonelist->zones being iterated
1770 	zonelist = &NODE_DATA(nid)->node_zonelists[ZONELIST_FALLBACK];  in movable_only_nodes()
1791 #define PFN_SECTION_SHIFT	(SECTION_SIZE_BITS - PAGE_SHIFT)
1796 #define PAGE_SECTION_MASK	(~(PAGES_PER_SECTION-1))
1799 	((1UL << (PFN_SECTION_SHIFT - pageblock_order)) * NR_PAGEBLOCK_BITS)
1814 #define SECTION_ALIGN_UP(pfn)	(((pfn) + PAGES_PER_SECTION - 1) & PAGE_SECTION_MASK)
1820 #define PFN_SUBSECTION_SHIFT (SUBSECTION_SHIFT - PAGE_SHIFT)
1822 #define PAGE_SUBSECTION_MASK (~(PAGES_PER_SUBSECTION-1))
1827 #define SUBSECTIONS_PER_SECTION (1UL << (SECTION_SIZE_BITS - SUBSECTION_SHIFT))
1871 	 * WARNING: mem_section must be a power-of-2 in size for the
1884 #define SECTION_ROOT_MASK	(SECTIONS_PER_ROOT - 1)
1894 	return ms->usage->pageblock_flags;  in section_to_usemap()
1915  * as mem_map - section_nr_to_pfn(pnum).  The result is
1917  *   1. All mem_map arrays are page-aligned.
1919  *      lowest bits.  PFN_SECTION_SHIFT is arch-specific
1920  *      (equal SECTION_SIZE_BITS - PAGE_SHIFT), and the
1947 #define SECTION_MAP_MASK		(~(BIT(SECTION_MAP_LAST_BIT) - 1))
1952 	unsigned long map = section->section_mem_map;  in __section_mem_map_addr()
1959 	return (section && (section->section_mem_map & SECTION_MARKED_PRESENT));  in present_section()
1969 	return (section && (section->section_mem_map & SECTION_HAS_MEM_MAP));  in valid_section()
1974 	return (section && (section->section_mem_map & SECTION_IS_EARLY));  in early_section()
1984 	return (section && (section->section_mem_map & SECTION_IS_ONLINE));  in online_section()
1992 	return section && ((section->section_mem_map & flags) == flags);  in online_device_section()
2027 	struct mem_section_usage *usage = READ_ONCE(ms->usage);  in pfn_section_valid()
2029 	return usage ? test_bit(idx, usage->subsection_map) : 0;  in pfn_section_valid()
2040  * pfn_valid - check if there is a valid memory map entry for a PFN
2074 	 * the entire section-sized span.  in pfn_valid()
2097 	return -1;  in next_present_section_nr()