Catcrafts/milos-linux
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

mm: page_isolation: introduce page_is_unmovable()

Browse source 
Patch series "mm: accelerate gigantic folio allocation".

Optimize pfn_range_valid_contig() and replace_free_hugepage_folios() in
alloc_contig_frozen_pages() to speed up gigantic folio allocation.  The
allocation time for 120*1G folios drops from 3.605s to 0.431s.


This patch (of 5):

Factor out the check if a page is unmovable into a new helper, and will be
reused in the following patch.

No functional change intended, the minor changes are as follows,
1) Avoid unnecessary calls by checking CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION
2) Directly call PageCompound since PageTransCompound may be dropped
3) Using folio_test_hugetlb()

Link: https://lkml.kernel.org/r/20260112150954.1802953-1-wangkefeng.wang@huawei.com
Link: https://lkml.kernel.org/r/20260112150954.1802953-2-wangkefeng.wang@huawei.com
Signed-off-by: Kefeng Wang <wangkefeng.wang@huawei.com>
Reviewed-by: Zi Yan <ziy@nvidia.com>
Reviewed-by: Oscar Salvador <osalvador@suse.de>
Cc: Brendan Jackman <jackmanb@google.com>
Cc: David Hildenbrand <david@kernel.org>
Cc: Jane Chu <jane.chu@oracle.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Sidhartha Kumar <sidhartha.kumar@oracle.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Kefeng Wang 2026-01-12 23:09:50 +08:00 • committed by Andrew Morton
commit c83109e95c
2 changed files with 101 additions and 88 deletions
Download patch file Download diff file Expand all files Collapse all files

2
include/linux/page-isolation.h
View file

@ -67,4 +67,6 @@ void undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn);
int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn, int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn,
enum pb_isolate_mode mode); enum pb_isolate_mode mode);
bool page_is_unmovable(struct zone *zone, struct page *page,
enum pb_isolate_mode mode, unsigned long *step);
#endif #endif

187
mm/page_isolation.c
View file

@ -15,6 +15,100 @@
#define CREATE_TRACE_POINTS #define CREATE_TRACE_POINTS
#include <trace/events/page_isolation.h> #include <trace/events/page_isolation.h>
bool page_is_unmovable(struct zone *zone, struct page *page,
enum pb_isolate_mode mode, unsigned long *step)
{
/*
* Both, bootmem allocations and memory holes are marked
* PG_reserved and are unmovable. We can even have unmovable
* allocations inside ZONE_MOVABLE, for example when
* specifying "movablecore".
*/
if (PageReserved(page))
return true;
/*
* If the zone is movable and we have ruled out all reserved
* pages then it should be reasonably safe to assume the rest
* is movable.
*/
if (zone_idx(zone) == ZONE_MOVABLE)
return false;
/*
* Hugepages are not in LRU lists, but they're movable.
* THPs are on the LRU, but need to be counted as #small pages.
* We need not scan over tail pages because we don't
* handle each tail page individually in migration.
*/
if (PageHuge(page) || PageCompound(page)) {
struct folio *folio = page_folio(page);
if (folio_test_hugetlb(folio)) {
struct hstate *h;
if (!IS_ENABLED(CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION))
return true;
/*
* The huge page may be freed so can not
* use folio_hstate() directly.
*/
h = size_to_hstate(folio_size(folio));
if (h && !hugepage_migration_supported(h))
return true;
} else if (!folio_test_lru(folio)) {
return true;
}
*step = folio_nr_pages(folio) - folio_page_idx(folio, page);
return false;
}
/*
* We can't use page_count without pin a page
* because another CPU can free compound page.
* This check already skips compound tails of THP
* because their page->_refcount is zero at all time.
*/
if (!page_ref_count(page)) {
if (PageBuddy(page))
*step = (1 << buddy_order(page));
return false;
}
/*
* The HWPoisoned page may be not in buddy system, and
* page_count() is not 0.
*/
if ((mode == PB_ISOLATE_MODE_MEM_OFFLINE) && PageHWPoison(page))
return false;
/*
* We treat all PageOffline() pages as movable when offlining
* to give drivers a chance to decrement their reference count
* in MEM_GOING_OFFLINE in order to indicate that these pages
* can be offlined as there are no direct references anymore.
* For actually unmovable PageOffline() where the driver does
* not support this, we will fail later when trying to actually
* move these pages that still have a reference count > 0.
* (false negatives in this function only)
*/
if ((mode == PB_ISOLATE_MODE_MEM_OFFLINE) && PageOffline(page))
return false;
if (PageLRU(page) || page_has_movable_ops(page))
return false;
/*
* If there are RECLAIMABLE pages, we need to check
* it. But now, memory offline itself doesn't call
* shrink_node_slabs() and it still to be fixed.
*/
return true;
}
/* /*
* This function checks whether the range [start_pfn, end_pfn) includes * This function checks whether the range [start_pfn, end_pfn) includes
* unmovable pages or not. The range must fall into a single pageblock and * unmovable pages or not. The range must fall into a single pageblock and
@ -35,7 +129,6 @@ static struct page *has_unmovable_pages(unsigned long start_pfn, unsigned long e
{ {
struct page *page = pfn_to_page(start_pfn); struct page *page = pfn_to_page(start_pfn);
struct zone *zone = page_zone(page); struct zone *zone = page_zone(page);
unsigned long pfn;
VM_BUG_ON(pageblock_start_pfn(start_pfn) != VM_BUG_ON(pageblock_start_pfn(start_pfn) !=
pageblock_start_pfn(end_pfn - 1)); pageblock_start_pfn(end_pfn - 1));
@ -52,96 +145,14 @@ static struct page *has_unmovable_pages(unsigned long start_pfn, unsigned long e
return page; return page;
} }
for (pfn = start_pfn; pfn < end_pfn; pfn++) { while (start_pfn < end_pfn) {
page = pfn_to_page(pfn); unsigned long step = 1;
/* page = pfn_to_page(start_pfn);
* Both, bootmem allocations and memory holes are marked if (page_is_unmovable(zone, page, mode, &step))
* PG_reserved and are unmovable. We can even have unmovable
* allocations inside ZONE_MOVABLE, for example when
* specifying "movablecore".
*/
if (PageReserved(page))
return page; return page;
/* start_pfn += step;
* If the zone is movable and we have ruled out all reserved
* pages then it should be reasonably safe to assume the rest
* is movable.
*/
if (zone_idx(zone) == ZONE_MOVABLE)
continue;
/*
* Hugepages are not in LRU lists, but they're movable.
* THPs are on the LRU, but need to be counted as #small pages.
* We need not scan over tail pages because we don't
* handle each tail page individually in migration.
*/
if (PageHuge(page) || PageTransCompound(page)) {
struct folio *folio = page_folio(page);
unsigned int skip_pages;
if (PageHuge(page)) {
struct hstate *h;
/*
* The huge page may be freed so can not
* use folio_hstate() directly.
*/
h = size_to_hstate(folio_size(folio));
if (h && !hugepage_migration_supported(h))
return page;
} else if (!folio_test_lru(folio)) {
return page;
}
skip_pages = folio_nr_pages(folio) - folio_page_idx(folio, page);
pfn += skip_pages - 1;
continue;
}
/*
* We can't use page_count without pin a page
* because another CPU can free compound page.
* This check already skips compound tails of THP
* because their page->_refcount is zero at all time.
*/
if (!page_ref_count(page)) {
if (PageBuddy(page))
pfn += (1 << buddy_order(page)) - 1;
continue;
}
/*
* The HWPoisoned page may be not in buddy system, and
* page_count() is not 0.
*/
if ((mode == PB_ISOLATE_MODE_MEM_OFFLINE) && PageHWPoison(page))
continue;
/*
* We treat all PageOffline() pages as movable when offlining
* to give drivers a chance to decrement their reference count
* in MEM_GOING_OFFLINE in order to indicate that these pages
* can be offlined as there are no direct references anymore.
* For actually unmovable PageOffline() where the driver does
* not support this, we will fail later when trying to actually
* move these pages that still have a reference count > 0.
* (false negatives in this function only)
*/
if ((mode == PB_ISOLATE_MODE_MEM_OFFLINE) && PageOffline(page))
continue;
if (PageLRU(page) || page_has_movable_ops(page))
continue;
/*
* If there are RECLAIMABLE pages, we need to check
* it. But now, memory offline itself doesn't call
* shrink_node_slabs() and it still to be fixed.
*/
return page;
} }
return NULL; return NULL;
} }