mirror of
https://github.com/torvalds/linux.git
synced 2026-04-18 06:44:00 -04:00
mm: rename the 'compound_head' field in the 'struct page' to 'compound_info'
The 'compound_head' field in the 'struct page' encodes whether the page is a tail and where to locate the head page. Bit 0 is set if the page is a tail, and the remaining bits in the field point to the head page. As preparation for changing how the field encodes information about the head page, rename the field to 'compound_info'. Link: https://lkml.kernel.org/r/20260227194302.274384-4-kas@kernel.org Signed-off-by: Kiryl Shutsemau <kas@kernel.org> Reviewed-by: Muchun Song <muchun.song@linux.dev> Reviewed-by: Zi Yan <ziy@nvidia.com> Acked-by: David Hildenbrand (arm) <david@kernel.org> Reviewed-by: Vlastimil Babka <vbabka@suse.cz> Cc: Albert Ou <aou@eecs.berkeley.edu> Cc: Alexandre Ghiti <alex@ghiti.fr> Cc: Baoquan He <bhe@redhat.com> Cc: Christoph Lameter <cl@gentwo.org> Cc: David Rientjes <rientjes@google.com> Cc: Frank van der Linden <fvdl@google.com> Cc: Harry Yoo <harry.yoo@oracle.com> Cc: Huacai Chen <chenhuacai@kernel.org> Cc: Johannes Weiner <hannes@cmpxchg.org> Cc: Jonathan Corbet <corbet@lwn.net> Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com> Cc: Matthew Wilcox (Oracle) <willy@infradead.org> Cc: Michal Hocko <mhocko@suse.com> Cc: Mike Rapoport <rppt@kernel.org> Cc: Oscar Salvador <osalvador@suse.de> Cc: Palmer Dabbelt <palmer@dabbelt.com> Cc: Paul Walmsley <paul.walmsley@sifive.com> Cc: Roman Gushchin <roman.gushchin@linux.dev> Cc: Usama Arif <usamaarif642@gmail.com> Cc: WANG Xuerui <kernel@xen0n.name> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
committed by
Andrew Morton
parent
f0369fb136
commit
d50569612c
@@ -141,7 +141,7 @@ nodemask_t
|
||||
The size of a nodemask_t type. Used to compute the number of online
|
||||
nodes.
|
||||
|
||||
(page, flags|_refcount|mapping|lru|_mapcount|private|compound_order|compound_head)
|
||||
(page, flags|_refcount|mapping|lru|_mapcount|private|compound_order|compound_info)
|
||||
----------------------------------------------------------------------------------
|
||||
|
||||
User-space tools compute their values based on the offset of these
|
||||
|
||||
@@ -24,7 +24,7 @@ For each base page, there is a corresponding ``struct page``.
|
||||
Within the HugeTLB subsystem, only the first 4 ``struct page`` are used to
|
||||
contain unique information about a HugeTLB page. ``__NR_USED_SUBPAGE`` provides
|
||||
this upper limit. The only 'useful' information in the remaining ``struct page``
|
||||
is the compound_head field, and this field is the same for all tail pages.
|
||||
is the compound_info field, and this field is the same for all tail pages.
|
||||
|
||||
By removing redundant ``struct page`` for HugeTLB pages, memory can be returned
|
||||
to the buddy allocator for other uses.
|
||||
@@ -124,10 +124,10 @@ Here is how things look before optimization::
|
||||
| |
|
||||
+-----------+
|
||||
|
||||
The value of page->compound_head is the same for all tail pages. The first
|
||||
The value of page->compound_info is the same for all tail pages. The first
|
||||
page of ``struct page`` (page 0) associated with the HugeTLB page contains the 4
|
||||
``struct page`` necessary to describe the HugeTLB. The only use of the remaining
|
||||
pages of ``struct page`` (page 1 to page 7) is to point to page->compound_head.
|
||||
pages of ``struct page`` (page 1 to page 7) is to point to page->compound_info.
|
||||
Therefore, we can remap pages 1 to 7 to page 0. Only 1 page of ``struct page``
|
||||
will be used for each HugeTLB page. This will allow us to free the remaining
|
||||
7 pages to the buddy allocator.
|
||||
|
||||
@@ -126,14 +126,14 @@ struct page {
|
||||
atomic_long_t pp_ref_count;
|
||||
};
|
||||
struct { /* Tail pages of compound page */
|
||||
unsigned long compound_head; /* Bit zero is set */
|
||||
unsigned long compound_info; /* Bit zero is set */
|
||||
};
|
||||
struct { /* ZONE_DEVICE pages */
|
||||
/*
|
||||
* The first word is used for compound_head or folio
|
||||
* The first word is used for compound_info or folio
|
||||
* pgmap
|
||||
*/
|
||||
void *_unused_pgmap_compound_head;
|
||||
void *_unused_pgmap_compound_info;
|
||||
void *zone_device_data;
|
||||
/*
|
||||
* ZONE_DEVICE private pages are counted as being
|
||||
@@ -409,7 +409,7 @@ struct folio {
|
||||
/* private: avoid cluttering the output */
|
||||
/* For the Unevictable "LRU list" slot */
|
||||
struct {
|
||||
/* Avoid compound_head */
|
||||
/* Avoid compound_info */
|
||||
void *__filler;
|
||||
/* public: */
|
||||
unsigned int mlock_count;
|
||||
@@ -510,7 +510,7 @@ struct folio {
|
||||
FOLIO_MATCH(flags, flags);
|
||||
FOLIO_MATCH(lru, lru);
|
||||
FOLIO_MATCH(mapping, mapping);
|
||||
FOLIO_MATCH(compound_head, lru);
|
||||
FOLIO_MATCH(compound_info, lru);
|
||||
FOLIO_MATCH(__folio_index, index);
|
||||
FOLIO_MATCH(private, private);
|
||||
FOLIO_MATCH(_mapcount, _mapcount);
|
||||
@@ -529,7 +529,7 @@ FOLIO_MATCH(_last_cpupid, _last_cpupid);
|
||||
static_assert(offsetof(struct folio, fl) == \
|
||||
offsetof(struct page, pg) + sizeof(struct page))
|
||||
FOLIO_MATCH(flags, _flags_1);
|
||||
FOLIO_MATCH(compound_head, _head_1);
|
||||
FOLIO_MATCH(compound_info, _head_1);
|
||||
FOLIO_MATCH(_mapcount, _mapcount_1);
|
||||
FOLIO_MATCH(_refcount, _refcount_1);
|
||||
#undef FOLIO_MATCH
|
||||
@@ -537,13 +537,13 @@ FOLIO_MATCH(_refcount, _refcount_1);
|
||||
static_assert(offsetof(struct folio, fl) == \
|
||||
offsetof(struct page, pg) + 2 * sizeof(struct page))
|
||||
FOLIO_MATCH(flags, _flags_2);
|
||||
FOLIO_MATCH(compound_head, _head_2);
|
||||
FOLIO_MATCH(compound_info, _head_2);
|
||||
#undef FOLIO_MATCH
|
||||
#define FOLIO_MATCH(pg, fl) \
|
||||
static_assert(offsetof(struct folio, fl) == \
|
||||
offsetof(struct page, pg) + 3 * sizeof(struct page))
|
||||
FOLIO_MATCH(flags, _flags_3);
|
||||
FOLIO_MATCH(compound_head, _head_3);
|
||||
FOLIO_MATCH(compound_info, _head_3);
|
||||
#undef FOLIO_MATCH
|
||||
|
||||
/**
|
||||
@@ -609,8 +609,8 @@ struct ptdesc {
|
||||
#define TABLE_MATCH(pg, pt) \
|
||||
static_assert(offsetof(struct page, pg) == offsetof(struct ptdesc, pt))
|
||||
TABLE_MATCH(flags, pt_flags);
|
||||
TABLE_MATCH(compound_head, pt_list);
|
||||
TABLE_MATCH(compound_head, _pt_pad_1);
|
||||
TABLE_MATCH(compound_info, pt_list);
|
||||
TABLE_MATCH(compound_info, _pt_pad_1);
|
||||
TABLE_MATCH(mapping, __page_mapping);
|
||||
TABLE_MATCH(__folio_index, pt_index);
|
||||
TABLE_MATCH(rcu_head, pt_rcu_head);
|
||||
|
||||
@@ -213,7 +213,7 @@ static __always_inline const struct page *page_fixed_fake_head(const struct page
|
||||
/*
|
||||
* Only addresses aligned with PAGE_SIZE of struct page may be fake head
|
||||
* struct page. The alignment check aims to avoid access the fields (
|
||||
* e.g. compound_head) of the @page[1]. It can avoid touch a (possibly)
|
||||
* e.g. compound_info) of the @page[1]. It can avoid touch a (possibly)
|
||||
* cold cacheline in some cases.
|
||||
*/
|
||||
if (IS_ALIGNED((unsigned long)page, PAGE_SIZE) &&
|
||||
@@ -223,7 +223,7 @@ static __always_inline const struct page *page_fixed_fake_head(const struct page
|
||||
* because the @page is a compound page composed with at least
|
||||
* two contiguous pages.
|
||||
*/
|
||||
unsigned long head = READ_ONCE(page[1].compound_head);
|
||||
unsigned long head = READ_ONCE(page[1].compound_info);
|
||||
|
||||
if (likely(head & 1))
|
||||
return (const struct page *)(head - 1);
|
||||
@@ -281,7 +281,7 @@ static __always_inline int page_is_fake_head(const struct page *page)
|
||||
|
||||
static __always_inline unsigned long _compound_head(const struct page *page)
|
||||
{
|
||||
unsigned long head = READ_ONCE(page->compound_head);
|
||||
unsigned long head = READ_ONCE(page->compound_info);
|
||||
|
||||
if (unlikely(head & 1))
|
||||
return head - 1;
|
||||
@@ -320,13 +320,13 @@ static __always_inline unsigned long _compound_head(const struct page *page)
|
||||
|
||||
static __always_inline int PageTail(const struct page *page)
|
||||
{
|
||||
return READ_ONCE(page->compound_head) & 1 || page_is_fake_head(page);
|
||||
return READ_ONCE(page->compound_info) & 1 || page_is_fake_head(page);
|
||||
}
|
||||
|
||||
static __always_inline int PageCompound(const struct page *page)
|
||||
{
|
||||
return test_bit(PG_head, &page->flags.f) ||
|
||||
READ_ONCE(page->compound_head) & 1;
|
||||
READ_ONCE(page->compound_info) & 1;
|
||||
}
|
||||
|
||||
#define PAGE_POISON_PATTERN -1l
|
||||
@@ -348,7 +348,7 @@ static const unsigned long *const_folio_flags(const struct folio *folio,
|
||||
{
|
||||
const struct page *page = &folio->page;
|
||||
|
||||
VM_BUG_ON_PGFLAGS(page->compound_head & 1, page);
|
||||
VM_BUG_ON_PGFLAGS(page->compound_info & 1, page);
|
||||
VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags.f), page);
|
||||
return &page[n].flags.f;
|
||||
}
|
||||
@@ -357,7 +357,7 @@ static unsigned long *folio_flags(struct folio *folio, unsigned n)
|
||||
{
|
||||
struct page *page = &folio->page;
|
||||
|
||||
VM_BUG_ON_PGFLAGS(page->compound_head & 1, page);
|
||||
VM_BUG_ON_PGFLAGS(page->compound_info & 1, page);
|
||||
VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags.f), page);
|
||||
return &page[n].flags.f;
|
||||
}
|
||||
@@ -873,12 +873,12 @@ static inline bool folio_test_large(const struct folio *folio)
|
||||
static __always_inline void set_compound_head(struct page *tail,
|
||||
const struct page *head, unsigned int order)
|
||||
{
|
||||
WRITE_ONCE(tail->compound_head, (unsigned long)head + 1);
|
||||
WRITE_ONCE(tail->compound_info, (unsigned long)head + 1);
|
||||
}
|
||||
|
||||
static __always_inline void clear_compound_head(struct page *page)
|
||||
{
|
||||
WRITE_ONCE(page->compound_head, 0);
|
||||
WRITE_ONCE(page->compound_info, 0);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
||||
|
||||
@@ -239,7 +239,7 @@ struct ustat {
|
||||
*
|
||||
* This guarantee is important for few reasons:
|
||||
* - future call_rcu_lazy() will make use of lower bits in the pointer;
|
||||
* - the structure shares storage space in struct page with @compound_head,
|
||||
* - the structure shares storage space in struct page with @compound_info,
|
||||
* which encode PageTail() in bit 0. The guarantee is needed to avoid
|
||||
* false-positive PageTail().
|
||||
*/
|
||||
|
||||
@@ -198,7 +198,7 @@ static int __init crash_save_vmcoreinfo_init(void)
|
||||
VMCOREINFO_OFFSET(page, lru);
|
||||
VMCOREINFO_OFFSET(page, _mapcount);
|
||||
VMCOREINFO_OFFSET(page, private);
|
||||
VMCOREINFO_OFFSET(page, compound_head);
|
||||
VMCOREINFO_OFFSET(page, compound_info);
|
||||
VMCOREINFO_OFFSET(pglist_data, node_zones);
|
||||
VMCOREINFO_OFFSET(pglist_data, nr_zones);
|
||||
#ifdef CONFIG_FLATMEM
|
||||
|
||||
@@ -731,7 +731,7 @@ static inline bool pcp_allowed_order(unsigned int order)
|
||||
* The first PAGE_SIZE page is called the "head page" and have PG_head set.
|
||||
*
|
||||
* The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded
|
||||
* in bit 0 of page->compound_head. The rest of bits is pointer to head page.
|
||||
* in bit 0 of page->compound_info. The rest of bits is pointer to head page.
|
||||
*
|
||||
* The first tail page's ->compound_order holds the order of allocation.
|
||||
* This usage means that zero-order pages may not be compound.
|
||||
|
||||
@@ -94,7 +94,7 @@ struct slab {
|
||||
#define SLAB_MATCH(pg, sl) \
|
||||
static_assert(offsetof(struct page, pg) == offsetof(struct slab, sl))
|
||||
SLAB_MATCH(flags, flags);
|
||||
SLAB_MATCH(compound_head, slab_cache); /* Ensure bit 0 is clear */
|
||||
SLAB_MATCH(compound_info, slab_cache); /* Ensure bit 0 is clear */
|
||||
SLAB_MATCH(_refcount, __page_refcount);
|
||||
#ifdef CONFIG_MEMCG
|
||||
SLAB_MATCH(memcg_data, obj_exts);
|
||||
|
||||
@@ -1276,7 +1276,7 @@ void snapshot_page(struct page_snapshot *ps, const struct page *page)
|
||||
again:
|
||||
memset(&ps->folio_snapshot, 0, sizeof(struct folio));
|
||||
memcpy(&ps->page_snapshot, page, sizeof(*page));
|
||||
head = ps->page_snapshot.compound_head;
|
||||
head = ps->page_snapshot.compound_info;
|
||||
if ((head & 1) == 0) {
|
||||
ps->idx = 0;
|
||||
foliop = (struct folio *)&ps->page_snapshot;
|
||||
|
||||
Reference in New Issue
Block a user