bybrooklyn/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-04-18 06:44:00 -04:00
Code Issues Packages Projects Releases Wiki Activity
Files
334fbe734e687404f346eba7d5d96ed2b44d35ab
linux/arch/loongarch/include/asm/pgtable.h
Kiryl Shutsemau 2969b42c8f LoongArch/mm: align vmemmap to maximal folio size 
The upcoming change to the HugeTLB vmemmap optimization (HVO) requires
struct pages of the head page to be naturally aligned with regard to the
folio size.

Align vmemmap to MAX_FOLIO_VMEMMAP_ALIGN.

Link: https://lkml.kernel.org/r/20260227194302.274384-7-kas@kernel.org
Signed-off-by: Kiryl Shutsemau <kas@kernel.org>
Cc: Albert Ou <aou@eecs.berkeley.edu>
Cc: Alexandre Ghiti <alex@ghiti.fr>
Cc: Baoquan He <bhe@redhat.com>
Cc: Christoph Lameter <cl@gentwo.org>
Cc: David Hildenbrand (arm) <david@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Frank van der Linden <fvdl@google.com>
Cc: Harry Yoo <harry.yoo@oracle.com>
Cc: Huacai Chen <chenhuacai@kernel.org>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Paul Walmsley <paul.walmsley@sifive.com>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Usama Arif <usamaarif642@gmail.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: WANG Xuerui <kernel@xen0n.name>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
2026-04-05 13:53:08 -07:00

635 lines
16 KiB
C
Raw Blame History

/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2020-2022 Loongson Technology Corporation Limited
*
* Derived from MIPS:
* Copyright (C) 1994, 95, 96, 97, 98, 99, 2000, 2003 Ralf Baechle
* Copyright (C) 1999, 2000, 2001 Silicon Graphics, Inc.
*/
#ifndef _ASM_PGTABLE_H
#define _ASM_PGTABLE_H
#include <linux/compiler.h>
#include <asm/addrspace.h>
#include <asm/asm.h>
#include <asm/page.h>
#include <asm/pgtable-bits.h>
#if CONFIG_PGTABLE_LEVELS == 2
#include <asm-generic/pgtable-nopmd.h>
#elif CONFIG_PGTABLE_LEVELS == 3
#include <asm-generic/pgtable-nopud.h>
#else
#include <asm-generic/pgtable-nop4d.h>
#endif
#if CONFIG_PGTABLE_LEVELS == 2
#define PGDIR_SHIFT (PAGE_SHIFT + (PAGE_SHIFT - PTRLOG))
#elif CONFIG_PGTABLE_LEVELS == 3
#define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT - PTRLOG))
#define PMD_SIZE (1UL << PMD_SHIFT)
#define PMD_MASK (~(PMD_SIZE-1))
#define PGDIR_SHIFT (PMD_SHIFT + (PAGE_SHIFT - PTRLOG))
#elif CONFIG_PGTABLE_LEVELS == 4
#define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT - PTRLOG))
#define PMD_SIZE (1UL << PMD_SHIFT)
#define PMD_MASK (~(PMD_SIZE-1))
#define PUD_SHIFT (PMD_SHIFT + (PAGE_SHIFT - PTRLOG))
#define PUD_SIZE (1UL << PUD_SHIFT)
#define PUD_MASK (~(PUD_SIZE-1))
#define PGDIR_SHIFT (PUD_SHIFT + (PAGE_SHIFT - PTRLOG))
#endif
#define PGDIR_SIZE (1UL << PGDIR_SHIFT)
#define PGDIR_MASK (~(PGDIR_SIZE-1))
#ifdef CONFIG_32BIT
#define VA_BITS 32
#else
#define VA_BITS (PGDIR_SHIFT + (PAGE_SHIFT - PTRLOG))
#endif
#define PTRS_PER_PGD (PAGE_SIZE >> PTRLOG)
#if CONFIG_PGTABLE_LEVELS > 3
#define PTRS_PER_PUD (PAGE_SIZE >> PTRLOG)
#endif
#if CONFIG_PGTABLE_LEVELS > 2
#define PTRS_PER_PMD (PAGE_SIZE >> PTRLOG)
#endif
#define PTRS_PER_PTE (PAGE_SIZE >> PTRLOG)
#ifdef CONFIG_32BIT
#define USER_PTRS_PER_PGD (TASK_SIZE / PGDIR_SIZE)
#else
#define USER_PTRS_PER_PGD ((TASK_SIZE64 / PGDIR_SIZE)?(TASK_SIZE64 / PGDIR_SIZE):1)
#endif
#ifndef __ASSEMBLER__
#include <linux/mm_types.h>
#include <linux/mmzone.h>
#include <asm/fixmap.h>
#include <asm/sparsemem.h>
struct mm_struct;
struct vm_area_struct;
#ifdef CONFIG_32BIT
#define VMALLOC_START (vm_map_base + PCI_IOSIZE + (2 * PAGE_SIZE))
#define VMALLOC_END (FIXADDR_START - (2 * PAGE_SIZE))
#endif
#ifdef CONFIG_64BIT
#define MODULES_VADDR (vm_map_base + PCI_IOSIZE + (2 * PAGE_SIZE))
#define MODULES_END (MODULES_VADDR + SZ_256M)
#ifdef CONFIG_KFENCE
#define KFENCE_AREA_SIZE (((CONFIG_KFENCE_NUM_OBJECTS + 1) * 2 + 2) * PAGE_SIZE)
#else
#define KFENCE_AREA_SIZE 0
#endif
#define VMALLOC_START MODULES_END
#ifndef CONFIG_KASAN
#define VMALLOC_END \
(vm_map_base + \
min(PTRS_PER_PGD * PTRS_PER_PUD * PTRS_PER_PMD * PTRS_PER_PTE * PAGE_SIZE, (1UL << cpu_vabits)) - PMD_SIZE - VMEMMAP_SIZE - KFENCE_AREA_SIZE)
#else
#define VMALLOC_END \
(vm_map_base + \
min(PTRS_PER_PGD * PTRS_PER_PUD * PTRS_PER_PMD * PTRS_PER_PTE * PAGE_SIZE, (1UL << cpu_vabits) / 2) - PMD_SIZE - VMEMMAP_SIZE - KFENCE_AREA_SIZE)
#endif
#define VMEMMAP_ALIGN max(PMD_SIZE, MAX_FOLIO_VMEMMAP_ALIGN)
#define vmemmap ((struct page *)(ALIGN(VMALLOC_END, VMEMMAP_ALIGN)))
#define VMEMMAP_END ((unsigned long)vmemmap + VMEMMAP_SIZE - 1)
#define KFENCE_AREA_START (VMEMMAP_END + 1)
#define KFENCE_AREA_END (KFENCE_AREA_START + KFENCE_AREA_SIZE - 1)
#endif
#define ptep_get(ptep) READ_ONCE(*(ptep))
#define pmdp_get(pmdp) READ_ONCE(*(pmdp))
#define pte_ERROR(e) \
pr_err("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e))
#ifndef __PAGETABLE_PMD_FOLDED
#define pmd_ERROR(e) \
pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e))
#endif
#ifndef __PAGETABLE_PUD_FOLDED
#define pud_ERROR(e) \
pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e))
#endif
#define pgd_ERROR(e) \
pr_err("%s:%d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e))
extern pte_t invalid_pte_table[PTRS_PER_PTE];
#ifndef __PAGETABLE_PUD_FOLDED
typedef struct { unsigned long pud; } pud_t;
#define pud_val(x) ((x).pud)
#define __pud(x) ((pud_t) { (x) })
extern pud_t invalid_pud_table[PTRS_PER_PUD];
/*
* Empty pgd/p4d entries point to the invalid_pud_table.
*/
static inline int p4d_none(p4d_t p4d)
{
return p4d_val(p4d) == (unsigned long)invalid_pud_table;
}
static inline int p4d_bad(p4d_t p4d)
{
return p4d_val(p4d) & ~PAGE_MASK;
}
static inline int p4d_present(p4d_t p4d)
{
return p4d_val(p4d) != (unsigned long)invalid_pud_table;
}
static inline pud_t *p4d_pgtable(p4d_t p4d)
{
return (pud_t *)p4d_val(p4d);
}
static inline void set_p4d(p4d_t *p4d, p4d_t p4dval)
{
WRITE_ONCE(*p4d, p4dval);
}
static inline void p4d_clear(p4d_t *p4dp)
{
set_p4d(p4dp, __p4d((unsigned long)invalid_pud_table));
}
#define p4d_phys(p4d) PHYSADDR(p4d_val(p4d))
#define p4d_page(p4d) (pfn_to_page(p4d_phys(p4d) >> PAGE_SHIFT))
#endif
#ifndef __PAGETABLE_PMD_FOLDED
typedef struct { unsigned long pmd; } pmd_t;
#define pmd_val(x) ((x).pmd)
#define __pmd(x) ((pmd_t) { (x) })
extern pmd_t invalid_pmd_table[PTRS_PER_PMD];
/*
* Empty pud entries point to the invalid_pmd_table.
*/
static inline int pud_none(pud_t pud)
{
return pud_val(pud) == (unsigned long)invalid_pmd_table;
}
static inline int pud_bad(pud_t pud)
{
return pud_val(pud) & ~PAGE_MASK;
}
static inline int pud_present(pud_t pud)
{
return pud_val(pud) != (unsigned long)invalid_pmd_table;
}
static inline pmd_t *pud_pgtable(pud_t pud)
{
return (pmd_t *)pud_val(pud);
}
static inline void set_pud(pud_t *pud, pud_t pudval)
{
WRITE_ONCE(*pud, pudval);
}
static inline void pud_clear(pud_t *pudp)
{
set_pud(pudp, __pud((unsigned long)invalid_pmd_table));
}
#define pud_phys(pud) PHYSADDR(pud_val(pud))
#define pud_page(pud) (pfn_to_page(pud_phys(pud) >> PAGE_SHIFT))
#endif
/*
* Empty pmd entries point to the invalid_pte_table.
*/
static inline int pmd_none(pmd_t pmd)
{
return pmd_val(pmd) == (unsigned long)invalid_pte_table;
}
static inline int pmd_bad(pmd_t pmd)
{
return (pmd_val(pmd) & ~PAGE_MASK);
}
static inline int pmd_present(pmd_t pmd)
{
if (unlikely(pmd_val(pmd) & _PAGE_HUGE))
return !!(pmd_val(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE | _PAGE_PRESENT_INVALID));
return pmd_val(pmd) != (unsigned long)invalid_pte_table;
}
static inline void set_pmd(pmd_t *pmd, pmd_t pmdval)
{
WRITE_ONCE(*pmd, pmdval);
}
static inline void pmd_clear(pmd_t *pmdp)
{
set_pmd(pmdp, __pmd((unsigned long)invalid_pte_table));
}
#define pmd_phys(pmd) PHYSADDR(pmd_val(pmd))
#ifndef CONFIG_TRANSPARENT_HUGEPAGE
#define pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT))
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#define pmd_page_vaddr(pmd) pmd_val(pmd)
extern void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmd);
#define pte_page(x) pfn_to_page(pte_pfn(x))
#define pte_pfn(x) ((unsigned long)(((x).pte & _PFN_MASK) >> PFN_PTE_SHIFT))
#define pfn_pte(pfn, prot) __pte(((pfn) << PFN_PTE_SHIFT) | pgprot_val(prot))
#define pfn_pmd(pfn, prot) __pmd(((pfn) << PFN_PTE_SHIFT) | pgprot_val(prot))
/*
* Initialize a new pgd / pud / pmd table with invalid pointers.
*/
extern void pgd_init(void *addr);
extern void pud_init(void *addr);
#define pud_init pud_init
extern void pmd_init(void *addr);
#define pmd_init pmd_init
extern void kernel_pte_init(void *addr);
#define kernel_pte_init kernel_pte_init
/*
* Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that
* are !pte_none() && !pte_present().
*
* Format of 32bit swap PTEs:
*
* 3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1
* 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0
* <------------ offset -------------> E <- type -> <-- zeroes -->
*
* E is the exclusive marker that is not stored in swap entries.
* The zero'ed bits include _PAGE_PRESENT.
*
* Format of 64bit swap PTEs:
*
* 6 6 6 6 5 5 5 5 5 5 5 5 5 5 4 4 4 4 4 4 4 4 4 4 3 3 3 3 3 3 3 3
* 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2
* <--------------------------- offset ---------------------------
*
* 3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1
* 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0
* --------------> E <--- type ---> <---------- zeroes ---------->
*
* E is the exclusive marker that is not stored in swap entries.
* The zero'ed bits include _PAGE_PRESENT and _PAGE_PROTNONE.
*/
#define __SWP_TYPE_BITS (IS_ENABLED(CONFIG_32BIT) ? 5 : 7)
#define __SWP_TYPE_MASK ((1UL << __SWP_TYPE_BITS) - 1)
#define __SWP_TYPE_SHIFT (IS_ENABLED(CONFIG_32BIT) ? 8 : 16)
#define __SWP_OFFSET_SHIFT (__SWP_TYPE_BITS + __SWP_TYPE_SHIFT + 1)
static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset)
{
pte_t pte;
pte_val(pte) = ((type & __SWP_TYPE_MASK) << __SWP_TYPE_SHIFT) | (offset << __SWP_OFFSET_SHIFT);
return pte;
}
#define __swp_type(x) (((x).val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK)
#define __swp_offset(x) ((x).val >> __SWP_OFFSET_SHIFT)
#define __swp_entry(type, offset) ((swp_entry_t) { pte_val(mk_swap_pte((type), (offset))) })
#define __swp_entry_to_pte(x) __pte((x).val)
#define __swp_entry_to_pmd(x) __pmd((x).val | _PAGE_HUGE)
#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) })
#define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) })
static inline bool pte_swp_exclusive(pte_t pte)
{
return pte_val(pte) & _PAGE_SWP_EXCLUSIVE;
}
static inline pte_t pte_swp_mkexclusive(pte_t pte)
{
pte_val(pte) |= _PAGE_SWP_EXCLUSIVE;
return pte;
}
static inline pte_t pte_swp_clear_exclusive(pte_t pte)
{
pte_val(pte) &= ~_PAGE_SWP_EXCLUSIVE;
return pte;
}
#define pte_none(pte) (!(pte_val(pte) & ~_PAGE_GLOBAL))
#define pte_present(pte) (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROTNONE))
#define pte_no_exec(pte) (pte_val(pte) & _PAGE_NO_EXEC)
static inline void set_pte(pte_t *ptep, pte_t pteval)
{
WRITE_ONCE(*ptep, pteval);
#ifdef CONFIG_SMP
if (pte_val(pteval) & _PAGE_GLOBAL)
DBAR(0b11000); /* o_wrw = 0b11000 */
#endif
}
static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
{
pte_t pte = ptep_get(ptep);
pte_val(pte) &= _PAGE_GLOBAL;
set_pte(ptep, pte);
}
#define PGD_T_LOG2 (__builtin_ffs(sizeof(pgd_t)) - 1)
#define PMD_T_LOG2 (__builtin_ffs(sizeof(pmd_t)) - 1)
#define PTE_T_LOG2 (__builtin_ffs(sizeof(pte_t)) - 1)
extern pgd_t swapper_pg_dir[];
extern pgd_t invalid_pg_dir[];
/*
* The following only work if pte_present() is true.
* Undefined behaviour if not..
*/
static inline int pte_write(pte_t pte) { return pte_val(pte) & _PAGE_WRITE; }
static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; }
static inline int pte_dirty(pte_t pte) { return pte_val(pte) & (_PAGE_DIRTY | _PAGE_MODIFIED); }
static inline pte_t pte_mkold(pte_t pte)
{
pte_val(pte) &= ~_PAGE_ACCESSED;
return pte;
}
static inline pte_t pte_mkyoung(pte_t pte)
{
pte_val(pte) |= _PAGE_ACCESSED;
return pte;
}
static inline pte_t pte_mkclean(pte_t pte)
{
pte_val(pte) &= ~(_PAGE_DIRTY | _PAGE_MODIFIED);
return pte;
}
static inline pte_t pte_mkdirty(pte_t pte)
{
pte_val(pte) |= _PAGE_MODIFIED;
if (pte_val(pte) & _PAGE_WRITE)
pte_val(pte) |= _PAGE_DIRTY;
return pte;
}
static inline pte_t pte_mkwrite_novma(pte_t pte)
{
pte_val(pte) |= _PAGE_WRITE;
if (pte_val(pte) & _PAGE_MODIFIED)
pte_val(pte) |= _PAGE_DIRTY;
return pte;
}
static inline pte_t pte_wrprotect(pte_t pte)
{
pte_val(pte) &= ~(_PAGE_WRITE | _PAGE_DIRTY);
return pte;
}
static inline int pte_huge(pte_t pte) { return pte_val(pte) & _PAGE_HUGE; }
static inline pte_t pte_mkhuge(pte_t pte)
{
pte_val(pte) |= _PAGE_HUGE;
return pte;
}
#if defined(CONFIG_ARCH_HAS_PTE_SPECIAL)
static inline int pte_special(pte_t pte) { return pte_val(pte) & _PAGE_SPECIAL; }
static inline pte_t pte_mkspecial(pte_t pte) { pte_val(pte) |= _PAGE_SPECIAL; return pte; }
#endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */
#define pte_accessible pte_accessible
static inline unsigned long pte_accessible(struct mm_struct *mm, pte_t a)
{
if (pte_val(a) & _PAGE_PRESENT)
return true;
if ((pte_val(a) & _PAGE_PROTNONE) &&
atomic_read(&mm->tlb_flush_pending))
return true;
return false;
}
static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
{
if (pte_val(pte) & _PAGE_DIRTY)
pte_val(pte) |= _PAGE_MODIFIED;
return __pte((pte_val(pte) & _PAGE_CHG_MASK) |
(pgprot_val(newprot) & ~_PAGE_CHG_MASK));
}
extern void __update_tlb(struct vm_area_struct *vma,
unsigned long address, pte_t *ptep);
static inline void update_mmu_cache_range(struct vm_fault *vmf,
struct vm_area_struct *vma, unsigned long address,
pte_t *ptep, unsigned int nr)
{
for (;;) {
__update_tlb(vma, address, ptep);
if (--nr == 0)
break;
address += PAGE_SIZE;
ptep++;
}
}
#define update_mmu_cache(vma, addr, ptep) \
update_mmu_cache_range(NULL, vma, addr, ptep, 1)
#define update_mmu_tlb_range(vma, addr, ptep, nr) \
update_mmu_cache_range(NULL, vma, addr, ptep, nr)
static inline void update_mmu_cache_pmd(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp)
{
__update_tlb(vma, address, (pte_t *)pmdp);
}
static inline unsigned long pmd_pfn(pmd_t pmd)
{
return (pmd_val(pmd) & _PFN_MASK) >> PFN_PTE_SHIFT;
}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
/* We don't have hardware dirty/accessed bits, generic_pmdp_establish is fine.*/
#define pmdp_establish generic_pmdp_establish
static inline int pmd_trans_huge(pmd_t pmd)
{
return !!(pmd_val(pmd) & _PAGE_HUGE) && pmd_present(pmd);
}
static inline pmd_t pmd_mkhuge(pmd_t pmd)
{
pmd_val(pmd) = (pmd_val(pmd) & ~(_PAGE_GLOBAL)) |
((pmd_val(pmd) & _PAGE_GLOBAL) << (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT));
pmd_val(pmd) |= _PAGE_HUGE;
return pmd;
}
#define pmd_write pmd_write
static inline int pmd_write(pmd_t pmd)
{
return !!(pmd_val(pmd) & _PAGE_WRITE);
}
static inline pmd_t pmd_mkwrite_novma(pmd_t pmd)
{
pmd_val(pmd) |= _PAGE_WRITE;
if (pmd_val(pmd) & _PAGE_MODIFIED)
pmd_val(pmd) |= _PAGE_DIRTY;
return pmd;
}
static inline pmd_t pmd_wrprotect(pmd_t pmd)
{
pmd_val(pmd) &= ~(_PAGE_WRITE | _PAGE_DIRTY);
return pmd;
}
#define pmd_dirty pmd_dirty
static inline int pmd_dirty(pmd_t pmd)
{
return !!(pmd_val(pmd) & (_PAGE_DIRTY | _PAGE_MODIFIED));
}
static inline pmd_t pmd_mkclean(pmd_t pmd)
{
pmd_val(pmd) &= ~(_PAGE_DIRTY | _PAGE_MODIFIED);
return pmd;
}
static inline pmd_t pmd_mkdirty(pmd_t pmd)
{
pmd_val(pmd) |= _PAGE_MODIFIED;
if (pmd_val(pmd) & _PAGE_WRITE)
pmd_val(pmd) |= _PAGE_DIRTY;
return pmd;
}
#define pmd_young pmd_young
static inline int pmd_young(pmd_t pmd)
{
return !!(pmd_val(pmd) & _PAGE_ACCESSED);
}
static inline pmd_t pmd_mkold(pmd_t pmd)
{
pmd_val(pmd) &= ~_PAGE_ACCESSED;
return pmd;
}
static inline pmd_t pmd_mkyoung(pmd_t pmd)
{
pmd_val(pmd) |= _PAGE_ACCESSED;
return pmd;
}
static inline struct page *pmd_page(pmd_t pmd)
{
if (pmd_trans_huge(pmd))
return pfn_to_page(pmd_pfn(pmd));
return pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT);
}
static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
{
if (pmd_val(pmd) & _PAGE_DIRTY)
pmd_val(pmd) |= _PAGE_MODIFIED;
return __pmd((pmd_val(pmd) & _HPAGE_CHG_MASK) |
(pgprot_val(newprot) & ~_HPAGE_CHG_MASK));
}
static inline pmd_t pmd_mkinvalid(pmd_t pmd)
{
pmd_val(pmd) |= _PAGE_PRESENT_INVALID;
pmd_val(pmd) &= ~(_PAGE_PRESENT | _PAGE_VALID | _PAGE_DIRTY | _PAGE_PROTNONE);
return pmd;
}
/*
* The generic version pmdp_huge_get_and_clear uses a version of pmd_clear() with a
* different prototype.
*/
#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
unsigned long address, pmd_t *pmdp)
{
pmd_t old = pmdp_get(pmdp);
pmd_clear(pmdp);
return old;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#ifdef CONFIG_NUMA_BALANCING
static inline long pte_protnone(pte_t pte)
{
return (pte_val(pte) & _PAGE_PROTNONE);
}
static inline long pmd_protnone(pmd_t pmd)
{
return (pmd_val(pmd) & _PAGE_PROTNONE);
}
#endif /* CONFIG_NUMA_BALANCING */
#define pmd_leaf(pmd) ((pmd_val(pmd) & _PAGE_HUGE) != 0)
#define pud_leaf(pud) ((pud_val(pud) & _PAGE_HUGE) != 0)
/*
* We provide our own get_unmapped area to cope with the virtual aliasing
* constraints placed on us by the cache architecture.
*/
#define HAVE_ARCH_UNMAPPED_AREA
#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
#endif /* !__ASSEMBLER__ */
#endif /* _ASM_PGTABLE_H */
Reference in New Issue View Git Blame Copy Permalink