--- v8
+++ v1
@@ -2,233 +2,322 @@
ptdesc equivalents, convert various page table functions to use ptdescs.
Some of the functions use the *get*page*() helper functions. Convert
-these to use pagetable_alloc() and ptdesc_address() instead to help
+these to use ptdesc_alloc() and ptdesc_address() instead to help
standardize page tables further.
Signed-off-by: Vishal Moola (Oracle) <vishal.moola@gmail.com>
---
- include/asm-generic/pgalloc.h | 88 +++++++++++++++++++++--------------
- 1 file changed, 52 insertions(+), 36 deletions(-)
+ arch/s390/include/asm/pgalloc.h | 4 +-
+ arch/s390/include/asm/tlb.h | 4 +-
+ arch/s390/mm/pgalloc.c | 108 ++++++++++++++++----------------
+ 3 files changed, 59 insertions(+), 57 deletions(-)
-diff --git a/include/asm-generic/pgalloc.h b/include/asm-generic/pgalloc.h
-index a7cf825befae..c75d4a753849 100644
---- a/include/asm-generic/pgalloc.h
-+++ b/include/asm-generic/pgalloc.h
-@@ -8,7 +8,7 @@
- #define GFP_PGTABLE_USER (GFP_PGTABLE_KERNEL | __GFP_ACCOUNT)
-
- /**
-- * __pte_alloc_one_kernel - allocate a page for PTE-level kernel page table
-+ * __pte_alloc_one_kernel - allocate memory for a PTE-level kernel page table
- * @mm: the mm_struct of the current context
- *
- * This function is intended for architectures that need
-@@ -18,12 +18,17 @@
- */
- static inline pte_t *__pte_alloc_one_kernel(struct mm_struct *mm)
- {
-- return (pte_t *)__get_free_page(GFP_PGTABLE_KERNEL);
-+ struct ptdesc *ptdesc = pagetable_alloc(GFP_PGTABLE_KERNEL &
-+ ~__GFP_HIGHMEM, 0);
-+
+diff --git a/arch/s390/include/asm/pgalloc.h b/arch/s390/include/asm/pgalloc.h
+index 17eb618f1348..9841481560ae 100644
+--- a/arch/s390/include/asm/pgalloc.h
++++ b/arch/s390/include/asm/pgalloc.h
+@@ -86,7 +86,7 @@ static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long vmaddr)
+ if (!table)
+ return NULL;
+ crst_table_init(table, _SEGMENT_ENTRY_EMPTY);
+- if (!pgtable_pmd_page_ctor(virt_to_page(table))) {
++ if (!ptdesc_pmd_ctor(virt_to_ptdesc(table))) {
+ crst_table_free(mm, table);
+ return NULL;
+ }
+@@ -97,7 +97,7 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
+ {
+ if (mm_pmd_folded(mm))
+ return;
+- pgtable_pmd_page_dtor(virt_to_page(pmd));
++ ptdesc_pmd_dtor(virt_to_ptdesc(pmd));
+ crst_table_free(mm, (unsigned long *) pmd);
+ }
+
+diff --git a/arch/s390/include/asm/tlb.h b/arch/s390/include/asm/tlb.h
+index b91f4a9b044c..1388c819b467 100644
+--- a/arch/s390/include/asm/tlb.h
++++ b/arch/s390/include/asm/tlb.h
+@@ -89,12 +89,12 @@ static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
+ {
+ if (mm_pmd_folded(tlb->mm))
+ return;
+- pgtable_pmd_page_dtor(virt_to_page(pmd));
++ ptdesc_pmd_dtor(virt_to_ptdesc(pmd));
+ __tlb_adjust_range(tlb, address, PAGE_SIZE);
+ tlb->mm->context.flush_mm = 1;
+ tlb->freed_tables = 1;
+ tlb->cleared_puds = 1;
+- tlb_remove_table(tlb, pmd);
++ tlb_remove_ptdesc(tlb, pmd);
+ }
+
+ /*
+diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c
+index 6b99932abc66..16a29d2cfe85 100644
+--- a/arch/s390/mm/pgalloc.c
++++ b/arch/s390/mm/pgalloc.c
+@@ -43,17 +43,17 @@ __initcall(page_table_register_sysctl);
+
+ unsigned long *crst_table_alloc(struct mm_struct *mm)
+ {
+- struct page *page = alloc_pages(GFP_KERNEL, CRST_ALLOC_ORDER);
++ struct ptdesc *ptdesc = ptdesc(GFP_KERNEL, CRST_ALLOC_ORDER);
+
+- if (!page)
++ if (!ptdesc)
+ return NULL;
+- arch_set_page_dat(page, CRST_ALLOC_ORDER);
+- return (unsigned long *) page_to_virt(page);
++ arch_set_page_dat(ptdesc_page(ptdesc), CRST_ALLOC_ORDER);
++ return (unsigned long *) ptdesc_to_virt(ptdesc);
+ }
+
+ void crst_table_free(struct mm_struct *mm, unsigned long *table)
+ {
+- free_pages((unsigned long)table, CRST_ALLOC_ORDER);
++ ptdesc_free(virt_to_ptdesc(table);
+ }
+
+ static void __crst_table_upgrade(void *arg)
+@@ -140,21 +140,21 @@ static inline unsigned int atomic_xor_bits(atomic_t *v, unsigned int bits)
+
+ struct page *page_table_alloc_pgste(struct mm_struct *mm)
+ {
+- struct page *page;
++ struct page *ptdesc;
+ u64 *table;
+
+- page = alloc_page(GFP_KERNEL);
+- if (page) {
+- table = (u64 *)page_to_virt(page);
++ ptdesc = ptdesc_alloc(GFP_KERNEL, 0);
++ if (ptdesc) {
++ table = (u64 *)ptdesc_to_virt(page);
+ memset64(table, _PAGE_INVALID, PTRS_PER_PTE);
+ memset64(table + PTRS_PER_PTE, 0, PTRS_PER_PTE);
+ }
+- return page;
++ return ptdesc_page(ptdesc);
+ }
+
+ void page_table_free_pgste(struct page *page)
+ {
+- __free_page(page);
++ ptdesc_free(page_ptdesc(page));
+ }
+
+ #endif /* CONFIG_PGSTE */
+@@ -230,7 +230,7 @@ void page_table_free_pgste(struct page *page)
+ unsigned long *page_table_alloc(struct mm_struct *mm)
+ {
+ unsigned long *table;
+- struct page *page;
++ struct ptdesc *ptdesc;
+ unsigned int mask, bit;
+
+ /* Try to get a fragment of a 4K page as a 2K page table */
+@@ -238,9 +238,9 @@ unsigned long *page_table_alloc(struct mm_struct *mm)
+ table = NULL;
+ spin_lock_bh(&mm->context.lock);
+ if (!list_empty(&mm->context.pgtable_list)) {
+- page = list_first_entry(&mm->context.pgtable_list,
+- struct page, lru);
+- mask = atomic_read(&page->pt_frag_refcount);
++ ptdesc = list_first_entry(&mm->context.pgtable_list,
++ struct ptdesc, pt_list);
++ mask = atomic_read(&ptdesc->pt_frag_refcount);
+ /*
+ * The pending removal bits must also be checked.
+ * Failure to do so might lead to an impossible
+@@ -253,13 +253,13 @@ unsigned long *page_table_alloc(struct mm_struct *mm)
+ */
+ mask = (mask | (mask >> 4)) & 0x03U;
+ if (mask != 0x03U) {
+- table = (unsigned long *) page_to_virt(page);
++ table = (unsigned long *) ptdesc_to_virt(ptdesc);
+ bit = mask & 1; /* =1 -> second 2K */
+ if (bit)
+ table += PTRS_PER_PTE;
+- atomic_xor_bits(&page->pt_frag_refcount,
++ atomic_xor_bits(&ptdesc->pt_frag_refcount,
+ 0x01U << bit);
+- list_del(&page->lru);
++ list_del(&ptdesc->pt_list);
+ }
+ }
+ spin_unlock_bh(&mm->context.lock);
+@@ -267,27 +267,27 @@ unsigned long *page_table_alloc(struct mm_struct *mm)
+ return table;
+ }
+ /* Allocate a fresh page */
+- page = alloc_page(GFP_KERNEL);
+- if (!page)
++ ptdesc = ptdesc_alloc(GFP_KERNEL, 0);
++ if (!ptdesc)
+ return NULL;
+- if (!pgtable_pte_page_ctor(page)) {
+- __free_page(page);
++ if (!ptdesc_pte_ctor(ptdesc)) {
++ ptdesc_free(ptdesc);
+ return NULL;
+ }
+- arch_set_page_dat(page, 0);
++ arch_set_page_dat(ptdesc_page(ptdesc), 0);
+ /* Initialize page table */
+- table = (unsigned long *) page_to_virt(page);
++ table = (unsigned long *) ptdesc_to_virt(ptdesc);
+ if (mm_alloc_pgste(mm)) {
+ /* Return 4K page table with PGSTEs */
+- atomic_xor_bits(&page->pt_frag_refcount, 0x03U);
++ atomic_xor_bits(&ptdesc->pt_frag_refcount, 0x03U);
+ memset64((u64 *)table, _PAGE_INVALID, PTRS_PER_PTE);
+ memset64((u64 *)table + PTRS_PER_PTE, 0, PTRS_PER_PTE);
+ } else {
+ /* Return the first 2K fragment of the page */
+- atomic_xor_bits(&page->pt_frag_refcount, 0x01U);
++ atomic_xor_bits(&ptdesc->pt_frag_refcount, 0x01U);
+ memset64((u64 *)table, _PAGE_INVALID, 2 * PTRS_PER_PTE);
+ spin_lock_bh(&mm->context.lock);
+- list_add(&page->lru, &mm->context.pgtable_list);
++ list_add(&ptdesc->pt_list, &mm->context.pgtable_list);
+ spin_unlock_bh(&mm->context.lock);
+ }
+ return table;
+@@ -309,9 +309,8 @@ static void page_table_release_check(struct page *page, void *table,
+ void page_table_free(struct mm_struct *mm, unsigned long *table)
+ {
+ unsigned int mask, bit, half;
+- struct page *page;
++ struct ptdesc *ptdesc = virt_to_ptdesc(table);
+
+- page = virt_to_page(table);
+ if (!mm_alloc_pgste(mm)) {
+ /* Free 2K page table fragment of a 4K page */
+ bit = ((unsigned long) table & ~PAGE_MASK)/(PTRS_PER_PTE*sizeof(pte_t));
+@@ -321,39 +320,38 @@ void page_table_free(struct mm_struct *mm, unsigned long *table)
+ * will happen outside of the critical section from this
+ * function or from __tlb_remove_table()
+ */
+- mask = atomic_xor_bits(&page->pt_frag_refcount, 0x11U << bit);
++ mask = atomic_xor_bits(&ptdesc->pt_frag_refcount, 0x11U << bit);
+ if (mask & 0x03U)
+- list_add(&page->lru, &mm->context.pgtable_list);
++ list_add(&ptdesc->pt_list, &mm->context.pgtable_list);
+ else
+- list_del(&page->lru);
++ list_del(&ptdesc->pt_list);
+ spin_unlock_bh(&mm->context.lock);
+- mask = atomic_xor_bits(&page->pt_frag_refcount, 0x10U << bit);
++ mask = atomic_xor_bits(&ptdesc->pt_frag_refcount, 0x10U << bit);
+ if (mask != 0x00U)
+ return;
+ half = 0x01U << bit;
+ } else {
+ half = 0x03U;
+- mask = atomic_xor_bits(&page->pt_frag_refcount, 0x03U);
++ mask = atomic_xor_bits(&ptdesc->pt_frag_refcount, 0x03U);
+ }
+
+- page_table_release_check(page, table, half, mask);
+- pgtable_pte_page_dtor(page);
+- __free_page(page);
++ page_table_release_check(ptdesc_page(ptdesc), table, half, mask);
++ ptdesc_pte_dtor(ptdesc);
++ ptdesc_free(ptdesc);
+ }
+
+ void page_table_free_rcu(struct mmu_gather *tlb, unsigned long *table,
+ unsigned long vmaddr)
+ {
+ struct mm_struct *mm;
+- struct page *page;
+ unsigned int bit, mask;
++ struct ptdesc *ptdesc = virt_to_ptdesc(table);
+
+ mm = tlb->mm;
+- page = virt_to_page(table);
+ if (mm_alloc_pgste(mm)) {
+ gmap_unlink(mm, table, vmaddr);
+ table = (unsigned long *) ((unsigned long)table | 0x03U);
+- tlb_remove_table(tlb, table);
++ tlb_remove_ptdesc(tlb, table);
+ return;
+ }
+ bit = ((unsigned long) table & ~PAGE_MASK) / (PTRS_PER_PTE*sizeof(pte_t));
+@@ -363,11 +361,11 @@ void page_table_free_rcu(struct mmu_gather *tlb, unsigned long *table,
+ * outside of the critical section from __tlb_remove_table() or from
+ * page_table_free()
+ */
+- mask = atomic_xor_bits(&page->pt_frag_refcount, 0x11U << bit);
++ mask = atomic_xor_bits(&ptdesc->pt_frag_refcount, 0x11U << bit);
+ if (mask & 0x03U)
+- list_add_tail(&page->lru, &mm->context.pgtable_list);
++ list_add_tail(&ptdesc->pt_list, &mm->context.pgtable_list);
+ else
+- list_del(&page->lru);
++ list_del(&ptdesc->pt_list);
+ spin_unlock_bh(&mm->context.lock);
+ table = (unsigned long *) ((unsigned long) table | (0x01U << bit));
+ tlb_remove_table(tlb, table);
+@@ -377,7 +375,7 @@ void __tlb_remove_table(void *_table)
+ {
+ unsigned int mask = (unsigned long) _table & 0x03U, half = mask;
+ void *table = (void *)((unsigned long) _table ^ mask);
+- struct page *page = virt_to_page(table);
++ struct ptdesc *ptdesc = virt_to_ptdesc(table);
+
+ switch (half) {
+ case 0x00U: /* pmd, pud, or p4d */
+@@ -385,18 +383,18 @@ void __tlb_remove_table(void *_table)
+ return;
+ case 0x01U: /* lower 2K of a 4K page table */
+ case 0x02U: /* higher 2K of a 4K page table */
+- mask = atomic_xor_bits(&page->pt_frag_refcount, mask << 4);
++ mask = atomic_xor_bits(&ptdesc->pt_frag_refcount, mask << 4);
+ if (mask != 0x00U)
+ return;
+ break;
+ case 0x03U: /* 4K page table with pgstes */
+- mask = atomic_xor_bits(&page->pt_frag_refcount, 0x03U);
++ mask = atomic_xor_bits(&ptdesc->pt_frag_refcount, 0x03U);
+ break;
+ }
+
+- page_table_release_check(page, table, half, mask);
+- pgtable_pte_page_dtor(page);
+- __free_page(page);
++ page_table_release_check(ptdesc_page(ptdesc), table, half, mask);
++ ptdesc_pte_dtor(ptdesc);
++ ptdesc_free(ptdesc);
+ }
+
+ /*
+@@ -424,16 +422,20 @@ static void base_pgt_free(unsigned long *table)
+ static unsigned long *base_crst_alloc(unsigned long val)
+ {
+ unsigned long *table;
++ struct ptdesc *ptdesc;
+
+- table = (unsigned long *)__get_free_pages(GFP_KERNEL, CRST_ALLOC_ORDER);
+- if (table)
+- crst_table_init(table, val);
++ ptdesc = ptdesc_alloc(GFP_KERNEL, CRST_ALLOC_ORDER);
+ if (!ptdesc)
+ return NULL;
-+ return ptdesc_address(ptdesc);
- }
-
- #ifndef __HAVE_ARCH_PTE_ALLOC_ONE_KERNEL
- /**
-- * pte_alloc_one_kernel - allocate a page for PTE-level kernel page table
-+ * pte_alloc_one_kernel - allocate memory for a PTE-level kernel page table
- * @mm: the mm_struct of the current context
- *
- * Return: pointer to the allocated memory or %NULL on error
-@@ -35,40 +40,40 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
- #endif
-
- /**
-- * pte_free_kernel - free PTE-level kernel page table page
-+ * pte_free_kernel - free PTE-level kernel page table memory
- * @mm: the mm_struct of the current context
- * @pte: pointer to the memory containing the page table
- */
- static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
- {
-- free_page((unsigned long)pte);
-+ pagetable_free(virt_to_ptdesc(pte));
- }
-
- /**
-- * __pte_alloc_one - allocate a page for PTE-level user page table
-+ * __pte_alloc_one - allocate memory for a PTE-level user page table
- * @mm: the mm_struct of the current context
- * @gfp: GFP flags to use for the allocation
- *
-- * Allocates a page and runs the pgtable_pte_page_ctor().
-+ * Allocate memory for a page table and ptdesc and runs pagetable_pte_ctor().
- *
- * This function is intended for architectures that need
- * anything beyond simple page allocation or must have custom GFP flags.
- *
-- * Return: `struct page` initialized as page table or %NULL on error
-+ * Return: `struct page` referencing the ptdesc or %NULL on error
- */
- static inline pgtable_t __pte_alloc_one(struct mm_struct *mm, gfp_t gfp)
- {
-- struct page *pte;
-+ struct ptdesc *ptdesc;
-
-- pte = alloc_page(gfp);
-- if (!pte)
-+ ptdesc = pagetable_alloc(gfp, 0);
-+ if (!ptdesc)
- return NULL;
-- if (!pgtable_pte_page_ctor(pte)) {
-- __free_page(pte);
-+ if (!pagetable_pte_ctor(ptdesc)) {
-+ pagetable_free(ptdesc);
- return NULL;
- }
-
-- return pte;
-+ return ptdesc_page(ptdesc);
- }
-
- #ifndef __HAVE_ARCH_PTE_ALLOC_ONE
-@@ -76,9 +81,9 @@ static inline pgtable_t __pte_alloc_one(struct mm_struct *mm, gfp_t gfp)
- * pte_alloc_one - allocate a page for PTE-level user page table
- * @mm: the mm_struct of the current context
- *
-- * Allocates a page and runs the pgtable_pte_page_ctor().
-+ * Allocate memory for a page table and ptdesc and runs pagetable_pte_ctor().
- *
-- * Return: `struct page` initialized as page table or %NULL on error
-+ * Return: `struct page` referencing the ptdesc or %NULL on error
- */
- static inline pgtable_t pte_alloc_one(struct mm_struct *mm)
- {
-@@ -92,14 +97,16 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm)
- */
-
- /**
-- * pte_free - free PTE-level user page table page
-+ * pte_free - free PTE-level user page table memory
- * @mm: the mm_struct of the current context
-- * @pte_page: the `struct page` representing the page table
-+ * @pte_page: the `struct page` referencing the ptdesc
- */
- static inline void pte_free(struct mm_struct *mm, struct page *pte_page)
- {
-- pgtable_pte_page_dtor(pte_page);
-- __free_page(pte_page);
-+ struct ptdesc *ptdesc = page_ptdesc(pte_page);
++ table = ptdesc_address(ptdesc);
+
-+ pagetable_pte_dtor(ptdesc);
-+ pagetable_free(ptdesc);
- }
-
-
-@@ -107,10 +114,11 @@ static inline void pte_free(struct mm_struct *mm, struct page *pte_page)
-
- #ifndef __HAVE_ARCH_PMD_ALLOC_ONE
- /**
-- * pmd_alloc_one - allocate a page for PMD-level page table
-+ * pmd_alloc_one - allocate memory for a PMD-level page table
- * @mm: the mm_struct of the current context
- *
-- * Allocates a page and runs the pgtable_pmd_page_ctor().
-+ * Allocate memory for a page table and ptdesc and runs pagetable_pmd_ctor().
-+ *
- * Allocations use %GFP_PGTABLE_USER in user context and
- * %GFP_PGTABLE_KERNEL in kernel context.
- *
-@@ -118,28 +126,30 @@ static inline void pte_free(struct mm_struct *mm, struct page *pte_page)
- */
- static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
- {
-- struct page *page;
-+ struct ptdesc *ptdesc;
- gfp_t gfp = GFP_PGTABLE_USER;
-
- if (mm == &init_mm)
- gfp = GFP_PGTABLE_KERNEL;
-- page = alloc_page(gfp);
-- if (!page)
-+ ptdesc = pagetable_alloc(gfp, 0);
-+ if (!ptdesc)
- return NULL;
-- if (!pgtable_pmd_page_ctor(page)) {
-- __free_page(page);
-+ if (!pagetable_pmd_ctor(ptdesc)) {
-+ pagetable_free(ptdesc);
- return NULL;
- }
-- return (pmd_t *)page_address(page);
-+ return ptdesc_address(ptdesc);
- }
- #endif
-
- #ifndef __HAVE_ARCH_PMD_FREE
- static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
- {
-+ struct ptdesc *ptdesc = virt_to_ptdesc(pmd);
-+
- BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
-- pgtable_pmd_page_dtor(virt_to_page(pmd));
-- free_page((unsigned long)pmd);
-+ pagetable_pmd_dtor(ptdesc);
-+ pagetable_free(ptdesc);
- }
- #endif
-
-@@ -150,19 +160,25 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
- static inline pud_t *__pud_alloc_one(struct mm_struct *mm, unsigned long addr)
- {
- gfp_t gfp = GFP_PGTABLE_USER;
-+ struct ptdesc *ptdesc;
-
- if (mm == &init_mm)
- gfp = GFP_PGTABLE_KERNEL;
-- return (pud_t *)get_zeroed_page(gfp);
-+ gfp &= ~__GFP_HIGHMEM;
-+
-+ ptdesc = pagetable_alloc(gfp, 0);
-+ if (!ptdesc)
-+ return NULL;
-+ return ptdesc_address(ptdesc);
- }
-
- #ifndef __HAVE_ARCH_PUD_ALLOC_ONE
- /**
-- * pud_alloc_one - allocate a page for PUD-level page table
-+ * pud_alloc_one - allocate memory for a PUD-level page table
- * @mm: the mm_struct of the current context
- *
-- * Allocates a page using %GFP_PGTABLE_USER for user context and
-- * %GFP_PGTABLE_KERNEL for kernel context.
-+ * Allocate memory for a page table using %GFP_PGTABLE_USER for user context
-+ * and %GFP_PGTABLE_KERNEL for kernel context.
- *
- * Return: pointer to the allocated memory or %NULL on error
- */
-@@ -175,7 +191,7 @@ static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
- static inline void __pud_free(struct mm_struct *mm, pud_t *pud)
- {
- BUG_ON((unsigned long)pud & (PAGE_SIZE-1));
-- free_page((unsigned long)pud);
-+ pagetable_free(virt_to_ptdesc(pud));
- }
-
- #ifndef __HAVE_ARCH_PUD_FREE
-@@ -190,7 +206,7 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pud)
- #ifndef __HAVE_ARCH_PGD_FREE
- static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
- {
-- free_page((unsigned long)pgd);
-+ pagetable_free(virt_to_ptdesc(pgd));
- }
- #endif
-
++ crst_table_init(table, val);
+ return table;
+ }
+
+ static void base_crst_free(unsigned long *table)
+ {
+- free_pages((unsigned long)table, CRST_ALLOC_ORDER);
++ ptdesc_free(virt_to_ptdesc(table));
+ }
+
+ #define BASE_ADDR_END_FUNC(NAME, SIZE) \
--
-2.40.1
+2.39.2