--- v11
+++ v7
@@ -42,10 +42,10 @@
Signed-off-by: John Hubbard <jhubbard@nvidia.com>
---
Documentation/core-api/index.rst | 1 +
- Documentation/core-api/pin_user_pages.rst | 232 ++++++++++++++++++++++
+ Documentation/core-api/pin_user_pages.rst | 233 ++++++++++++++++++++++
include/linux/mm.h | 63 ++++--
- mm/gup.c | 161 +++++++++++++--
- 4 files changed, 423 insertions(+), 34 deletions(-)
+ mm/gup.c | 153 ++++++++++++--
+ 4 files changed, 416 insertions(+), 34 deletions(-)
create mode 100644 Documentation/core-api/pin_user_pages.rst
diff --git a/Documentation/core-api/index.rst b/Documentation/core-api/index.rst
@@ -62,10 +62,10 @@
boot-time-mm
diff --git a/Documentation/core-api/pin_user_pages.rst b/Documentation/core-api/pin_user_pages.rst
new file mode 100644
-index 000000000000..71849830cd48
+index 000000000000..4f26637a5005
--- /dev/null
+++ b/Documentation/core-api/pin_user_pages.rst
-@@ -0,0 +1,232 @@
+@@ -0,0 +1,233 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+====================================================
@@ -77,11 +77,11 @@
+Overview
+========
+
-+This document describes the following functions::
-+
-+ pin_user_pages()
-+ pin_user_pages_fast()
-+ pin_user_pages_remote()
++This document describes the following functions: ::
++
++ pin_user_pages
++ pin_user_pages_fast
++ pin_user_pages_remote
+
+Basic description of FOLL_PIN
+=============================
@@ -112,7 +112,7 @@
+The FOLL_PIN implementation is nearly the same as FOLL_GET, except that FOLL_PIN
+uses a different reference counting technique.
+
-+FOLL_PIN is a prerequisite to FOLL_LONGTERM. Another way of saying that is,
++FOLL_PIN is a prerequisite to FOLL_LONGTGERM. Another way of saying that is,
+FOLL_LONGTERM is a specific case, more restrictive case of FOLL_PIN.
+
+Which flags are set by each wrapper
@@ -278,7 +278,8 @@
+has the following new calls to exercise the new pin*() wrapper functions:
+
+* PIN_FAST_BENCHMARK (./gup_benchmark -a)
-+* PIN_BENCHMARK (./gup_benchmark -b)
++* PIN_LONGTERM_BENCHMARK (./gup_benchmark -a)
++* PIN_BENCHMARK (./gup_benchmark -a)
+
+You can monitor how many total dma-pinned pages have been acquired and released
+since the system was booted, via two new /proc/vmstat entries: ::
@@ -299,10 +300,10 @@
+
+John Hubbard, October, 2019
diff --git a/include/linux/mm.h b/include/linux/mm.h
-index 77a4df06c8a7..0fb9929e00af 100644
+index 96228376139c..568cbb895f03 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
-@@ -1048,16 +1048,14 @@ static inline void put_page(struct page *page)
+@@ -1075,16 +1075,14 @@ static inline void put_page(struct page *page)
* put_user_page() - release a gup-pinned page
* @page: pointer to page to be released
*
@@ -324,7 +325,7 @@
*/
static inline void put_user_page(struct page *page)
{
-@@ -1515,9 +1513,16 @@ long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
+@@ -1542,9 +1540,16 @@ long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages,
struct vm_area_struct **vmas, int *locked);
@@ -341,7 +342,7 @@
long get_user_pages_locked(unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages, int *locked);
long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
-@@ -1525,6 +1530,8 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
+@@ -1552,6 +1557,8 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
int get_user_pages_fast(unsigned long start, int nr_pages,
unsigned int gup_flags, struct page **pages);
@@ -350,7 +351,7 @@
int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc);
int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc,
-@@ -2588,13 +2595,15 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
+@@ -2610,13 +2617,15 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
#define FOLL_ANON 0x8000 /* don't do file mappings */
#define FOLL_LONGTERM 0x10000 /* mapping lifetime is indefinite: see below */
#define FOLL_SPLIT_PMD 0x20000 /* split huge pmd before returning */
@@ -369,7 +370,7 @@
*
* FIXME: For pages which are part of a filesystem, mappings are subject to the
* lifetime enforced by the filesystem and we need guarantees that longterm
-@@ -2609,11 +2618,39 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
+@@ -2631,11 +2640,39 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
* Currently only get_user_pages() and get_user_pages_fast() support this flag
* and calls to get_user_pages_[un]locked are specifically not allowed. This
* is due to an incompatibility with the FS DAX check and
@@ -413,7 +414,7 @@
static inline int vm_fault_to_errno(vm_fault_t vm_fault, int foll_flags)
diff --git a/mm/gup.c b/mm/gup.c
-index 958ab0757389..4862ff982bc3 100644
+index cce2c9676853..f72d7a1635b4 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -201,6 +201,10 @@ static struct page *follow_page_pte(struct vm_area_struct *vma,
@@ -427,7 +428,7 @@
retry:
if (unlikely(pmd_bad(*pmd)))
return no_page_table(vma, flags);
-@@ -818,7 +822,7 @@ static long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
+@@ -812,7 +816,7 @@ static long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
start = untagged_addr(start);
@@ -436,7 +437,7 @@
/*
* If FOLL_FORCE is set then do not force a full fault as the hinting
-@@ -1042,7 +1046,16 @@ static __always_inline long __get_user_pages_locked(struct task_struct *tsk,
+@@ -1036,7 +1040,16 @@ static __always_inline long __get_user_pages_locked(struct task_struct *tsk,
BUG_ON(*locked != 1);
}
@@ -454,7 +455,7 @@
flags |= FOLL_GET;
pages_done = 0;
-@@ -1185,6 +1198,13 @@ long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
+@@ -1173,6 +1186,13 @@ long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
unsigned int gup_flags, struct page **pages,
struct vm_area_struct **vmas, int *locked)
{
@@ -468,22 +469,7 @@
/*
* Parts of FOLL_LONGTERM behavior are incompatible with
* FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
-@@ -1400,6 +1420,14 @@ static long __get_user_pages_locked(struct task_struct *tsk,
- finish_or_fault:
- return i ? : -EFAULT;
- }
-+
-+long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
-+ unsigned long start, unsigned long nr_pages,
-+ unsigned int gup_flags, struct page **pages,
-+ struct vm_area_struct **vmas, int *locked)
-+{
-+ return 0;
-+}
- #endif /* !CONFIG_MMU */
-
- #if defined(CONFIG_FS_DAX) || defined (CONFIG_CMA)
-@@ -1654,6 +1682,13 @@ long get_user_pages(unsigned long start, unsigned long nr_pages,
+@@ -1640,6 +1660,13 @@ long get_user_pages(unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages,
struct vm_area_struct **vmas)
{
@@ -497,7 +483,7 @@
return __gup_longterm_locked(current, current->mm, start, nr_pages,
pages, vmas, gup_flags | FOLL_TOUCH);
}
-@@ -2392,30 +2427,15 @@ static int __gup_longterm_unlocked(unsigned long start, int nr_pages,
+@@ -2386,29 +2413,14 @@ static int __gup_longterm_unlocked(unsigned long start, int nr_pages,
return ret;
}
@@ -526,13 +512,12 @@
unsigned long addr, len, end;
int nr = 0, ret = 0;
- if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM |
-- FOLL_FORCE)))
-+ FOLL_FORCE | FOLL_PIN)))
+- if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM)))
++ if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM | FOLL_PIN)))
return -EINVAL;
start = untagged_addr(start) & PAGE_MASK;
-@@ -2455,4 +2475,103 @@ int get_user_pages_fast(unsigned long start, int nr_pages,
+@@ -2448,4 +2460,103 @@ int get_user_pages_fast(unsigned long start, int nr_pages,
return ret;
}
@@ -637,5 +622,5 @@
+}
+EXPORT_SYMBOL(pin_user_pages);
--
-2.24.1
-
+2.24.0
+