diff --git a/include/linux/page_frag_cache.h b/include/linux/page_frag_cache.h
index 5f9971c1be74..50466f5b71ea 100644
--- a/include/linux/page_frag_cache.h
+++ b/include/linux/page_frag_cache.h
@@ -100,12 +100,90 @@ static inline void *page_frag_alloc_va_align(struct page_frag_cache *nc,
return __page_frag_alloc_va_align(nc, fragsz, gfp_mask, -align);
}
+static inline unsigned int page_frag_cache_page_offset(const struct page_frag_cache *nc)
+{
+ return __page_frag_cache_page_offset(nc->encoded_va, nc->remaining);
+}
+
static inline void *page_frag_alloc_va(struct page_frag_cache *nc,
unsigned int fragsz, gfp_t gfp_mask)
{
return __page_frag_alloc_va_align(nc, fragsz, gfp_mask, ~0u);
}
+void *page_frag_alloc_va_prepare(struct page_frag_cache *nc, unsigned int *fragsz,
+ gfp_t gfp);
+
+static inline void *page_frag_alloc_va_prepare_align(struct page_frag_cache *nc,
+ unsigned int *fragsz,
+ gfp_t gfp,
+ unsigned int align)
+{
+ WARN_ON_ONCE(!is_power_of_2(align) || align > PAGE_SIZE);
+ nc->remaining = nc->remaining & -align;
+ return page_frag_alloc_va_prepare(nc, fragsz, gfp);
+}
+
+struct page *page_frag_alloc_pg_prepare(struct page_frag_cache *nc,
+ unsigned int *offset,
+ unsigned int *fragsz, gfp_t gfp);
+
+struct page *page_frag_alloc_prepare(struct page_frag_cache *nc,
+ unsigned int *offset,
+ unsigned int *fragsz,
+ void **va, gfp_t gfp);
+
+static inline struct encoded_va *__page_frag_alloc_probe(struct page_frag_cache *nc,
+ unsigned int *offset,
+ unsigned int *fragsz,
+ void **va)
+{
+ struct encoded_va *encoded_va;
+
+ *fragsz = nc->remaining;
+ encoded_va = nc->encoded_va;
+ *offset = __page_frag_cache_page_offset(encoded_va, *fragsz);
+ *va = encoded_page_address(encoded_va) + *offset;
+
+ return encoded_va;
+}
+
+#define page_frag_alloc_probe(nc, offset, fragsz, va) \
+({ \
+ struct page *__page = NULL; \
+ \
+ VM_BUG_ON(!*(fragsz)); \
+ if (likely((nc)->remaining >= *(fragsz))) \
+ __page = virt_to_page(__page_frag_alloc_probe(nc, \
+ offset, \
+ fragsz, \
+ va)); \
+ \
+ __page; \
+})
+
+static inline void page_frag_alloc_commit(struct page_frag_cache *nc,
+ unsigned int fragsz)
+{
+ VM_BUG_ON(fragsz > nc->remaining || !nc->pagecnt_bias);
+ nc->pagecnt_bias--;
+ nc->remaining -= fragsz;
+}
+
+static inline void page_frag_alloc_commit_noref(struct page_frag_cache *nc,
+ unsigned int fragsz)
+{
+ VM_BUG_ON(fragsz > nc->remaining);
+ nc->remaining -= fragsz;
+}
+
+static inline void page_frag_alloc_abort(struct page_frag_cache *nc,
+ unsigned int fragsz)
+{
+ nc->pagecnt_bias++;
+ nc->remaining += fragsz;
+}
+
void page_frag_free_va(void *addr);
#endifdiff --git a/mm/page_frag_cache.c b/mm/page_frag_cache.c
index 97cbc2dac67f..273ccd715eae 100644
--- a/mm/page_frag_cache.c
+++ b/mm/page_frag_cache.c
@@ -54,6 +54,127 @@ static struct page *__page_frag_cache_refill(struct page_frag_cache *nc,
return page;
}
+static struct page *page_frag_cache_refill(struct page_frag_cache *nc,
+ gfp_t gfp_mask)
+{
+ struct encoded_va *encoded_va = nc->encoded_va;
+ struct page *page;
+
+ if (unlikely(!encoded_va)) {
+refill:
+ page = __page_frag_cache_refill(nc, gfp_mask);
+ if (!page)
+ return NULL;
+
+ page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE);
+ nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
+ return page;
+ }
+
+ page = virt_to_page(encoded_va);
+
+ if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
+ goto refill;
+
+ if (unlikely(encoded_page_pfmemalloc(encoded_va))) {
+ free_unref_page(page, compound_order(page));
+ goto refill;
+ }
+
+ /* OK, page count is 0, we can safely set it */
+ set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1);
+
+ /* reset page count bias and offset to start of new frag */
+ nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
+ nc->remaining = page_frag_cache_page_size(encoded_va);
+
+ return page;
+}
+
+void *page_frag_alloc_va_prepare(struct page_frag_cache *nc,
+ unsigned int *fragsz, gfp_t gfp)
+{
+ struct encoded_va *encoded_va;
+ unsigned int remaining;
+
+ remaining = nc->remaining;
+ if (unlikely(*fragsz > remaining)) {
+ if (WARN_ON_ONCE(*fragsz > PAGE_SIZE) ||
+ !page_frag_cache_refill(nc, gfp))
+ return NULL;
+
+ remaining = nc->remaining;
+ }
+
+ encoded_va = nc->encoded_va;
+ *fragsz = remaining;
+ return encoded_page_address(encoded_va) +
+ __page_frag_cache_page_offset(encoded_va, remaining);
+}
+EXPORT_SYMBOL(page_frag_alloc_va_prepare);
+
+struct page *page_frag_alloc_pg_prepare(struct page_frag_cache *nc,
+ unsigned int *offset,
+ unsigned int *fragsz, gfp_t gfp)
+{
+ struct encoded_va *encoded_va;
+ unsigned int remaining;
+ struct page *page;
+
+ remaining = nc->remaining;
+ if (unlikely(*fragsz > remaining)) {
+ if (WARN_ON_ONCE(*fragsz > PAGE_SIZE)) {
+ *fragsz = 0;
+ return NULL;
+ }
+
+ page = page_frag_cache_refill(nc, gfp);
+ remaining = nc->remaining;
+ encoded_va = nc->encoded_va;
+ } else {
+ encoded_va = nc->encoded_va;
+ page = virt_to_page(encoded_va);
+ }
+
+ *offset = __page_frag_cache_page_offset(encoded_va, remaining);
+ *fragsz = remaining;
+
+ return page;
+}
+EXPORT_SYMBOL(page_frag_alloc_pg_prepare);
+
+struct page *page_frag_alloc_prepare(struct page_frag_cache *nc,
+ unsigned int *offset,
+ unsigned int *fragsz,
+ void **va, gfp_t gfp)
+{
+ struct encoded_va *encoded_va;
+ unsigned int remaining;
+ struct page *page;
+
+ remaining = nc->remaining;
+ if (unlikely(*fragsz > remaining)) {
+ if (WARN_ON_ONCE(*fragsz > PAGE_SIZE)) {
+ *fragsz = 0;
+ return NULL;
+ }
+
+ page = page_frag_cache_refill(nc, gfp);
+ remaining = nc->remaining;
+ encoded_va = nc->encoded_va;
+ } else {
+ encoded_va = nc->encoded_va;
+ page = virt_to_page(encoded_va);
+ }
+
+ *offset = __page_frag_cache_page_offset(encoded_va, remaining);
+ *fragsz = remaining;
+ *va = encoded_page_address(encoded_va) + *offset;
+
+ return page;
+}
+EXPORT_SYMBOL(page_frag_alloc_prepare);
+
void page_frag_cache_drain(struct page_frag_cache *nc)
{
if (!nc->encoded_va)--
2.33.0