[PATCH 3/6] Protectable Memory

@@ -0,0 +1,281 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * pmalloc.h: Header for Protectable Memory Allocator
+ *
+ * (C) Copyright 2017-18 Huawei Technologies Co. Ltd.
+ * Author: Igor Stoppa <igor.stoppa@huawei.com>
+ */
+
+#ifndef _LINUX_PMALLOC_H
+#define _LINUX_PMALLOC_H
+
+
+#include <linux/string.h>
+
+/*
+ * Library for dynamic allocation of pools of protectable memory.
+ * A pool is a single linked list of vmap_area structures.
+ * Whenever a pool is protected, all the areas it contain at that point
+ * are write protected.
+ * More areas can be added and protected, in the same way.
+ * Memory in a pool cannot be individually unprotected, but the pool can
+ * be destroyed.
+ * Upon destruction of a certain pool, all the related memory is released,
+ * including its metadata.
+ *
+ * Pmalloc memory is intended to complement __read_only_after_init.
+ * It can be used, for example, where there is a write-once variable, for
+ * which it is not possible to know the initialization value before init
+ * is completed (which is what __read_only_after_init requires).
+ *
+ * It can be useful also where the amount of data to protect is not known
+ * at compile time and the memory can only be allocated dynamically.
+ *
+ * Finally, it can be useful also when it is desirable to control
+ * dynamically (for example throguh the command line) if something ought
+ * to be protected or not, without having to rebuild the kernel (like in
+ * the build used for a linux distro).
+ */
+
+
+#define PMALLOC_REFILL_DEFAULT (0)
+#define PMALLOC_ALIGN_DEFAULT (-1)
+
+struct pmalloc_pool *pmalloc_create_custom_pool(unsigned long int refill,
+						short int align_order);
+
+/**
+ * pmalloc_create_pool() - create a protectable memory pool
+ *
+ * Shorthand for pmalloc_create_custom_pool() with default arguments:
+ * * refill is set to PMALLOC_REFILL_DEFAULT, which is one memory page
+ * * align_order is set to PMALLOC_ALIGN_DEFAULT, which is size_of(size_t)
+ *
+ * Return:
+ * * pointer to the new pool	- success
+ * * NULL			- error
+ */
+static inline struct pmalloc_pool *pmalloc_create_pool(void)
+{
+	return pmalloc_create_custom_pool(PMALLOC_REFILL_DEFAULT,
+					  PMALLOC_ALIGN_DEFAULT);
+}
+
+
+//bool pmalloc_expand_pool(struct gen_pool *pool, size_t size);
+
+
+void *pmalloc_align(struct pmalloc_pool *pool, size_t size,
+		    short int align_order);
+
+
+/**
+ * pmalloc() - allocates protectable memory from a pool
+ * @pool: handle to the pool to be used for memory allocation
+ * @size: amount of memory (in bytes) requested
+ *
+ * Shorthand for pmalloc_align() with default argument:
+ * align_order = PMALLOC_ALIGN_DEFAULT, value set when creating the pool
+ *
+ * Return:
+ * * pointer to the new pool	- success
+ * * NULL			- error
+ */
+static inline void *pmalloc(struct pmalloc_pool *pool, size_t size)
+{
+	return pmalloc_align(pool, size, PMALLOC_ALIGN_DEFAULT);
+}
+
+
+/**
+ * pzalloc_align() - zero-initialized version of pmalloc_align
+ * @pool: handle to the pool to be used for memory allocation
+ * @size: amount of memory (in bytes) requested
+ * @align_order: log2 of the alignment of the allocation
+ *               Setting it to PMALLOC_ALIGN_DEFAULT will use the value
+ *               specified when the pool was created.
+ *
+ * Executes pmalloc_align, initializing the memory requested to 0,
+ * before returning its address.
+ *
+ * Return:
+ * * pointer to the memory requested	- success
+ * * NULL				- error
+ */
+static inline void *pzalloc_align(struct pmalloc_pool *pool, size_t size,
+				  short int align_order)
+{
+	void *ptr = pmalloc_align(pool, size, align_order);
+
+	if (likely(ptr))
+		memset(ptr, 0, size);
+
+	return ptr;
+}
+
+
+/**
+ * pzalloc() - zero-initialized version of pmalloc()
+ * @pool: handle to the pool to be used for memory allocation
+ * @size: amount of memory (in bytes) requested
+ *
+ * Shorthand for pmalloc_align(), with align set to PMALLOC_ALIGN_DEFAULT
+ *
+ * Return:
+ * * pointer to the memory requested	- success
+ * * NULL				- error
+ */
+static inline void *pzalloc(struct pmalloc_pool *pool, size_t size)
+{
+	return pzalloc_align(pool, size, PMALLOC_ALIGN_DEFAULT);
+}
+
+
+/**
+ * pmalloc_array_align() - array version of pmalloc_align
+ * @pool: handle to the pool to be used for memory allocation
+ * @n: number of elements in the array
+ * @size: amount of memory (in bytes) requested for each element
+ * @align_order: log2 of the alignment of the allocation
+ *               Setting it to PMALLOC_ALIGN_DEFAULT will use the value
+ *               specified when the pool was created.
+ *
+ * Executes pmalloc_align(), on an array.
+ *
+ * Return:
+ * * the pmalloc result	- success
+ * * NULL		- error
+ */
+static inline void *pmalloc_array_align(struct pmalloc_pool *pool,
+					size_t n, size_t size,
+					short int align_order)
+{
+	return pmalloc_align(pool, n * size, align_order);
+}
+
+
+/**
+ * pmalloc_array() - array version of pmalloc()
+ * @pool: handle to the pool to be used for memory allocation
+ * @n: number of elements in the array
+ * @size: amount of memory (in bytes) requested for each element
+ *
+ * Executes pmalloc_align(), on an array.
+ *
+ * Return:
+ * * the pmalloc result	- success
+ * * NULL		- error
+ */
+static inline void *pmalloc_array(struct pmalloc_pool *pool, size_t n,
+				  size_t size)
+{
+	return pmalloc_array_align(pool, n, size, PMALLOC_ALIGN_DEFAULT);
+}
+
+
+/**
+ * pcalloc_align() - array version of pzalloc_align()
+ * @pool: handle to the pool to be used for memory allocation
+ * @n: number of elements in the array
+ * @size: amount of memory (in bytes) requested for each element
+ * @align_order: log2 of the alignment of the allocation
+ *               Setting it to PMALLOC_ALIGN_DEFAULT will use the value
+ *               specified when the pool was created.
+ *
+ * Executes pzalloc_align(), on an array.
+ *
+ * Return:
+ * * the pmalloc result	- success
+ * * NULL		- error
+ */
+static inline void *pcalloc_align(struct pmalloc_pool *pool, size_t n,
+				  size_t size, short int align_order)
+{
+	return pzalloc_align(pool, n * size, align_order);
+}
+
+
+/**
+ * pcalloc() - array version of pzalloc()
+ * @pool: handle to the pool to be used for memory allocation
+ * @n: number of elements in the array
+ * @size: amount of memory (in bytes) requested for each element
+ *
+ * Executes pzalloc(), on an array.
+ *
+ * Return:
+ * * the pmalloc result	- success
+ * * NULL		- error
+ */
+static inline void *pcalloc(struct pmalloc_pool *pool, size_t n,
+			    size_t size)
+{
+	return pzalloc_align(pool, n * size, PMALLOC_ALIGN_DEFAULT);
+}
+
+/**
+ * pstrdup_align() - duplicate a string, using pmalloc_align()
+ * @pool: handle to the pool to be used for memory allocation
+ * @s: string to duplicate
+ * @align_order: log2 of the alignment of the allocation
+ *               Setting it to PMALLOC_ALIGN_DEFAULT will use the value
+ *               specified when the pool was created.
+ *
+ * Generates a copy of the given string, allocating sufficient memory
+ * from the given pmalloc pool.
+ *
+ * Return:
+ * * pointer to the replica	- success
+ * * NULL			- error
+ */
+static inline char *pstrdup_align(struct pmalloc_pool *pool,
+				  const char *s, short int align_order)
+{
+	size_t len;
+	char *buf;
+
+	len = strlen(s) + 1;
+	buf = pmalloc_align(pool, len, align_order);
+	if (likely(buf))
+		strncpy(buf, s, len);
+	return buf;
+}
+
+
+/**
+ * pstrdup() - duplicate a string, using pmalloc()
+ * @pool: handle to the pool to be used for memory allocation
+ * @s: string to duplicate
+ *
+ * Generates a copy of the given string, allocating sufficient memory
+ * from the given pmalloc pool.
+ *
+ * Return:
+ * * pointer to the replica	- success
+ * * NULL			- error
+ */
+static inline char *pstrdup(struct pmalloc_pool *pool, const char *s)
+{
+	return pstrdup_align(pool, s, PMALLOC_ALIGN_DEFAULT);
+}
+
+
+void pmalloc_protect_pool(struct pmalloc_pool *pool);
+
+
+void pmalloc_destroy_pool(struct pmalloc_pool *pool);
+
+
+int is_pmalloc_object(const void *ptr, const unsigned long n);
+
+
+unsigned long pmalloc_get_offset(struct pmalloc_pool *pool);
+
+
+unsigned long pmalloc_get_align(struct pmalloc_pool *pool);
+
+
+unsigned long pmalloc_get_refill(struct pmalloc_pool *pool);
+
+
+#endif

@@ -20,6 +20,8 @@ struct notifier_block;		/* in notifier.h */
 #define VM_UNINITIALIZED	0x00000020	/* vm_struct is not fully initialized */
 #define VM_NO_GUARD		0x00000040      /* don't add guard page */
 #define VM_KASAN		0x00000080      /* has allocated kasan shadow memory */
+#define VM_PMALLOC		0x00000100	/* pmalloc area - see docs */
+#define VM_PMALLOC_PROTECTED	0x00000200	/* protected area - see docs */
 /* bits [20..32] reserved for arch specific ioremap internals */
 
 /*

@@ -133,6 +135,7 @@ extern struct vm_struct *__get_vm_area_caller(unsigned long size,
 					const void *caller);
 extern struct vm_struct *remove_vm_area(const void *addr);
 extern struct vm_struct *find_vm_area(const void *addr);
+extern struct vmap_area *find_vmap_area(unsigned long addr);
 
 extern int map_vm_area(struct vm_struct *area, pgprot_t prot,
 			struct page **pages);

@@ -760,3 +760,9 @@ config GUP_BENCHMARK
 	  performance of get_user_pages_fast().
 
 	  See tools/testing/selftests/vm/gup_benchmark.c
+
+config PROTECTABLE_MEMORY
+    bool
+    depends on MMU
+    depends on ARCH_HAS_SET_MEMORY
+    default y

@@ -65,6 +65,7 @@ obj-$(CONFIG_SPARSEMEM)	+= sparse.o
 obj-$(CONFIG_SPARSEMEM_VMEMMAP) += sparse-vmemmap.o
 obj-$(CONFIG_SLOB) += slob.o
 obj-$(CONFIG_MMU_NOTIFIER) += mmu_notifier.o
+obj-$(CONFIG_PROTECTABLE_MEMORY) += pmalloc.o
 obj-$(CONFIG_KSM) += ksm.o
 obj-$(CONFIG_PAGE_POISONING) += page_poison.o
 obj-$(CONFIG_SLAB) += slab.o

@@ -0,0 +1,321 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * pmalloc.c: Protectable Memory Allocator
+ *
+ * (C) Copyright 2017-2018 Huawei Technologies Co. Ltd.
+ * Author: Igor Stoppa <igor.stoppa@huawei.com>
+ */
+
+#include <linux/printk.h>
+#include <linux/init.h>
+#include <linux/mm.h>
+#include <linux/vmalloc.h>
+#include <linux/kernel.h>
+#include <linux/log2.h>
+#include <linux/slab.h>
+#include <linux/set_memory.h>
+#include <linux/bug.h>
+#include <linux/mutex.h>
+#include <linux/llist.h>
+#include <asm/cacheflush.h>
+#include <asm/page.h>
+
+#include <linux/pmalloc.h>
+
+#define MAX_ALIGN_ORDER (ilog2(sizeof(void *)))
+struct pmalloc_pool {
+	struct mutex mutex;
+	struct list_head pool_node;
+	struct llist_head vm_areas;
+	unsigned long refill;
+	unsigned long offset;
+	unsigned long align;
+};
+
+static LIST_HEAD(pools_list);
+static DEFINE_MUTEX(pools_mutex);
+
+static inline void tag_area(struct vmap_area *area)
+{
+	area->vm->flags |= VM_PMALLOC;
+}
+
+static inline void untag_area(struct vmap_area *area)
+{
+	area->vm->flags &= ~VM_PMALLOC;
+}
+
+static inline struct vmap_area *current_area(struct pmalloc_pool *pool)
+{
+	return llist_entry(pool->vm_areas.first, struct vmap_area,
+			   area_list);
+}
+
+static inline bool is_area_protected(struct vmap_area *area)
+{
+	return area->vm->flags & VM_PMALLOC_PROTECTED;
+}
+
+static inline bool protect_area(struct vmap_area *area)
+{
+	if (unlikely(is_area_protected(area)))
+		return false;
+	set_memory_ro(area->va_start, area->vm->nr_pages);
+	area->vm->flags |= VM_PMALLOC_PROTECTED;
+	return true;
+}
+
+static inline void destroy_area(struct vmap_area *area)
+{
+	WARN(!is_area_protected(area), "Destroying unprotected area.");
+	set_memory_rw(area->va_start, area->vm->nr_pages);
+	vfree((void *)area->va_start);
+}
+
+static inline bool empty(struct pmalloc_pool *pool)
+{
+	return unlikely(llist_empty(&pool->vm_areas));
+}
+
+static inline bool protected(struct pmalloc_pool *pool)
+{
+	return is_area_protected(current_area(pool));
+}
+
+static inline unsigned long get_align(struct pmalloc_pool *pool,
+				      short int align_order)
+{
+	if (likely(align_order < 0))
+		return pool->align;
+	return 1UL << align_order;
+}
+
+static inline bool exhausted(struct pmalloc_pool *pool, size_t size,
+			     short int align_order)
+{
+	unsigned long align = get_align(pool, align_order);
+	unsigned long space_before = round_down(pool->offset, align);
+	unsigned long space_after = pool->offset - space_before;
+
+	return unlikely(space_after < size && space_before < size);
+}
+
+static inline bool space_needed(struct pmalloc_pool *pool, size_t size,
+				short int align_order)
+{
+	return empty(pool) || protected(pool) ||
+		exhausted(pool, size, align_order);
+}
+
+#define DEFAULT_REFILL_SIZE PAGE_SIZE
+/**
+ * pmalloc_create_custom_pool() - create a new protectable memory pool
+ * @refill: the minimum size to allocate when in need of more memory.
+ *          It will be rounded up to a multiple of PAGE_SIZE
+ *          The value of 0 gives the default amount of PAGE_SIZE.
+ * @align_order: log2 of the alignment to use when allocating memory
+ *               Negative values give log2(sizeof(size_t)).
+ *
+ * Creates a new (empty) memory pool for allocation of protectable
+ * memory. Memory will be allocated upon request (through pmalloc).
+ *
+ * Return:
+ * * pointer to the new pool	- success
+ * * NULL			- error
+ */
+struct pmalloc_pool *pmalloc_create_custom_pool(unsigned long refill,
+						short int align_order)
+{
+	struct pmalloc_pool *pool;
+
+	pool = kzalloc(sizeof(struct pmalloc_pool), GFP_KERNEL);
+	if (WARN(!pool, "Could not allocate pool meta data."))
+		return NULL;
+
+	pool->refill = refill ? PAGE_ALIGN(refill) : DEFAULT_REFILL_SIZE;
+	if (align_order < 0)
+		pool->align = sizeof(size_t);
+	else
+		pool->align = 1UL << align_order;
+	mutex_init(&pool->mutex);
+
+	mutex_lock(&pools_mutex);
+	list_add(&pool->pool_node, &pools_list);
+	mutex_unlock(&pools_mutex);
+	return pool;
+}
+
+
+static int grow(struct pmalloc_pool *pool, size_t size,
+		short int align_order)
+{
+	void *addr;
+	struct vmap_area *area;
+
+	addr = vmalloc(max(size, pool->refill));
+	if (WARN(!addr, "Failed to allocate %zd bytes", PAGE_ALIGN(size)))
+		return -ENOMEM;
+
+	area = find_vmap_area((unsigned long)addr);
+	tag_area(area);
+	pool->offset = area->vm->nr_pages * PAGE_SIZE;
+	llist_add(&area->area_list, &pool->vm_areas);
+	return 0;
+}
+
+static unsigned long reserve_mem(struct pmalloc_pool *pool, size_t size,
+				 short int align_order)
+{
+	unsigned long align;
+
+	align = get_align(pool, align_order);
+	pool->offset = round_down(pool->offset - size, align);
+	return current_area(pool)->va_start + pool->offset;
+
+}
+
+/**
+ * pmalloc_align() - allocate protectable memory from a pool
+ * @pool: handle to the pool to be used for memory allocation
+ * @size: amount of memory (in bytes) requested
+ * @align_order: log2 of the alignment of the allocation
+ *               Setting it to PMALLOC_ALIGN_DEFAULT will use the value
+ *               specified when the pool was created.
+ *
+ * Allocates memory from a pool.
+ * If needed, the pool will automatically allocate enough memory to
+ * either satisfy the request or meet the "refill" parameter received
+ * upon creation.
+ * New allocation can happen also if the current memory in the pool is
+ * already write protected.
+ *
+ * Return:
+ * * pointer to the memory requested	- success
+ * * NULL				- error
+ */
+void *pmalloc_align(struct pmalloc_pool *pool, size_t size,
+		    short int align_order)
+{
+	unsigned long retval = 0;
+
+	mutex_lock(&pool->mutex);
+	if (space_needed(pool, size, align_order))
+		if (unlikely(grow(pool, size, align_order)))
+			goto out;
+	retval = reserve_mem(pool, size, align_order);
+out:
+	mutex_unlock(&pool->mutex);
+	return (void *)retval;
+}
+
+/**
+ * pmalloc_protect_pool() - write-protects the memory in the pool
+ * @pool: the pool associated tothe memory to write-protect
+ *
+ * Write-protects all the memory areas currently assigned to the pool
+ * that are still unprotected.
+ * This does not prevent further allocation of additional memory, that
+ * can be initialized and protected.
+ * The catch is that protecting a pool will make unavailable whatever
+ * free memory it might still contain.
+ * Successive allocations will grab more free pages.
+ */
+void pmalloc_protect_pool(struct pmalloc_pool *pool)
+{
+	struct vmap_area *area;
+
+	mutex_lock(&pool->mutex);
+	llist_for_each_entry(area, pool->vm_areas.first, area_list)
+		if (unlikely(!protect_area(area)))
+			break;
+	mutex_unlock(&pool->mutex);
+}
+
+
+/**
+ * is_pmalloc_object() - test if the given range is within a pmalloc pool
+ * @ptr: the base address of the range
+ * @n: the size of the range
+ *
+ * Return:
+ * * true	- the range given is fully within a pmalloc pool
+ * * false	- the range given is not fully within a pmalloc pool
+ */
+int is_pmalloc_object(const void *ptr, const unsigned long n)
+{
+	struct vm_struct *area;
+
+	if (likely(!is_vmalloc_addr(ptr)))
+		return false;
+
+	area = vmalloc_to_page(ptr)->area;
+	if (unlikely(!(area->flags & VM_PMALLOC)))
+		return false;
+
+	return ((n + (unsigned long)ptr) <=
+		(area->nr_pages * PAGE_SIZE + (unsigned long)area->addr));
+
+}
+
+
+/**
+ * pmalloc_destroy_pool() - destroys a pool and all the associated memory
+ * @pool: the pool to destroy
+ *
+ * All the memory associated to the pool will be freed, including the
+ * metadata used for the pool.
+ */
+void pmalloc_destroy_pool(struct pmalloc_pool *pool)
+{
+	struct vmap_area *area;
+	struct llist_node *tmp;
+
+	mutex_lock(&pools_mutex);
+	list_del(&pool->pool_node);
+	mutex_unlock(&pools_mutex);
+
+	mutex_lock(&pool->mutex);
+	while (pool->vm_areas.first) {
+		tmp = pool->vm_areas.first;
+		pool->vm_areas.first = pool->vm_areas.first->next;
+		area = llist_entry(tmp, struct vmap_area, area_list);
+		destroy_area(area);
+	}
+	mutex_unlock(&pool->mutex);
+	kfree(pool);
+}
+
+/**
+ * pmalloc_get_offset() - returns the offset in a pool
+ * @pool: the pool from which to return the offset
+ *
+ * Return: the offset inside the active vmap_area
+ */
+unsigned long pmalloc_get_offset(struct pmalloc_pool *pool)
+{
+	return pool->offset;
+}
+
+
+/**
+ * pmalloc_get_align() - returns the align in a pool
+ * @pool: the pool from which to return the align
+ *
+ * Return: the align inside the active vmap_area
+ */
+unsigned long pmalloc_get_align(struct pmalloc_pool *pool)
+{
+	return pool->align;
+}
+
+
+/**
+ * pmalloc_get_refill() - returns the refill in a pool
+ * @pool: the pool from which to return the refill
+ *
+ * Return: the refill inside the active vmap_area
+ */
+unsigned long pmalloc_get_refill(struct pmalloc_pool *pool)
+{
+	return pool->refill;
+}

@@ -240,6 +240,36 @@ static inline void check_heap_object(const void *ptr, unsigned long n,
 	}
 }
 
+#ifdef CONFIG_PROTECTABLE_MEMORY
+
+int is_pmalloc_object(const void *ptr, const unsigned long n);
+
+static void check_pmalloc_object(const void *ptr, unsigned long n,
+				 bool to_user)
+{
+	int retv;
+
+	retv = is_pmalloc_object(ptr, n);
+	if (unlikely(retv)) {
+		if (unlikely(!to_user))
+			usercopy_abort("pmalloc",
+				       "trying to write to pmalloc object",
+				       to_user, (const unsigned long)ptr, n);
+		if (retv < 0)
+			usercopy_abort("pmalloc",
+				       "invalid pmalloc object",
+				       to_user, (const unsigned long)ptr, n);
+	}
+}
+
+#else
+
+static void check_pmalloc_object(const void *ptr, unsigned long n,
+				 bool to_user)
+{
+}
+#endif
+
 /*
  * Validates that the given object is:
  * - not bogus address

@@ -277,5 +307,8 @@ void __check_object_size(const void *ptr, unsigned long n, bool to_user)
 
 	/* Check for object in kernel to avoid text exposure. */
 	check_kernel_text_object((const unsigned long)ptr, n, to_user);
+
+	/* Check if object is from a pmalloc chunk. */
+	check_pmalloc_object(ptr, n, to_user);
 }
 EXPORT_SYMBOL(__check_object_size);

@@ -759,7 +759,7 @@ static void free_unmap_vmap_area(struct vmap_area *va)
 	free_vmap_area_noflush(va);
 }
 
-static struct vmap_area *find_vmap_area(unsigned long addr)
+struct vmap_area *find_vmap_area(unsigned long addr)
 {
 	struct vmap_area *va;