[PATCH] mm/hugetlb: account for allowed nodes when gathering surplus pages

Hugetlb reservations are accounted globally, but hugetlb_acct_memory()
also verifies that the current cpuset and MPOL_BIND policy contain enough
free huge pages to add a new reservation.

gather_surplus_pages() calculates its allocation shortfall from the global
free and reserved counters. If the global pool has enough free pages, but
those pages reside outside the nodes allowed by the task, it allocates no
surplus pages. The subsequent allowed_mems_nr() check then rejects the
reservation and mmap() fails with ENOMEM, even when
nr_overcommit_hugepages permits allocating surplus pages on the allowed
nodes.

Calculate both the global shortfall and the shortfall within the allowed
nodes, and allocate the larger of the two. Include surplus pages allocated
outside hugetlb_lock in both calculations when rechecking after reacquiring
the lock. These pages are constrained by alloc_nodemask, so they satisfy
both shortages.

Easy way to reproduce this issue with 2+ NUMA nodes system:

  # echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepage
  # echo 3 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages
  # echo 1 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_overcommit_hugepages
  # cd tools/testing/selftests/mm
  # numactl --membind=1 ./hugetlb-mmap 2 21
  TAP version 13
  # [INFO] detected hugetlb page size: 2048 KiB
  # [INFO] detected hugetlb page size: 1048576 KiB
  # 2048 kB hugepages
  1..2
  # Mapping 2 Mbytes
  Bail out! mmap: Cannot allocate memory (12)
  # Planned tests != run tests (2 != 0)
  # Totals: pass:0 fail:0 xfail:0 xpass:0 skip:0 error:0

This fixes hugetlb mappings when, for example, a task runs with
MPOL_BIND on Node 1 while the existing free huge pages are on Node 0.

Similar issue also could be found in ltp if the free pages of global pool
reside outside the nodes allowed by the application.
  # cd ltp/testcases/kernel/mem/hugetlb/hugemmap/
  # numactl --cpunodebind=0 --membind=1 ./hugemmap10

Signed-off-by: Huaisheng Ye <redacted>
---
 mm/hugetlb.c | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 7857728457952..b078953099fe5 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c

@@ -116,6 +116,7 @@ struct mutex *hugetlb_fault_mutex_table __ro_after_init;
 
 /* Forward declaration */
 static int hugetlb_acct_memory(struct hstate *h, long delta);
+static unsigned int allowed_mems_nr(struct hstate *h);
 static void hugetlb_vma_lock_free(struct vm_area_struct *vma);
 static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma);
 static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma);

@@ -2224,6 +2225,19 @@ static nodemask_t *policy_mbind_nodemask(gfp_t gfp)
 	return NULL;
 }
 
+/*
+ * Reservations are globally accounted, but they must also be backed by free
+ * pages on nodes allowed by the current cpuset and MPOL_BIND policy.
+ */
+static long surplus_pages_needed(struct hstate *h, long delta, long allocated)
+{
+	long global_free = (long)h->free_huge_pages + allocated;
+	long allowed_free = (long)allowed_mems_nr(h) + allocated;
+
+	return max((long)h->resv_huge_pages + delta - global_free,
+		   delta - allowed_free);
+}
+
 /*
  * Increase the hugetlb pool such that it can accommodate a reservation
  * of size 'delta'.

@@ -2246,7 +2260,7 @@ static int gather_surplus_pages(struct hstate *h, long delta)
 		alloc_nodemask = cpuset_current_mems_allowed;
 
 	lockdep_assert_held(&hugetlb_lock);
-	needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
+	needed = surplus_pages_needed(h, delta, 0);
 	if (needed <= 0) {
 		h->resv_huge_pages += delta;
 		return 0;

@@ -2277,11 +2291,10 @@ static int gather_surplus_pages(struct hstate *h, long delta)
 
 	/*
 	 * After retaking hugetlb_lock, we need to recalculate 'needed'
-	 * because either resv_huge_pages or free_huge_pages may have changed.
+	 * because either resv_huge_pages or the free page counts may have changed.
 	 */
 	spin_lock_irq(&hugetlb_lock);
-	needed = (h->resv_huge_pages + delta) -
-			(h->free_huge_pages + allocated);
+	needed = surplus_pages_needed(h, delta, allocated);
 	if (needed > 0) {
 		if (alloc_ok)
 			goto retry;

-- 
2.52.0