This patch series implementate THP COW for private executable file
mmap. It's major designed to increase the iTLB cache hit rate for
hot patching application, and we add a new sysfs knob to disable or
enable it.
Zhang Qilong (2):
mm/huge_memory: Implementation of THP COW for executable file mmap
mm/huge_memory: Add sysfs knob for executable THP COW
Documentation/admin-guide/mm/transhuge.rst | 8 ++
include/linux/huge_mm.h | 5 +
mm/huge_memory.c | 105 ++++++++++++++++++++-
mm/memory.c | 13 +++
4 files changed, 130 insertions(+), 1 deletion(-)
--
2.43.0
During the user-space hot patching, the involved executable file
segments of private mapping will be modified. If the modification
meets THP mapping, the PMD entry will be cleared at first and do
page COW fault handle.
Currently, khugepaged may attempt to merge scattered file pages
into THP. However, due to the single page COW, the modified
executable segments can not be mapped in THP once again for hot
patched process. Hence it can not benefit form khugepaged efforts.
The executable segment mapped in page granularity may reduce the
iTLB cache hit rate compared with the original THP mapping.
For user-space hot patching, we introduce THP COW support for the
executable mapping. If the exec COW meets THP mapping, it will
allocate a anonymous THP and map it to remain PMD mapping.
Signed-off-by: Zhang Qilong <redacted>
---
include/linux/huge_mm.h | 1 +
mm/huge_memory.c | 87 +++++++++++++++++++++++++++++++++++++++++
mm/memory.c | 12 ++++++
3 files changed, 100 insertions(+)
mm/memory.c:6134:21: error: implicit declaration of function 'vma_is_special_huge'; did you mean 'vma_is_special_mapping'? [-Werror=implicit-function-declaration]
6134 | if (vma_is_special_huge(vma) || vma_is_shmem(vma))
| ^~~~~~~~~~~~~~~~~~~
| vma_is_special_mapping
cc1: some warnings being treated as errors
vim +6134 mm/memory.c
6104
6105 /* `inline' is required to avoid gcc 4.1.2 build error */
6106 static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf)
6107 {
6108 struct vm_area_struct *vma = vmf->vma;
6109 const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
6110 vm_fault_t ret;
6111
6112 if (vma_is_anonymous(vma)) {
6113 if (likely(!unshare) &&
6114 userfaultfd_huge_pmd_wp(vma, vmf->orig_pmd)) {
6115 if (userfaultfd_wp_async(vmf->vma))
6116 goto split;
6117 return handle_userfault(vmf, VM_UFFD_WP);
6118 }
6119 return do_huge_pmd_wp_page(vmf);
6120 }
6121
6122 if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
6123 if (vma->vm_ops->huge_fault) {
6124 ret = vma->vm_ops->huge_fault(vmf, PMD_ORDER);
6125 if (!(ret & VM_FAULT_FALLBACK))
6126 return ret;
6127 }
6128 }
6129
6130
6131 if (is_exec_mapping(vma->vm_flags) &&
6132 is_cow_mapping(vma->vm_flags)) {
6133 /* Skip special and shmem */
6134 if (vma_is_special_huge(vma) || vma_is_shmem(vma))
6135 goto split;
6136
6137 ret = do_huge_pmd_exec_cow(vmf);
6138 if (!(ret & VM_FAULT_FALLBACK))
6139 return ret;
6140 }
6141
6142 split:
6143 /* COW or write-notify handled on pte level: split pmd. */
6144 __split_huge_pmd(vma, vmf->pmd, vmf->address, false);
6145
6146 return VM_FAULT_FALLBACK;
6147 }
6148
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
mm/memory.c:6134:7: error: call to undeclared function 'vma_is_special_huge'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
6134 | if (vma_is_special_huge(vma) || vma_is_shmem(vma))
| ^
1 error generated.
vim +/vma_is_special_huge +6134 mm/memory.c
6104
6105 /* `inline' is required to avoid gcc 4.1.2 build error */
6106 static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf)
6107 {
6108 struct vm_area_struct *vma = vmf->vma;
6109 const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
6110 vm_fault_t ret;
6111
6112 if (vma_is_anonymous(vma)) {
6113 if (likely(!unshare) &&
6114 userfaultfd_huge_pmd_wp(vma, vmf->orig_pmd)) {
6115 if (userfaultfd_wp_async(vmf->vma))
6116 goto split;
6117 return handle_userfault(vmf, VM_UFFD_WP);
6118 }
6119 return do_huge_pmd_wp_page(vmf);
6120 }
6121
6122 if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
6123 if (vma->vm_ops->huge_fault) {
6124 ret = vma->vm_ops->huge_fault(vmf, PMD_ORDER);
6125 if (!(ret & VM_FAULT_FALLBACK))
6126 return ret;
6127 }
6128 }
6129
6130
6131 if (is_exec_mapping(vma->vm_flags) &&
6132 is_cow_mapping(vma->vm_flags)) {
6133 /* Skip special and shmem */
6134 if (vma_is_special_huge(vma) || vma_is_shmem(vma))
6135 goto split;
6136
6137 ret = do_huge_pmd_exec_cow(vmf);
6138 if (!(ret & VM_FAULT_FALLBACK))
6139 return ret;
6140 }
6141
6142 split:
6143 /* COW or write-notify handled on pte level: split pmd. */
6144 __split_huge_pmd(vma, vmf->pmd, vmf->address, false);
6145
6146 return VM_FAULT_FALLBACK;
6147 }
6148
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
mm/memory.c:6134:7: error: call to undeclared function 'vma_is_special_huge'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
6134 | if (vma_is_special_huge(vma) || vma_is_shmem(vma))
| ^
1 error generated.
vim +/vma_is_special_huge +6134 mm/memory.c
6104
6105 /* `inline' is required to avoid gcc 4.1.2 build error */
6106 static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf)
6107 {
6108 struct vm_area_struct *vma = vmf->vma;
6109 const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
6110 vm_fault_t ret;
6111
6112 if (vma_is_anonymous(vma)) {
6113 if (likely(!unshare) &&
6114 userfaultfd_huge_pmd_wp(vma, vmf->orig_pmd)) {
6115 if (userfaultfd_wp_async(vmf->vma))
6116 goto split;
6117 return handle_userfault(vmf, VM_UFFD_WP);
6118 }
6119 return do_huge_pmd_wp_page(vmf);
6120 }
6121
6122 if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
6123 if (vma->vm_ops->huge_fault) {
6124 ret = vma->vm_ops->huge_fault(vmf, PMD_ORDER);
6125 if (!(ret & VM_FAULT_FALLBACK))
6126 return ret;
6127 }
6128 }
6129
6130
6131 if (is_exec_mapping(vma->vm_flags) &&
6132 is_cow_mapping(vma->vm_flags)) {
6133 /* Skip special and shmem */
6134 if (vma_is_special_huge(vma) || vma_is_shmem(vma))
6135 goto split;
6136
6137 ret = do_huge_pmd_exec_cow(vmf);
6138 if (!(ret & VM_FAULT_FALLBACK))
6139 return ret;
6140 }
6141
6142 split:
6143 /* COW or write-notify handled on pte level: split pmd. */
6144 __split_huge_pmd(vma, vmf->pmd, vmf->address, false);
6145
6146 return VM_FAULT_FALLBACK;
6147 }
6148
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
Although THP-granularity exec COW can reduce the number of faults
and improve iTLB hit rates, but after enabling it, the THP folio
allocating and copying operations may introduce higher latency,
and it consumes more memory compared to page COW handling. These
side effects may be unacceptable in certain scenarios.
Therefore, we add use_exec_cow sysfs knob for THP COW of executable
private file mmap. It's enabled by default, kernel will try to
allocate PMD page and map it. If it's disabled, it will fallback to
split PMD mapping and do pte fault handle.
Signed-off-by: Zhang Qilong <redacted>
---
Documentation/admin-guide/mm/transhuge.rst | 8 ++++++++
include/linux/huge_mm.h | 4 ++++
mm/huge_memory.c | 18 +++++++++++++++++-
mm/memory.c | 3 ++-
4 files changed, 31 insertions(+), 2 deletions(-)
@@ -201,10 +201,18 @@ page fault to anonymous mapping. It's possible to disable huge zero page by writing 0 or enable it back by writing 1:: echo 0 >/sys/kernel/mm/transparent_hugepage/use_zero_page echo 1 >/sys/kernel/mm/transparent_hugepage/use_zero_page+By default kernel tries to use huge, PMD-mappable page on private+executable file THP mmap fault handle. It's possible to disable+THP COW of private executable mmap by writing 0 or enable it back+by writing 1::++ echo 0 >/sys/kernel/mm/transparent_hugepage/use_exec_cow+ echo 1 >/sys/kernel/mm/transparent_hugepage/use_exec_cow+ Some userspace (such as a test program, or an optimized memory allocation library) may want to know the size (in bytes) of a PMD-mappable transparent hugepage:: cat /sys/kernel/mm/transparent_hugepage/hpage_pmd_size
mm/memory.c:6131:6: error: call to undeclared function 'transparent_hugepage_use_exec_cow'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
6131 | if (transparent_hugepage_use_exec_cow() &&
| ^
mm/memory.c:6135:7: error: call to undeclared function 'vma_is_special_huge'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
6135 | if (vma_is_special_huge(vma) || vma_is_shmem(vma))
| ^
2 errors generated.
vim +/transparent_hugepage_use_exec_cow +6131 mm/memory.c
6104
6105 /* `inline' is required to avoid gcc 4.1.2 build error */
6106 static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf)
6107 {
6108 struct vm_area_struct *vma = vmf->vma;
6109 const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
6110 vm_fault_t ret;
6111
6112 if (vma_is_anonymous(vma)) {
6113 if (likely(!unshare) &&
6114 userfaultfd_huge_pmd_wp(vma, vmf->orig_pmd)) {
6115 if (userfaultfd_wp_async(vmf->vma))
6116 goto split;
6117 return handle_userfault(vmf, VM_UFFD_WP);
6118 }
6119 return do_huge_pmd_wp_page(vmf);
6120 }
6121
6122 if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
6123 if (vma->vm_ops->huge_fault) {
6124 ret = vma->vm_ops->huge_fault(vmf, PMD_ORDER);
6125 if (!(ret & VM_FAULT_FALLBACK))
6126 return ret;
6127 }
6128 }
6129
6130
6131 if (transparent_hugepage_use_exec_cow() &&
6132 is_exec_mapping(vma->vm_flags) &&
6133 is_cow_mapping(vma->vm_flags)) {
6134 /* Skip special and shmem */
6135 if (vma_is_special_huge(vma) || vma_is_shmem(vma))
6136 goto split;
6137
6138 ret = do_huge_pmd_exec_cow(vmf);
6139 if (!(ret & VM_FAULT_FALLBACK))
6140 return ret;
6141 }
6142
6143 split:
6144 /* COW or write-notify handled on pte level: split pmd. */
6145 __split_huge_pmd(vma, vmf->pmd, vmf->address, false);
6146
6147 return VM_FAULT_FALLBACK;
6148 }
6149
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
From: Matthew Wilcox <willy@infradead.org> Date: 2025-12-15 14:00:50
On Mon, Dec 15, 2025 at 08:34:05PM +0800, Zhang Qilong wrote:
This patch series implementate THP COW for private executable file
mmap. It's major designed to increase the iTLB cache hit rate for
hot patching application, and we add a new sysfs knob to disable or
enable it.
You're going to have to provide data to get this patch in. We've
deliberately not done this in the past due to memory consumption overhead.
So you need to prove that's now the wrong decision to make.
Microbenchmarks would be a bare minimum, but what are really needed are
numbers from actual workloads.
On Mon, Dec 15, 2025 at 08:34:05PM +0800, Zhang Qilong wrote:
quoted
This patch series implementate THP COW for private executable file
mmap. It's major designed to increase the iTLB cache hit rate for
hot patching application, and we add a new sysfs knob to disable or
enable it.
You're going to have to provide data to get this patch in. We've
deliberately not done this in the past due to memory consumption overhead.
So you need to prove that's now the wrong decision to make.
Microbenchmarks would be a bare minimum, but what are really needed are
numbers from actual workloads.
In addition, the sysfs toggle is rather horrible. It's rather clear that
this is not a system-wide setting to be made, as you likely only want
that behavior (if at all ...) for a handful of special processes I assume?
--
Cheers
David