---
 include/linux/kvm_host.h |  4 ++++
 virt/kvm/guest_memfd.c   | 22 +++++++++++++++++++---
 virt/kvm/kvm_main.c      | 12 +++++++++++-
 3 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 27687fb9d5201..acb552745b428 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h

@@ -2560,6 +2560,8 @@ static inline bool kvm_mem_range_is_private(struct kvm *kvm, gfn_t start,
 #endif  /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */

 #ifdef kvm_arch_has_private_mem
+extern bool gmem_in_place_conversion;
+
 typedef bool (kvm_mem_is_private_t)(struct kvm *kvm, gfn_t gfn);
 DECLARE_STATIC_CALL(__kvm_mem_is_private, kvm_mem_is_private_t);

@@ -2568,6 +2570,8 @@ static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn)
        return static_call(__kvm_mem_is_private)(kvm, gfn);
 }
 #else
+#define gmem_in_place_conversion false
+
 static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn)
 {
        return false;

diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c
index bca912db5be6e..e0e544ef47d69 100644
--- a/virt/kvm/guest_memfd.c
+++ b/virt/kvm/guest_memfd.c

@@ -926,6 +926,24 @@ int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot,
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gmem_get_pfn);

 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_POPULATE
+static bool kvm_gmem_range_is_private(struct file *file, pgoff_t index,
+                                     size_t nr_pages, struct kvm *kvm, gfn_t gfn)
+{
+       struct maple_tree *mt = &GMEM_I(file_inode(file))->attributes;
+       pgoff_t end = index + nr_pages - 1;
+       void *entry;
+
+       if (!gmem_in_place_conversion)
+               return kvm_range_has_vm_memory_attributes(kvm, gfn, gfn + nr_pages,
+                                                         KVM_MEMORY_ATTRIBUTE_PRIVATE,
+                                                         KVM_MEMORY_ATTRIBUTE_PRIVATE);
+
+       mt_for_each(mt, entry, index, end) {
+               if (xa_to_value(entry) != KVM_MEMORY_ATTRIBUTE_PRIVATE)
+                       return false;
+       }
+       return true;
+}

 static long __kvm_gmem_populate(struct kvm *kvm, struct kvm_memory_slot *slot,
                                struct file *file, gfn_t gfn, struct page *src_page,

@@ -946,9 +964,7 @@ static long __kvm_gmem_populate(struct kvm *kvm, struct kvm_memory_slot *slot,

        folio_unlock(folio);

-       if (!kvm_range_has_vm_memory_attributes(kvm, gfn, gfn + 1,
-                                               KVM_MEMORY_ATTRIBUTE_PRIVATE,
-                                               KVM_MEMORY_ATTRIBUTE_PRIVATE)) {
+       if (!kvm_gmem_range_is_private(file, index, 1, kvm, gfn)) {
                ret = -EINVAL;
                goto out_put_folio;
        }

diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 8b238e461b854..01761f6e25d25 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c

@@ -101,6 +101,10 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(halt_poll_ns_shrink);
 static bool __ro_after_init allow_unsafe_mappings;
 module_param(allow_unsafe_mappings, bool, 0444);

+#ifdef kvm_arch_has_private_mem
+bool __ro_after_init gmem_in_place_conversion = false;
+#endif
+
 /*
  * Ordering of locks:
  *

@@ -2422,6 +2426,9 @@ static int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm,
 static u64 kvm_supported_vm_mem_attributes(struct kvm *kvm)
 {
 #ifdef kvm_arch_has_private_mem
+       if (gmem_in_place_conversion)
+               return 0;
+
        if (!kvm || kvm_arch_has_private_mem(kvm))
                return KVM_MEMORY_ATTRIBUTE_PRIVATE;
 #endif

@@ -2633,8 +2640,11 @@ EXPORT_STATIC_CALL_GPL(__kvm_mem_is_private);

 static void kvm_init_memory_attributes(void)
 {
+       if (gmem_in_place_conversion)
+               static_call_update(__kvm_mem_is_private, kvm_gmem_is_private);
 #ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES
-       static_call_update(__kvm_mem_is_private, kvm_vm_mem_is_private);
+       else
+               static_call_update(__kvm_mem_is_private, kvm_vm_mem_is_private);
 #endif
 }
 #else

--

2.55.0.rc0.738.g0c8ab3ebcc-goog