diff --git a/drivers/hv/hv_proc.c b/drivers/hv/hv_proc.c
index f811bb5c42a1..b63f8da1ecc1 100644
--- a/drivers/hv/hv_proc.c
+++ b/drivers/hv/hv_proc.c
@@ -7,12 +7,65 @@
#include <linux/cpuhotplug.h>
#include <linux/minmax.h>
#include <linux/export.h>
+#include <linux/set_memory.h>
#include <asm/mshyperv.h>
+#include <asm/tlbflush.h>
#define HV_DEPOSIT_MAX 512
#define HV_DEPOSIT_INP_MAX ((HV_HYP_PAGE_SIZE - \
offsetof(struct hv_deposit_memory, gpa_page_list)) / sizeof(u64))
+static inline int find_consecutive_pfns(const u64 *pfns, int count, int start)
+{
+ int end;
+
+ if (start >= count)
+ return count;
+
+ for (end = start + 1; end < count; ++end) {
+ if (pfns[end] != pfns[end - 1] + 1)
+ break;
+ }
+
+ return end;
+}
+
+/*
+ * Remove pages from the direct map before depositing them. On failure,
+ * restore the mapping and free all supplied pages, leaking pages whose
+ * mappings cannot be restored.
+ */
+static int hv_remove_deposit_mapping(const u64 *pfns, int count)
+{
+ int i, end, ret;
+ unsigned long addr, size;
+
+ for (i = 0; i < count; i = end) {
+ end = find_consecutive_pfns(pfns, count, i);
+
+ ret = set_direct_map_invalid_noflush(pfn_to_page(pfns[i]),
+ end - i);
+ if (ret)
+ goto restore;
+ }
+
+ for (i = 0; i < count; i = end) {
+ addr = (unsigned long)page_address(pfn_to_page(pfns[i]));
+
+ end = find_consecutive_pfns(pfns, count, i);
+
+ size = (unsigned long)(end - i) << PAGE_SHIFT;
+ flush_tlb_kernel_range(addr, addr + size);
+ }
+
+ return 0;
+
+restore:
+ /* The failing run may have been partially modified too. */
+ hv_restore_withdrawn_pages(pfns, count);
+ return ret;
+}
+
/*
* Allocate free pages for deposit to hypervisor. pfna[] must be large enough
* to hold HV_DEPOSIT_INP_MAX (511) pages. If num_pages is 512, return last@@ -84,7 +137,7 @@ static int hv_alloc_dep_pages(int node, u64 *pfna, u64 *lastpfnp, int num_pages,
static int hv_call_deposit_pages(int node, u64 partition_id, bool contiguous)
{
struct hv_deposit_memory *hc_input;
- int i, rc, num_pages;
+ int i, rc, num_pages, completed;
u64 status, *pfna, lastpfn = 0;
bool trunc_extra = false;
@@ -125,6 +178,21 @@ static int hv_call_deposit_pages(int node, u64 partition_id, bool contiguous)
num_pages = HV_MAX_CONTIGUOUS_ALLOCATION_PAGES;
}
+ rc = hv_remove_deposit_mapping(pfna, num_pages);
+ if (rc) {
+ if (lastpfn)
+ __free_page(pfn_to_page(lastpfn));
+ goto out_free;
+ }
+
+ if (lastpfn) {
+ rc = hv_remove_deposit_mapping(&lastpfn, 1);
+ if (rc) {
+ hv_restore_withdrawn_pages(pfna, num_pages);
+ goto out_free;
+ }
+ }
+
/* Not using hyperv_pcpu_input_arg, so no need to disable interrupts */
status = hv_do_rep_hypercall(HVCALL_DEPOSIT_MEMORY, num_pages, 0,@@ -139,7 +207,7 @@ static int hv_call_deposit_pages(int node, u64 partition_id, bool contiguous)
if (!hv_result_success(status) && hv_repcomp(status) == 0)
/* We deposited lot earlier, so give it a go */
- __free_page(pfn_to_page(lastpfn));
+ hv_restore_withdrawn_pages(&lastpfn, 1);
}
free_page((unsigned long)hc_input);
@@ -148,17 +216,51 @@ static int hv_call_deposit_pages(int node, u64 partition_id, bool contiguous)
err_free_dep_pages:
hv_status_err(status, "\n");
rc = hv_result_to_errno(status);
+ completed = hv_repcomp(status);
- for (i = hv_repcomp(status); i < num_pages; i++)
- __free_page(pfn_to_page(pfna[i]));
+ hv_restore_withdrawn_pages(pfna + completed, num_pages - completed);
if (lastpfn)
- __free_page(pfn_to_page(lastpfn));
+ hv_restore_withdrawn_pages(&lastpfn, 1);
out_free:
free_page((unsigned long)hc_input);
return rc;
}
+/*
+ * Put withdrawn pages back in the direct map and free them. Counterpart to
+ * the direct map removal done by hv_call_deposit_pages(). Leak the pages
+ * that failed to be restored.
+ */
+void hv_restore_withdrawn_pages(const u64 *pfns, int count)
+{
+ int i, j, end, n, ret;
+ struct page *page;
+
+ for (i = 0; i < count; i = end) {
+ page = pfn_to_page(pfns[i]);
+
+ end = find_consecutive_pfns(pfns, count, i);
+
+ n = end - i;
+ ret = set_direct_map_default_noflush(page, n);
+
+ for (j = i; j < end; ++j) {
+ page = pfn_to_page(pfns[j]);
+ /*
+ * Retry individually on failed batch. We don't want
+ * to return pages to the allocator that we couldn't mark
+ * valid in the direct map.
+ */
+ if (ret &&
+ WARN_ON_ONCE(set_direct_map_default_noflush(page, 1)))
+ continue;
+ __free_page(page);
+ }
+ }
+}
+EXPORT_SYMBOL_GPL(hv_restore_withdrawn_pages);
+
int hv_deposit_memory_node(int node, u64 pt_id, u64 hv_status)
{
int result = hv_result(hv_status);diff --git a/drivers/hv/mshv_root_hv_call.c b/drivers/hv/mshv_root_hv_call.c
index b8d199f95299..80256ddaa768 100644
--- a/drivers/hv/mshv_root_hv_call.c
+++ b/drivers/hv/mshv_root_hv_call.c
@@ -44,7 +44,6 @@ int hv_call_withdraw_memory(u64 count, int node, u64 partition_id)
struct page *page;
u16 completed;
u64 status, withdrawn = 0;
- int i;
unsigned long flags;
page = alloc_page(GFP_KERNEL);
@@ -67,8 +66,7 @@ int hv_call_withdraw_memory(u64 count, int node, u64 partition_id)
completed = hv_repcomp(status);
- for (i = 0; i < completed; i++)
- __free_page(pfn_to_page(output_page->gpa_page_list[i]));
+ hv_restore_withdrawn_pages(output_page->gpa_page_list, completed);
if (!hv_result_success(status)) {
if (hv_result(status) == HV_STATUS_NO_RESOURCES)diff --git a/include/asm-generic/mshyperv.h b/include/asm-generic/mshyperv.h
index 4c9f3353074a..b4dc9cbf0fa6 100644
--- a/include/asm-generic/mshyperv.h
+++ b/include/asm-generic/mshyperv.h
@@ -347,6 +347,7 @@ static inline bool hv_parent_partition(void)
bool hv_result_needs_memory(u64 status);
int hv_deposit_memory_node(int node, u64 partition_id, u64 status);
+void hv_restore_withdrawn_pages(const u64 *pfns, int count);
int hv_call_add_logical_proc(int node, u32 lp_index, u32 acpi_id);
int hv_call_notify_all_processors_started(void);
bool hv_lp_exists(u32 lp_index);
@@ -361,6 +362,9 @@ static inline int hv_deposit_memory_node(int node, u64 partition_id, u64 status)
{
return -EOPNOTSUPP;
}
+
+static inline void hv_restore_withdrawn_pages(const u64 *pfns, int count) { }
+
static inline int hv_call_add_logical_proc(int node, u32 lp_index, u32 acpi_id)
{
return -EOPNOTSUPP;--
2.34.1