Thread (13 messages) 13 messages, 1 author, 16h ago
HOTtoday

[PATCH v9 07/12] iommu/arm-smmu-v3-kexec: Add ASID/VMID reservation helpers

From: Nicolin Chen <hidden>
Date: 2026-07-21 19:49:51
Also in: linux-iommu, lkml
Subsystem: arm smmu drivers, iommu subsystem, the rest · Maintainers: Will Deacon, Joerg Roedel, Linus Torvalds

An adopted stream table keeps translating in-flight DMAs, so the SMMU also
keeps caching the TLB entries tagged with the previous kernel's ASIDs and
VMIDs. Any kexec flavor retaining those translations must reserve all the
in-use IDs, so that this kernel cannot ever give any of its own domains an
overlapping ID that would alias the previous kernel's cached TLB entries.

Add the reservation helpers to arm-smmu-v3-kexec.c:
 - arm_smmu_kexec_scan_and_resv_ids()
 - arm_smmu_kexec_unresv_ids()

At adoption time, arm_smmu_kexec_scan_and_resv_ids() runs a one-off scan:
walking the stream table set up in the strtab_cfg and every CD table behind
an S1 STE, it reserves the ASIDs in the asid xarray and the S2VMIDs in the
vmid ida. The scan memremaps each table transiently, which is a necessity
even for a live update: the ID reservation must be complete by the end of
the SMMU probe itself, before this kernel starts allocating any ID for its
own domains, yet the CD tables are only claimed when their masters get to
re-probe, long after that point. Thus, the scan cannot rely on any claimed
mapping and must map each table by itself.

The scan walks untrusted tables, yet every loop is strictly index-bounded:
the iteration counts derive from the log2size and s1cdmax fields, which are
validated against this kernel's own sid_bits and ssid_bits, so a corrupted
table cannot extend the walk.

Since both kernels allocate ASIDs from 1, reserving any in-use ASID would
often find it already taken in the xarray, either by another scan or by a
live domain. A reservation made by another scan is permanent, so it is safe
to rely on. A live domain will free its ASID for reuse at some point, and
the scan must then fail and fall back. To tell these two cases apart, store
the per-scan ID as an xarray value entry: a value entry identifies another
scan's reservation and a pointer entry identifies a live domain. Note that
the scan ID scopes a failing scan's rollback to exactly the entries that it
inserted, so a rollback cannot erase another scan's reservations.

Also add an arm_smmu_kexec_resv_lock mutex serializing entire scans and any
rollback of failing ones, so that a concurrently probing SMMU can only rely
on the reservations that will persist.

Assisted-by: Claude:claude-fable-5
Signed-off-by: Nicolin Chen <redacted>
---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h   |   4 +
 .../iommu/arm/arm-smmu-v3/arm-smmu-v3-kexec.c | 260 ++++++++++++++++++
 2 files changed, 264 insertions(+)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 183c617ecb273..5c03de5aed27b 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -1241,6 +1241,8 @@ tegra241_cmdqv_probe(struct arm_smmu_device *smmu)
 #endif /* CONFIG_TEGRA241_CMDQV */
 
 #ifdef CONFIG_ARM_SMMU_V3_KEXEC
+extern struct mutex arm_smmu_kexec_resv_lock;
+
 int arm_smmu_kexec_parse_strtab_2lvl(struct arm_smmu_device *smmu, u32 cfg_reg,
 				     phys_addr_t base, u32 *num_l1_ents);
 int arm_smmu_kexec_parse_strtab_linear(struct arm_smmu_device *smmu,
@@ -1252,6 +1254,8 @@ int arm_smmu_kexec_check_strtab_l1_desc(struct arm_smmu_device *smmu,
 int arm_smmu_kexec_check_ste_cdtab(struct arm_smmu_device *smmu, u64 ste0,
 				   phys_addr_t *cdtab, u32 *s1fmt,
 				   u32 *max_contexts);
+int arm_smmu_kexec_scan_and_resv_ids(struct arm_smmu_device *smmu);
+void arm_smmu_kexec_unresv_ids(struct arm_smmu_device *smmu);
 #endif /* CONFIG_ARM_SMMU_V3_KEXEC */
 
 #ifdef CONFIG_CRASH_DUMP
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kexec.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kexec.c
index 5ad1fe1922eec..582192ab8390c 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kexec.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kexec.c
@@ -199,3 +199,263 @@ int arm_smmu_kexec_check_ste_cdtab(struct arm_smmu_device *smmu, u64 ste0,
 	*max_contexts = 1U << s1cdmax;
 	return 0;
 }
+
+/*
+ * Identifies the ASID reservations of an arm_smmu_kexec_scan_and_resv_ids call.
+ *
+ * A failing scan can only roll back its own insertions. Two scans of the same
+ * SMMU (e.g. a probe retry) must not share an ID, as the retried scan may fail
+ * and roll back, while the prior one's committed reservations must persist. So
+ * it must not be a per-SMMU value, but a per-scan one.
+ *
+ * Serialized by arm_smmu_kexec_resv_lock.
+ */
+static unsigned long arm_smmu_kexec_scan_id;
+DEFINE_MUTEX(arm_smmu_kexec_resv_lock);
+
+static int arm_smmu_kexec_resv_asid(struct arm_smmu_device *smmu, u32 asid)
+{
+	int ret;
+
+	/* A valid CD never has ASID 0; both kernels share the same HW limit */
+	if (!asid || asid >= 1UL << smmu->asid_bits)
+		return -EINVAL;
+
+	guard(mutex)(&arm_smmu_asid_lock);
+
+	/* The value entry marks the ASID as in-use and identifies its scan */
+	ret = xa_insert(&arm_smmu_asid_xa, asid,
+			xa_mk_value(arm_smmu_kexec_scan_id), GFP_KERNEL);
+	/*
+	 * An -EBUSY against a value entry safely shares a permanent reservation
+	 * made by another scan. A pointer entry means a live domain that will
+	 * free its ASID for reuse eventually: keep -EBUSY to fail the scan.
+	 */
+	if (ret == -EBUSY && xa_is_value(xa_load(&arm_smmu_asid_xa, asid)))
+		ret = 0;
+	return ret;
+}
+
+static int arm_smmu_kexec_resv_vmid(struct arm_smmu_device *smmu, u32 vmid)
+{
+	int ret;
+
+	/* A translating STE never has VMID 0, which is reserved for bypass */
+	if (!vmid || vmid >= 1UL << smmu->vmid_bits)
+		return -EINVAL;
+
+	ret = ida_alloc_range(&smmu->vmid_map, vmid, vmid, GFP_KERNEL);
+	if (ret < 0 && ret != -ENOSPC) /* -ENOSPC means already reserved */
+		return ret;
+	return 0;
+}
+
+static int arm_smmu_kexec_resv_cd_asids(struct arm_smmu_device *smmu,
+					struct arm_smmu_cd *cds, u32 num_cds)
+{
+	int ret = 0;
+	u32 i;
+
+	for (i = 0; i < num_cds; i++) {
+		u64 val = le64_to_cpu(cds[i].data[0]);
+		u32 asid = FIELD_GET(CTXDESC_CD_0_ASID, val);
+
+		if (!(val & CTXDESC_CD_0_V))
+			continue;
+		ret = arm_smmu_kexec_resv_asid(smmu, asid);
+		if (ret)
+			break;
+	}
+	return ret;
+}
+
+/*
+ * Reserve the ASIDs of all the valid CDs of an S1 STE in the previous kernel's
+ * CD tables. The CD tables are transiently memremapped for the scan.
+ */
+static int arm_smmu_kexec_resv_s1_asids(struct arm_smmu_device *smmu, u64 ste0)
+{
+	struct arm_smmu_cdtab_l1 *l1tab;
+	u32 num_l1_ents, num_cds, i;
+	u32 max_contexts, s1fmt;
+	phys_addr_t cdtab;
+	int ret;
+
+	ret = arm_smmu_kexec_check_ste_cdtab(smmu, ste0, &cdtab, &s1fmt,
+					     &max_contexts);
+	if (ret)
+		return ret;
+
+	if (s1fmt == STRTAB_STE_0_S1FMT_LINEAR) {
+		struct arm_smmu_cd *cds;
+
+		cds = memremap(cdtab, max_contexts * sizeof(*cds), MEMREMAP_WB);
+		if (!cds)
+			return -ENOMEM;
+		ret = arm_smmu_kexec_resv_cd_asids(smmu, cds, max_contexts);
+		memunmap(cds);
+		return ret;
+	}
+
+	num_l1_ents = DIV_ROUND_UP(max_contexts, CTXDESC_L2_ENTRIES);
+	l1tab = memremap(cdtab, num_l1_ents * sizeof(*l1tab), MEMREMAP_WB);
+	if (!l1tab)
+		return -ENOMEM;
+
+	/* max_contexts being under a full leaf makes the only leaf partial */
+	num_cds = min_t(u32, max_contexts, CTXDESC_L2_ENTRIES);
+
+	/* Aliased L2 tables cannot extend the walk; they only repeat a scan */
+	for (i = 0; i < num_l1_ents; i++) {
+		u64 l1_desc = le64_to_cpu(l1tab[i].l2ptr);
+		phys_addr_t l2_base = l1_desc & CTXDESC_L1_DESC_L2PTR_MASK;
+		struct arm_smmu_cdtab_l2 *l2;
+
+		if (!(l1_desc & CTXDESC_L1_DESC_V))
+			continue;
+
+		/* A valid descriptor never carries a null pointer */
+		if (!l2_base) {
+			ret = -EINVAL;
+			break;
+		}
+
+		l2 = memremap(l2_base, num_cds * sizeof(*l2->cds), MEMREMAP_WB);
+		if (!l2) {
+			ret = -ENOMEM;
+			break;
+		}
+		ret = arm_smmu_kexec_resv_cd_asids(smmu, l2->cds, num_cds);
+		memunmap(l2);
+		if (ret)
+			break;
+	}
+	memunmap(l1tab);
+	return ret;
+}
+
+static int arm_smmu_kexec_resv_ste_ids(struct arm_smmu_device *smmu,
+				       struct arm_smmu_ste *ste)
+{
+	u32 vmid = FIELD_GET(STRTAB_STE_2_S2VMID, le64_to_cpu(ste->data[2]));
+	u64 ste0 = le64_to_cpu(ste->data[0]);
+
+	if (!(ste0 & STRTAB_STE_0_V))
+		return 0;
+
+	switch (FIELD_GET(STRTAB_STE_0_CFG, ste0)) {
+	case STRTAB_STE_0_CFG_ABORT:
+	case STRTAB_STE_0_CFG_BYPASS:
+		return 0;
+	case STRTAB_STE_0_CFG_S1_TRANS:
+		return arm_smmu_kexec_resv_s1_asids(smmu, ste0);
+	case STRTAB_STE_0_CFG_NESTED:
+		/*
+		 * A guest-owned CD table is in the IPA space, unreachable. Its
+		 * ASIDs are only tagged with the S2VMID reserved below, so they
+		 * cannot alias this kernel's VMID-0 or EL2 S1 domains.
+		 */
+		fallthrough;
+	case STRTAB_STE_0_CFG_S2_TRANS:
+		return arm_smmu_kexec_resv_vmid(smmu, vmid);
+	default:
+		return -EINVAL;
+	}
+}
+
+/**
+ * arm_smmu_kexec_scan_and_resv_ids() - Reserve a stream table's in-use IDs
+ * @smmu: SMMU device of this kernel, with an adopted or restored strtab_cfg
+ *
+ * Scan the stream table set up in the strtab_cfg and every CD table behind an
+ * S1 STE, reserving all of the in-use ASIDs and VMIDs. The caller must hold the
+ * arm_smmu_kexec_resv_lock, so that a failing scan can roll back, prior to any
+ * concurrent scan, via arm_smmu_kexec_unresv_ids().
+ *
+ * Note that the scan selects the linear or 2-level walk per this kernel's own
+ * ARM_SMMU_FEAT_2_LVL_STRTAB, so the caller must have matched the feature bit
+ * to the format of the adopted stream table in the strtab_cfg.
+ *
+ * Return: 0 on success, -EINVAL on any malformed table entry, or -ENOMEM on a
+ * memory shortage
+ */
+int arm_smmu_kexec_scan_and_resv_ids(struct arm_smmu_device *smmu)
+{
+	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
+	int ret = 0;
+	u32 i, j;
+
+	lockdep_assert_held(&arm_smmu_kexec_resv_lock);
+
+	/* Allocate a new ID for this scan, to scope its rollback */
+	arm_smmu_kexec_scan_id++;
+
+	if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)) {
+		for (i = 0; i < cfg->linear.num_ents; i++) {
+			ret = arm_smmu_kexec_resv_ste_ids(
+				smmu, &cfg->linear.table[i]);
+			if (ret)
+				return ret;
+		}
+		return 0;
+	}
+
+	/* Aliased L2 tables cannot extend the scan; they only repeat a scan */
+	for (i = 0; i < cfg->l2.num_l1_ents; i++) {
+		u64 l1_desc = le64_to_cpu(cfg->l2.l1tab[i].l2ptr);
+		struct arm_smmu_strtab_l2 *l2;
+		phys_addr_t base;
+
+		ret = arm_smmu_kexec_check_strtab_l1_desc(smmu, l1_desc, i,
+							  &base);
+		if (ret == 1)
+			continue;
+		if (ret)
+			return ret;
+
+		/*
+		 * This kernel will map the previous kernel's L2 tables lazily
+		 * or not at all. Here, take a transient view for this scan.
+		 */
+		l2 = memremap(base, sizeof(*l2), MEMREMAP_WB);
+		if (!l2)
+			return -ENOMEM;
+		for (j = 0; j < ARRAY_SIZE(l2->stes); j++) {
+			ret = arm_smmu_kexec_resv_ste_ids(smmu, &l2->stes[j]);
+			if (ret)
+				break;
+		}
+		memunmap(l2);
+		if (ret)
+			return ret;
+	}
+	return 0;
+}
+
+/**
+ * arm_smmu_kexec_unresv_ids() - Roll back a failing reservation scan
+ * @smmu: SMMU device of this kernel that failed its reservation scan
+ *
+ * Roll back the current scan for a failing arm_smmu_kexec_scan_and_resv_ids()
+ * call, typically to a full reset.
+ *
+ * arm_smmu_kexec_resv_lock must be held for arm_smmu_kexec_scan_and_resv_ids()
+ * and the roll back.
+ */
+void arm_smmu_kexec_unresv_ids(struct arm_smmu_device *smmu)
+{
+	unsigned long index;
+	void *entry;
+
+	lockdep_assert_held(&arm_smmu_kexec_resv_lock);
+
+	mutex_lock(&arm_smmu_asid_lock);
+	xa_for_each(&arm_smmu_asid_xa, index, entry) {
+		if (entry == xa_mk_value(arm_smmu_kexec_scan_id))
+			xa_erase(&arm_smmu_asid_xa, index);
+	}
+	mutex_unlock(&arm_smmu_asid_lock);
+
+	/* No domain exists yet, so the ida holds only the reservations */
+	ida_destroy(&smmu->vmid_map);
+}
-- 
2.43.0

Keyboard shortcuts
hback out one level
jnext message in thread
kprevious message in thread
ldrill in
Escclose help / fold thread tree
?toggle this help