[PATCH 2/6] soc: qcom: qpace: Add Qualcomm Page Compression Engine driver

@@ -288,6 +288,20 @@ config QCOM_PBS
 	  This module provides the APIs to the client drivers that wants to send the
 	  PBS trigger event to the PBS RAM.
 
+config QCOM_PAGE_COMPRESSION_ENGINE
+	tristate "Qualcomm Page Compression Engine (QPaCE)"
+	depends on ARM64
+	depends on ARCH_QCOM || COMPILE_TEST
+	depends on OF
+	depends on INTERCONNECT
+	help
+	  Enable support for the Qualcomm Page Compression Engine (QPaCE),
+	  a hardware accelerator that provides high-throughput page compression,
+	  decompression, and DMA copy operations.
+
+	  The engine is used as a hardware backend for compressed-memory
+	  subsystems such as zram. If unsure, say N.
+
 endif
 
 # Options selected by other drivers from different subsystems must be outside

@@ -42,3 +42,4 @@ qcom_ice-objs			+= ice.o
 obj-$(CONFIG_QCOM_INLINE_CRYPTO_ENGINE)	+= qcom_ice.o
 obj-$(CONFIG_QCOM_PBS) +=	qcom-pbs.o
 obj-$(CONFIG_QCOM_UBWC_CONFIG) += ubwc_config.o
+obj-$(CONFIG_QCOM_PAGE_COMPRESSION_ENGINE) += qpace.o

@@ -0,0 +1,764 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <dt-bindings/interconnect/qcom,icc.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/export.h>
+#include <linux/interconnect.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/pm_qos.h>
+#include <linux/soc/qcom/qpace.h>
+#include <linux/workqueue.h>
+
+#include "qpace_internal.h"
+
+#define QPACE_REG_PAGE_SIZE 4096
+#define QPACE_CORE_CACHEINDEX_SIZE 4
+
+struct qpace_priv {
+	void __iomem *gen_regs;
+	void __iomem *gen_core_regs;
+	void __iomem *comp_core_regs;
+	void __iomem *decomp_core_regs;
+	void __iomem *urg_regs;
+
+	struct device *dev;
+	u32 hw_version;
+	bool suspended;
+
+	struct icc_path *interconnect;
+	struct pm_qos_request qos_req;
+
+	int active_rings;
+	struct completion no_active_refs;
+
+	bool broken;
+	struct work_struct disable_work;
+};
+
+static inline u32 qpace_read_gen(struct qpace_priv *p, u32 offset)
+{
+	return readl(p->gen_regs + offset);
+}
+
+static inline void qpace_write_gen(struct qpace_priv *p, u32 offset, u32 val)
+{
+	writel(val, p->gen_regs + offset);
+}
+
+static inline u32 qpace_read_gen_core(struct qpace_priv *p, u32 offset)
+{
+	return readl(p->gen_core_regs + offset);
+}
+
+static inline void qpace_write_gen_core(struct qpace_priv *p, u32 offset, u32 val)
+{
+	writel(val, p->gen_core_regs + offset);
+}
+
+static inline u32 qpace_read_comp_core(struct qpace_priv *p, u32 offset)
+{
+	return readl(p->comp_core_regs + offset);
+}
+
+static inline void qpace_write_comp_core(struct qpace_priv *p, u32 offset, u32 val)
+{
+	writel(val, p->comp_core_regs + offset);
+}
+
+static inline u32 qpace_read_urg_cacheindex(struct qpace_priv *p, int idx, u32 offset)
+{
+	return readl(p->gen_regs + idx * QPACE_CORE_CACHEINDEX_SIZE + offset);
+}
+
+static inline void qpace_write_urg_cacheindex(struct qpace_priv *p, int idx, u32 offset, u32 val)
+{
+	writel(val, p->gen_regs + idx * QPACE_CORE_CACHEINDEX_SIZE + offset);
+}
+
+static inline u32 qpace_read_decomp_core(struct qpace_priv *p, u32 offset)
+{
+	return readl(p->decomp_core_regs + offset);
+}
+
+static inline void qpace_write_decomp_core(struct qpace_priv *p, u32 offset, u32 val)
+{
+	writel(val, p->decomp_core_regs + offset);
+}
+
+static inline u32 qpace_read_urg_cmd(struct qpace_priv *p, int idx, u32 offset)
+{
+	return readl(p->urg_regs + idx * QPACE_REG_PAGE_SIZE + offset);
+}
+
+static inline void qpace_write_urg_cmd(struct qpace_priv *p, int idx, u32 offset, u32 val)
+{
+	writel(val, p->urg_regs + idx * QPACE_REG_PAGE_SIZE + offset);
+}
+
+static inline void qpace_write_urg_cmd_ctx(struct qpace_priv *p, u32 offset,
+					   int urg_reg_num, int ctx_num, u32 val)
+{
+	writel(val, p->urg_regs + urg_reg_num * QPACE_REG_PAGE_SIZE +
+	       offset + ctx_num * URG_CMD_CONTEXT_SPACING);
+}
+
+static struct qpace_priv *qpace_priv;
+
+static DEFINE_STATIC_KEY_FALSE(qpace_drv_probed);
+
+static void qpace_disable_work_fn(struct work_struct *work)
+{
+	static_branch_disable(&qpace_drv_probed);
+}
+
+enum urg_reg_cxts {
+	LZ4_URG_COMP_CNTXT,
+	LZ4_URG_DECOMP_CNTXT,
+};
+
+const struct qpace_algorithm qpace_lz4_algorithm = {
+	.name = "qpace-lz4",
+	.comp_opcode = LZ4_COMP,
+	.decomp_opcode = LZ4_DECOMP,
+	.urg_comp_cntxt = LZ4_URG_COMP_CNTXT,
+	.urg_decomp_cntxt = LZ4_URG_DECOMP_CNTXT,
+};
+EXPORT_SYMBOL_GPL(qpace_lz4_algorithm);
+
+bool qpace_is_valid_algorithm(const char *algo_name)
+{
+	if (!static_branch_likely(&qpace_drv_probed))
+		return false;
+	if (READ_ONCE(qpace_priv->broken))
+		return false;
+
+	return algo_name && !strcmp(algo_name, qpace_lz4_algorithm.name);
+}
+EXPORT_SYMBOL_GPL(qpace_is_valid_algorithm);
+
+static inline void program_urg_comp_context(int urg_reg_num, const struct qpace_algorithm *algo)
+{
+	u32 urg_cmd_settings = 0;
+	int page_cnt = (1 << (PAGE_SHIFT - 12)) - 1;
+
+	/* Set the limit of the compression output size */
+	urg_cmd_settings = FIELD_PREP(URG_CMD_0_CFG_CNTXT_SIZE_SIZE, PAGE_SIZE - 1);
+	qpace_write_urg_cmd_ctx(qpace_priv, QPACE_URG_CMD_0_CFG_CNTXT_SIZE_n_OFFSET,
+				urg_reg_num, algo->urg_comp_cntxt,
+				urg_cmd_settings);
+
+	urg_cmd_settings = FIELD_PREP(URG_CMD_0_CFG_CNTXT_MISC_OPER, algo->comp_opcode);
+
+	/* Set page count for urgent compression input */
+	urg_cmd_settings |= FIELD_PREP(URG_CMD_0_CFG_CNTXT_MISC_PAGE_CNT, page_cnt);
+
+	/* Program part of the SMMU input and output SIDs */
+	urg_cmd_settings |= FIELD_PREP(URG_CMD_0_CFG_CNTXT_MISC_SCTX,
+				       DEFAULT_SMMU_CONTEXT);
+	urg_cmd_settings |= FIELD_PREP(URG_CMD_0_CFG_CNTXT_MISC_DCTX,
+				       DEFAULT_SMMU_CONTEXT);
+
+	/* Cache accesses in the system cache during compression */
+	urg_cmd_settings |= FIELD_PREP(URG_CMD_0_CFG_CNTXT_MISC_WA, 1);
+
+	qpace_write_urg_cmd_ctx(qpace_priv, QPACE_URG_CMD_0_CFG_CNTXT_MISC_n_OFFSET,
+				urg_reg_num, algo->urg_comp_cntxt,
+				urg_cmd_settings);
+}
+
+static inline void program_urg_decomp_context(int urg_reg_num, const struct qpace_algorithm *algo)
+{
+	u32 urg_cmd_settings = 0;
+	int page_cnt = (1 << (PAGE_SHIFT - 12)) - 1;
+
+	/* The input size will be programmed by a requester */
+	urg_cmd_settings = FIELD_PREP(URG_CMD_0_CFG_CNTXT_SIZE_SIZE, 0);
+	qpace_write_urg_cmd_ctx(qpace_priv, QPACE_URG_CMD_0_CFG_CNTXT_SIZE_n_OFFSET,
+				urg_reg_num, algo->urg_decomp_cntxt,
+				urg_cmd_settings);
+
+	urg_cmd_settings = FIELD_PREP(URG_CMD_0_CFG_CNTXT_MISC_OPER, algo->decomp_opcode);
+
+	/* Configure page count which will limit decompression output size */
+	urg_cmd_settings |= FIELD_PREP(URG_CMD_0_CFG_CNTXT_MISC_PAGE_CNT, page_cnt);
+
+	/* Program part of the SMMU input and output SIDs */
+	urg_cmd_settings |= FIELD_PREP(URG_CMD_0_CFG_CNTXT_MISC_SCTX,
+				       DEFAULT_SMMU_CONTEXT);
+	urg_cmd_settings |= FIELD_PREP(URG_CMD_0_CFG_CNTXT_MISC_DCTX,
+				       DEFAULT_SMMU_CONTEXT);
+
+	/* Cache accesses in the system cache during decompression */
+	urg_cmd_settings |= FIELD_PREP(URG_CMD_0_CFG_CNTXT_MISC_WA, 1);
+
+	qpace_write_urg_cmd_ctx(qpace_priv, QPACE_URG_CMD_0_CFG_CNTXT_MISC_n_OFFSET,
+				urg_reg_num, algo->urg_decomp_cntxt,
+				urg_cmd_settings);
+}
+
+static void program_urg_cmd_cacheindex(int urg_reg_num)
+{
+	u32 reg_val;
+
+	reg_val = qpace_read_urg_cacheindex(qpace_priv, urg_reg_num,
+					    QPACE_CORE_URG_CMD_n_CACHEINDEX_CFG_OFFSET);
+	reg_val = u32_replace_bits(reg_val, 0x24, CORE_URG_CMD_n_CACHEINDEX__ENGINE_CACHEINDEX);
+	reg_val = u32_replace_bits(reg_val, 0x1,
+				   CORE_URG_CMD_n_CACHEINDEX__ENGINE_CACHEINDEX_OVERRIDE);
+	qpace_write_urg_cacheindex(qpace_priv, urg_reg_num,
+				   QPACE_CORE_URG_CMD_n_CACHEINDEX_CFG_OFFSET, reg_val);
+}
+
+static void program_decomp_core_cfg(void)
+{
+	u32 reg_val;
+
+	reg_val = qpace_read_decomp_core(qpace_priv, QPACE_DECOMP_CORE_CFG_OFFSET);
+	reg_val = u32_replace_bits(reg_val, 0x8, DECOMP_CORE_CFG_DMA_RD_MAX_OT);
+	reg_val = u32_replace_bits(reg_val, 0x16, DECOMP_CORE_CFG_DMA_WR_MAX_OT);
+	/*
+	 * MEM_MNG_PAGE_SIZE controls the prefetch boundary:
+	 * 0: 4 KB, 1: 8 KB, 2: 16 KB, 3: 32 KB, 4: 64 KB
+	 */
+	reg_val = u32_replace_bits(reg_val, 0x0, DECOMP_CORE_CFG_DECOMP_DMA_MEM_MNG_PAGE_SIZE);
+	qpace_write_decomp_core(qpace_priv, QPACE_DECOMP_CORE_CFG_OFFSET, reg_val);
+}
+
+static void program_urg_command_contexts_v2(void)
+{
+	int urg_reg_num;
+
+	/* Configure the needed contexts for each of the urgent command registers */
+	for (urg_reg_num = 0; urg_reg_num < NUM_TRS_ERS_URG_CMD_REGS; urg_reg_num++) {
+		program_urg_cmd_cacheindex(urg_reg_num);
+		program_urg_comp_context(urg_reg_num, &qpace_lz4_algorithm);
+		program_urg_decomp_context(urg_reg_num, &qpace_lz4_algorithm);
+	}
+}
+
+static inline int qpace_urgent_command_trigger(dma_addr_t input_addr,
+					       dma_addr_t output_addr,
+					       int urg_reg_num,
+					       enum urg_reg_cxts command)
+{
+	void __iomem *td_dst_src_reg = qpace_priv->urg_regs +
+				       (urg_reg_num * QPACE_REG_PAGE_SIZE) +
+					QPACE_URG_CMD_0_TD_DST_ADDR_L_CFG_CNTXT_OFFSET;
+	u64 urg_addr_field_lower, urg_addr_field_upper;
+	u32 stat_reg;
+	unsigned long ret;
+
+	urg_addr_field_lower = FIELD_PREP(URG_CMD_0_TD_DST_ADDR_L__CMD_CFG_CNTXT,
+					  command);
+	urg_addr_field_lower |= GENMASK(63, 8) & output_addr;
+
+	urg_addr_field_upper = input_addr;
+
+	/* Clear a stale request-on-active error before ringing the doorbell. */
+	qpace_write_urg_cmd(qpace_priv, urg_reg_num, QPACE_URG_CMD_0_STAT_CLR_OFFSET,
+			    URG_CMD_0_STAT_CLR_REQ_ON_ACTIVE_ERR);
+
+	/* Ensure that preceding stores that QPaCE will depend on are done executing */
+	dma_wmb();
+
+	/*
+	 * Ring the doorbell with a single 128-bit atomic store. QPaCE
+	 * triggers processing when it observes the (dst_addr, src_addr)
+	 * pair land together.
+	 */
+	asm volatile("stp %0, %1, [%2]\n" :
+		     : "r" (urg_addr_field_lower), "r" (urg_addr_field_upper), "r" (td_dst_src_reg)
+		     : "memory");
+
+	ret = readl_poll_timeout_atomic(qpace_priv->urg_regs +
+					urg_reg_num * QPACE_REG_PAGE_SIZE +
+					QPACE_URG_CMD_0_ED_STAT_OFFSET,
+					stat_reg,
+					FIELD_GET(URG_CMD_0_ED_STAT_COMP_CODE,
+						  stat_reg) != OP_URG_ONGOING,
+					1, 100 * USEC_PER_MSEC);
+	if (ret) {
+		dev_err(qpace_priv->dev, "QPaCE urgent command timed out\n");
+		return -ETIMEDOUT;
+	}
+
+	return stat_reg;
+}
+
+int qpace_urgent_compress(dma_addr_t input_addr,
+			  dma_addr_t output_addr,
+			  struct qpace_algorithm *algo)
+{
+	int urg_reg_num;
+	int stat_reg;
+	u32 stat_reg_val;
+	int ret;
+
+	ret = qpace_get();
+	if (ret)
+		return ret;
+
+	urg_reg_num = get_cpu() % NUM_TRS_ERS_URG_CMD_REGS;
+	stat_reg = qpace_urgent_command_trigger(input_addr, output_addr, urg_reg_num,
+						algo->urg_comp_cntxt);
+	put_cpu();
+
+	qpace_put();
+
+	if (stat_reg < 0) {
+		ret = stat_reg;
+		goto out;
+	}
+
+	stat_reg_val = FIELD_GET(URG_CMD_0_ED_STAT_COMP_CODE, stat_reg);
+
+	if (stat_reg_val == OP_COMP_TOO_BIG) {
+		ret = -E2BIG;
+		goto out;
+	}
+
+	if (stat_reg_val != OP_OK) {
+		pr_err("%s: register %d failed with %u\n",
+		       __func__, urg_reg_num, stat_reg_val);
+		ret = -EINVAL;
+		goto out;
+	}
+
+	ret = FIELD_GET(URG_CMD_0_ED_STAT_SIZE, stat_reg);
+out:
+	return ret;
+}
+EXPORT_SYMBOL_GPL(qpace_urgent_compress);
+
+int qpace_urgent_decompress(dma_addr_t input_addr,
+			    dma_addr_t output_addr,
+			    size_t input_size,
+			    struct qpace_algorithm *algo)
+{
+	int urg_reg_num;
+	int stat_reg;
+	u32 stat_reg_val;
+	int ret;
+
+	ret = qpace_get();
+	if (ret)
+		goto out;
+
+	urg_reg_num = get_cpu() % NUM_TRS_ERS_URG_CMD_REGS;
+	qpace_write_urg_cmd_ctx(qpace_priv, QPACE_URG_CMD_0_CFG_CNTXT_SIZE_n_OFFSET,
+				urg_reg_num, algo->urg_decomp_cntxt,
+				FIELD_PREP(URG_CMD_0_CFG_CNTXT_SIZE_SIZE, input_size));
+	stat_reg = qpace_urgent_command_trigger(input_addr, output_addr, urg_reg_num,
+						algo->urg_decomp_cntxt);
+	put_cpu();
+
+	qpace_put();
+
+	if (stat_reg < 0) {
+		ret = stat_reg;
+		goto out;
+	}
+
+	stat_reg_val = FIELD_GET(URG_CMD_0_ED_STAT_COMP_CODE, stat_reg);
+	if (stat_reg_val != OP_OK) {
+		pr_err("%s: register %d failed with %u\n",
+		       __func__, urg_reg_num, stat_reg_val);
+		ret = -EINVAL;
+		goto out;
+	}
+
+	ret = FIELD_GET(URG_CMD_0_ED_STAT_SIZE, stat_reg);
+out:
+	return ret;
+}
+EXPORT_SYMBOL_GPL(qpace_urgent_decompress);
+
+static DEFINE_MUTEX(qpace_ref_lock);
+
+static void _get_qpace(void)
+{
+	lockdep_assert_held(&qpace_ref_lock);
+	if (!qpace_priv->active_rings) {
+		reinit_completion(&qpace_priv->no_active_refs);
+		pm_stay_awake(qpace_priv->dev);
+		cpu_latency_qos_update_request(&qpace_priv->qos_req, 300);
+		program_urg_command_contexts_v2();
+		program_decomp_core_cfg();
+	}
+	qpace_priv->active_rings++;
+}
+
+static void _put_qpace(void)
+{
+	lockdep_assert_held(&qpace_ref_lock);
+	if (!--qpace_priv->active_rings) {
+		cpu_latency_qos_update_request(&qpace_priv->qos_req, PM_QOS_DEFAULT_VALUE);
+		pm_relax(qpace_priv->dev);
+		complete(&qpace_priv->no_active_refs);
+	}
+}
+
+int qpace_get(void)
+{
+	int ret = 0;
+
+	mutex_lock(&qpace_ref_lock);
+	if (qpace_priv->suspended || READ_ONCE(qpace_priv->broken))
+		ret = -EBUSY;
+	else
+		_get_qpace();
+	mutex_unlock(&qpace_ref_lock);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(qpace_get);
+
+void qpace_put(void)
+{
+	mutex_lock(&qpace_ref_lock);
+	_put_qpace();
+	mutex_unlock(&qpace_ref_lock);
+}
+EXPORT_SYMBOL_GPL(qpace_put);
+
+static irqreturn_t urgent_interrupt_handler(int irq, void *unused)
+{
+	pr_debug("Urgent interrupt handled\n");
+	return IRQ_HANDLED;
+}
+
+static int qpace_hw_init(void)
+{
+	u32 reg_val;
+
+	/* Select CPU SCID for our system cache slice. */
+	reg_val = qpace_read_gen(qpace_priv, QPACE_CORE_QNS4_CFG_OFFSET);
+	reg_val = u32_replace_bits(reg_val, 0x1, CORE_QNS4_CFG_CACHEINDEX);
+	qpace_write_gen(qpace_priv, QPACE_CORE_QNS4_CFG_OFFSET, reg_val);
+
+	/* QMB2 register configurations. */
+	reg_val = qpace_read_gen(qpace_priv, QPACE_CORE_GEN_CFG_OFFSET);
+	reg_val = u32_replace_bits(reg_val, 0x48, CORE_GEN_CFG_QMB2_MAX_RD_OUTST_LIMIT);
+	reg_val = u32_replace_bits(reg_val, 0x48, CORE_GEN_CFG_QMB2_MAX_WR_OUTST_LIMIT);
+	qpace_write_gen(qpace_priv, QPACE_CORE_GEN_CFG_OFFSET, reg_val);
+
+	/* DECOMP_CORE_CFG init steps. */
+	program_decomp_core_cfg();
+
+	/* Below settings help save power since all decomp cores are set to sync. */
+	reg_val = qpace_read_gen_core(qpace_priv, QPACE_CORE_OPER_CFG_OFFSET);
+	reg_val |= CORE_OPER_CFG_COMP_MEM_PWR_DWN_1;
+	qpace_write_gen_core(qpace_priv, QPACE_CORE_OPER_CFG_OFFSET, reg_val);
+
+	reg_val = qpace_read_comp_core(qpace_priv, QPACE_COMP_CORE_CFG_OFFSET);
+	reg_val = u32_replace_bits(reg_val, 0x8, COMP_CORE_CFG_DMA_RD_MAX_OT);
+	reg_val = u32_replace_bits(reg_val, 0x8, COMP_CORE_CFG_DMA_WR_MAX_OT);
+	qpace_write_comp_core(qpace_priv, QPACE_COMP_CORE_CFG_OFFSET, reg_val);
+
+	/* Set all COMP engines to bulk mode. */
+	reg_val = qpace_read_comp_core(qpace_priv, QPACE_COMP_CORE_BULK_MODE_OFFSET);
+	reg_val |= COMP_CORE_BULK_MODE_ALL_CORES;
+	qpace_write_comp_core(qpace_priv, QPACE_COMP_CORE_BULK_MODE_OFFSET, reg_val);
+
+	/* URG CMD register configurations. */
+	program_urg_command_contexts_v2();
+
+	return 0;
+}
+
+enum qpace_interrupts {
+	QPACE_IRQ_URGENT
+};
+
+static int qpace_register_interrupts(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	int irq, ret;
+
+	irq = platform_get_irq(pdev, QPACE_IRQ_URGENT);
+	if (irq < 0)
+		return irq;
+
+	ret = devm_request_irq(dev, irq, urgent_interrupt_handler,
+			       0, "qpace-urgent-irq", NULL);
+	if (ret)
+		dev_err(dev, "failed to request urgent interrupt\n");
+
+	return ret;
+}
+
+static inline bool _qpace_power_on(void)
+{
+	u32 ready_status;
+
+	qpace_write_gen_core(qpace_priv, QPACE_CORE_OPER_CORE_RUN_STOP_OFFSET, QPACE_RUN);
+
+	if (readl_poll_timeout(qpace_priv->gen_core_regs +
+			       QPACE_CORE_OPER_CORE_READY_OFFSET,
+			       ready_status, ready_status,
+			       1000, 5 * QPACE_STATE_CHANGE_TIMEOUT_US)) {
+		pr_err("Timeout in waiting for QPaCE to turn on\n");
+		return false;
+	}
+
+	return true;
+}
+
+static int qpace_power_on(struct device *dev)
+{
+	int ret, ret2;
+
+	qpace_priv->interconnect = devm_of_icc_get(dev, "qpace-mem");
+	if (IS_ERR_OR_NULL(qpace_priv->interconnect)) {
+		ret = PTR_ERR_OR_ZERO(qpace_priv->interconnect);
+		pr_err("%s: devm_of_icc_get() failed with %d\n", __func__, ret);
+		return qpace_priv->interconnect ? ret : -EINVAL;
+	}
+
+	ret = device_init_wakeup(dev, true);
+	if (ret) {
+		pr_err("%s: device_init_wakeup() failed with %d\n", __func__, ret);
+		return ret;
+	}
+
+	cpu_latency_qos_add_request(&qpace_priv->qos_req, PM_QOS_DEFAULT_VALUE);
+
+	icc_set_tag(qpace_priv->interconnect, QCOM_ICC_TAG_ACTIVE_ONLY);
+
+	ret = icc_set_bw(qpace_priv->interconnect, 0, 1);
+	if (ret) {
+		pr_err("Failed to turn on QPaCE VCD: %d\n", ret);
+		goto rm_qos;
+	}
+
+	if (!_qpace_power_on()) {
+		pr_err("Failed to start QPaCE\n");
+		ret = -EINVAL;
+		goto rm_bw;
+	}
+
+	return 0;
+
+rm_bw:
+	ret2 = icc_set_bw(qpace_priv->interconnect, 0, 0);
+	if (ret2)
+		pr_err("Failed to remove QPaCE VCD vote: %d\n", ret2);
+rm_qos:
+	cpu_latency_qos_remove_request(&qpace_priv->qos_req);
+	device_init_wakeup(dev, false);
+
+	return ret;
+}
+
+static inline bool _qpace_power_off(void)
+{
+	u32 ready_status;
+
+	qpace_write_gen_core(qpace_priv, QPACE_CORE_OPER_CORE_RUN_STOP_OFFSET, QPACE_STOP);
+
+	if (readl_poll_timeout(qpace_priv->gen_core_regs +
+			       QPACE_CORE_OPER_CORE_READY_OFFSET,
+			       ready_status, !ready_status,
+			       1000, 5 * QPACE_STATE_CHANGE_TIMEOUT_US)) {
+		pr_err("Timeout in waiting for QPaCE to turn off\n");
+		return false;
+	}
+
+	return true;
+}
+
+static void qpace_power_off(struct device *dev)
+{
+	int ret;
+
+	/* If this fails we can still remove our vote for the VCD to turn QPaCE off */
+	if (!_qpace_power_off())
+		pr_err("Failed to stop QPaCE\n");
+
+	ret = icc_set_bw(qpace_priv->interconnect, 0, 0);
+	if (ret)
+		pr_err("Failed to turn off QPaCE VCD: %d\n", ret);
+
+	cpu_latency_qos_remove_request(&qpace_priv->qos_req);
+
+	device_init_wakeup(dev, false);
+}
+
+static inline int qpace_register_ioremap(struct platform_device *pdev)
+{
+	qpace_priv->gen_regs = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(qpace_priv->gen_regs))
+		return PTR_ERR(qpace_priv->gen_regs);
+
+	qpace_priv->gen_core_regs = qpace_priv->gen_regs + QPACE_GEN_CORE_REGS_OFFSET;
+	qpace_priv->comp_core_regs = qpace_priv->gen_regs + QPACE_COMP_CORE_REGS_OFFSET;
+	qpace_priv->decomp_core_regs = qpace_priv->gen_regs + QPACE_DECOMP_CORE_REGS_OFFSET;
+	qpace_priv->urg_regs = qpace_priv->gen_regs + QPACE_URG_REGS_OFFSET;
+
+	return 0;
+}
+
+bool qpace_is_dev_available(void)
+{
+	return static_branch_likely(&qpace_drv_probed) &&
+	       !READ_ONCE(qpace_priv->broken);
+}
+EXPORT_SYMBOL_GPL(qpace_is_dev_available);
+
+struct device *qpace_get_dma_dev(void)
+{
+	return qpace_priv->dev;
+}
+EXPORT_SYMBOL_GPL(qpace_get_dma_dev);
+
+static int qpace_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct qpace_priv *priv;
+	int ret;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->dev = dev;
+	/* Starts already complete since active_rings == 0 at init. */
+	init_completion(&priv->no_active_refs);
+	complete(&priv->no_active_refs);
+	INIT_WORK(&priv->disable_work, qpace_disable_work_fn);
+	qpace_priv = priv;
+	platform_set_drvdata(pdev, priv);
+
+	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to set DMA mask\n");
+
+	ret = qpace_register_ioremap(pdev);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to map QPaCE registers\n");
+
+	ret = qpace_power_on(dev);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to power on QPaCE\n");
+
+	/* Get QPaCE HW version. */
+	qpace_priv->hw_version = qpace_read_gen(qpace_priv, QPACE_CORE_HW_VERSION_OFFSET);
+	if (qpace_priv->hw_version != QPACE_HW_VERSION_V2) {
+		dev_err(dev, "Unsupported QPaCE HW version returned: 0x%x\n",
+			qpace_priv->hw_version);
+		ret = -EINVAL;
+		goto power_off;
+	}
+
+	ret = qpace_hw_init();
+	if (ret) {
+		dev_err(dev, "init failed: (%d)\n", ret);
+		goto power_off;
+	}
+
+	ret = qpace_register_interrupts(pdev);
+	if (ret) {
+		dev_err(dev, "failed to register interrupts\n");
+		goto power_off;
+	}
+
+	static_branch_enable(&qpace_drv_probed);
+
+	return ret;
+
+power_off:
+	qpace_power_off(dev);
+
+	return ret;
+}
+
+static void qpace_remove(struct platform_device *pdev)
+{
+	mutex_lock(&qpace_ref_lock);
+	qpace_priv->suspended = true;
+	mutex_unlock(&qpace_ref_lock);
+
+	static_branch_disable(&qpace_drv_probed);
+
+	wait_for_completion(&qpace_priv->no_active_refs);
+
+	/* No callers remain; tear down the hardware. */
+	cancel_work_sync(&qpace_priv->disable_work);
+	qpace_power_off(&pdev->dev);
+}
+
+static const struct of_device_id qpace_match_table[] = {
+	{ .compatible = "qcom,hawi-qpace" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, qpace_match_table);
+
+static int qpace_suspend(struct device *dev)
+{
+	mutex_lock(&qpace_ref_lock);
+
+	qpace_priv->suspended = true;
+
+	if (qpace_priv->active_rings) {
+		dev_err(dev, "active_rings not 0, abort suspend\n");
+		qpace_priv->suspended = false;
+		mutex_unlock(&qpace_ref_lock);
+		return -EBUSY;
+	}
+
+	mutex_unlock(&qpace_ref_lock);
+	return 0;
+}
+
+static int qpace_resume(struct device *dev)
+{
+	int ret;
+
+	mutex_lock(&qpace_ref_lock);
+
+	if (qpace_priv->active_rings)
+		dev_err(dev, "active_rings not 0, unexpected case\n");
+
+	ret = icc_set_bw(qpace_priv->interconnect, 0, 1);
+	if (ret)
+		goto out_unlock;
+
+	program_urg_command_contexts_v2();
+	program_decomp_core_cfg();
+
+	qpace_priv->suspended = false;
+
+out_unlock:
+	if (ret)
+		dev_err(dev, "failed to resume QPaCE: %d\n", ret);
+	mutex_unlock(&qpace_ref_lock);
+	return ret;
+}
+
+static DEFINE_SIMPLE_DEV_PM_OPS(qpace_pm_ops, qpace_suspend, qpace_resume);
+
+static struct platform_driver qpace_driver = {
+	.probe = qpace_probe,
+	.remove = qpace_remove,
+	.driver = {
+		.name = "qcom-qpace",
+		.of_match_table = qpace_match_table,
+		.pm = pm_sleep_ptr(&qpace_pm_ops),
+	},
+};
+
+module_platform_driver(qpace_driver);
+
+MODULE_DESCRIPTION("Qualcomm Page Compression Engine driver");
+MODULE_LICENSE("GPL");

@@ -0,0 +1,84 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef __QPACE_INTERNAL_H__
+#define __QPACE_INTERNAL_H__
+
+#include <linux/bits.h>
+#include <linux/bitfield.h>
+
+/* hardware definitions and opcode values */
+#define NUM_TRS_ERS_URG_CMD_REGS 8
+#define QPACE_HW_VERSION_V2 0x2000000
+#define DEFAULT_SMMU_CONTEXT 0
+
+#define LZ4_COMP 0x9
+#define LZ4_DECOMP 0xa
+
+#define OP_OK 0x0
+#define OP_BUS_ERROR 0x1
+#define OP_COMP_TOO_BIG 0x2
+#define OP_TIMED_OUT 0x5
+#define OP_URG_ONGOING 0xf
+
+#define LLC_ALLOC 0x1
+
+#define URG_CMD_CONTEXT_SPACING 0x10
+
+#define QPACE_RUN 0x1
+#define QPACE_STOP 0x0
+#define QPACE_STATE_CHANGE_TIMEOUT_US 10000
+
+/* register offsets */
+#define QPACE_GEN_CORE_REGS_OFFSET 0x1000
+#define QPACE_COMP_CORE_REGS_OFFSET 0x2000
+#define QPACE_DECOMP_CORE_REGS_OFFSET 0x6000
+#define QPACE_URG_REGS_OFFSET 0x10000
+
+#define QPACE_CORE_HW_VERSION_OFFSET 0x0
+#define QPACE_CORE_QNS4_CFG_OFFSET 0x14
+#define CORE_QNS4_CFG_CACHEINDEX GENMASK(17, 12)
+
+#define QPACE_CORE_GEN_CFG_OFFSET 0x18
+#define CORE_GEN_CFG_QMB2_MAX_RD_OUTST_LIMIT GENMASK(13, 7)
+#define CORE_GEN_CFG_QMB2_MAX_WR_OUTST_LIMIT GENMASK(6, 0)
+
+#define QPACE_CORE_URG_CMD_n_CACHEINDEX_CFG_OFFSET 0x2fc
+#define CORE_URG_CMD_n_CACHEINDEX__ENGINE_CACHEINDEX_OVERRIDE BIT(6)
+#define CORE_URG_CMD_n_CACHEINDEX__ENGINE_CACHEINDEX GENMASK(5, 0)
+
+#define QPACE_CORE_OPER_CORE_RUN_STOP_OFFSET 0x0
+#define QPACE_CORE_OPER_CORE_READY_OFFSET 0x4
+#define QPACE_CORE_OPER_CFG_OFFSET 0x8
+#define CORE_OPER_CFG_COMP_MEM_PWR_DWN_1 BIT(3)
+
+#define QPACE_COMP_CORE_CFG_OFFSET 0x0
+#define COMP_CORE_CFG_DMA_WR_MAX_OT GENMASK(17, 13)
+#define COMP_CORE_CFG_DMA_RD_MAX_OT GENMASK(12, 8)
+#define QPACE_COMP_CORE_BULK_MODE_OFFSET 0x4
+#define COMP_CORE_BULK_MODE_ALL_CORES GENMASK(5, 0)
+
+#define QPACE_DECOMP_CORE_CFG_OFFSET 0x0
+#define DECOMP_CORE_CFG_DECOMP_DMA_MEM_MNG_PAGE_SIZE GENMASK(12, 10)
+#define DECOMP_CORE_CFG_DMA_WR_MAX_OT GENMASK(9, 5)
+#define DECOMP_CORE_CFG_DMA_RD_MAX_OT GENMASK(4, 0)
+
+#define QPACE_URG_CMD_0_TD_DST_ADDR_L_CFG_CNTXT_OFFSET 0x0
+#define URG_CMD_0_TD_DST_ADDR_L__CMD_CFG_CNTXT GENMASK(3, 0)
+#define QPACE_URG_CMD_0_ED_STAT_OFFSET 0x18
+#define URG_CMD_0_ED_STAT_COMP_CODE GENMASK(23, 20)
+#define URG_CMD_0_ED_STAT_SIZE GENMASK(19, 0)
+#define QPACE_URG_CMD_0_STAT_CLR_OFFSET 0x28
+#define URG_CMD_0_STAT_CLR_REQ_ON_ACTIVE_ERR BIT(1)
+#define QPACE_URG_CMD_0_CFG_CNTXT_SIZE_n_OFFSET 0x40
+#define URG_CMD_0_CFG_CNTXT_SIZE_SIZE GENMASK(19, 0)
+#define QPACE_URG_CMD_0_CFG_CNTXT_MISC_n_OFFSET 0x48
+#define URG_CMD_0_CFG_CNTXT_MISC_WA BIT(21)
+#define URG_CMD_0_CFG_CNTXT_MISC_DCTX GENMASK(17, 16)
+#define URG_CMD_0_CFG_CNTXT_MISC_SCTX GENMASK(15, 14)
+#define URG_CMD_0_CFG_CNTXT_MISC_PAGE_CNT GENMASK(13, 8)
+#define URG_CMD_0_CFG_CNTXT_MISC_OPER GENMASK(7, 4)
+
+#endif /* __QPACE_INTERNAL_H__ */

@@ -0,0 +1,154 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef __LINUX_SOC_QCOM_QPACE_H
+#define __LINUX_SOC_QCOM_QPACE_H
+
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/errno.h>
+#include <linux/kconfig.h>
+#include <linux/types.h>
+
+/**
+ * struct qpace_algorithm - Descriptor for a QPaCE compression algorithm.
+ * @name:            Algorithm name string (e.g. "qpace-lz4").
+ * @comp_opcode:     Hardware opcode for compression.
+ * @decomp_opcode:   Hardware opcode for decompression.
+ * @unused:          Reserved bits.
+ * @urg_comp_cntxt:  Urgent-path context index for compression.
+ * @urg_decomp_cntxt: Urgent-path context index for decompression.
+ */
+struct qpace_algorithm {
+	char *name;
+	u32 comp_opcode:4;
+	u32 decomp_opcode:4;
+	u32 unused:24;
+	int urg_comp_cntxt;
+	int urg_decomp_cntxt;
+};
+
+#if IS_ENABLED(CONFIG_QCOM_PAGE_COMPRESSION_ENGINE)
+
+/**
+ * qpace_is_valid_algorithm() - Check if a name identifies a supported QPaCE algorithm.
+ * @algo_name: Algorithm name string to check.
+ *
+ * Returns true if @algo_name matches a QPaCE algorithm and the driver has
+ * probed successfully, false otherwise.
+ */
+bool qpace_is_valid_algorithm(const char *algo_name);
+
+/**
+ * qpace_lz4_algorithm - Algorithm descriptor for QPaCE LZ4.
+ *
+ * Pass to qpace_urgent_compress() and qpace_urgent_decompress() to select
+ * the LZ4 algorithm.
+ */
+extern const struct qpace_algorithm qpace_lz4_algorithm;
+
+/**
+ * qpace_is_dev_available() - Check whether the QPaCE hardware is ready.
+ *
+ * Returns true if the driver has probed successfully and the hardware is
+ * available for use, false otherwise.
+ */
+bool qpace_is_dev_available(void);
+
+/**
+ * qpace_urgent_compress() - Compress a page synchronously via the urgent path.
+ * @input_addr:  DMA address of the page to compress.
+ * @output_addr: DMA address to write the compressed output to.
+ * @algo:        Algorithm descriptor to use.
+ *
+ * Blocks the calling CPU until compression completes. May sleep; must not be
+ * called from atomic or IRQ-disabled context.
+ *
+ * Return: Compressed size in bytes on success, negative error code on failure.
+ */
+int qpace_urgent_compress(dma_addr_t input_addr,
+			  dma_addr_t output_addr,
+			  struct qpace_algorithm *algo);
+
+/**
+ * qpace_urgent_decompress() - Decompress a page synchronously via the urgent path.
+ * @input_addr:  DMA address of the compressed input.
+ * @output_addr: DMA address to write the decompressed output to.
+ * @input_size:  Size of the compressed input in bytes.
+ * @algo:        Algorithm descriptor to use.
+ *
+ * Blocks the calling CPU until decompression completes. May sleep; must not
+ * be called from atomic or IRQ-disabled context.
+ *
+ * Return: Decompressed size in bytes on success, negative error code on failure.
+ */
+int qpace_urgent_decompress(dma_addr_t input_addr,
+			    dma_addr_t output_addr,
+			    size_t input_size,
+			    struct qpace_algorithm *algo);
+
+/**
+ * qpace_get() - Acquire a reference to QPaCE, preventing runtime power collapse.
+ *
+ * Must be paired with qpace_put() on success. Callers that hold a reference
+ * will keep the hardware awake and the urgent command contexts programmed.
+ *
+ * Return: 0 on success, -EBUSY if the device is suspended.
+ */
+int qpace_get(void);
+
+/**
+ * qpace_put() - Release a reference acquired with qpace_get().
+ *
+ * When the last reference is dropped, QPaCE may enter a low-power state.
+ */
+void qpace_put(void);
+
+/**
+ * qpace_get_dma_dev() - Return the device to use for QPaCE DMA mappings.
+ *
+ * Return: Pointer to the QPaCE platform device, or NULL if unavailable.
+ */
+struct device *qpace_get_dma_dev(void);
+
+#else /* !CONFIG_QCOM_PAGE_COMPRESSION_ENGINE */
+
+static inline bool qpace_is_valid_algorithm(const char *algo_name)
+{
+	return false;
+}
+
+static const struct qpace_algorithm qpace_lz4_algorithm;
+
+static inline bool qpace_is_dev_available(void)
+{
+	return false;
+}
+
+static inline int qpace_urgent_compress(dma_addr_t input_addr,
+					dma_addr_t output_addr,
+					struct qpace_algorithm *algo)
+{
+	return -EINVAL;
+}
+
+static inline int qpace_urgent_decompress(dma_addr_t input_addr,
+					  dma_addr_t output_addr,
+					  size_t input_size,
+					  struct qpace_algorithm *algo)
+{
+	return -EINVAL;
+}
+
+static inline int qpace_get(void) { return -ENODEV; }
+static inline void qpace_put(void) {}
+
+static inline struct device *qpace_get_dma_dev(void)
+{
+	return NULL;
+}
+
+#endif /* CONFIG_QCOM_PAGE_COMPRESSION_ENGINE */
+#endif /* __LINUX_SOC_QCOM_QPACE_H */