[PATCH 11/15] yunsilicon: add RDMA CQ/SRQ/AH

@@ -6,4 +6,4 @@ ccflags-y := -I$(srctree)/drivers/net/ethernet/yunsilicon/xsc
 
 obj-$(CONFIG_INFINIBAND_XSC) += xsc_ib.o
 
-xsc_ib-y := main.o mem.o mr.o
+xsc_ib-y := main.o cq.o qp.o mem.o mr.o srq.o ah.o

@@ -0,0 +1,144 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2021 - 2026, Shanghai Yunsilicon Technology Co., Ltd.
+ * All rights reserved.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/random.h>
+#include "xsc_ib.h"
+#include <rdma/xsc-abi.h>
+
+static u32 xsc_calc_roce_udp_flow_label(void)
+{
+	/* Generate a non-zero 20 bit flow label. */
+	return get_random_u32_below(IB_GRH_FLOWLABEL_MASK) + 1;
+}
+
+static u16 xsc_ah_get_udp_sport(const struct xsc_ib_dev *dev,
+				struct rdma_ah_attr *ah_attr)
+{
+	enum ib_gid_type gid_type = ah_attr->grh.sgid_attr->gid_type;
+	u16 sport = 0;
+	u32 fl = 0;
+
+	if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP &&
+	    (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) &&
+	    (ah_attr->grh.flow_label & IB_GRH_FLOWLABEL_MASK)) {
+		fl = ah_attr->grh.flow_label;
+	} else {
+		/*generate a 20bit flow_label and output to user layer*/
+		fl = xsc_calc_roce_udp_flow_label();
+		ah_attr->grh.flow_label = fl;
+	}
+
+	sport = rdma_flow_label_to_udp_sport(fl);
+	ibdev_dbg(&dev->ib_dev, "fl=0x%x,sport=0x%x\n", fl, sport);
+	return sport;
+}
+
+static struct ib_ah *create_ib_ah(struct xsc_ib_dev *dev,
+				  struct xsc_ib_ah *ah,
+				  struct rdma_ah_attr *ah_attr)
+{
+	enum ib_gid_type gid_type;
+
+	if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) {
+		const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
+
+		memcpy(ah->av.rgid, &grh->dgid, 16);
+		ah->av.grh_gid_fl = cpu_to_be32(grh->flow_label |
+						(1 << 30) |
+						grh->sgid_index << 20);
+		ah->av.hop_limit = grh->hop_limit;
+		ah->av.tclass = grh->traffic_class;
+	}
+
+	ah->av.stat_rate_sl = (rdma_ah_get_static_rate(ah_attr) << 4);
+
+	if (ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) {
+		gid_type = ah_attr->grh.sgid_attr->gid_type;
+
+		memcpy(ah->av.rmac, ah_attr->roce.dmac,
+		       sizeof(ah_attr->roce.dmac));
+
+		ah->av.udp_sport = xsc_ah_get_udp_sport(dev, ah_attr);
+		ah->av.stat_rate_sl |= (rdma_ah_get_sl(ah_attr) & 0x7) << 1;
+		if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)
+#define XSC_ECN_ENABLED BIT(1)
+			ah->av.tclass |= XSC_ECN_ENABLED;
+	} else {
+		ah->av.rlid = cpu_to_be16(rdma_ah_get_dlid(ah_attr));
+		ah->av.fl_mlid = rdma_ah_get_path_bits(ah_attr) & 0x7f;
+		ah->av.stat_rate_sl |= (rdma_ah_get_sl(ah_attr) & 0xf);
+	}
+
+	return &ah->ibah;
+}
+
+int xsc_ib_create_ah(struct ib_ah *ibah,
+		     struct rdma_ah_init_attr *init_attr,
+		      struct ib_udata *udata)
+{
+	struct xsc_ib_ah *ah = to_xah(ibah);
+	struct xsc_ib_dev *dev = to_xdev(ibah->device);
+	struct rdma_ah_attr *ah_attr = init_attr->ah_attr;
+	enum rdma_ah_attr_type ah_type = ah_attr->type;
+
+	if (ah_type == RDMA_AH_ATTR_TYPE_ROCE &&
+	    !(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH))
+		return -EINVAL;
+
+	if (ah_type == RDMA_AH_ATTR_TYPE_ROCE && udata) {
+		int err;
+		struct xsc_ib_create_ah_resp resp = {};
+		u32 min_resp_len = offsetof(typeof(resp), dmac) +
+				   sizeof(resp.dmac);
+
+		if (udata->outlen < min_resp_len)
+			return -EINVAL;
+
+		resp.response_length = min_resp_len;
+		memcpy(resp.dmac, ah_attr->roce.dmac, ETH_ALEN);
+		err = ib_copy_to_udata(udata, &resp, resp.response_length);
+		if (err)
+			return err;
+	}
+
+	create_ib_ah(dev, ah, ah_attr); /* never fails */
+	return 0;
+}
+
+int xsc_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)
+{
+	struct xsc_ib_ah *ah = to_xah(ibah);
+	u32 tmp;
+
+	memset(ah_attr, 0, sizeof(*ah_attr));
+	ah_attr->type = ibah->type;
+
+	tmp = be32_to_cpu(ah->av.grh_gid_fl);
+	if (tmp & (1 << 30)) {
+		rdma_ah_set_grh(ah_attr, NULL, tmp & IB_GRH_FLOWLABEL_MASK,
+				(tmp >> 20) & 0xff, ah->av.hop_limit,
+				ah->av.tclass);
+		rdma_ah_set_dgid_raw(ah_attr, ah->av.rgid);
+	}
+
+	rdma_ah_set_static_rate(ah_attr, ah->av.stat_rate_sl >> 4);
+	if (ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) {
+		rdma_ah_set_sl(ah_attr, (ah->av.stat_rate_sl >> 1) & 0x7);
+	} else {
+		rdma_ah_set_dlid(ah_attr, be16_to_cpu(ah->av.rlid));
+		rdma_ah_set_path_bits(ah_attr, ah->av.fl_mlid & 0x7f);
+		rdma_ah_set_sl(ah_attr, ah->av.stat_rate_sl & 0xf);
+	}
+
+	return 0;
+}
+
+int xsc_ib_destroy_ah(struct ib_ah *ibah, u32 flags)
+{
+	return 0;
+}

@@ -0,0 +1,895 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2021 - 2026, Shanghai Yunsilicon Technology Co., Ltd.
+ * All rights reserved.
+ */
+
+#include <linux/kref.h>
+#include <rdma/ib_umem.h>
+#include "xsc_ib.h"
+#include <rdma/xsc-abi.h>
+#include "common/xsc_hsi.h"
+#include <linux/dma-direct.h>
+
+#ifndef IB_UVERBS_CQ_FLAGS_IGNORE_OVERRUN
+#define IB_UVERBS_CQ_FLAGS_IGNORE_OVERRUN BIT(1)
+#endif
+
+enum {
+	CQ_OK					=  0,
+	CQ_EMPTY				= -1,
+	CQ_POLL_ERR				= -2
+};
+
+enum {
+	XSC_CQE_APP_TAG_MATCHING = 1,
+};
+
+enum {
+	XSC_CQE_APP_OP_TM_CONSUMED = 0x1,
+	XSC_CQE_APP_OP_TM_EXPECTED = 0x2,
+	XSC_CQE_APP_OP_TM_UNEXPECTED = 0x3,
+	XSC_CQE_APP_OP_TM_NO_TAG = 0x4,
+	XSC_CQE_APP_OP_TM_APPEND = 0x5,
+	XSC_CQE_APP_OP_TM_REMOVE = 0x6,
+	XSC_CQE_APP_OP_TM_NOOP = 0x7,
+	XSC_CQE_APP_OP_TM_CONSUMED_SW_RDNV = 0x9,
+	XSC_CQE_APP_OP_TM_CONSUMED_MSG = 0xA,
+	XSC_CQE_APP_OP_TM_CONSUMED_MSG_SW_RDNV = 0xB,
+	XSC_CQE_APP_OP_TM_MSG_COMPLETION_CANCELED = 0xC,
+};
+
+static const u32 xsc_cqe_opcode[] = {
+	[XSC_OPCODE_RDMA_REQ_SEND]		= IB_WC_SEND,
+	[XSC_OPCODE_RDMA_REQ_SEND_IMMDT]	= IB_WC_SEND,
+	[XSC_OPCODE_RDMA_RSP_RECV]		= IB_WC_RECV,
+	[XSC_OPCODE_RDMA_RSP_RECV_IMMDT]	= IB_WC_RECV,
+	[XSC_OPCODE_RDMA_REQ_WRITE]		= IB_WC_RDMA_WRITE,
+	[XSC_OPCODE_RDMA_REQ_WRITE_IMMDT]	= IB_WC_RDMA_WRITE,
+	[XSC_OPCODE_RDMA_RSP_WRITE_IMMDT]	= IB_WC_RECV_RDMA_WITH_IMM,
+	[XSC_OPCODE_RDMA_REQ_READ]		= IB_WC_RDMA_READ,
+	[XSC_OPCODE_RDMA_MAD_REQ_SEND]		= IB_WC_SEND,
+	[XSC_OPCODE_RDMA_MAD_RSP_RECV]		= IB_WC_RECV,
+};
+
+static void xsc_ib_cq_comp(struct xsc_core_cq *cq)
+{
+	struct ib_cq *ibcq = &core_to_xcq(cq)->ibcq;
+
+	ibcq->comp_handler(ibcq, ibcq->cq_context);
+}
+
+static void xsc_ib_cq_event(struct xsc_core_cq *xcq, enum xsc_event type)
+{
+	struct xsc_ib_cq *cq = container_of(xcq, struct xsc_ib_cq, xcq);
+	struct xsc_ib_dev *dev = to_xdev(cq->ibcq.device);
+	struct ib_cq *ibcq = &cq->ibcq;
+	struct ib_event event;
+
+	if (type != XSC_EVENT_TYPE_CQ_ERROR) {
+		ibdev_err(&dev->ib_dev, "Unexpected event type %d on CQ %06x\n",
+			  type, xcq->cqn);
+		return;
+	}
+
+	if (ibcq->event_handler) {
+		event.device     = &dev->ib_dev;
+		event.event      = IB_EVENT_CQ_ERR;
+		event.element.cq = ibcq;
+		ibcq->event_handler(&event, ibcq->cq_context);
+		pr_info("xsc_ib: xsc async event 0x%x, ib event %d\n",
+			XSC_EVENT_TYPE_CQ_ERROR, IB_EVENT_CQ_ERR);
+	}
+}
+
+static void *get_cqe_from_buf(struct xsc_ib_cq_buf *buf, int n, int size)
+{
+	return xsc_buf_offset(&buf->buf, n * size);
+}
+
+static void *get_cqe(struct xsc_ib_cq *cq, int n)
+{
+	return get_cqe_from_buf(&cq->buf, n, cq->xcq.cqe_sz);
+}
+
+static void *get_sw_cqe(struct xsc_ib_cq *cq, int n)
+{
+	struct xsc_cqe *cqe;
+
+	cqe = (struct xsc_cqe *)get_cqe(cq, n & (cq->ibcq.cqe - 1));
+
+	return ((cqe->owner & XSC_CQE_OWNER_MASK) ^
+		!!(n & cq->ibcq.cqe)) ? NULL : cqe;
+}
+
+static inline void handle_good_req(struct ib_wc *wc,
+				   struct xsc_cqe *cqe,
+				   u8 opcode)
+{
+	wc->opcode = xsc_cqe_opcode[opcode];
+	if (opcode == XSC_OPCODE_RDMA_REQ_READ)
+		wc->byte_len  = RD_LE_32(cqe->msg_len);
+	wc->status = IB_WC_SUCCESS;
+}
+
+static void handle_responder(struct ib_wc *wc, struct xsc_cqe *cqe,
+			     struct xsc_ib_qp *qp, u8 opcode)
+{
+	struct xsc_ib_wq *wq = &qp->rq;
+	u16 idx;
+
+	wc->byte_len = RD_LE_32(cqe->msg_len);
+	wc->opcode = xsc_cqe_opcode[opcode];
+	wc->status = IB_WC_SUCCESS;
+
+	idx = wq->tail & (wq->wqe_cnt - 1);
+	wc->wr_id = wq->wrid[idx];
+	atomic_dec(&wq->flush_wqe_cnt);
+	++wq->tail;
+}
+
+static void *get_wqe(struct xsc_ib_qp *qp, int offset)
+{
+	return xsc_buf_offset(&qp->buf, offset);
+}
+
+static void *get_recv_wqe(struct xsc_ib_qp *qp, int n)
+{
+	return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift));
+}
+
+static void *get_seg_wqe(void *first, int n)
+{
+	return first + (n << XSC_BASE_WQE_SHIFT);
+}
+
+static void xsc_handle_rdma_mad_resp_recv(struct xsc_ib_cq *cq,
+					  struct xsc_ib_qp **cur_qp,
+					  struct ib_wc *wc,
+					  struct xsc_cqe *cqe,
+					  u8 opcode)
+{
+	struct xsc_ib_dev *dev = to_xdev(cq->ibcq.device);
+	void *recv;
+	struct xsc_wqe_data_seg *data_seg;
+	struct iphdr *ip4h = NULL;
+	struct ipv6hdr *ip6h;
+	struct udphdr *udph;
+	struct ib_unpacked_eth *eth;
+	struct ib_unpacked_vlan *vlan;
+	struct ib_grh *grh;
+	struct ib_mad *mad;
+	struct rxe_bth *bth;
+	struct rxe_deth *deth;
+	unsigned int pading_sz = 0;
+	struct xsc_ib_wq *wq;
+	int idx;
+	u16 eth_type;
+	void *l3_start;
+
+	wq = &(*cur_qp)->rq;
+	idx = wq->tail & (wq->wqe_cnt - 1);
+
+	handle_responder(wc, cqe, *cur_qp, opcode);
+
+	data_seg = get_seg_wqe(get_recv_wqe(*cur_qp, idx), 0);
+	recv = xsc_ib_recv_mad_sg_virt_addr(&dev->ib_dev, wc, data_seg->va);
+
+	eth = (struct ib_unpacked_eth *)recv;
+	grh = (struct ib_grh *)recv;
+	if (eth->type == htons(ETH_P_8021Q)) {
+		vlan = (struct ib_unpacked_vlan *)(eth + 1);
+		eth_type = ntohs(vlan->type);
+		l3_start = vlan + 1;
+
+		wc->vlan_id = ntohs(vlan->tag) & 0x0fff;
+		wc->sl = (ntohs(vlan->tag) >> 13) & 0x7;
+		wc->wc_flags |= IB_WC_WITH_VLAN;
+	} else {
+		eth_type = ntohs(eth->type);
+		l3_start = eth + 1;
+	}
+
+	if (eth_type == ETH_P_IP) {
+		ip4h = (struct iphdr *)l3_start;
+		udph = (struct udphdr *)(ip4h + 1);
+	} else {
+		ip6h = (struct ipv6hdr *)l3_start;
+		udph = (struct udphdr *)(ip6h + 1);
+	}
+	bth = (struct rxe_bth *)(udph + 1);
+	deth = (struct rxe_deth *)(bth + 1);
+	mad = (struct ib_mad *)(deth + 1);
+
+	if (eth_type == ETH_P_IP) {
+		pading_sz = sizeof(*grh) - sizeof(*ip4h);
+		memmove((u8 *)(grh + 1) - sizeof(*ip4h), ip4h, sizeof(*ip4h));
+		memset(grh, 0, pading_sz);
+	} else {
+		memmove(grh, ip6h, sizeof(*ip6h));
+	}
+	memmove(grh + 1, mad, sizeof(*mad));
+
+	wc->wc_flags |= IB_WC_GRH;
+
+	ibdev_dbg(&dev->ib_dev, "recv cqe idx:%u, len:%u\n", wq->tail, wc->byte_len);
+	ibdev_dbg(&dev->ib_dev, "qp[%d] recv MAD packet, msg_len=%d\n",
+		  (*cur_qp)->xqp.qpn, wc->byte_len);
+	wc->status = IB_WC_SUCCESS;
+}
+
+static void xsc_generate_lost_cqe(struct xsc_ib_dev *dev, struct xsc_ib_wq *wq,
+				  int qpn, int idx, struct ib_wc *wc)
+{
+	int lost_cqe_idx = wq->prev_ce_wqe[idx];
+
+	ibdev_err(&dev->ib_dev, "detected lost cqe, cur wqe idx of qp%d is %d\n", qpn, idx);
+
+	while (wq->prev_ce_wqe[lost_cqe_idx] != wq->last_comp_ce_wqe)
+		lost_cqe_idx = wq->prev_ce_wqe[lost_cqe_idx];
+	ibdev_err(&dev->ib_dev, "generate lost cqe idx of qp%d is %d\n", qpn, lost_cqe_idx);
+
+	wq->last_comp_ce_wqe = lost_cqe_idx;
+	wc->wr_id = wq->wrid[lost_cqe_idx];
+	wq->tail = wq->wqe_head[lost_cqe_idx] + 1;
+	atomic_dec(&wq->flush_wqe_cnt);
+	wq->need_flush[lost_cqe_idx] = 0;
+	wc->status = IB_WC_GENERAL_ERR;
+}
+
+static int xsc_poll_one(struct xsc_ib_cq *cq,
+			struct xsc_ib_qp **cur_qp,
+			struct ib_wc *wc)
+{
+	struct xsc_ib_dev *dev = to_xdev(cq->ibcq.device);
+	struct xsc_core_qp *xqp;
+	struct xsc_ib_wq *wq;
+	u8 opcode;
+	u32 qpn;
+	int idx;
+	struct xsc_cqe *cqe;
+	u32 *p = NULL;
+	struct xsc_ib_srq *xsrq;
+
+	cqe = get_sw_cqe(cq, cq->xcq.cons_index);
+	if (!cqe)
+		return -EAGAIN;
+
+	++cq->xcq.cons_index;
+
+	/* Make sure we read CQ entry contents after we've checked the
+	 * ownership bit.
+	 */
+	rmb();
+
+	p = (u32 *)cqe;
+
+	qpn = cqe->qp_id;
+	qpn = le32_to_cpu(qpn);
+	if (!*cur_qp || (qpn != (*cur_qp)->ibqp.qp_num)) {
+		/* We do not have to take the QP table lock here,
+		 * because CQs will be locked while QPs are removed
+		 * from the table.
+		 */
+		xqp = __xsc_qp_lookup(dev->xdev, qpn);
+		if (unlikely(!xqp)) {
+			ibdev_warn(&dev->ib_dev, "CQE@CQ %d for unknown QPN %d\n",
+				   cq->xcq.cqn, qpn);
+			return -EINVAL;
+		}
+
+		*cur_qp = core_to_xqp(xqp);
+	}
+
+	memset(wc, 0, sizeof(*wc));
+	wc->qp = &(*cur_qp)->ibqp;
+	opcode = xsc_get_cqe_opcode(dev->xdev, cqe);
+	switch (opcode) {
+	case XSC_OPCODE_RDMA_REQ_SEND_IMMDT:
+	case XSC_OPCODE_RDMA_REQ_WRITE_IMMDT:
+		wc->wc_flags |= IB_WC_WITH_IMM;
+		fallthrough;
+	case XSC_OPCODE_RDMA_REQ_SEND:
+	case XSC_OPCODE_RDMA_REQ_WRITE:
+	case XSC_OPCODE_RDMA_REQ_READ:
+	case XSC_OPCODE_RDMA_MAD_REQ_SEND:
+		wq = &(*cur_qp)->sq;
+		idx = xsc_get_cqe_wqe_id(dev->xdev, cqe) >> (wq->wqe_shift - XSC_BASE_WQE_SHIFT);
+		idx &= (wq->wqe_cnt - 1);
+		if (wq->prev_ce_wqe[idx] != wq->last_comp_ce_wqe) {
+			xsc_generate_lost_cqe(dev, wq, qpn, idx, wc);
+			cq->invalid = 1;
+			break;
+		}
+		wq->last_comp_ce_wqe = idx;
+		handle_good_req(wc, cqe, opcode);
+		wc->wr_id = wq->wrid[idx];
+		wq->tail = wq->wqe_head[idx] + 1;
+		if (opcode != XSC_OPCODE_RDMA_MAD_REQ_SEND)
+			atomic_dec(&wq->flush_wqe_cnt);
+		wq->need_flush[idx] = 0;
+		ibdev_dbg(&dev->ib_dev, "wqeid:%u, wq tail:%u qpn:%u\n", idx, wq->tail, qpn);
+		wc->status = IB_WC_SUCCESS;
+		break;
+	case XSC_OPCODE_RDMA_RSP_RECV_IMMDT:
+	case XSC_OPCODE_RDMA_RSP_WRITE_IMMDT:
+		wc->wc_flags |= IB_WC_WITH_IMM;
+		WR_BE_32(wc->ex.imm_data, RD_LE_32(cqe->imm_data));
+		fallthrough;
+	case XSC_OPCODE_RDMA_RSP_RECV:
+		wq = &(*cur_qp)->rq;
+		handle_responder(wc, cqe, *cur_qp, opcode);
+		if ((*cur_qp)->ibqp.srq) {
+			xsrq = to_xsrq((*cur_qp)->ibqp.srq);
+			xsc_free_one_cache_srq_wr(xsrq, *cur_qp, wc->wr_id);
+		}
+		ibdev_dbg(&dev->ib_dev, "recv cqe idx:%u, len:%u, qpn:%u\n",
+			  wq->tail, wc->byte_len, qpn);
+		wc->status = IB_WC_SUCCESS;
+		break;
+
+	case XSC_OPCODE_RDMA_MAD_RSP_RECV:
+		ibdev_dbg(&dev->ib_dev, "recv MAD, qpn:%u\n", qpn);
+		xsc_handle_rdma_mad_resp_recv(cq, cur_qp, wc, cqe, opcode);
+		break;
+
+	case XSC_OPCODE_RDMA_REQ_ERROR:
+		wq = &(*cur_qp)->sq;
+		idx = xsc_get_cqe_wqe_id(dev->xdev, cqe) >> (wq->wqe_shift - XSC_BASE_WQE_SHIFT);
+		idx &= (wq->wqe_cnt - 1);
+
+		if (!wq->need_flush[idx]) {
+			unsigned int i;
+			unsigned int tmp;
+			unsigned int prev = -1;
+
+			for (i = wq->tail; i != wq->head; i++) {
+				tmp = i & (wq->wqe_cnt - 1);
+				if (tmp == idx)
+					break;
+				if (wq->need_flush[tmp])
+					prev = tmp;
+			}
+
+			if (prev != -1) {
+				if (wq->prev_ce_wqe[prev] != wq->last_comp_ce_wqe) {
+					xsc_generate_lost_cqe(dev, wq, qpn, prev, wc);
+					cq->invalid = 1;
+					break;
+				}
+			}
+		} else {
+			if (wq->prev_ce_wqe[idx] != wq->last_comp_ce_wqe) {
+				xsc_generate_lost_cqe(dev, wq, qpn, idx, wc);
+				cq->invalid = 1;
+				break;
+			}
+		}
+
+		wc->wr_id = wq->wrid[idx];
+		wq->tail = wq->wqe_head[idx] + 1;
+		if (wq->need_flush[idx])
+			atomic_dec(&wq->flush_wqe_cnt);
+		wq->need_flush[idx] = 0;
+		ibdev_err(&dev->ib_dev, "req error\n%08x %08x %08x %08x %08x %08x\n",
+			  p[0], p[1], p[2], p[3], p[5], p[6]);
+		wc->status = IB_WC_GENERAL_ERR;
+		xsc_ib_err_state_qp(dev, *cur_qp, (*cur_qp)->state, XSC_QP_STATE_ERR);
+		break;
+	case XSC_OPCODE_RDMA_RSP_ERROR:
+		wq = &(*cur_qp)->rq;
+		idx = wq->tail & (wq->wqe_cnt - 1);
+		wc->wr_id = wq->wrid[idx];
+		wq->tail++;
+		atomic_dec(&wq->flush_wqe_cnt);
+		if ((*cur_qp)->ibqp.srq) {
+			xsrq = to_xsrq((*cur_qp)->ibqp.srq);
+			xsc_free_one_cache_srq_wr(xsrq, *cur_qp, wc->wr_id);
+		}
+		ibdev_err(&dev->ib_dev, "rsp error\n%08x %08x %08x %08x %08x %08x\n",
+			  p[0], p[1], p[2], p[3], p[5], p[6]);
+		wc->status = IB_WC_GENERAL_ERR;
+		xsc_ib_err_state_qp(dev, *cur_qp, (*cur_qp)->state, XSC_QP_STATE_ERR);
+		break;
+
+	default:
+		ibdev_err(&dev->ib_dev, "completion error\n%08x %08x %08x %08x %08x %08x\n",
+			  p[0], p[1], p[2], p[3], p[5], p[6]);
+		wc->status = IB_WC_GENERAL_ERR;
+		wc->wr_id = 0;
+		break;
+	}
+
+	return 0;
+}
+
+static inline void gen_flush_err_cqe(struct xsc_err_state_qp_node *err_node,
+				     struct ib_qp *ibqp, struct xsc_ib_wq *wq, u32 idx,
+				     struct ib_wc *wc)
+{
+	memset(wc, 0, sizeof(*wc));
+	if (err_node->is_sq) {
+		switch (wq->wr_opcode[idx]) {
+		case IB_WR_SEND:
+		case IB_WR_SEND_WITH_IMM:
+			wc->opcode = IB_WC_SEND;
+			break;
+		case IB_WR_RDMA_WRITE:
+		case IB_WR_RDMA_WRITE_WITH_IMM:
+			wc->opcode = IB_WC_RDMA_WRITE;
+			break;
+		case IB_WR_RDMA_READ:
+			wc->opcode = IB_WC_RDMA_READ;
+		}
+	} else {
+		wc->opcode = IB_WC_RECV;
+	}
+
+	wc->qp = ibqp;
+	wc->status = IB_WC_WR_FLUSH_ERR;
+	wc->vendor_err = XSC_ERR_CODE_FLUSH;
+	wc->wr_id = wq->wrid[idx];
+	wq->tail++;
+	atomic_dec(&wq->flush_wqe_cnt);
+	if (err_node->is_sq)
+		wq->need_flush[idx] = 0;
+}
+
+static inline int xsc_generate_flush_err_cqe(struct ib_cq *ibcq,
+					     int ne, int *npolled, struct ib_wc *wc)
+{
+	u32 qp_id = 0;
+	int flush_wqe_cnt = 0;
+	int sw_npolled = 0;
+	u32 idx = 0;
+	struct xsc_err_state_qp_node *err_qp_node;
+	struct xsc_core_qp *xqp;
+	struct xsc_ib_cq *cq = to_xcq(ibcq);
+	struct xsc_ib_wq *wq;
+	struct xsc_ib_dev *dev = to_xdev(cq->ibcq.device);
+
+	list_for_each_entry(err_qp_node, &cq->err_state_qp_list, entry) {
+		if (!err_qp_node)
+			break;
+
+		sw_npolled = 0;
+		qp_id = err_qp_node->qp_id;
+		xqp = __xsc_qp_lookup(dev->xdev, qp_id);
+		if (unlikely(!xqp)) {
+			ibdev_warn(&dev->ib_dev, "CQE@CQ %d for unknown QPN %d\n",
+				   cq->xcq.cqn, qp_id);
+			continue;
+		}
+		wq = err_qp_node->is_sq ? &(core_to_xqp(xqp)->sq) : &(core_to_xqp(xqp)->rq);
+		flush_wqe_cnt = atomic_read(&wq->flush_wqe_cnt);
+		ibdev_dbg(&dev->ib_dev, "is_sq %d, flush_wq_cnt %d, ne %d, npolled %d, qp_id %d, cqn %d\n",
+			  err_qp_node->is_sq, flush_wqe_cnt, ne, *npolled, qp_id, cq->xcq.cqn);
+
+		while (wq->tail != wq->head) {
+			idx = wq->tail & (wq->wqe_cnt - 1);
+			if (err_qp_node->is_sq && !wq->need_flush[idx]) {
+				wq->tail++;
+				continue;
+			} else {
+				gen_flush_err_cqe(err_qp_node, &(core_to_xqp(xqp)->ibqp), wq,
+						  idx, wc + *npolled + sw_npolled);
+				++sw_npolled;
+			}
+
+			if (sw_npolled == ne - *npolled)
+				break;
+		}
+		*npolled += sw_npolled;
+		if (*npolled == ne)
+			break;
+	}
+
+	return 0;
+}
+
+int xsc_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
+{
+	struct xsc_ib_cq *cq = to_xcq(ibcq);
+	struct xsc_core_cq *xcq = &cq->xcq;
+	struct xsc_ib_qp *cur_qp = NULL;
+	int npolled = 0;
+	int err = 0;
+	unsigned long flags;
+	u32 next_cid;
+
+	spin_lock_irqsave(&cq->lock, flags);
+	next_cid = xcq->cons_index;
+
+	for (npolled = 0; npolled < num_entries; npolled++) {
+		if (cq->invalid) {
+			err = -EINVAL;
+			break;
+		}
+		err = xsc_poll_one(cq, &cur_qp, wc + npolled);
+		if (err)
+			break;
+	}
+
+	if (err) {
+		if (npolled < num_entries && !(list_empty(&cq->err_state_qp_list)))
+			xsc_generate_flush_err_cqe(ibcq, num_entries, &npolled, wc);
+	}
+
+	if (next_cid != xcq->cons_index)
+		xsc_update_cq_ci(xcq->dev, xcq->cqn, xcq->cons_index);
+
+	spin_unlock_irqrestore(&cq->lock, flags);
+
+	return npolled;
+}
+
+static int xsc_cqe_is_empty(struct xsc_ib_cq *cq)
+{
+	struct xsc_cqe *cqe = get_sw_cqe(cq, cq->xcq.cons_index);
+
+	if (cq->invalid)
+		return 1;
+
+	if (!cqe)
+		return 1;
+
+	return 0;
+}
+
+int xsc_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
+{
+	struct xsc_ib_cq *xcq = to_xcq(ibcq);
+	struct xsc_core_cq *cq = &xcq->xcq;
+	int ret = 0;
+	unsigned long irq_flags;
+	u8 solicited = 0;
+
+	spin_lock_irqsave(&xcq->lock, irq_flags);
+	if (flags & IB_CQ_NEXT_COMP)
+		solicited = 0;
+	else if (flags & IB_CQ_SOLICITED)
+		solicited = 1;/* arm next:0 arm solicited:1 */
+
+	if ((flags & IB_CQ_REPORT_MISSED_EVENTS) && (!xsc_cqe_is_empty(xcq))) {
+		ret = 1;
+		goto out;
+	}
+
+	xsc_arm_cq(cq->dev, cq->cqn, cq->cons_index, solicited);
+out:
+	spin_unlock_irqrestore(&xcq->lock, irq_flags);
+	return ret;
+}
+
+static int alloc_cq_buf(struct xsc_ib_dev *dev, struct xsc_ib_cq_buf *buf,
+			int nent, int cqe_size)
+{
+	int err;
+
+	err = xsc_buf_alloc(dev->xdev, nent * cqe_size,
+			    PAGE_SIZE, &buf->buf);
+	if (err)
+		return err;
+
+	buf->cqe_size = cqe_size;
+
+	return 0;
+}
+
+static void free_cq_buf(struct xsc_ib_dev *dev, struct xsc_ib_cq_buf *buf)
+{
+	xsc_buf_free(dev->xdev, &buf->buf);
+}
+
+static struct ib_umem *cq_umem_get(struct xsc_ib_dev *dev,
+				   struct ib_udata *udata, struct ib_ucontext *context,
+	int entries, struct xsc_ib_create_cq *ucmd)
+{
+	struct ib_umem *umem;
+
+	umem = ib_umem_get_va(&dev->ib_dev, ucmd->buf_addr,
+			      entries * ucmd->cqe_size,
+			      IB_ACCESS_LOCAL_WRITE);
+	return umem;
+}
+
+static int create_cq_user(struct xsc_ib_dev *dev, struct ib_udata *udata,
+			  struct ib_ucontext *context, struct xsc_ib_cq *cq,
+			  int entries, struct xsc_create_cq_ex_mbox_in **cqb,
+			  int *cqe_size, int *index, int *inlen)
+{
+	struct xsc_ib_create_cq ucmd;
+	int page_shift;
+	int npages;
+	int ncont;
+	int err;
+	int log_cq_sz;
+	int hw_npages;
+	struct ib_umem_dmabuf *umem_dmabuf;
+
+	if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)))
+		return -EFAULT;
+
+	*cqe_size = ucmd.cqe_size;
+
+	if (ucmd.dmabuf_fd != -1) {
+		umem_dmabuf = ib_umem_dmabuf_get_pinned(&dev->ib_dev, 0, ucmd.dmabuf_sz,
+							ucmd.dmabuf_fd, IB_ACCESS_LOCAL_WRITE);
+
+		if (IS_ERR(umem_dmabuf)) {
+			ibdev_warn(&dev->ib_dev, "umem_dmabuf get failed by fd %d (%ld)\n",
+				   ucmd.dmabuf_fd, PTR_ERR(umem_dmabuf));
+			return -ENOMEM;
+		}
+
+		cq->buf.umem = &umem_dmabuf->umem;
+		ibdev_dbg(&dev->ib_dev, "get umem_dmabuf success by fd %d, umem_dmabuf_addr %p\n",
+			  ucmd.dmabuf_fd, umem_dmabuf);
+	} else {
+		cq->buf.umem = cq_umem_get(dev, udata, context, entries, &ucmd);
+	}
+
+	if (IS_ERR(cq->buf.umem)) {
+		err = PTR_ERR(cq->buf.umem);
+		return err;
+	}
+
+	xsc_ib_cont_pages(cq->buf.umem, ucmd.buf_addr, &npages, &page_shift,
+			  &ncont);
+	if (ncont != npages) {
+		ibdev_dbg(&dev->ib_dev, "bad page_shift:%d, ncont:%d\n", page_shift, ncont);
+		/* amber doesn't support compound pages */
+		page_shift = PAGE_SHIFT;
+		ncont = npages;
+		ibdev_dbg(&dev->ib_dev, "overwrite to page_shift:%d, ncont:%d\n",
+			  page_shift, ncont);
+	}
+	log_cq_sz = ilog2(entries);
+	hw_npages = DIV_ROUND_UP((1 << log_cq_sz) * (*cqe_size), PAGE_SIZE_4K);
+	ibdev_dbg(&dev->ib_dev, "addr 0x%llx, entries %d, npages %d, hw_npages %d\n",
+		  ucmd.buf_addr, entries, npages, hw_npages);
+
+	*inlen = sizeof(**cqb) + sizeof(*(*cqb)->pas) * hw_npages;
+	*cqb = xsc_vzalloc(*inlen);
+	if (!*cqb) {
+		err = -ENOMEM;
+		goto err_umem;
+	}
+	xsc_ib_populate_pas(dev, cq->buf.umem, page_shift, (*cqb)->pas, hw_npages, true);
+	(*cqb)->ctx_ex.ctx.pa_num = cpu_to_be16(hw_npages);
+	(*cqb)->ctx_ex.pa_num_ex = cpu_to_be32(hw_npages);
+
+	return 0;
+
+err_umem:
+	ib_umem_release(cq->buf.umem);
+	return err;
+}
+
+static void destroy_cq_user(struct xsc_ib_cq *cq, struct ib_udata *udata)
+{
+	ib_umem_release(cq->buf.umem);
+}
+
+static int create_cq_kernel(struct xsc_ib_dev *dev, struct xsc_ib_cq *cq,
+			    int entries, int cqe_size,
+			    struct xsc_create_cq_ex_mbox_in **cqb,
+			    int *index, int *inlen)
+{
+	int err;
+	int i = 0;
+	struct xsc_cqe *cqe = NULL;
+	int hw_npages;
+
+	cq->xcq.cqe_sz = cqe_size;
+
+	err = alloc_cq_buf(dev, &cq->buf, entries, cqe_size);
+	if (err)
+		return err;
+
+	for (i = 0; i < entries; i++) {
+		cqe = (struct xsc_cqe *)get_cqe(cq, i);
+		cqe->owner = 1;
+	}
+
+	hw_npages = DIV_ROUND_UP(entries * cqe_size, PAGE_SIZE_4K);
+	*inlen = sizeof(**cqb) + sizeof(*(*cqb)->pas) * hw_npages;
+	*cqb = xsc_vzalloc(*inlen);
+	if (!*cqb) {
+		err = -ENOMEM;
+		goto err_buf;
+	}
+	xsc_fill_page_array(&cq->buf.buf, (*cqb)->pas, hw_npages);
+	(*cqb)->ctx_ex.ctx.pa_num = cpu_to_be16(hw_npages);
+	(*cqb)->ctx_ex.pa_num_ex = cpu_to_be32(hw_npages);
+
+	return 0;
+
+err_buf:
+	free_cq_buf(dev, &cq->buf);
+	return err;
+}
+
+static void destroy_cq_kernel(struct xsc_ib_dev *dev, struct xsc_ib_cq *cq)
+{
+	free_cq_buf(dev, &cq->buf);
+}
+
+int xsc_ib_create_cq(struct ib_cq *ibcq,
+		     const struct ib_cq_init_attr *attr,
+		     struct uverbs_attr_bundle *attrs)
+{
+	struct ib_device *ibdev = ibcq->device;
+	struct ib_udata *udata = attrs ? &attrs->driver_udata : NULL;
+	int entries = attr->cqe;
+	int vector = attr->comp_vector;
+	struct xsc_create_cq_ex_mbox_in *cqb = NULL;
+	struct xsc_ib_dev *dev = to_xdev(ibdev);
+	struct xsc_ib_cq *cq;
+	int index;
+	int inlen = 0;
+	int cqe_size;
+	int irqn;
+	int err;
+	unsigned int eqn;
+
+	entries = roundup_pow_of_two(entries);
+
+	ibdev_dbg(&dev->ib_dev, "entries:%d, vector:%d, max_cqes:%d\n", entries, vector,
+		  dev->xdev->caps.max_cqes);
+
+	if (entries > dev->xdev->caps.max_cqes)
+		entries = dev->xdev->caps.max_cqes;
+	cq = to_xcq(ibcq);
+	cq->ibcq.cqe = entries;
+	mutex_init(&cq->resize_mutex);
+	spin_lock_init(&cq->lock);
+	cq->resize_buf = NULL;
+	cq->resize_umem = NULL;
+	cq->create_flags = attr->flags;
+	cq->invalid = 0;
+
+	if (udata) {
+		err = create_cq_user(dev, udata, NULL, cq, entries,
+				     &cqb, &cqe_size, &index, &inlen);
+		if (err)
+			goto err_create;
+	} else {
+		cqe_size = is_support_cqe64(dev->xdev) ?
+			sizeof(struct xsc_cqe64) : sizeof(struct xsc_cqe);
+		err = create_cq_kernel(dev, cq, entries, cqe_size, &cqb, &index, &inlen);
+		if (err)
+			goto err_create;
+	}
+
+	cq->cqe_size = cqe_size;
+	cqb->ctx_ex.ctx.log_cq_sz = ilog2(entries);
+	cqb->ctx_ex.ctx.glb_func_id = cpu_to_be16(dev->xdev->glb_func_id);
+
+	err = xsc_vector2eqn(dev->xdev, vector, &eqn, &irqn);
+	if (err)
+		goto err_cqb;
+
+	cqb->ctx_ex.ctx.eqn = eqn;
+	cqb->ctx_ex.ctx.eqn = cpu_to_be16(cqb->ctx_ex.ctx.eqn);
+	cqb->ctx_ex.page_shift = PAGE_SHIFT;
+	if (cq->create_flags & IB_UVERBS_CQ_FLAGS_IGNORE_OVERRUN)
+		cqb->ctx_ex.overrun_ignore = 1;
+
+	err = xsc_core_create_cq(dev->xdev, &cq->xcq, cqb, inlen);
+	if (err)
+		goto err_cqb;
+
+	ibdev_dbg(&dev->ib_dev, "succeeded to create cqn %d, vector=%d, cq_sz=%d, eqn=%d\n",
+		  cq->xcq.cqn, vector, entries, eqn);
+	cq->xcq.irqn = irqn;
+	cq->xcq.comp  = xsc_ib_cq_comp;
+	cq->xcq.event = xsc_ib_cq_event;
+
+	if (udata) {
+		if (ib_copy_to_udata(udata, &cq->xcq.cqn, sizeof(__u32))) {
+			err = -EFAULT;
+			goto err_cmd;
+		}
+	}
+
+	INIT_LIST_HEAD(&cq->send_qp_list);
+	INIT_LIST_HEAD(&cq->recv_qp_list);
+	INIT_LIST_HEAD(&cq->err_state_qp_list);
+	xsc_vfree(cqb);
+
+	return 0;
+
+err_cmd:
+	xsc_core_destroy_cq(dev->xdev, &cq->xcq);
+
+err_cqb:
+	xsc_vfree(cqb);
+	if (udata)
+		destroy_cq_user(cq, udata);
+	else
+		destroy_cq_kernel(dev, cq);
+
+err_create:
+
+	return err;
+}
+
+int xsc_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
+{
+	struct xsc_ib_dev *dev = to_xdev(cq->device);
+	struct xsc_ib_cq *xcq = to_xcq(cq);
+	struct xsc_err_state_qp_node *tmp = NULL, *err_qp_node = NULL;
+
+	list_for_each_entry_safe(err_qp_node, tmp, &xcq->err_state_qp_list, entry) {
+		list_del(&err_qp_node->entry);
+		kfree(err_qp_node);
+	}
+
+	xsc_core_destroy_cq(dev->xdev, &xcq->xcq);
+	if (udata)
+		destroy_cq_user(xcq, udata);
+	else
+		destroy_cq_kernel(dev, xcq);
+
+	return 0;
+}
+
+static int is_equal_rsn(struct xsc_cqe *cqe, u32 rsn)
+{
+	u32 qpn = le32_to_cpu(cqe->qp_id);
+	return rsn == qpn;
+}
+
+void __xsc_ib_cq_clean(struct xsc_ib_cq *cq, u32 rsn)
+{
+	struct xsc_cqe *cqe, *dest;
+	u32 prod_index;
+	int nfreed = 0;
+	u8 owner_bit;
+
+	if (!cq)
+		return;
+
+	/* First we need to find the current producer index, so we
+	 * know where to start cleaning from.  It doesn't matter if HW
+	 * adds new entries after this loop -- the QP we're worried
+	 * about is already in RESET, so the new entries won't come
+	 * from our QP and therefore don't need to be checked.
+	 */
+	for (prod_index = cq->xcq.cons_index; get_sw_cqe(cq, prod_index); prod_index++)
+		if (prod_index == cq->xcq.cons_index + cq->ibcq.cqe)
+			break;
+
+	/* Now sweep backwards through the CQ, removing CQ entries
+	 * that match our QP by copying older entries on top of them.
+	 */
+	while ((int)(--prod_index) - (int)cq->xcq.cons_index >= 0) {
+		cqe = (struct xsc_cqe *)get_cqe(cq, prod_index & (cq->ibcq.cqe - 1));
+		if (is_equal_rsn(cqe, rsn)) {
+			++nfreed;
+		} else if (nfreed) {
+			dest = (struct xsc_cqe *)get_cqe(cq, (prod_index + nfreed) &
+							 (cq->ibcq.cqe - 1));
+			owner_bit = dest->owner & XSC_CQE_OWNER_MASK;
+			memcpy(dest, cqe, cq->xcq.cqe_sz);
+			dest->owner = owner_bit |
+				(dest->owner & ~XSC_CQE_OWNER_MASK);
+		}
+	}
+
+	if (nfreed) {
+		cq->xcq.cons_index += nfreed;
+		xsc_update_cq_ci(cq->xcq.dev, cq->xcq.cqn, cq->xcq.cons_index);
+	}
+}
+
+void xsc_ib_cq_clean(struct xsc_ib_cq *cq, u32 qpn)
+{
+	if (!cq)
+		return;
+
+	spin_lock_irq(&cq->lock);
+	__xsc_ib_cq_clean(cq, qpn);
+	spin_unlock_irq(&cq->lock);
+}