[PATCH 12/15] yunsilicon: add RDMA QP verbs

@@ -570,6 +570,11 @@ static void xsc_ib_dev_setting(struct xsc_ib_dev *dev)
 	dev->ib_dev.ops.get_link_layer	= xsc_ib_port_link_layer;
 	dev->ib_dev.ops.get_netdev		= xsc_get_netdev;
 
+	dev->ib_dev.ops.create_qp		= xsc_ib_create_qp;
+	dev->ib_dev.ops.modify_qp		= xsc_ib_modify_qp;
+	dev->ib_dev.ops.query_qp		= xsc_ib_query_qp;
+	dev->ib_dev.ops.destroy_qp		= xsc_ib_destroy_qp;
+	dev->ib_dev.ops.post_recv		= xsc_ib_post_recv;
 	dev->ib_dev.ops.create_cq		= xsc_ib_create_cq;
 	dev->ib_dev.ops.destroy_cq		= xsc_ib_destroy_cq;
 	dev->ib_dev.ops.poll_cq		= xsc_ib_poll_cq;

@@ -592,6 +597,7 @@ static void xsc_ib_dev_setting(struct xsc_ib_dev *dev)
 	dev->ib_dev.ops INIT_RDMA_OBJ_SIZE(ib_ah, xsc_ib_ah, ibah);
 	dev->ib_dev.ops INIT_RDMA_OBJ_SIZE(ib_cq, xsc_ib_cq, ibcq);
 	dev->ib_dev.ops INIT_RDMA_OBJ_SIZE(ib_pd, xsc_ib_pd, ibpd);
+	dev->ib_dev.ops INIT_RDMA_OBJ_SIZE(ib_qp, xsc_ib_qp, ibqp);
 	dev->ib_dev.ops INIT_RDMA_OBJ_SIZE(ib_srq, xsc_ib_srq, ibsrq);
 }
 

@@ -28,11 +28,21 @@ enum {
 #define UDP_SPORT_MASK			0xffff
 #define UDP_SPORT_OFFSET		0
 
+static int is_qp0(enum ib_qp_type qp_type)
+{
+	return qp_type == IB_QPT_SMI;
+}
+
 static int is_qp1(enum ib_qp_type qp_type)
 {
 	return qp_type == IB_QPT_GSI;
 }
 
+static int is_sqp(enum ib_qp_type qp_type)
+{
+	return is_qp0(qp_type) || is_qp1(qp_type);
+}
+
 static void *get_wqe(struct xsc_ib_qp *qp, int offset)
 {
 	return xsc_buf_offset(&qp->buf, offset);

@@ -48,6 +58,258 @@ static void *get_seg_wqe(void *first, int n)
 	return first + (n << XSC_BASE_WQE_SHIFT);
 }
 
+static void xsc_ib_qp_event(struct xsc_core_qp *qp, int type)
+{
+	struct ib_qp *ibqp = &core_to_xqp(qp)->ibqp;
+	struct ib_event event;
+
+	if (ibqp->event_handler) {
+		event.device     = ibqp->device;
+		event.element.qp = ibqp;
+		switch (type) {
+		case XSC_EVENT_TYPE_WQ_CATAS_ERROR:
+			event.event = IB_EVENT_QP_FATAL;
+			break;
+		case XSC_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
+			event.event = IB_EVENT_QP_REQ_ERR;
+			break;
+		case XSC_EVENT_TYPE_WQ_ACCESS_ERROR:
+			event.event = IB_EVENT_QP_ACCESS_ERR;
+			break;
+		default:
+			pr_warn("xsc_ib: Unexpected event type %d on QP %06x\n", type, qp->qpn);
+			return;
+		}
+
+		ibqp->event_handler(&event, ibqp->qp_context);
+		pr_info("xsc_ib: xsc async event 0x%x, ib event %d\n", type, event.event);
+	}
+}
+
+static int set_rq_size(struct xsc_ib_dev *dev, struct ib_qp_init_attr *init_attr,
+		       int has_rq, struct xsc_ib_qp *qp, struct xsc_ib_create_qp *ucmd)
+{
+	bool check_res = false;
+	u32 wqe_cnt = init_attr->cap.max_recv_wr ?
+		roundup_pow_of_two(init_attr->cap.max_recv_wr) : 0;
+
+	if (init_attr->srq)
+		wqe_cnt = to_xsrq(init_attr->srq)->xsrq.max;
+
+	/* Sanity check RQ size before proceeding */
+	check_res = xsc_check_max_qp_depth(dev->xdev, &wqe_cnt, dev->xdev->caps.max_wqes);
+	if (check_res) {
+		ibdev_err(&dev->ib_dev, "max_recv_wr:%d exceed max rq depth\n",
+			  init_attr->cap.max_recv_wr);
+		return -EINVAL;
+	}
+
+	if (!has_rq) {
+		qp->rq.max_gs = 0;
+		qp->rq.wqe_cnt = 0;
+		qp->rq.wqe_shift = 0;
+	} else {
+		if (ucmd) {
+			qp->rq.wqe_cnt = ucmd->rq_wqe_count;
+			qp->rq.wqe_shift = ucmd->rq_wqe_shift;
+			qp->rq.max_gs = 1;
+			qp->rq.max_post = qp->rq.wqe_cnt;
+		} else {
+			qp->rq.wqe_cnt = wqe_cnt;
+			if (qp->xqp.qp_type == IB_QPT_GSI)
+				qp->rq.wqe_shift = ilog2(xsc_get_recv_ds_num(dev->xdev))
+					+ XSC_RECV_WQE_SHIFT;
+			else
+				qp->rq.wqe_shift = dev->xdev->caps.recv_wqe_shift;
+			qp->rq.max_gs = dev->xdev->caps.recv_ds_num;
+			qp->rq.max_post = qp->rq.wqe_cnt;
+		}
+	}
+
+	return 0;
+}
+
+static int calc_sq_size(struct xsc_ib_dev *dev, struct ib_qp_init_attr *attr,
+			struct xsc_ib_qp *qp)
+{
+	int wqe_size;
+	int wq_size;
+	bool check_res = false;
+
+	if (!attr->cap.max_send_wr) {
+		ibdev_err(&dev->ib_dev, "invalid max_send_wr:%d\n", attr->cap.max_send_wr);
+		return -1;
+	}
+
+	wqe_size = 1 << dev->xdev->caps.send_wqe_shift;
+	qp->max_inline_data = (dev->xdev->caps.send_ds_num - 2) * sizeof(struct xsc_wqe_data_seg);
+	attr->cap.max_inline_data = qp->max_inline_data;
+
+	attr->cap.max_send_wr = min_t(u32, attr->cap.max_send_wr, dev->xdev->caps.max_wqes);
+	qp->sq.wqe_cnt = roundup_pow_of_two(attr->cap.max_send_wr);
+
+	check_res = xsc_check_max_qp_depth(dev->xdev, &qp->sq.wqe_cnt, dev->xdev->caps.max_wqes);
+	if (check_res) {
+		ibdev_err(&dev->ib_dev, "max_send_wr:%d exceed max sq depth\n",
+			  attr->cap.max_send_wr);
+		return -EINVAL;
+	}
+	qp->sq.ds_cnt = qp->sq.wqe_cnt << (dev->xdev->caps.send_wqe_shift - XSC_BASE_WQE_SHIFT);
+	wq_size = qp->sq.wqe_cnt * wqe_size;
+	qp->sq.wqe_shift = ilog2(wqe_size);
+	qp->sq.max_gs = dev->xdev->caps.send_ds_num - XSC_CTRL_SEG_NUM - XSC_RADDR_SEG_NUM;
+	qp->sq.max_post = qp->sq.wqe_cnt;
+
+	return wq_size;
+}
+
+static int qp_has_rq(struct ib_qp_init_attr *attr)
+{
+	if (!attr->srq && !attr->cap.max_recv_wr)
+		return 0;
+
+	return 1;
+}
+
+static enum xsc_qp_state to_xsc_state(enum ib_qp_state state)
+{
+	switch (state) {
+	case IB_QPS_RESET:	return XSC_QP_STATE_RST;
+	case IB_QPS_INIT:	return XSC_QP_STATE_INIT;
+	case IB_QPS_RTR:	return XSC_QP_STATE_RTR;
+	case IB_QPS_RTS:	return XSC_QP_STATE_RTS;
+	case IB_QPS_SQD:	return XSC_QP_STATE_SQD;
+	case IB_QPS_SQE:	return XSC_QP_STATE_SQER;
+	case IB_QPS_ERR:	return XSC_QP_STATE_ERR;
+	default:		return -1;
+	}
+}
+
+static char *qp_state_to_str(enum ib_qp_state state)
+{
+	switch (state) {
+	case IB_QPS_RESET:	return "RST";
+	case IB_QPS_INIT:	return "INIT";
+	case IB_QPS_RTR:	return "RTR";
+	case IB_QPS_RTS:	return "RTS";
+	case IB_QPS_SQD:	return "SQD";
+	case IB_QPS_SQE:	return "SQE";
+	case IB_QPS_ERR:	return "ERR";
+	default:		return "UNKNOWN";
+	}
+}
+
+static struct ib_umem *qp_umem_get(struct xsc_ib_dev *dev, struct xsc_ib_qp *qp,
+				   struct ib_udata *udata, struct ib_pd *pd,
+	struct xsc_ib_create_qp *ucmd)
+{
+	struct ib_umem *umem;
+
+	umem = ib_umem_get_va(&dev->ib_dev, ucmd->buf_addr, qp->buf_size,
+			      IB_ACCESS_LOCAL_WRITE);
+	return umem;
+}
+
+static int create_user_qp(struct xsc_ib_dev *dev, struct ib_pd *pd,
+			  struct xsc_ib_qp *qp, struct ib_udata *udata,
+			  struct xsc_create_qp_mbox_in **in,
+			  struct xsc_ib_create_qp_resp *resp, int *inlen)
+{
+	struct xsc_ib_create_qp ucmd;
+	int page_shift;
+	int npages;
+	u32 offset;
+	int ncont;
+	int err;
+	int hw_npages;
+	struct ib_umem_dmabuf *umem_dmabuf;
+
+	err = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd));
+	if (err) {
+		ibdev_err(&dev->ib_dev, "failed to copy from udata, err=%d\n", err);
+		return err;
+	}
+	ibdev_dbg(&dev->ib_dev, "buf_addr:0x%lx db_addr:0x%lx sq cnt:%u, rq cnt:%u, rq shift:%u\n",
+		  (uintptr_t)ucmd.buf_addr, (uintptr_t)ucmd.db_addr,
+		    ucmd.sq_wqe_count, ucmd.rq_wqe_count, ucmd.rq_wqe_shift);
+
+	qp->sq.ds_cnt = ucmd.sq_wqe_count;
+	qp->sq.wqe_cnt = ucmd.sq_wqe_count;
+	qp->sq.wqe_shift = XSC_BASE_WQE_SHIFT;
+	qp->rq.ds_cnt = ucmd.rq_wqe_count;
+	qp->rq.wqe_cnt = ucmd.rq_wqe_count;
+	qp->rq.wqe_shift = XSC_BASE_WQE_SHIFT;
+
+	qp->buf_size = (qp->sq.wqe_cnt << qp->sq.wqe_shift) + (qp->rq.wqe_cnt << qp->rq.wqe_shift);
+
+	if (ucmd.dmabuf_fd != -1) {
+		umem_dmabuf = ib_umem_dmabuf_get_pinned(&dev->ib_dev, 0, ucmd.dmabuf_sz,
+							ucmd.dmabuf_fd, IB_ACCESS_LOCAL_WRITE);
+
+		if (IS_ERR(umem_dmabuf)) {
+			ibdev_warn(&dev->ib_dev, "qp umem_dmabuf get failed (%ld)\n",
+				   PTR_ERR(umem_dmabuf));
+			return -ENOMEM;
+		}
+
+		qp->umem = &umem_dmabuf->umem;
+	} else {
+		qp->umem = qp_umem_get(dev, qp, udata, pd, &ucmd);
+	}
+
+	if (IS_ERR(qp->umem)) {
+		ibdev_err(&dev->ib_dev, "umem_get failed\n");
+		err = PTR_ERR(qp->umem);
+		goto err_uuar;
+	}
+
+	xsc_ib_cont_pages(qp->umem, ucmd.buf_addr, &npages, &page_shift,
+			  &ncont);
+	if (ncont != npages) {
+		/* TODO: peer memory support failed */
+		page_shift = PAGE_SHIFT;
+		ncont = npages;
+	}
+
+	hw_npages = DIV_ROUND_UP(qp->buf_size, PAGE_SIZE_4K);
+	err = xsc_ib_get_buf_offset(ucmd.buf_addr, page_shift, &offset);
+	if (err) {
+		ibdev_err(&dev->ib_dev, "bad offset:%d\n", offset);
+		goto err_umem;
+	}
+	ibdev_dbg(&dev->ib_dev, "npage:%d, page_shift:%d, ncont:%d, offset:%d, hw_npages %d\n",
+		  npages, page_shift, ncont, offset, hw_npages);
+
+	*inlen = sizeof(**in) + sizeof(*((*in)->req.pas)) * hw_npages;
+	*in = xsc_vzalloc(*inlen);
+	if (!*in) {
+		err = -ENOMEM;
+		goto err_umem;
+	}
+	xsc_ib_populate_pas(dev, qp->umem, page_shift, (*in)->req.pas, hw_npages, true);
+	(*in)->req.pa_num = cpu_to_be16(hw_npages);
+
+	err = ib_copy_to_udata(udata, resp, sizeof(*resp));
+	if (err) {
+		ibdev_err(&dev->ib_dev, "failed to copy to udata, err=%d\n", err);
+		goto err_umem;
+	}
+	qp->create_type = XSC_QP_USER;
+
+	return 0;
+
+err_umem:
+	ib_umem_release(qp->umem);
+
+err_uuar:
+	return err;
+}
+
+static void destroy_qp_user(struct ib_pd *__maybe_unused pd, struct xsc_ib_qp *qp)
+{
+	ib_umem_release(qp->umem);
+}
+
 #define MAX_QP1_SQ_HDR_SIZE_V2	512
 #define MAX_QP1_SQ_HDR_SIZE	86
 	/* Ethernet header	=  14 */

@@ -57,6 +319,690 @@ static void *get_seg_wqe(void *first, int n)
 	/* iCRC			=   4 */
 #define MAX_QP1_RQ_HDR_SIZE_V2	512
 
+static int create_kernel_qp(struct xsc_ib_dev *dev,
+			    struct ib_qp_init_attr *init_attr,
+			    struct xsc_ib_qp *qp,
+			    struct xsc_create_qp_mbox_in **in, int *inlen)
+{
+	int err;
+	int sq_size;
+	int hw_npages;
+
+	sq_size = calc_sq_size(dev, init_attr, qp);
+	if (sq_size < 0) {
+		err = -EINVAL;
+		ibdev_err(&dev->ib_dev, "err %d\n", err);
+		return err;
+	}
+
+	qp->rq.ds_cnt = qp->rq.wqe_cnt << (qp->rq.wqe_shift - XSC_BASE_WQE_SHIFT);
+	qp->rq.offset = 0;
+	qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
+	qp->buf_size = qp->sq.offset + sq_size;
+	qp->send_psn = 0;
+
+	err = xsc_buf_alloc(dev->xdev, qp->buf_size, PAGE_SIZE, &qp->buf);
+	if (err) {
+		ibdev_err(&dev->ib_dev, "failed to alloc qp buffer,err=%d\n", err);
+		return err;
+	}
+
+	qp->sq.qend = qp->buf.direct.buf + qp->sq.offset + sq_size;
+	hw_npages = DIV_ROUND_UP(qp->buf_size, PAGE_SIZE_4K);
+	*inlen = sizeof(**in) + sizeof(*(*in)->req.pas) * hw_npages;
+	*in = xsc_vzalloc(*inlen);
+	if (!*in) {
+		err = -ENOMEM;
+		goto err_buf;
+	}
+
+	xsc_fill_page_array(&qp->buf, (*in)->req.pas, hw_npages);
+	(*in)->req.pa_num = cpu_to_be16(hw_npages);
+
+	qp->sq.wrid = kcalloc(qp->sq.wqe_cnt, sizeof(*qp->sq.wrid), GFP_KERNEL);
+	qp->sq.wr_data = kcalloc(qp->sq.wqe_cnt, sizeof(*qp->sq.wr_data), GFP_KERNEL);
+	qp->rq.wrid = kcalloc(qp->rq.wqe_cnt, sizeof(*qp->rq.wrid), GFP_KERNEL);
+	qp->sq.w_list = kcalloc(qp->sq.wqe_cnt, sizeof(*qp->sq.w_list), GFP_KERNEL);
+	qp->sq.wqe_head = kcalloc(qp->sq.wqe_cnt, sizeof(*qp->sq.wqe_head), GFP_KERNEL);
+	qp->sq.wr_opcode = kcalloc(qp->sq.wqe_cnt, sizeof(*qp->sq.wr_opcode), GFP_KERNEL);
+	qp->sq.need_flush = kcalloc(qp->sq.wqe_cnt, sizeof(*qp->sq.need_flush), GFP_KERNEL);
+	qp->sq.prev_ce_wqe = kcalloc(qp->sq.wqe_cnt, sizeof(*qp->sq.prev_ce_wqe), GFP_KERNEL);
+	qp->sq.last_comp_ce_wqe = -1;
+	qp->sq.last_post_ce_wqe = -1;
+
+	if (!qp->sq.wrid || !qp->sq.wr_data || !qp->rq.wrid || !qp->sq.prev_ce_wqe ||
+	    !qp->sq.w_list || !qp->sq.wqe_head || !qp->sq.wr_opcode || !qp->sq.need_flush) {
+		err = -ENOMEM;
+		goto err_wrid;
+	}
+	qp->create_type = XSC_QP_KERNEL;
+
+	if (init_attr->qp_type == IB_QPT_GSI) {
+		qp->sq.mad_index = 0;
+		qp->sq.mad_queue_depth = MAD_QUEUE_DEPTH;
+		qp->sq.hdr_size = MAX_QP1_SQ_HDR_SIZE_V2 * MAD_QUEUE_DEPTH;
+		qp->sq.hdr_buf = dma_alloc_coherent(dev->ib_dev.dma_device,
+						    qp->sq.hdr_size,
+						    &qp->sq.hdr_dma,
+						    GFP_KERNEL);
+		if (!qp->sq.hdr_buf) {
+			err = -ENOMEM;
+			ibdev_err(&dev->ib_dev, "Failed to create sq_hdr_buf");
+			goto err_wrid;
+		}
+	}
+
+	return 0;
+
+err_wrid:
+	kfree(qp->sq.prev_ce_wqe);
+	kfree(qp->sq.need_flush);
+	kfree(qp->sq.wr_opcode);
+	kfree(qp->sq.wqe_head);
+	kfree(qp->sq.w_list);
+	kfree(qp->sq.wrid);
+	kfree(qp->sq.wr_data);
+	kfree(qp->rq.wrid);
+
+err_buf:
+	xsc_buf_free(dev->xdev, &qp->buf);
+	return err;
+}
+
+static void destroy_qp_kernel(struct xsc_ib_dev *dev, struct xsc_ib_qp *qp)
+{
+	if (qp->sq.hdr_buf)
+		dma_free_coherent(dev->ib_dev.dma_device, qp->sq.hdr_size,
+				  qp->sq.hdr_buf, qp->sq.hdr_dma);
+	kfree(qp->sq.prev_ce_wqe);
+	kfree(qp->sq.need_flush);
+	kfree(qp->sq.wr_opcode);
+	kfree(qp->sq.wqe_head);
+	kfree(qp->sq.w_list);
+	kfree(qp->sq.wrid);
+	kfree(qp->sq.wr_data);
+	kfree(qp->rq.wrid);
+	xsc_buf_free(dev->xdev, &qp->buf);
+}
+
+static u8 ib_to_xsc_qp_type(enum ib_qp_type qp_type, __u32 flags)
+{
+	if (qp_type == IB_QPT_RC) {
+		if (flags & XSC_QP_FLAG_WORC)
+			return XSC_QUEUE_TYPE_WORC;
+		else
+			return XSC_QUEUE_TYPE_RDMA_RC;
+	} else if ((qp_type == IB_QPT_GSI) || (qp_type == IB_QPT_SMI)) {
+		return XSC_QUEUE_TYPE_RDMA_MAD;
+	} else if (qp_type == IB_QPT_RAW_PACKET) {
+		if (flags & XSC_QP_FLAG_RAWPACKET_TSO)
+			return XSC_QUEUE_TYPE_RAW_TSO;
+		else if (flags & XSC_QP_FLAG_RAWPACKET_TX)
+			return XSC_QUEUE_TYPE_RAW_TX;
+		else if (flags & XSC_QP_FLAG_RAWPACKET_SNIFFER)
+			return XSC_QUEUE_TYPE_SNIFFER;
+		else
+			return XSC_QUEUE_TYPE_RAW;
+	} else {
+		return XSC_QUEUE_TYPE_INVALID;
+	}
+}
+
+static void get_cqs(enum ib_qp_type qp_type,
+		    struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq,
+		    struct xsc_ib_cq **send_cq, struct xsc_ib_cq **recv_cq)
+{
+	switch (qp_type) {
+	case IB_QPT_SMI:
+	case IB_QPT_GSI:
+	case IB_QPT_RC:
+	case IB_QPT_UC:
+	case IB_QPT_UD:
+	case IB_QPT_RAW_PACKET:
+		*send_cq = ib_send_cq ? to_xcq(ib_send_cq) : NULL;
+		*recv_cq = ib_recv_cq ? to_xcq(ib_recv_cq) : NULL;
+		break;
+	default:
+		*send_cq = NULL;
+		*recv_cq = NULL;
+		break;
+	}
+}
+
+static void xsc_ib_lock_cqs(struct xsc_ib_cq *send_cq, struct xsc_ib_cq *recv_cq)
+	__acquires(&send_cq->lock) __acquires(&recv_cq->lock)
+{
+	if (send_cq) {
+		if (recv_cq) {
+			if (send_cq->xcq.cqn < recv_cq->xcq.cqn)  {
+				spin_lock_irq(&send_cq->lock);
+				spin_lock_nested(&recv_cq->lock,
+						 SINGLE_DEPTH_NESTING);
+			} else if (send_cq->xcq.cqn == recv_cq->xcq.cqn) {
+				spin_lock_irq(&send_cq->lock);
+				__acquire(&recv_cq->lock);
+			} else {
+				spin_lock_irq(&recv_cq->lock);
+				spin_lock_nested(&send_cq->lock,
+						 SINGLE_DEPTH_NESTING);
+			}
+		} else {
+			spin_lock_irq(&send_cq->lock);
+		}
+	} else if (recv_cq) {
+		spin_lock_irq(&recv_cq->lock);
+	}
+}
+
+static void xsc_ib_unlock_cqs(struct xsc_ib_cq *send_cq, struct xsc_ib_cq *recv_cq)
+	__releases(&send_cq->lock) __releases(&recv_cq->lock)
+{
+	if (send_cq) {
+		if (recv_cq) {
+			if (send_cq->xcq.cqn < recv_cq->xcq.cqn)  {
+				spin_unlock(&recv_cq->lock);
+				spin_unlock_irq(&send_cq->lock);
+			} else if (send_cq->xcq.cqn == recv_cq->xcq.cqn) {
+				__release(&recv_cq->lock);
+				spin_unlock_irq(&send_cq->lock);
+			} else {
+				spin_unlock(&send_cq->lock);
+				spin_unlock_irq(&recv_cq->lock);
+			}
+		} else {
+			spin_unlock_irq(&send_cq->lock);
+		}
+	} else if (recv_cq) {
+		spin_unlock_irq(&recv_cq->lock);
+	}
+}
+
+static int create_qp_common(struct xsc_ib_dev *dev, struct ib_pd *pd,
+			    struct ib_qp_init_attr *init_attr,
+			    struct ib_udata *udata, struct xsc_ib_qp *qp)
+{
+	struct xsc_ib_create_qp_resp resp;
+	struct xsc_create_qp_mbox_in *in = NULL;
+	struct xsc_ib_create_qp ucmd;
+	struct xsc_ib_cq *send_cq;
+	struct xsc_ib_cq *recv_cq;
+	struct xsc_srq_qp_node *qp_node;
+	int inlen = sizeof(*in);
+	int err;
+	char buf[256];
+	char *ptr = buf;
+	int ret = 0;
+
+	mutex_init(&qp->mutex);
+	spin_lock_init(&qp->sq.lock);
+	spin_lock_init(&qp->rq.lock);
+	spin_lock_init(&qp->lock);
+
+	if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
+		qp->sq_signal_bits = XSC_WQE_CTRL_CQ_UPDATE;
+
+	if (pd->uobject) {
+		if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {
+			ibdev_err(&dev->ib_dev, "failed to copy from udata\n");
+			return -EFAULT;
+		}
+	}
+
+	qp->has_rq = qp_has_rq(init_attr);
+
+	err = set_rq_size(dev, init_attr, qp->has_rq,
+			  qp, pd->uobject ? &ucmd : NULL);
+	if (err) {
+		ibdev_err(&dev->ib_dev, "failed to set rq size %d\n", err);
+		return err;
+	}
+
+	if (pd->uobject) {
+		err = create_user_qp(dev, pd, qp, udata, &in, &resp, &inlen);
+		if (err)
+			ibdev_err(&dev->ib_dev, "failed to create user qp, err = %d\n", err);
+	} else {
+		err = create_kernel_qp(dev, init_attr, qp, &in, &inlen);
+		if (err)
+			ibdev_err(&dev->ib_dev, "failed to create kernel qp, err = %d\n", err);
+		else
+			qp->pa_lkey = to_xpd(pd)->pa_lkey;
+	}
+
+	if (err)
+		return err;
+
+	if (is_sqp(init_attr->qp_type))
+		qp->port = init_attr->port_num;
+
+	if (is_qp1(init_attr->qp_type))
+		in->req.input_qpn = cpu_to_be16(1);
+
+	in->req.pdn = cpu_to_be32(to_xpd(pd)->pdn);
+
+	if (qp->rq.ds_cnt)
+		in->req.log_rq_sz = ilog2(qp->rq.ds_cnt);
+
+	if (qp->sq.ds_cnt)
+		in->req.log_sq_sz = ilog2(qp->sq.ds_cnt);
+	else
+		in->req.log_sq_sz = ilog2(0x80);
+
+	if (init_attr->send_cq) {
+		qp->send_cq = init_attr->send_cq;
+		in->req.cqn_send = to_xcq(init_attr->send_cq)->xcq.cqn;
+		in->req.cqn_send = cpu_to_be16(in->req.cqn_send);
+	}
+
+	if (init_attr->recv_cq) {
+		qp->recv_cq = init_attr->recv_cq;
+		in->req.cqn_recv = to_xcq(init_attr->recv_cq)->xcq.cqn;
+		in->req.cqn_recv = cpu_to_be16(in->req.cqn_recv);
+	}
+
+	in->req.qp_type = ib_to_xsc_qp_type(init_attr->qp_type, pd->uobject ?
+					    ucmd.flags : 0);
+
+	if (in->req.qp_type == XSC_QUEUE_TYPE_INVALID) {
+		ibdev_err(&dev->ib_dev, "invalid qp type:%d\n", init_attr->qp_type);
+		goto err_create;
+	}
+	in->req.glb_funcid = cpu_to_be16(dev->xdev->glb_func_id);
+	in->req.page_shift = PAGE_SHIFT;
+
+	qp->xqp.qp_type_internal = in->req.qp_type;
+
+	err = xsc_core_create_qp(dev->xdev, &qp->xqp, in, inlen);
+	if (err) {
+		ibdev_err(&dev->ib_dev, "create qp failed, err=%d\n", err);
+		goto err_create;
+	}
+
+	qp->doorbell_qpn = qp->xqp.qpn;
+
+	qp->xqp.event = xsc_ib_qp_event;
+	qp->xqp.qp_type = init_attr->qp_type;
+	ret += snprintf(ptr + ret, 256 - ret, "pdn=%d,", to_xpd(pd)->pdn);
+	ret += snprintf(ptr + ret, 256 - ret, "log_rq_sz=%d,", in->req.log_rq_sz);
+	ret += snprintf(ptr + ret, 256 - ret, "log_sq_sz=%d,", in->req.log_sq_sz);
+	ret += snprintf(ptr + ret, 256 - ret, "scqn=%d,", to_xcq(init_attr->send_cq)->xcq.cqn);
+	ret += snprintf(ptr + ret, 256 - ret, "rcqn=%d", to_xcq(init_attr->recv_cq)->xcq.cqn);
+
+	get_cqs(init_attr->qp_type, init_attr->send_cq, init_attr->recv_cq, &send_cq, &recv_cq);
+	xsc_ib_lock_cqs(send_cq, recv_cq);
+	if (send_cq)
+		list_add_tail(&qp->cq_send_list, &send_cq->send_qp_list);
+	if (recv_cq)
+		list_add_tail(&qp->cq_recv_list, &recv_cq->recv_qp_list);
+	xsc_ib_unlock_cqs(send_cq, recv_cq);
+
+	if (!pd->uobject && init_attr->srq) {
+		qp_node = kvzalloc_obj(*qp_node, GFP_KERNEL);
+		if (!qp_node)
+			goto err_create;
+		qp_node->ibqp = &qp->ibqp;
+		qp_node->qp_id = qp->xqp.qpn;
+		ibdev_dbg(&dev->ib_dev, "qpn:%u add into srq_qp_list\n", qp_node->qp_id);
+		list_add_tail(&qp_node->entry, &to_xsrq(init_attr->srq)->xsrq.srq_qp_list);
+	}
+
+	qp->xqp.qp_protocol_mode = dev->xdev->rdma_proto_mode;
+
+	ibdev_dbg(&dev->ib_dev, "succeeded to create qp:%d, %s\n", qp->xqp.qpn, buf);
+
+	xsc_vfree(in);
+
+	return 0;
+
+err_create:
+	if (qp->create_type == XSC_QP_USER)
+		destroy_qp_user(pd, qp);
+	else if (qp->create_type == XSC_QP_KERNEL)
+		destroy_qp_kernel(dev, qp);
+
+	xsc_vfree(in);
+	return err;
+}
+
+static struct xsc_ib_pd *get_pd(struct xsc_ib_qp *qp)
+{
+	return to_xpd(qp->ibqp.pd);
+}
+
+static void destroy_qp_common(struct xsc_ib_dev *dev, struct xsc_ib_qp *qp)
+{
+	struct xsc_ib_cq *send_cq = NULL;
+	struct xsc_ib_cq *recv_cq = NULL;
+	struct xsc_modify_qp_mbox_in *in;
+	int err;
+
+	in = kzalloc_obj(*in, GFP_KERNEL);
+	if (!in)
+		return;
+
+	if (qp->xqp.qp_type_internal == XSC_QUEUE_TYPE_RAW ||
+	    qp->xqp.qp_type_internal == XSC_QUEUE_TYPE_RAW_TSO ||
+	    qp->xqp.qp_type_internal == XSC_QUEUE_TYPE_RAW_TX ||
+	    qp->state != IB_QPS_RESET)
+		if (xsc_core_qp_modify(dev->xdev, to_xsc_state(qp->state),
+				       XSC_QP_STATE_RST, in, sizeof(*in), &qp->xqp))
+			ibdev_warn(&dev->ib_dev, "modify QP %06x to RESET failed\n", qp->xqp.qpn);
+
+	get_cqs(qp->xqp.qp_type, qp->ibqp.send_cq, qp->ibqp.recv_cq, &send_cq, &recv_cq);
+	xsc_ib_lock_cqs(send_cq, recv_cq);
+	if (send_cq)
+		list_del(&qp->cq_send_list);
+
+	if (recv_cq)
+		list_del(&qp->cq_recv_list);
+
+	if (qp->create_type == XSC_QP_KERNEL) {
+		__xsc_ib_cq_clean(recv_cq, qp->xqp.qpn);
+		if (send_cq != recv_cq)
+			__xsc_ib_cq_clean(send_cq, qp->xqp.qpn);
+	}
+	xsc_ib_unlock_cqs(send_cq, recv_cq);
+
+	err = xsc_core_destroy_qp(dev->xdev, &qp->xqp);
+	if (err)
+		ibdev_warn(&dev->ib_dev, "failed to destroy QP 0x%x\n", qp->xqp.qpn);
+	kfree(in);
+
+	if (qp->create_type == XSC_QP_KERNEL)
+		destroy_qp_kernel(dev, qp);
+	else if (qp->create_type == XSC_QP_USER)
+		destroy_qp_user(&get_pd(qp)->ibpd, qp);
+}
+
+static const char *ib_qp_type_str(enum ib_qp_type type)
+{
+	switch (type) {
+	case IB_QPT_SMI:
+		return "IB_QPT_SMI";
+	case IB_QPT_GSI:
+		return "IB_QPT_GSI";
+	case IB_QPT_RC:
+		return "IB_QPT_RC";
+	case IB_QPT_UC:
+		return "IB_QPT_UC";
+	case IB_QPT_UD:
+		return "IB_QPT_UD";
+	case IB_QPT_RAW_IPV6:
+		return "IB_QPT_RAW_IPV6";
+	case IB_QPT_RAW_ETHERTYPE:
+		return "IB_QPT_RAW_ETHERTYPE";
+	case IB_QPT_RAW_PACKET:
+		return "IB_QPT_RAW_PACKET";
+	case IB_QPT_MAX:
+	default:
+		return "Invalid QP type";
+	}
+}
+
+int xsc_ib_create_qp(struct ib_qp *ibqp,
+		     struct ib_qp_init_attr *init_attr,
+		     struct ib_udata *udata)
+{
+	struct xsc_ib_dev *dev;
+	struct xsc_ib_qp *qp;
+	struct ib_pd *pd = ibqp->pd;
+	int err;
+
+	if (pd) {
+		dev = to_xdev(pd->device);
+	} else {
+		pr_warn("%s: no PD for transport %s\n", __func__,
+			ib_qp_type_str(init_attr->qp_type));
+		return -EINVAL;
+	}
+
+	if (init_attr->qp_type != IB_QPT_RAW_PACKET) {
+		if (!is_support_rdma(dev->xdev) ||
+		    (is_qp1(init_attr->qp_type) && !is_support_rdma_cm(dev->xdev))) {
+			return -EPROTONOSUPPORT;
+		}
+	}
+
+	qp = to_xqp(ibqp);
+
+	qp->xqp.qp_type = init_attr->qp_type;
+
+	switch (init_attr->qp_type) {
+	case IB_QPT_RC:
+	case IB_QPT_SMI:
+	case IB_QPT_GSI:
+	case IB_QPT_RAW_PACKET:
+		err = create_qp_common(dev, pd, init_attr, udata, qp);
+		if (err) {
+			ibdev_err(&dev->ib_dev, "create_qp_common failed\n");
+			return err;
+		}
+
+		if (is_qp0(init_attr->qp_type)) {
+			qp->ibqp.qp_num = 0;
+		} else if (is_qp1(init_attr->qp_type)) {
+			qp->ibqp.qp_num = 1;
+			dev->xdev->gsi_qpn = qp->xqp.qpn;
+		} else {
+			qp->ibqp.qp_num = qp->xqp.qpn;
+		}
+
+		break;
+
+	case IB_QPT_RAW_IPV6:
+	case IB_QPT_RAW_ETHERTYPE:
+	case IB_QPT_MAX:
+	default:
+		ibdev_err(&dev->ib_dev, "unsupported qp type %d\n",
+			  init_attr->qp_type);
+		/* Don't support raw QPs */
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int xsc_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)
+{
+	struct xsc_ib_dev *dev = to_xdev(qp->device);
+	struct xsc_ib_qp *xqp = to_xqp(qp);
+	struct xsc_err_state_qp_node *tmp = NULL, *err_rq_node = NULL, *err_sq_node = NULL;
+	struct xsc_srq_qp_node *qp_node = NULL, *tmp_qp_node = NULL;
+	unsigned long flags;
+	struct xsc_ib_srq *xsrq;
+
+	if (qp->qp_type == IB_QPT_RC) {
+		if (xqp->recv_cq) {
+			spin_lock_irq(&to_xcq(xqp->recv_cq)->lock);
+			list_for_each_entry_safe(err_rq_node, tmp,
+						 &to_xcq(xqp->recv_cq)->err_state_qp_list, entry) {
+				if (err_rq_node->qp_id == xqp->xqp.qpn && !err_rq_node->is_sq) {
+					list_del(&err_rq_node->entry);
+					kfree(err_rq_node);
+				}
+			}
+			spin_unlock_irq(&to_xcq(xqp->recv_cq)->lock);
+		}
+
+		if (xqp->send_cq) {
+			spin_lock_irq(&to_xcq(xqp->send_cq)->lock);
+			list_for_each_entry_safe(err_sq_node, tmp,
+						 &to_xcq(xqp->send_cq)->err_state_qp_list, entry) {
+				if (err_sq_node->qp_id == xqp->xqp.qpn && err_sq_node->is_sq) {
+					list_del(&err_sq_node->entry);
+					kfree(err_sq_node);
+				}
+			}
+			spin_unlock_irq(&to_xcq(xqp->send_cq)->lock);
+		}
+	}
+
+	if (xqp->create_type == XSC_QP_KERNEL && qp->srq) {
+		if (!to_xsrq(qp->srq) || list_empty(&to_xsrq(qp->srq)->xsrq.srq_qp_list))
+			ibdev_warn(&dev->ib_dev, "srq_qp_list is empty!\n");
+
+		xsrq = to_xsrq(qp->srq);
+		spin_lock_irqsave(&xsrq->lock, flags);
+		xsc_restore_cache_srq_wrs(xsrq, xqp);
+		list_for_each_entry_safe(qp_node, tmp_qp_node,
+					 &to_xsrq(qp->srq)->xsrq.srq_qp_list, entry) {
+			if (qp_node->qp_id == xqp->xqp.qpn) {
+				to_xsrq(qp->srq)->qp_num--;
+				ibdev_dbg(&dev->ib_dev, "srq_qp_list del node qpn:%u qp_num:%d\n",
+					  qp_node->qp_id, to_xsrq(qp->srq)->qp_num);
+				list_del_init(&qp_node->entry);
+				kvfree(qp_node);
+				break;
+			}
+		}
+		spin_unlock_irqrestore(&xsrq->lock, flags);
+	}
+
+	destroy_qp_common(dev, xqp);
+
+	return 0;
+}
+
+static inline u16 xsc_calc_udp_sport(u32 lqpn, u32 rqpn)
+{
+	unsigned char *p;
+	u8 ports[2];
+	u16 sport;
+	u64 tqpn;
+
+	tqpn = ((u64)(lqpn & 0xffffff)) * ((u64)(rqpn & 0xffffff));
+	p = (unsigned char *)&tqpn;
+	ports[0] = p[0] ^ p[2] ^ p[4];
+	ports[1] = p[1] ^ p[3] ^ p[5];
+	sport = *((u16 *)ports) | 0xC000;
+
+	return sport;
+}
+
+static inline void xsc_path_set_udp_sport(struct xsc_qp_path *path,
+					  const struct rdma_ah_attr *ah,
+					  u32 lqpn, u32 rqpn)
+{
+	if ((ah->grh.flow_label & UDP_SPORT_MASK) != 0) {
+		if ((ah->grh.flow_label & UDP_SPORT_MASK_EN) == 0)
+			path->sport = cpu_to_be16(rdma_flow_label_to_udp_sport(ah->grh.flow_label));
+		else
+			path->sport = cpu_to_be16((ah->grh.flow_label & UDP_SPORT_MASK) >>
+							UDP_SPORT_OFFSET);
+	} else {
+		path->sport = cpu_to_be16(xsc_calc_udp_sport(lqpn, rqpn));
+	}
+}
+
+static int xsc_set_path(struct xsc_ib_dev *dev, const struct rdma_ah_attr *ah,
+			struct xsc_qp_path *path, u8 port, int attr_mask,
+			u32 path_flags, const struct ib_qp_attr *attr, struct xsc_ib_qp *qp)
+{
+	struct ib_global_route *grh = rdma_ah_retrieve_grh((struct rdma_ah_attr *)ah);
+	union ib_gid *dgid = &grh->dgid;
+	const struct ib_gid_attr *sgid_attr = grh->sgid_attr;
+	union ib_gid *sgid = &((struct ib_gid_attr *)sgid_attr)->gid;
+	union {
+		struct sockaddr     _sockaddr;
+		struct sockaddr_in  _sockaddr_in;
+		struct sockaddr_in6 _sockaddr_in6;
+	} sgid_addr, dgid_addr;
+	int force_pcp, force_dscp;
+	char buf[256] = {0};
+	char *ptr = buf;
+	int ret = 0;
+
+	if (ah->type == RDMA_AH_ATTR_TYPE_ROCE) {
+		if (!(rdma_ah_get_ah_flags(ah) & IB_AH_GRH))
+			return -EINVAL;
+
+		if (qp->ibqp.qp_type == IB_QPT_RC ||
+		    qp->ibqp.qp_type == IB_QPT_UC) {
+			if (((ah->grh.flow_label & UDP_SPORT_MASK_EN) != 0) &&
+			    qp->xqp.qp_protocol_mode == RDMA_PROTO_VEROCE) {
+				ibdev_err(&dev->ib_dev,
+					  "qp type is  RDMA_PROTO_VEROCE, using flow_label to set the UDP sport %d is not allowed.",
+					   (ah->grh.flow_label & UDP_SPORT_MASK) >>
+					   UDP_SPORT_OFFSET);
+				return -EINVAL;
+			}
+			xsc_path_set_udp_sport(path, ah, qp->ibqp.qp_num, attr->dest_qp_num);
+		}
+
+		if (sgid_attr->gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP) {
+			ibdev_err(&dev->ib_dev, "gid type not ROCEv2\n");
+			return -EINVAL;
+		}
+
+		force_dscp = dev->xdev->priv.force_dscp;
+		if (force_dscp == DSCP_PCP_UNSET)
+			path->ecn_dscp = (grh->traffic_class >> 2) & 0x3f;
+		else
+			path->ecn_dscp = force_dscp;
+		path->hop_limit = grh->hop_limit;
+
+		rdma_gid2ip((struct sockaddr *)&sgid_addr, sgid);
+		rdma_gid2ip((struct sockaddr *)&dgid_addr, dgid);
+
+		if (sgid_addr._sockaddr.sa_family == AF_INET &&
+		    dgid_addr._sockaddr.sa_family == AF_INET) {
+			memcpy(path->sip, &sgid_addr._sockaddr_in.sin_addr.s_addr,
+			       sizeof(struct in_addr));
+			memcpy(path->dip, &dgid_addr._sockaddr_in.sin_addr.s_addr,
+			       sizeof(struct in_addr));
+			path->af_type = AF_INET;
+			ret += snprintf(ptr + ret, 256 - ret, "sip=%#x,",
+					be32_to_cpu(path->sip[0]));
+			ret += snprintf(ptr + ret, 256 - ret, "dip=%#x,",
+					be32_to_cpu(path->dip[0]));
+		} else if (sgid_addr._sockaddr.sa_family == AF_INET6 &&
+			   dgid_addr._sockaddr.sa_family == AF_INET6) {
+			memcpy(path->sip, &sgid_addr._sockaddr_in6.sin6_addr.s6_addr,
+			       sizeof(path->sip));
+			memcpy(path->dip, &dgid_addr._sockaddr_in6.sin6_addr.s6_addr,
+			       sizeof(path->dip));
+			path->af_type = AF_INET6;
+			ret += snprintf(ptr + ret, 256 - ret, "sip=%08x%08x%08x%08x,",
+					be32_to_cpu(path->sip[0]), be32_to_cpu(path->sip[1]),
+					be32_to_cpu(path->sip[2]), be32_to_cpu(path->sip[3]));
+			ret += snprintf(ptr + ret, 256 - ret, "dip=%08x%08x%08x%08x,",
+					be32_to_cpu(path->dip[0]), be32_to_cpu(path->dip[1]),
+					be32_to_cpu(path->dip[2]), be32_to_cpu(path->dip[3]));
+		} else {
+			return -EINVAL;
+		}
+
+		ether_addr_copy(path->smac, dev->netdev->dev_addr);
+
+		memcpy(path->dmac, ah->roce.dmac, sizeof(ah->roce.dmac));
+		ret += snprintf(ptr + ret, 256 - ret, "smac=%02x%02x%02x%02x%02x%02x,",
+				path->smac[0], path->smac[1], path->smac[2],
+				path->smac[3], path->smac[4], path->smac[5]);
+		ret += snprintf(ptr + ret, 256 - ret, "dmac=%02x%02x%02x%02x%02x%02x",
+				path->dmac[0], path->dmac[1], path->dmac[2],
+				path->dmac[3], path->dmac[4], path->dmac[5]);
+		if (is_zero_ether_addr(path->dmac))
+			ibdev_err(&dev->ib_dev, "path dmac is zero!\n");
+		else
+			ibdev_dbg(&dev->ib_dev, "ib path info:%s\n", buf);
+
+		if (is_vlan_dev(sgid_attr->ndev)) {
+			path->vlan_valid = 1;
+			path->vlan_id = cpu_to_be16(vlan_dev_vlan_id(sgid_attr->ndev));
+
+			force_pcp = dev->xdev->priv.force_pcp;
+			if (force_pcp == DSCP_PCP_UNSET)
+				path->dci_cfi_prio_sl = (ah->sl & 0x7);
+			else
+				path->dci_cfi_prio_sl = force_pcp;
+		} else {
+			path->vlan_valid = 0;
+		}
+	}
+	ibdev_dbg(&dev->ib_dev, "path dscp %d pcp %d\n", path->ecn_dscp, path->dci_cfi_prio_sl);
+	return 0;
+}
+
 static inline void xsc_ib_trigger_cq_comp_handler(struct ib_cq *send_cq, struct ib_cq *recv_cq)
 {
 	if (send_cq)