Lines Matching refs:dev
87 struct qedr_dev *dev = get_qedr_dev(ibdev);
90 ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
92 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
100 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
101 struct qedr_device_attr *qattr = &dev->attr;
114 struct qedr_dev *dev = get_qedr_dev(ibdev);
115 struct qedr_device_attr *qattr = &dev->attr;
117 if (!dev->rdma_ctx) {
118 DP_ERR(dev,
120 dev->rdma_ctx);
140 if (!rdma_protocol_iwarp(&dev->ibdev, 1))
150 attr->atomic_cap = dev->atomic_cap;
217 struct qedr_dev *dev;
220 dev = get_qedr_dev(ibdev);
222 if (!dev->rdma_ctx) {
223 DP_ERR(dev, "rdma_ctx is NULL\n");
227 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
243 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
244 attr->active_mtu = iboe_get_mtu(dev->iwarp_max_mtu);
247 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
268 struct qedr_dev *dev = get_qedr_dev(ibdev);
279 DP_ERR(dev, "Problem copying data from user space\n");
287 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
289 DP_ERR(dev,
309 entry->dev = dev;
318 if (!dev->user_dpm_enabled)
320 else if (rdma_protocol_iwarp(&dev->ibdev, 1))
338 uresp.max_send_wr = dev->attr.max_sqe;
339 uresp.max_recv_wr = dev->attr.max_rqe;
340 uresp.max_srq_wr = dev->attr.max_srq_wr;
343 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
350 ctx->dev = dev;
352 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
358 dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi);
369 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
378 struct qedr_dev *dev = entry->dev;
383 dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi);
390 struct ib_device *dev = ucontext->device;
397 ibdev_dbg(dev,
403 ibdev_dbg(dev, "pgoff[%#lx] does not have valid entry\n",
408 ibdev_dbg(dev,
428 ibdev_dbg(dev,
439 struct qedr_dev *dev = get_qedr_dev(ibdev);
444 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
447 if (!dev->rdma_ctx) {
448 DP_ERR(dev, "invalid RDMA context\n");
452 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
467 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
468 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
481 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
484 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
485 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
492 struct qedr_dev *dev = get_qedr_dev(ibxrcd->device);
495 return dev->ops->rdma_alloc_xrcd(dev->rdma_ctx, &xrcd->xrcd_id);
500 struct qedr_dev *dev = get_qedr_dev(ibxrcd->device);
503 dev->ops->rdma_dealloc_xrcd(dev->rdma_ctx, xrcd_id);
506 static void qedr_free_pbl(struct qedr_dev *dev,
509 struct pci_dev *pdev = dev->pdev;
515 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
529 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
533 struct pci_dev *pdev = dev->pdev;
545 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,
565 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
568 qedr_free_pbl(dev, pbl_info, pbl_table);
573 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
583 DP_ERR(dev, "prepare pbl table: too many pages %d\n",
614 DP_DEBUG(dev, QEDR_MSG_MR,
621 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
643 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
672 static int qedr_db_recovery_add(struct qedr_dev *dev,
679 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
683 return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data,
687 static void qedr_db_recovery_del(struct qedr_dev *dev,
692 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
699 dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data);
702 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
719 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
744 struct qedr_dev *dev, struct qedr_userq *q,
760 DP_ERR(dev, "get_zeroed_page failed\n");
791 struct qedr_dev *dev,
802 q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access);
804 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
810 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
815 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
820 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
831 return qedr_init_user_db_rec(udata, dev, q, requires_db_rec);
842 struct qedr_dev *dev, int vector,
853 params->dpi = (ctx) ? ctx->dpi : dev->dpi;
870 struct qedr_dev *dev;
872 dev = get_qedr_dev(ibcq->device);
875 DP_ERR(dev,
916 struct qedr_dev *dev = get_qedr_dev(ibdev);
929 DP_DEBUG(dev, QEDR_MSG_INIT,
937 DP_ERR(dev,
953 DP_ERR(dev,
959 DP_ERR(dev,
966 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
980 rc = dev->ops->common->chain_alloc(dev->cdev, &cq->pbl,
990 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
993 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, ¶ms, &icid);
1002 rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset);
1006 rc = qedr_db_recovery_add(dev, cq->q.db_addr,
1016 cq->db_addr = dev->db_addr + db_offset;
1027 rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data,
1033 DP_DEBUG(dev, QEDR_MSG_CQ,
1041 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
1045 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1050 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1061 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1067 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
1073 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1078 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
1079 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1082 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1086 qedr_db_recovery_del(dev, cq->q.db_addr,
1091 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1190 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1194 struct qedr_device_attr *qattr = &dev->attr;
1201 DP_DEBUG(dev, QEDR_MSG_QP,
1208 DP_ERR(dev,
1215 DP_ERR(dev,
1222 DP_ERR(dev,
1229 DP_ERR(dev,
1238 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created) &&
1246 DP_ERR(dev,
1255 static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1265 DP_ERR(dev, "create srq: problem copying data to user space\n");
1270 static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1275 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1290 static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1297 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1307 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1316 qedr_copy_sq_uresp(dev, uresp, qp);
1319 qedr_copy_rq_uresp(dev, uresp, qp);
1321 uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1326 DP_ERR(dev,
1342 static void qedr_set_common_qp_params(struct qedr_dev *dev,
1348 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1362 qp->dev = dev;
1367 DP_DEBUG(dev, QEDR_MSG_QP,
1379 DP_DEBUG(dev, QEDR_MSG_QP,
1384 DP_DEBUG(dev, QEDR_MSG_QP,
1388 DP_DEBUG(dev, QEDR_MSG_QP,
1393 static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1398 qp->sq.db = dev->db_addr +
1401 rc = qedr_db_recovery_add(dev, qp->sq.db, &qp->sq.db_data,
1408 qp->rq.db = dev->db_addr +
1411 rc = qedr_db_recovery_add(dev, qp->rq.db, &qp->rq.db_data,
1414 qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
1420 static int qedr_check_srq_params(struct qedr_dev *dev,
1424 struct qedr_device_attr *qattr = &dev->attr;
1427 DP_ERR(dev,
1434 DP_ERR(dev,
1440 DP_ERR(dev, "XRC SRQs are not supported in kernel-space\n");
1449 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1457 struct qedr_dev *dev = srq->dev;
1459 dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1461 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1474 rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
1482 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1484 DP_ERR(srq->dev,
1497 struct qedr_dev *dev,
1512 va = dma_alloc_coherent(&dev->pdev->dev,
1516 DP_ERR(dev,
1527 rc = dev->ops->common->chain_alloc(dev->cdev, &hw_srq->pbl, ¶ms);
1536 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1546 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1556 DP_DEBUG(dev, QEDR_MSG_QP,
1564 rc = qedr_check_srq_params(dev, init_attr, udata);
1568 srq->dev = dev;
1579 DP_ERR(dev,
1595 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1620 rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1627 rc = qedr_copy_srq_uresp(dev, srq, udata);
1632 rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);
1636 DP_DEBUG(dev, QEDR_MSG_SRQ,
1643 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1656 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1659 xa_erase_irq(&dev->srqs, srq->srq_id);
1662 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1669 DP_DEBUG(dev, QEDR_MSG_SRQ,
1679 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1684 DP_ERR(dev,
1692 DP_ERR(dev,
1700 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1707 DP_DEBUG(dev, QEDR_MSG_SRQ,
1728 qedr_init_common_qp_in_params(struct qedr_dev *dev,
1746 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1765 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1767 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1782 qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1789 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1796 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1800 static void qedr_cleanup_user(struct qedr_dev *dev,
1814 if (rdma_protocol_roce(&dev->ibdev, 1)) {
1815 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1816 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1823 qedr_db_recovery_del(dev, qp->usq.db_addr,
1829 qedr_db_recovery_del(dev, qp->urq.db_addr,
1834 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1835 qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr,
1839 static int qedr_create_user_qp(struct qedr_dev *dev,
1849 int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
1865 DP_ERR(dev, "Problem copying data from user space\n");
1872 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
1880 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
1885 if (rdma_protocol_roce(&dev->ibdev, 1)) {
1886 qedr_free_pbl(dev, &qp->usq.pbl_info,
1896 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1921 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1929 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1930 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1936 rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp);
1945 rc = qedr_db_recovery_add(dev, qp->usq.db_addr,
1956 rc = qedr_db_recovery_add(dev, qp->urq.db_addr,
1964 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1974 rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr,
1981 qedr_qp_user_print(dev, qp);
1984 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1986 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1989 qedr_cleanup_user(dev, ctx, qp);
1993 static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1997 qp->sq.db = dev->db_addr +
2001 rc = qedr_db_recovery_add(dev, qp->sq.db,
2008 qp->rq.db = dev->db_addr +
2011 qp->rq.iwarp_db2 = dev->db_addr +
2016 rc = qedr_db_recovery_add(dev, qp->rq.db,
2023 rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2,
2031 qedr_roce_create_kernel_qp(struct qedr_dev *dev,
2047 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, ¶ms);
2058 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, ¶ms);
2065 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2074 return qedr_set_roce_db_info(dev, qp);
2078 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
2099 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2113 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, ¶ms);
2123 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, ¶ms);
2130 return qedr_set_iwarp_db_info(dev, qp);
2133 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2138 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
2140 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
2143 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
2150 qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
2153 qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data);
2155 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2156 qedr_db_recovery_del(dev, qp->rq.iwarp_db2,
2161 static int qedr_create_kernel_qp(struct qedr_dev *dev,
2186 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
2187 dev->attr.max_sqe);
2192 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
2210 DP_ERR(dev,
2216 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
2219 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
2225 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2226 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
2229 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
2232 qedr_cleanup_kernel(dev, qp);
2237 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
2246 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2252 qedr_cleanup_user(dev, ctx, qp);
2254 qedr_cleanup_kernel(dev, qp);
2265 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2277 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
2280 rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
2284 DP_DEBUG(dev, QEDR_MSG_QP,
2292 qedr_set_common_qp_params(dev, qp, pd, attrs);
2295 return qedr_create_gsi_qp(dev, attrs, qp);
2298 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
2300 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
2307 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
2308 rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
2316 qedr_free_qp_resources(dev, qp, udata);
2362 static int qedr_update_qp_state(struct qedr_dev *dev,
2389 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2445 DP_NOTICE(dev,
2470 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
2476 DP_DEBUG(dev, QEDR_MSG_QP,
2489 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2492 DP_ERR(dev,
2538 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2550 (dev->ndev->mtu)));
2555 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2570 DP_ERR(dev,
2576 rc = qedr_get_dmac(dev, &attr->ah_attr,
2582 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2584 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2587 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2590 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2603 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2649 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2651 DP_ERR(dev,
2654 dev->attr.max_qp_req_rd_atomic_resc);
2678 dev->attr.max_qp_resp_rd_atomic_resc) {
2679 DP_ERR(dev,
2682 dev->attr.max_qp_resp_rd_atomic_resc);
2713 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2718 rc = qedr_update_qp_state(dev, qp, cur_state,
2747 struct qedr_dev *dev = qp->dev;
2755 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, ¶ms);
2776 qp_attr->cap.max_inline_data = dev->attr.max_inline;
2804 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2814 struct qedr_dev *dev = qp->dev;
2818 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2821 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2857 qedr_destroy_gsi_qp(dev);
2863 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2864 xa_erase(&dev->qps, qp->qp_id);
2866 qedr_free_qp_resources(dev, qp, udata);
2868 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
2894 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2907 qedr_free_pbl(dev, &info->pbl_info, pbl);
2911 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2920 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2925 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2931 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2937 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2939 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2945 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2949 free_mr_info(dev, info);
2957 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2963 DP_DEBUG(dev, QEDR_MSG_MR,
2982 rc = init_mr_info(dev, &mr->info,
2987 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2990 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2993 DP_ERR(dev, "Out of MR resources\n");
2995 DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
3019 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3021 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3030 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
3035 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3037 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
3046 struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
3049 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
3053 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3056 free_mr_info(dev, &mr->info);
3070 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3074 DP_DEBUG(dev, QEDR_MSG_MR,
3082 mr->dev = dev;
3085 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
3089 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3092 DP_ERR(dev, "Out of MR resources\n");
3094 DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
3117 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3119 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3126 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
3130 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3132 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
3162 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
3166 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
3181 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
3185 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
3208 handle_completed_mrs(mr->dev, &mr->info);
3214 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3225 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3228 DP_ERR(dev, "Out of MR resources\n");
3230 DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
3245 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3247 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3256 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
3260 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3289 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
3300 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
3396 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3412 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3419 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3431 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3506 struct qedr_dev *dev = qp->dev;
3515 DP_ERR(dev,
3522 DP_ERR(dev,
3530 DP_ERR(dev,
3543 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3581 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3592 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3605 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3618 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3627 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3638 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3651 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3668 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3715 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3722 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3731 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3749 DP_ERR(dev, "POST SEND FAILED\n");
3758 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3770 if (rdma_protocol_roce(&dev->ibdev, 1)) {
3776 DP_DEBUG(dev, QEDR_MSG_CQ,
3836 struct qedr_dev *dev = srq->dev;
3852 DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n",
3870 DP_DEBUG(dev, QEDR_MSG_SRQ,
3881 DP_DEBUG(dev, QEDR_MSG_SRQ,
3905 DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3916 struct qedr_dev *dev = qp->dev;
3931 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n",
3995 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
4052 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
4107 static int qedr_poll_cq_req(struct qedr_dev *dev,
4116 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
4121 DP_DEBUG(dev, QEDR_MSG_CQ,
4124 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
4130 cnt = process_req(dev, qp, cq, num_entries, wc,
4139 DP_ERR(dev,
4145 DP_ERR(dev,
4151 DP_ERR(dev,
4157 DP_ERR(dev,
4163 DP_ERR(dev,
4169 DP_ERR(dev,
4175 DP_ERR(dev,
4181 DP_ERR(dev,
4187 DP_ERR(dev,
4193 DP_ERR(dev,
4199 DP_ERR(dev,
4204 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
4264 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4274 DP_ERR(dev,
4281 DP_ERR(dev,
4293 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4312 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4318 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4324 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4367 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4374 cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4380 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4392 cnt = process_resp_one(dev, qp, cq, wc, resp);
4411 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4420 DP_ERR(dev,
4449 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4454 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4458 cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4464 DP_ERR(dev, "Error: invalid CQE type = %d\n",