Lines Matching refs:rdi
64 static void rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp,
145 void rvt_wss_exit(struct rvt_dev_info *rdi)
147 struct rvt_wss *wss = rdi->wss;
155 kfree(rdi->wss);
156 rdi->wss = NULL;
164 int rvt_wss_init(struct rvt_dev_info *rdi)
166 unsigned int sge_copy_mode = rdi->dparms.sge_copy_mode;
167 unsigned int wss_threshold = rdi->dparms.wss_threshold;
168 unsigned int wss_clean_period = rdi->dparms.wss_clean_period;
174 int node = rdi->dparms.node;
177 rdi->wss = NULL;
181 rdi->wss = kzalloc_node(sizeof(*rdi->wss), GFP_KERNEL, node);
182 if (!rdi->wss)
184 wss = rdi->wss;
221 rvt_wss_exit(rdi);
328 static int init_qpn_table(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt)
334 if (!(rdi->dparms.qpn_res_end >= rdi->dparms.qpn_res_start))
339 qpt->last = rdi->dparms.qpn_start;
340 qpt->incr = rdi->dparms.qpn_inc << rdi->dparms.qos_shift;
350 qpt->nmaps = rdi->dparms.qpn_res_start / RVT_BITS_PER_PAGE;
353 offset = rdi->dparms.qpn_res_start & RVT_BITS_PER_PAGE_MASK;
358 rvt_pr_info(rdi, "Reserving QPNs from 0x%x to 0x%x for non-verbs use\n",
359 rdi->dparms.qpn_res_start, rdi->dparms.qpn_res_end);
360 for (i = rdi->dparms.qpn_res_start; i <= rdi->dparms.qpn_res_end; i++) {
394 * @rdi: rvt dev strucutre
398 int rvt_driver_qp_init(struct rvt_dev_info *rdi)
403 if (!rdi->dparms.qp_table_size)
410 if (!rdi->driver_f.free_all_qps ||
411 !rdi->driver_f.qp_priv_alloc ||
412 !rdi->driver_f.qp_priv_free ||
413 !rdi->driver_f.notify_qp_reset ||
414 !rdi->driver_f.notify_restart_rc)
418 rdi->qp_dev = kzalloc_node(sizeof(*rdi->qp_dev), GFP_KERNEL,
419 rdi->dparms.node);
420 if (!rdi->qp_dev)
424 rdi->qp_dev->qp_table_size = rdi->dparms.qp_table_size;
425 rdi->qp_dev->qp_table_bits = ilog2(rdi->dparms.qp_table_size);
426 rdi->qp_dev->qp_table =
427 kmalloc_array_node(rdi->qp_dev->qp_table_size,
428 sizeof(*rdi->qp_dev->qp_table),
429 GFP_KERNEL, rdi->dparms.node);
430 if (!rdi->qp_dev->qp_table)
433 for (i = 0; i < rdi->qp_dev->qp_table_size; i++)
434 RCU_INIT_POINTER(rdi->qp_dev->qp_table[i], NULL);
436 spin_lock_init(&rdi->qp_dev->qpt_lock);
439 if (init_qpn_table(rdi, &rdi->qp_dev->qpn_table))
442 spin_lock_init(&rdi->n_qps_lock);
447 kfree(rdi->qp_dev->qp_table);
448 free_qpn_table(&rdi->qp_dev->qpn_table);
451 kfree(rdi->qp_dev);
467 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
470 rvt_reset_qp(rdi, qp, qp->ibqp.qp_type);
478 * @rdi: rvt device info structure
484 static unsigned rvt_free_all_qps(struct rvt_dev_info *rdi)
488 qp_inuse += rvt_mcast_tree_empty(rdi);
490 rvt_qp_iter(rdi, (u64)&qp_inuse, rvt_free_qp_cb);
497 * @rdi: rvt dev structure
501 void rvt_qp_exit(struct rvt_dev_info *rdi)
503 u32 qps_inuse = rvt_free_all_qps(rdi);
506 rvt_pr_err(rdi, "QP memory leak! %u still in use\n",
509 kfree(rdi->qp_dev->qp_table);
510 free_qpn_table(&rdi->qp_dev->qpn_table);
511 kfree(rdi->qp_dev);
523 * @rdi: rvt device info structure
530 static int alloc_qpn(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt,
539 if (rdi->driver_f.alloc_qpn)
540 return rdi->driver_f.alloc_qpn(rdi, qpt, type, port_num);
603 WARN_ON(rdi->dparms.qos_shift > 1 &&
604 offset & ((BIT(rdi->dparms.qos_shift - 1) - 1) << 1));
622 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
644 for (n = 0; qp->s_ack_queue && n < rvt_max_atomic(rdi); n++) {
706 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
708 for (i = 0; qp->s_ack_queue && i < rvt_max_atomic(rdi); i++) {
763 * @rdi: rvt dev struct
769 static void rvt_remove_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp)
771 struct rvt_ibport *rvp = rdi->ports[qp->port_num - 1];
772 u32 n = hash_32(qp->ibqp.qp_num, rdi->qp_dev->qp_table_bits);
776 spin_lock_irqsave(&rdi->qp_dev->qpt_lock, flags);
779 lockdep_is_held(&rdi->qp_dev->qpt_lock)) == qp) {
782 lockdep_is_held(&rdi->qp_dev->qpt_lock)) == qp) {
789 qpp = &rdi->qp_dev->qp_table[n];
791 lockdep_is_held(&rdi->qp_dev->qpt_lock))) != NULL;
796 lockdep_is_held(&rdi->qp_dev->qpt_lock)));
804 spin_unlock_irqrestore(&rdi->qp_dev->qpt_lock, flags);
862 static void rvt_init_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp,
913 static void _rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp,
926 rdi->driver_f.flush_qp_waiters(qp);
934 rdi->driver_f.stop_send_queue(qp);
937 rdi->driver_f.quiesce_qp(qp);
940 rvt_remove_qp(rdi, qp);
952 rdi->driver_f.notify_qp_reset(qp);
954 rvt_init_qp(rdi, qp, type);
962 * @rdi: the device info
969 static void rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp,
975 _rvt_reset_qp(rdi, qp, type);
1076 struct rvt_dev_info *rdi = ib_to_rvt(ibpd->device);
1081 if (!rdi)
1084 if (init_attr->cap.max_send_sge > rdi->dparms.props.max_send_sge ||
1085 init_attr->cap.max_send_wr > rdi->dparms.props.max_qp_wr ||
1093 rdi->dparms.props.max_recv_sge ||
1094 init_attr->cap.max_recv_wr > rdi->dparms.props.max_qp_wr)
1105 rdi->dparms.reserved_operations;
1117 swq = vzalloc_node(array_size(sz, sqsize), rdi->dparms.node);
1133 rdi->dparms.node);
1141 kcalloc_node(rvt_max_atomic(rdi),
1144 rdi->dparms.node);
1158 priv = rdi->driver_f.qp_priv_alloc(rdi, qp);
1175 rdi->dparms.node, udata);
1200 err = alloc_ud_wq_attr(qp, rdi->dparms.node);
1209 err = alloc_qpn(rdi, &rdi->qp_dev->qpn_table,
1221 rvt_init_qp(rdi, qp, init_attr->qp_type);
1222 if (rdi->driver_f.qp_priv_init) {
1223 err = rdi->driver_f.qp_priv_init(rdi, qp, init_attr);
1255 qp->ip = rvt_create_mmap_info(rdi, s, udata,
1272 spin_lock(&rdi->n_qps_lock);
1273 if (rdi->n_qps_allocated == rdi->dparms.props.max_qp) {
1274 spin_unlock(&rdi->n_qps_lock);
1279 rdi->n_qps_allocated++;
1290 rdi->n_rc_qps++;
1291 rdi->busy_jiffies = rdi->n_rc_qps / RC_QP_SCALING_INTERVAL;
1293 spin_unlock(&rdi->n_qps_lock);
1296 spin_lock_irq(&rdi->pending_lock);
1297 list_add(&qp->ip->pending_mmaps, &rdi->pending_mmaps);
1298 spin_unlock_irq(&rdi->pending_lock);
1310 rvt_free_qpn(&rdi->qp_dev->qpn_table, qp->ibqp.qp_num);
1319 rdi->driver_f.qp_priv_free(rdi, qp);
1346 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
1363 rdi->driver_f.notify_error_qp(qp);
1367 rdi->driver_f.schedule_send(qp);
1428 static void rvt_insert_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp)
1430 struct rvt_ibport *rvp = rdi->ports[qp->port_num - 1];
1434 spin_lock_irqsave(&rdi->qp_dev->qpt_lock, flags);
1439 u32 n = hash_32(qp->ibqp.qp_num, rdi->qp_dev->qp_table_bits);
1441 qp->next = rdi->qp_dev->qp_table[n];
1442 rcu_assign_pointer(rdi->qp_dev->qp_table[n], qp);
1446 spin_unlock_irqrestore(&rdi->qp_dev->qpt_lock, flags);
1461 struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device);
1483 if (rdi->driver_f.check_modify_qp &&
1484 rdi->driver_f.check_modify_qp(qp, attr, attr_mask, udata))
1515 if (attr->alt_pkey_index >= rvt_get_npkeys(rdi))
1520 if (attr->pkey_index >= rvt_get_npkeys(rdi))
1555 pmtu = rdi->driver_f.get_pmtu_from_attr(rdi, qp, attr);
1579 if (attr->max_dest_rd_atomic > rdi->dparms.max_rdma_atomic)
1585 _rvt_reset_qp(rdi, qp, ibqp->qp_type);
1624 qp->s_next_psn = attr->sq_psn & rdi->dparms.psn_modify_mask;
1632 qp->r_psn = attr->rq_psn & rdi->dparms.psn_modify_mask;
1658 qp->pmtu = rdi->driver_f.mtu_from_qp(rdi, qp, pmtu);
1689 if (rdi->driver_f.modify_qp)
1690 rdi->driver_f.modify_qp(qp, attr, attr_mask, udata);
1697 rvt_insert_qp(rdi, qp);
1732 struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device);
1734 rvt_reset_qp(rdi, qp, ibqp->qp_type);
1738 rvt_free_qpn(&rdi->qp_dev->qpn_table, qp->ibqp.qp_num);
1740 spin_lock(&rdi->n_qps_lock);
1741 rdi->n_qps_allocated--;
1743 rdi->n_rc_qps--;
1744 rdi->busy_jiffies = rdi->n_rc_qps / RC_QP_SCALING_INTERVAL;
1746 spin_unlock(&rdi->n_qps_lock);
1751 rdi->driver_f.qp_priv_free(rdi, qp);
1774 struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device);
1778 attr->path_mtu = rdi->driver_f.mtu_to_path_mtu(qp->pmtu);
1781 attr->rq_psn = qp->r_psn & rdi->dparms.psn_mask;
1782 attr->sq_psn = qp->s_next_psn & rdi->dparms.psn_mask;
1786 rdi->dparms.reserved_operations;
1950 * @rdi: the rdmavt device
1963 struct rvt_dev_info *rdi,
1975 if (reserved_used >= rdi->dparms.reserved_operations)
1991 (rdi->dparms.reserved_operations - reserved_used);
1997 (qp->s_size - 1 - rdi->dparms.reserved_operations)))
1998 rvt_pr_err(rdi,
2022 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
2035 ret = rvt_qp_valid_operation(qp, rdi->post_parms, wr);
2051 if ((rdi->post_parms[wr->opcode].flags & RVT_OPERATION_LOCAL)) {
2077 reserved_op = rdi->post_parms[wr->opcode].flags &
2080 ret = rvt_qp_is_avail(qp, rdi, reserved_op);
2087 rkt = &rdi->lkey_table;
2131 if (rdi->post_parms[wr->opcode].flags & RVT_OPERATION_LOCAL) {
2149 if (rdi->driver_f.setup_wqe) {
2150 ret = rdi->driver_f.setup_wqe(qp, wqe, call_send);
2155 if (!(rdi->post_parms[wr->opcode].flags & RVT_OPERATION_LOCAL))
2198 struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device);
2238 rdi->driver_f.do_send(qp);
2240 rdi->driver_f.schedule_send_no_lock(qp);
2325 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
2327 rkt = &rdi->lkey_table;
2561 struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device);
2566 qp->s_timer.expires = jiffies + rdi->busy_jiffies +
2641 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
2647 struct rvt_ibport *rvp = rdi->ports[qp->port_num - 1];
2653 if (rdi->driver_f.notify_restart_rc)
2654 rdi->driver_f.notify_restart_rc(qp,
2657 rdi->driver_f.schedule_send(qp);
2669 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
2675 rdi->driver_f.schedule_send(qp);
2683 * @rdi: rvt devinfo
2700 struct rvt_qp_iter *rvt_qp_iter_init(struct rvt_dev_info *rdi,
2710 i->rdi = rdi;
2712 i->specials = rdi->ibdev.phys_port_cnt * 2;
2739 struct rvt_dev_info *rdi = iter->rdi;
2751 * n = iter->specials..rdi->qp_dev->qp_table_size+iter->specials are
2755 for (; n < rdi->qp_dev->qp_table_size + iter->specials; n++) {
2763 pidx = n % rdi->ibdev.phys_port_cnt;
2764 rvp = rdi->ports[pidx];
2768 rdi->qp_dev->qp_table[
2785 * @rdi: rvt devinfo
2799 void rvt_qp_iter(struct rvt_dev_info *rdi,
2805 .rdi = rdi,
2806 .specials = rdi->ibdev.phys_port_cnt * 2,
2833 struct rvt_dev_info *rdi;
2837 rdi = ib_to_rvt(qp->ibqp.device);
2841 last = rvt_qp_complete_swqe(qp, wqe, rdi->wc_opcode[wqe->wr.opcode],
2871 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
2872 struct rvt_wss *wss = rdi->wss;
2873 unsigned int sge_copy_mode = rdi->dparms.sge_copy_mode;
2956 struct rvt_dev_info *rdi = ib_to_rvt(sqp->ibqp.device);
2971 rvp = rdi->ports[sqp->port_num - 1];