Lines Matching refs:qpriv
341 struct hfi1_qp_priv *qpriv = qp->priv;
344 qpriv->rcd = qp_to_rcd(rdi, qp);
346 spin_lock_init(&qpriv->opfn.lock);
347 INIT_WORK(&qpriv->opfn.opfn_work, opfn_send_conn_request);
348 INIT_WORK(&qpriv->tid_rdma.trigger_work, tid_rdma_trigger_resume);
349 qpriv->flow_state.psn = 0;
350 qpriv->flow_state.index = RXE_NUM_TID_FLOWS;
351 qpriv->flow_state.last_index = RXE_NUM_TID_FLOWS;
352 qpriv->flow_state.generation = KERN_GENERATION_RESERVED;
353 qpriv->s_state = TID_OP(WRITE_RESP);
354 qpriv->s_tid_cur = HFI1_QP_WQE_INVALID;
355 qpriv->s_tid_head = HFI1_QP_WQE_INVALID;
356 qpriv->s_tid_tail = HFI1_QP_WQE_INVALID;
357 qpriv->rnr_nak_state = TID_RNR_NAK_INIT;
358 qpriv->r_tid_head = HFI1_QP_WQE_INVALID;
359 qpriv->r_tid_tail = HFI1_QP_WQE_INVALID;
360 qpriv->r_tid_ack = HFI1_QP_WQE_INVALID;
361 qpriv->r_tid_alloc = HFI1_QP_WQE_INVALID;
362 atomic_set(&qpriv->n_requests, 0);
363 atomic_set(&qpriv->n_tid_requests, 0);
364 timer_setup(&qpriv->s_tid_timer, hfi1_tid_timeout, 0);
365 timer_setup(&qpriv->s_tid_retry_timer, hfi1_tid_retry_timeout, 0);
366 INIT_LIST_HEAD(&qpriv->tid_wait);
369 struct hfi1_devdata *dd = qpriv->rcd->dd;
371 qpriv->pages = kzalloc_node(TID_RDMA_MAX_PAGES *
372 sizeof(*qpriv->pages),
374 if (!qpriv->pages)
415 struct hfi1_qp_priv *qpriv = qp->priv;
433 cancel_work_sync(&qpriv->opfn.opfn_work);
434 kfree(qpriv->pages);
435 qpriv->pages = NULL;
782 struct hfi1_qp_priv *qpriv = (struct hfi1_qp_priv *)qp->priv;
783 struct tid_flow_state *fs = &qpriv->flow_state;
822 struct hfi1_qp_priv *qpriv = (struct hfi1_qp_priv *)qp->priv;
823 struct tid_flow_state *fs = &qpriv->flow_state;
1467 struct hfi1_qp_priv *qpriv = req->qp->priv;
1489 if (kern_get_phys_blocks(flow, qpriv->pages, ss, last)) {
1517 * qpriv getting out of sync.
1520 flow->idx = qpriv->flow_state.index;
1521 flow->flow_state.generation = qpriv->flow_state.generation;
1522 flow->flow_state.spsn = qpriv->flow_state.psn;
1526 qpriv->flow_state.psn += flow->npkts;
1657 struct hfi1_qp_priv *qpriv = qp->priv;
1670 req->rcd = qpriv->rcd;
1710 struct hfi1_qp_priv *qpriv = qp->priv;
1744 remote = rcu_dereference(qpriv->tid_rdma.remote);
1758 cpu_to_be32(qpriv->tid_rdma.local.qp |
1761 qpriv->rcd->ctxt);
1774 qpriv->pending_tid_r_segs++;
1792 struct hfi1_qp_priv *qpriv = qp->priv;
1808 if (qpriv->pending_tid_r_segs)
1812 qpriv->s_flags &= ~HFI1_R_TID_SW_PSN;
1839 if ((qpriv->flow_state.psn + npkts) > MAX_TID_FLOW_PSN - 1) {
1845 if (hfi1_kern_setup_hw_flow(qpriv->rcd, qp))
1895 struct hfi1_qp_priv *qpriv = qp->priv;
1969 req->n_flows = qpriv->tid_rdma.local.max_read;
1975 req->seg_len = qpriv->tid_rdma.local.max_len;
1992 struct hfi1_qp_priv *qpriv = qp->priv;
2128 qpriv->pending_tid_w_segs -=
2135 qpriv->pending_tid_w_segs +=
2172 qpriv->pending_tid_w_segs -=
2180 qpriv->s_flags &= ~HFI1_R_TID_WAIT_INTERLCK;
2198 if (qpriv->rnr_nak_state) {
2200 qpriv->rnr_nak_state = TID_RNR_NAK_INIT;
2237 struct hfi1_qp_priv *qpriv = qp->priv;
2263 if (!len || len & ~PAGE_MASK || len > qpriv->tid_rdma.local.max_len)
2313 * queue, qpriv->r_tid_alloc follows qp->r_head_ack_queue. It is ok to
2317 qpriv->r_tid_alloc = qp->r_head_ack_queue;
2350 struct hfi1_qp_priv *qpriv = qp->priv;
2369 remote = rcu_dereference(qpriv->tid_rdma.remote);
2861 struct hfi1_qp_priv *qpriv;
2935 qpriv = qp->priv;
2936 if (qpriv->r_tid_tail == HFI1_QP_WQE_INVALID ||
2937 qpriv->r_tid_tail == qpriv->r_tid_head)
2939 e = &qp->s_ack_queue[qpriv->r_tid_tail];
2957 if (!(qpriv->s_flags & HFI1_R_TID_SW_PSN)) {
2958 qpriv->s_flags |= HFI1_R_TID_SW_PSN;
2962 qpriv->r_next_psn_kdeth =
2981 qpriv->s_nak_state = 0;
2992 qpriv->r_next_psn_kdeth =
3031 if (!qpriv->s_nak_state) {
3032 qpriv->s_nak_state = IB_NAK_PSN_ERROR;
3034 qpriv->s_nak_psn = mask_psn(flow->flow_state.r_next_psn);
3051 struct hfi1_qp_priv *qpriv = qp->priv;
3107 rvt_skip_sge(&qpriv->tid_ss, (req->cur_seg * req->seg_len) +
3138 i = qpriv->s_tid_tail;
3148 if (i == qpriv->s_tid_cur)
3166 struct hfi1_qp_priv *qpriv = qp->priv;
3176 fs = &qpriv->flow_state;
3178 hfi1_kern_clear_hw_flow(qpriv->rcd, qp);
3284 struct hfi1_qp_priv *qpriv = (struct hfi1_qp_priv *)qp->priv;
3289 struct hfi1_pportdata *ppd = qpriv->rcd->ppd;
3294 if (qpriv->hdr_type != HFI1_PKT_TYPE_9B)
3298 remote = rcu_dereference(qpriv->tid_rdma.remote);
3338 qpriv->tid_r_reqs++;
3342 atomic_inc(&qpriv->n_requests);
3370 struct hfi1_qp_priv *qpriv = qp->priv;
3375 remote = rcu_dereference(qpriv->tid_rdma.remote);
3415 static u32 position_in_queue(struct hfi1_qp_priv *qpriv,
3418 return qpriv->tid_enqueue - queue->dequeue;
3428 struct hfi1_qp_priv *qpriv = qp->priv;
3433 bytes_per_us = active_egress_rate(qpriv->rcd->ppd) / 8;
3460 * [request: qpriv->r_tid_alloc, segment: req->alloc_seg]
3467 struct hfi1_qp_priv *qpriv = qp->priv;
3468 struct hfi1_ctxtdata *rcd = qpriv->rcd;
3469 struct tid_rdma_params *local = &qpriv->tid_rdma.local;
3492 if (qpriv->rnr_nak_state == TID_RNR_NAK_SEND)
3496 if (qpriv->r_tid_alloc == qpriv->r_tid_head) {
3498 if (qpriv->flow_state.index < RXE_NUM_TID_FLOWS &&
3499 !qpriv->alloc_w_segs) {
3501 qpriv->s_flags &= ~HFI1_R_TID_SW_PSN;
3506 e = &qp->s_ack_queue[qpriv->r_tid_alloc];
3517 if (qpriv->alloc_w_segs >= local->max_write)
3521 if (qpriv->sync_pt && qpriv->alloc_w_segs)
3525 if (qpriv->sync_pt && !qpriv->alloc_w_segs) {
3527 qpriv->sync_pt = false;
3528 qpriv->s_flags &= ~HFI1_R_TID_SW_PSN;
3532 if (qpriv->flow_state.index >= RXE_NUM_TID_FLOWS) {
3533 ret = hfi1_kern_setup_hw_flow(qpriv->rcd, qp);
3536 position_in_queue(qpriv,
3548 if (qpriv->flow_state.psn + npkts > MAX_TID_FLOW_PSN - 1) {
3549 qpriv->sync_pt = true;
3571 to_seg = position_in_queue(qpriv, &rcd->rarr_queue);
3575 qpriv->alloc_w_segs++;
3580 if (++qpriv->r_tid_alloc >
3582 qpriv->r_tid_alloc = 0;
3609 qp->r_head_ack_queue = qpriv->r_tid_alloc + 1;
3612 qpriv->r_tid_head = qp->r_head_ack_queue;
3628 * qpriv->rnr_nak_state is used to determine when the scheduled RNR NAK
3634 qpriv->rnr_nak_state = TID_RNR_NAK_SEND;
3666 struct hfi1_qp_priv *qpriv = qp->priv;
3692 num_segs = DIV_ROUND_UP(len, qpriv->tid_rdma.local.max_len);
3704 if (qpriv->rnr_nak_state)
3723 if (qpriv->rnr_nak_state) {
3726 qpriv->rnr_nak_state = TID_RNR_NAK_INIT;
3757 req->n_flows = min_t(u16, num_segs, qpriv->tid_rdma.local.max_write);
3764 req->seg_len = qpriv->tid_rdma.local.max_len;
3788 if (qpriv->r_tid_tail == HFI1_QP_WQE_INVALID) {
3789 qpriv->r_tid_tail = qp->r_head_ack_queue;
3790 } else if (qpriv->r_tid_tail == qpriv->r_tid_head) {
3793 e = &qp->s_ack_queue[qpriv->r_tid_tail];
3798 if (qpriv->r_tid_tail == qpriv->r_tid_ack)
3799 qpriv->r_tid_ack = qp->r_head_ack_queue;
3800 qpriv->r_tid_tail = qp->r_head_ack_queue;
3805 qpriv->r_tid_head = qp->r_head_ack_queue;
3842 struct hfi1_qp_priv *qpriv = qp->priv;
3869 if (qpriv->rnr_nak_state == TID_RNR_NAK_SENT)
3914 remote = rcu_dereference(qpriv->tid_rdma.remote);
3925 cpu_to_be32(qpriv->tid_rdma.local.qp |
3928 qpriv->rcd->ctxt);
3933 qpriv->pending_tid_w_segs++;
3940 struct hfi1_qp_priv *qpriv = qp->priv;
3943 if (!(qpriv->s_flags & HFI1_R_TID_RSC_TIMER)) {
3944 qpriv->s_flags |= HFI1_R_TID_RSC_TIMER;
3945 qpriv->s_tid_timer.expires = jiffies +
3946 qpriv->tid_timer_timeout_jiffies;
3947 add_timer(&qpriv->s_tid_timer);
3953 struct hfi1_qp_priv *qpriv = qp->priv;
3956 qpriv->s_flags |= HFI1_R_TID_RSC_TIMER;
3957 mod_timer(&qpriv->s_tid_timer, jiffies +
3958 qpriv->tid_timer_timeout_jiffies);
3963 struct hfi1_qp_priv *qpriv = qp->priv;
3967 if (qpriv->s_flags & HFI1_R_TID_RSC_TIMER) {
3968 rval = del_timer(&qpriv->s_tid_timer);
3969 qpriv->s_flags &= ~HFI1_R_TID_RSC_TIMER;
3976 struct hfi1_qp_priv *qpriv = qp->priv;
3978 del_timer_sync(&qpriv->s_tid_timer);
3979 qpriv->s_flags &= ~HFI1_R_TID_RSC_TIMER;
3984 struct hfi1_qp_priv *qpriv = from_timer(qpriv, t, s_tid_timer);
3985 struct rvt_qp *qp = qpriv->owner;
3992 if (qpriv->s_flags & HFI1_R_TID_RSC_TIMER) {
3997 (u64)qpriv->tid_timer_timeout_jiffies);
4003 hfi1_kern_clear_hw_flow(qpriv->rcd, qp);
4042 struct hfi1_qp_priv *qpriv = qp->priv;
4081 wqe = rvt_get_swqe_ptr(qp, qpriv->s_tid_cur);
4178 if (qpriv->s_tid_cur != qpriv->s_tid_head &&
4180 for (i = qpriv->s_tid_cur + 1; ; i++) {
4184 if (i == qpriv->s_tid_head)
4189 qpriv->s_tid_cur = i;
4213 struct hfi1_qp_priv *qpriv = qp->priv;
4234 remote = rcu_dereference(qpriv->tid_rdma.remote);
4430 struct hfi1_qp_priv *qpriv = qp->priv;
4431 struct tid_flow_state *fs = &qpriv->flow_state;
4437 remote = rcu_dereference(qpriv->tid_rdma.remote);
4443 if (qpriv->resync) {
4447 } else if (qpriv->s_nak_state) {
4448 *bth2 = mask_psn(qpriv->s_nak_psn);
4451 (qpriv->s_nak_state <<
4459 cpu_to_be32(qpriv->tid_rdma.local.qp |
4462 qpriv->rcd->ctxt);
4468 if (qpriv->resync) {
4475 if (hfi1_tid_rdma_is_resync_psn(qpriv->r_next_psn_kdeth - 1)) {
4477 cpu_to_be32(qpriv->r_next_psn_kdeth_save);
4485 qpriv->r_next_psn_kdeth_save =
4486 qpriv->r_next_psn_kdeth - 1;
4488 cpu_to_be32(qpriv->r_next_psn_kdeth_save);
4489 qpriv->r_next_psn_kdeth = mask_psn(*bth2 + 1);
4491 qpriv->resync = false;
4501 struct hfi1_qp_priv *qpriv = qp->priv;
4521 cmp_psn(psn, qpriv->s_resync_psn))
4568 atomic_dec(&qpriv->n_tid_requests);
4583 if (qpriv->s_flags & RVT_S_WAIT_ACK)
4584 qpriv->s_flags &= ~RVT_S_WAIT_ACK;
4607 qpriv->s_flags &= ~RVT_S_SEND_ONE;
4610 if ((qp->s_acked == qpriv->s_tid_tail &&
4613 qpriv->s_state = TID_OP(WRITE_DATA_LAST);
4618 qpriv->s_state = TID_OP(WRITE_DATA);
4700 if (++last_acked == qpriv->s_tid_cur + 1)
4708 qpriv->s_tid_tail = qp->s_acked;
4709 qpriv->s_state = TID_OP(WRITE_REQ);
4713 qpriv->s_retry = qp->s_retry_cnt;
4731 qpriv->s_tid_tail = qp->s_acked;
4732 qpriv->s_state = TID_OP(WRITE_REQ);
4733 qpriv->s_retry = qp->s_retry_cnt;
4847 struct hfi1_qp_priv *qpriv = qp->priv;
4854 remote = rcu_dereference(qpriv->tid_rdma.remote);
4862 qpriv->s_resync_psn = *bth2;
4873 struct hfi1_qp_priv *qpriv = qp->priv;
4874 struct hfi1_ctxtdata *rcd = qpriv->rcd;
4879 struct tid_flow_state *fs = &qpriv->flow_state;
4900 if (qpriv->resync)
4920 qpriv->s_flags &= ~HFI1_R_TID_SW_PSN;
4928 for (idx = qpriv->r_tid_tail; ; idx++) {
4969 qpriv->resync = true;
4971 qpriv->s_nak_state = 0;
5191 struct hfi1_qp_priv *qpriv = qp->priv;
5208 e = &qp->s_ack_queue[qpriv->r_tid_ack];
5218 * to pull back qpriv->r_tid_ack, not the segment
5222 if (qpriv->resync) {
5224 qpriv->r_tid_ack = !qpriv->r_tid_ack ?
5226 qpriv->r_tid_ack - 1;
5227 e = &qp->s_ack_queue[qpriv->r_tid_ack];
5238 if (!qpriv->s_nak_state && !qpriv->resync &&
5266 next = qpriv->r_tid_ack + 1;
5269 qpriv->r_tid_ack = next;
5277 e = &qp->s_ack_queue[qpriv->r_tid_ack];
5282 * At this point qpriv->r_tid_ack == qpriv->r_tid_tail but e and
5285 if (qpriv->s_nak_state ||
5286 (qpriv->resync &&
5287 !hfi1_tid_rdma_is_resync_psn(qpriv->r_next_psn_kdeth - 1) &&
5288 (cmp_psn(qpriv->r_next_psn_kdeth - 1,
5294 * segment for the request at qpriv->r_tid_ack (same at
5296 * qpriv->r_tid_tail request)
5298 e = &qp->s_ack_queue[qpriv->r_tid_ack];
5302 qpriv->s_flags & HFI1_R_TID_WAIT_INTERLCK)
5303 qpriv->s_flags &= ~HFI1_R_TID_WAIT_INTERLCK;
5311 qpriv->s_flags &= ~RVT_S_ACK_PENDING;
5313 ps->s_txreq->sde = qpriv->s_sde;
5326 qpriv->s_flags &= ~RVT_S_ACK_PENDING;