Lines Matching refs:req
79 static int user_sdma_send_pkts(struct user_sdma_request *req, u16 maxpkts);
82 static void user_sdma_free_request(struct user_sdma_request *req);
83 static int check_header_template(struct user_sdma_request *req,
86 static int set_txreq_header(struct user_sdma_request *req,
88 static int set_txreq_header_ahg(struct user_sdma_request *req,
116 static int add_system_pages_to_sdma_packet(struct user_sdma_request *req,
351 struct user_sdma_request *req;
358 if (iovec[idx].iov_len < sizeof(info) + sizeof(req->hdr)) {
363 iovec[idx].iov_len, sizeof(info) + sizeof(req->hdr));
413 req = pq->reqs + info.comp_idx;
414 req->data_iovs = req_iovcnt(info.ctrl) - 1; /* subtract header vector */
415 req->data_len = 0;
416 req->pq = pq;
417 req->cq = cq;
418 req->ahg_idx = -1;
419 req->iov_idx = 0;
420 req->sent = 0;
421 req->seqnum = 0;
422 req->seqcomp = 0;
423 req->seqsubmitted = 0;
424 req->tids = NULL;
425 req->has_error = 0;
426 INIT_LIST_HEAD(&req->txps);
428 memcpy(&req->info, &info, sizeof(info));
435 if (req->data_iovs < 2) {
436 SDMA_DBG(req,
441 req->data_iovs--;
444 if (!info.npkts || req->data_iovs > MAX_VECTORS_PER_REQ) {
445 SDMA_DBG(req, "Too many vectors (%u/%u)", req->data_iovs,
452 ret = copy_from_user(&req->hdr, iovec[idx].iov_base + sizeof(info),
453 sizeof(req->hdr));
455 SDMA_DBG(req, "Failed to copy header template (%d)", ret);
462 req->hdr.pbc[2] = 0;
465 opcode = (be32_to_cpu(req->hdr.bth[0]) >> 24) & 0xff;
468 SDMA_DBG(req, "Invalid opcode (%d)", opcode);
477 vl = (le16_to_cpu(req->hdr.pbc[0]) >> 12) & 0xF;
478 sc = (((be16_to_cpu(req->hdr.lrh[0]) >> 12) & 0xF) |
479 (((le16_to_cpu(req->hdr.pbc[1]) >> 14) & 0x1) << 4));
482 SDMA_DBG(req, "Invalid SC(%u)/VL(%u)", sc, vl);
488 pkey = (u16)be32_to_cpu(req->hdr.bth[0]);
489 slid = be16_to_cpu(req->hdr.lrh[3]);
500 if ((be16_to_cpu(req->hdr.lrh[0]) & 0x3) == HFI1_LRH_GRH) {
501 SDMA_DBG(req, "User tried to pass in a GRH");
506 req->koffset = le32_to_cpu(req->hdr.kdeth.swdata[6]);
511 req->tidoffset = KDETH_GET(req->hdr.kdeth.ver_tid_offset, OFFSET) *
512 (KDETH_GET(req->hdr.kdeth.ver_tid_offset, OM) ?
515 info.comp_idx, req->tidoffset);
519 for (i = 0; i < req->data_iovs; i++) {
520 req->iovs[i].offset = 0;
521 INIT_LIST_HEAD(&req->iovs[i].list);
522 memcpy(&req->iovs[i].iov,
524 sizeof(req->iovs[i].iov));
525 if (req->iovs[i].iov.iov_len == 0) {
529 req->data_len += req->iovs[i].iov.iov_len;
532 info.comp_idx, req->data_len);
533 if (pcount > req->info.npkts)
534 pcount = req->info.npkts;
543 if (req_opcode(req->info.ctrl) == EXPECTED) {
544 u16 ntids = iovec[idx].iov_len / sizeof(*req->tids);
559 ntids * sizeof(*req->tids));
562 SDMA_DBG(req, "Failed to copy %d TIDs (%d)",
566 req->tids = tmp;
567 req->n_tids = ntids;
568 req->tididx = 0;
572 dlid = be16_to_cpu(req->hdr.lrh[1]);
575 req->sde = sdma_select_user_engine(dd, selector, vl);
577 if (!req->sde || !sdma_running(req->sde)) {
583 if (req->info.npkts > 1 && HFI1_CAP_IS_USET(SDMA_AHG))
584 req->ahg_idx = sdma_ahg_alloc(req->sde);
595 while (req->seqsubmitted != req->info.npkts) {
596 ret = user_sdma_send_pkts(req, pcount);
620 if (req->seqsubmitted < req->info.npkts) {
621 if (req->seqsubmitted)
623 (req->seqcomp == req->seqsubmitted - 1));
624 user_sdma_free_request(req);
631 static inline u32 compute_data_length(struct user_sdma_request *req,
648 if (!req->seqnum) {
649 if (req->data_len < sizeof(u32))
650 len = req->data_len;
652 len = ((be16_to_cpu(req->hdr.lrh[2]) << 2) -
654 } else if (req_opcode(req->info.ctrl) == EXPECTED) {
655 u32 tidlen = EXP_TID_GET(req->tids[req->tididx], LEN) *
661 len = min(tidlen - req->tidoffset, (u32)req->info.fragsize);
663 if (unlikely(!len) && ++req->tididx < req->n_tids &&
664 req->tids[req->tididx]) {
665 tidlen = EXP_TID_GET(req->tids[req->tididx],
667 req->tidoffset = 0;
668 len = min_t(u32, tidlen, req->info.fragsize);
675 len = min(len, req->data_len - req->sent);
677 len = min(req->data_len - req->sent, (u32)req->info.fragsize);
679 trace_hfi1_sdma_user_compute_length(req->pq->dd,
680 req->pq->ctxt,
681 req->pq->subctxt,
682 req->info.comp_idx,
700 static int user_sdma_txadd_ahg(struct user_sdma_request *req,
705 u16 pbclen = le16_to_cpu(req->hdr.pbc[0]);
706 u32 lrhlen = get_lrh_len(req->hdr, pad_len(datalen));
707 struct hfi1_user_sdma_pkt_q *pq = req->pq;
717 memcpy(&tx->hdr, &req->hdr, sizeof(tx->hdr));
722 ret = check_header_template(req, &tx->hdr, lrhlen, datalen);
726 sizeof(tx->hdr) + datalen, req->ahg_idx,
736 static int user_sdma_send_pkts(struct user_sdma_request *req, u16 maxpkts)
745 if (!req->pq)
748 pq = req->pq;
751 if (READ_ONCE(req->has_error))
757 if (unlikely(req->seqnum == req->info.npkts)) {
758 if (!list_empty(&req->txps))
763 if (!maxpkts || maxpkts > req->info.npkts - req->seqnum)
764 maxpkts = req->info.npkts - req->seqnum;
774 if (READ_ONCE(req->has_error))
782 tx->req = req;
789 if (req->seqnum == req->info.npkts - 1)
798 if (req->data_len) {
799 iovec = &req->iovs[req->iov_idx];
801 if (++req->iov_idx == req->data_iovs) {
805 iovec = &req->iovs[req->iov_idx];
809 datalen = compute_data_length(req, tx);
820 SDMA_DBG(req,
829 if (req->ahg_idx >= 0) {
830 if (!req->seqnum) {
831 ret = user_sdma_txadd_ahg(req, tx, datalen);
837 changes = set_txreq_header_ahg(req, tx,
845 ret = sdma_txinit(&tx->txreq, 0, sizeof(req->hdr) +
855 ret = set_txreq_header(req, tx, datalen);
860 req->koffset += datalen;
861 if (req_opcode(req->info.ctrl) == EXPECTED)
862 req->tidoffset += datalen;
863 req->sent += datalen;
865 ret = add_system_pages_to_sdma_packet(req, tx, iovec,
869 iovec = &req->iovs[req->iov_idx];
871 list_add_tail(&tx->txreq.list, &req->txps);
877 tx->seqnum = req->seqnum++;
881 ret = sdma_send_txlist(req->sde,
883 &req->txps, &count);
884 req->seqsubmitted += count;
885 if (req->seqsubmitted == req->info.npkts) {
892 if (req->ahg_idx >= 0)
893 sdma_ahg_free(req->sde, req->ahg_idx);
915 static int check_header_template(struct user_sdma_request *req,
929 if (req->info.fragsize % PIO_BLOCK_SIZE || lrhlen & 0x3 ||
930 lrhlen > get_lrh_len(*hdr, req->info.fragsize))
933 if (req_opcode(req->info.ctrl) == EXPECTED) {
940 u32 tidval = req->tids[req->tididx],
948 (KDETH_GET(req->hdr.kdeth.ver_tid_offset, OM) ?
985 static int set_txreq_header(struct user_sdma_request *req,
988 struct hfi1_user_sdma_pkt_q *pq = req->pq;
996 memcpy(hdr, &req->hdr, sizeof(*hdr));
1013 if (unlikely(req->seqnum == 2)) {
1021 req->hdr.pbc[0] = hdr->pbc[0];
1022 req->hdr.lrh[2] = hdr->lrh[2];
1030 if (unlikely(!req->seqnum)) {
1031 ret = check_header_template(req, hdr, lrhlen, datalen);
1039 (req_opcode(req->info.ctrl) == EXPECTED),
1040 req->seqnum));
1047 hdr->kdeth.swdata[6] = cpu_to_le32(req->koffset);
1049 if (req_opcode(req->info.ctrl) == EXPECTED) {
1050 tidval = req->tids[req->tididx];
1055 if ((req->tidoffset) == (EXP_TID_GET(tidval, LEN) *
1057 req->tidoffset = 0;
1062 if (++req->tididx > req->n_tids - 1 ||
1063 !req->tids[req->tididx]) {
1066 tidval = req->tids[req->tididx];
1085 pq->dd, pq->ctxt, pq->subctxt, req->info.comp_idx,
1086 req->tidoffset, req->tidoffset >> omfactor,
1089 req->tidoffset >> omfactor);
1095 req->info.comp_idx, hdr, tidval);
1099 static int set_txreq_header_ahg(struct user_sdma_request *req,
1105 struct hfi1_user_sdma_pkt_q *pq = req->pq;
1106 struct hfi1_pkt_header *hdr = &req->hdr;
1128 val32 = (be32_to_cpu(hdr->bth[2]) + req->seqnum) &
1142 (__force u16)cpu_to_le16(req->koffset & 0xffff));
1146 (__force u16)cpu_to_le16(req->koffset >> 16));
1149 if (req_opcode(req->info.ctrl) == EXPECTED) {
1152 tidval = req->tids[req->tididx];
1158 if ((req->tidoffset) == (EXP_TID_GET(tidval, LEN) *
1160 req->tidoffset = 0;
1165 if (++req->tididx > req->n_tids - 1 ||
1166 !req->tids[req->tididx])
1168 tidval = req->tids[req->tididx];
1178 ((req->tidoffset >> omfactor)
1205 req->info.comp_idx, req->sde->this_idx,
1206 req->ahg_idx, ahg, idx, tidval);
1209 datalen, req->ahg_idx, idx,
1210 ahg, sizeof(req->hdr),
1230 struct user_sdma_request *req;
1235 if (!tx->req)
1238 req = tx->req;
1239 pq = req->pq;
1240 cq = req->cq;
1243 SDMA_DBG(req, "SDMA completion with error %d",
1245 WRITE_ONCE(req->has_error, 1);
1249 req->seqcomp = tx->seqnum;
1253 if (req->seqcomp != req->info.npkts - 1)
1256 user_sdma_free_request(req);
1257 set_comp_state(pq, cq, req->info.comp_idx, state, status);
1267 static void user_sdma_free_request(struct user_sdma_request *req)
1269 if (!list_empty(&req->txps)) {
1272 list_for_each_entry_safe(t, p, &req->txps, list) {
1276 sdma_txclean(req->pq->dd, t);
1277 kmem_cache_free(req->pq->txreq_cache, tx);
1281 kfree(req->tids);
1282 clear_bit(req->info.comp_idx, req->pq->req_in_use);
1342 static int pin_system_pages(struct user_sdma_request *req,
1346 struct hfi1_user_sdma_pkt_q *pq = req->pq;
1357 SDMA_DBG(req, "Evicting: nlocked %u npages %u",
1364 SDMA_DBG(req, "Acquire user pages start_address %lx node->npages %u npages %u",
1371 SDMA_DBG(req, "pinned %d", pinned);
1376 SDMA_DBG(req, "npages %u pinned %d", npages, pinned);
1384 SDMA_DBG(req, "done. pinned %d", pinned);
1395 static int add_system_pinning(struct user_sdma_request *req,
1400 struct hfi1_user_sdma_pkt_q *pq = req->pq;
1415 ret = pin_system_pages(req, start, len, node, PFN_DOWN(len));
1430 static int get_system_cache_entry(struct user_sdma_request *req,
1434 struct hfi1_user_sdma_pkt_q *pq = req->pq;
1441 SDMA_DBG(req,
1447 SDMA_DBG(req, "req_start %lx req_len %lu", req_start, req_len);
1454 SDMA_DBG(req, "node %p start %llx end %llu", node, start, end);
1456 ret = add_system_pinning(req, node_p, start,
1477 SDMA_DBG(req, "prepend: node->rb.addr %lx, node->rb.refcount %d",
1488 ret = add_system_pinning(req, node_p, start, prepend_len);
1511 static int add_mapping_to_sdma_packet(struct user_sdma_request *req,
1517 struct hfi1_user_sdma_pkt_q *pq = req->pq;
1533 SDMA_DBG(req,
1565 SDMA_DBG(req,
1576 static int add_system_iovec_to_sdma_packet(struct user_sdma_request *req,
1588 ret = get_system_cache_entry(req, &cache_entry, start,
1591 SDMA_DBG(req, "pin system segment failed %d", ret);
1599 ret = add_mapping_to_sdma_packet(req, tx, cache_entry, start,
1610 SDMA_DBG(req, "add system segment failed %d", ret);
1621 static int add_system_pages_to_sdma_packet(struct user_sdma_request *req,
1644 req->iov_idx++;
1648 ret = add_system_iovec_to_sdma_packet(req, tx, cur_iovec,