Lines Matching refs:seg
31 * @seg: Current WQE position (16B aligned).
35 static inline void handle_post_send_edge(struct mlx5_ib_wq *sq, void **seg,
40 if (likely(*seg != *cur_edge))
46 *seg = mlx5_frag_buf_get_wqe(&sq->fbc, idx);
50 * pointers. At the end @seg is aligned to 16B regardless the copied size.
53 * @seg: Current WQE position (16B aligned).
59 void **seg, u32 *wqe_sz, const void *src,
63 size_t leftlen = *cur_edge - *seg;
67 memcpy(*seg, src, copysz);
72 *seg += stride;
74 handle_post_send_edge(sq, seg, *wqe_sz, cur_edge);
105 void **seg, int *size, void **cur_edge)
107 struct mlx5_wqe_eth_seg *eseg = *seg;
135 *seg += stride;
138 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
141 memcpy_send_wqe(&qp->sq, cur_edge, seg, size, pdata,
148 *seg += sizeof(struct mlx5_wqe_eth_seg);
373 static void set_reg_mkey_seg(struct mlx5_mkey_seg *seg,
379 memset(seg, 0, sizeof(*seg));
382 seg->log2_page_size = ilog2(mr->ibmr.page_size);
387 seg->flags = get_umr_flags(access) | mr->access_mode;
388 seg->qpn_mkey7_0 = cpu_to_be32((key & 0xff) | 0xffffff00);
389 seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL);
390 seg->start_addr = cpu_to_be64(mr->ibmr.iova);
391 seg->len = cpu_to_be64(mr->ibmr.length);
392 seg->xlt_oct_size = cpu_to_be32(ndescs);
395 static void set_linv_mkey_seg(struct mlx5_mkey_seg *seg)
397 memset(seg, 0, sizeof(*seg));
398 seg->status = MLX5_MKEY_STATUS_FREE;
402 struct mlx5_mkey_seg *seg,
407 memset(seg, 0, sizeof(*seg));
409 MLX5_SET(mkc, seg, free, 1);
411 MLX5_SET(mkc, seg, a,
413 MLX5_SET(mkc, seg, rw,
415 MLX5_SET(mkc, seg, rr, !!(umrwr->access_flags & IB_ACCESS_REMOTE_READ));
416 MLX5_SET(mkc, seg, lw, !!(umrwr->access_flags & IB_ACCESS_LOCAL_WRITE));
417 MLX5_SET(mkc, seg, lr, 1);
419 MLX5_SET(mkc, seg, relaxed_ordering_write,
422 MLX5_SET(mkc, seg, relaxed_ordering_read,
426 MLX5_SET(mkc, seg, pd, to_mpd(umrwr->pd)->pdn);
429 MLX5_SET(mkc, seg, length64, 1);
431 MLX5_SET64(mkc, seg, start_addr, umrwr->virt_addr);
432 MLX5_SET64(mkc, seg, len, umrwr->length);
433 MLX5_SET(mkc, seg, log_page_size, umrwr->page_shift);
434 MLX5_SET(mkc, seg, qpn, 0xffffff);
435 MLX5_SET(mkc, seg, mkey_7_0, mlx5_mkey_variant(umrwr->mkey));
484 struct mlx5_wqe_inline_seg *seg;
489 seg = *wqe;
490 *wqe += sizeof(*seg);
491 offset = sizeof(*seg);
521 seg->byte_count = cpu_to_be32(inl | MLX5_INLINE_SEG);
523 *wqe_sz += ALIGN(inl + sizeof(seg->byte_count), 16) / 16;
640 struct mlx5_ib_qp *qp, void **seg, int *size,
678 struct mlx5_klm *data_klm = *seg;
704 sblock_ctrl = *seg;
733 *seg += wqe_size;
735 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
737 bsf = *seg;
742 *seg += sizeof(*bsf);
744 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
749 static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg,
756 memset(seg, 0, sizeof(*seg));
758 seg->flags = get_umr_flags(access_flags) | MLX5_MKC_ACCESS_MODE_KLMS;
759 seg->qpn_mkey7_0 = cpu_to_be32((sig_key & 0xff) | 0xffffff00);
760 seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL | sigerr << 26 |
762 seg->len = cpu_to_be64(length);
763 seg->xlt_oct_size = cpu_to_be32(get_xlt_octo(size));
764 seg->bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE);
779 struct mlx5_ib_qp *qp, void **seg, int *size,
809 set_sig_umr_segment(*seg, xlt_size);
810 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
812 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
814 set_sig_mkey_segment(*seg, wr->mr, wr->access, xlt_size, region_len,
816 *seg += sizeof(struct mlx5_mkey_seg);
818 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
820 ret = set_sig_data_segment(send_wr, wr->mr, sig_attrs, qp, seg, size,
830 u32 psv_idx, void **seg, int *size)
832 struct mlx5_seg_set_psv *psv_seg = *seg;
850 *seg += sizeof(*psv_seg);
858 void **seg, int *size, void **cur_edge,
888 set_reg_umr_seg(*seg, mr, flags, atomic);
889 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
891 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
893 set_reg_mkey_seg(*seg, mr, wr->key, wr->access);
894 *seg += sizeof(struct mlx5_mkey_seg);
896 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
899 memcpy_send_wqe(&qp->sq, cur_edge, seg, size, mr->descs,
903 set_reg_data_seg(*seg, mr, pd);
904 *seg += sizeof(struct mlx5_wqe_data_seg);
910 static void set_linv_wr(struct mlx5_ib_qp *qp, void **seg, int *size,
913 set_linv_umr_seg(*seg);
914 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
916 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
917 set_linv_mkey_seg(*seg);
918 *seg += sizeof(struct mlx5_mkey_seg);
920 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
942 static int __begin_wqe(struct mlx5_ib_qp *qp, void **seg,
952 *seg = mlx5_frag_buf_get_wqe(&qp->sq.fbc, *idx);
953 *ctrl = *seg;
954 *(uint32_t *)(*seg + 8) = 0;
960 *seg += sizeof(**ctrl);
967 static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,
972 return __begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq,
979 void *seg, u8 size, void *cur_edge,
1001 seg = PTR_ALIGN(seg, MLX5_SEND_WQE_BB);
1002 qp->sq.cur_edge = (unlikely(seg == cur_edge)) ?
1008 static void handle_rdma_op(const struct ib_send_wr *wr, void **seg, int *size)
1010 set_raddr_seg(*seg, rdma_wr(wr)->remote_addr, rdma_wr(wr)->rkey);
1011 *seg += sizeof(struct mlx5_wqe_raddr_seg);
1016 struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
1021 set_linv_wr(qp, seg, size, cur_edge);
1025 struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
1030 return set_reg_wr(qp, reg_wr(wr), seg, size, cur_edge, true);
1035 struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
1045 err = __begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq,
1052 err = set_psv_wr(domain, psv_index, seg, size);
1057 finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id, nreq,
1067 struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
1094 err = set_reg_wr(qp, ®_pi_wr, seg, size, cur_edge, false);
1098 finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id,
1101 err = begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq);
1125 err = set_pi_umr_wr(wr, qp, seg, size, cur_edge);
1130 finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id, nreq,
1134 err = handle_psv(dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq,
1140 err = handle_psv(dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq,
1154 struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
1164 handle_rdma_op(wr, seg, size);
1175 handle_local_inv(qp, wr, ctrl, seg, size, cur_edge, *idx);
1180 err = handle_reg_mr(qp, wr, ctrl, seg, size, cur_edge, *idx);
1187 err = handle_reg_mr_integrity(dev, qp, wr, ctrl, seg, size,
1203 static void handle_qpt_uc(const struct ib_send_wr *wr, void **seg, int *size)
1208 handle_rdma_op(wr, seg, size);
1216 const struct ib_send_wr *wr, void **seg,
1219 set_datagram_seg(*seg, wr);
1220 *seg += sizeof(struct mlx5_wqe_datagram_seg);
1222 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1226 void **seg, int *size, void **cur_edge)
1228 set_datagram_seg(*seg, wr);
1229 *seg += sizeof(struct mlx5_wqe_datagram_seg);
1231 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1237 pad = *seg;
1239 *seg += sizeof(struct mlx5_wqe_eth_pad);
1241 set_eth_seg(wr, qp, seg, size, cur_edge);
1242 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1248 struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
1261 err = set_reg_umr_segment(dev, *seg, wr);
1264 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
1266 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1267 set_reg_mkey_segment(dev, *seg, wr);
1268 *seg += sizeof(struct mlx5_mkey_seg);
1270 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1290 void *seg;
1326 err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, &cur_edge,
1352 xrc = seg;
1353 seg += sizeof(*xrc);
1357 err = handle_qpt_rc(dev, qp, wr, &ctrl, &seg, &size,
1369 handle_qpt_uc(wr, &seg, &size);
1380 handle_qpt_hw_gsi(qp, wr, &seg, &size, &cur_edge);
1383 handle_qpt_ud(qp, wr, &seg, &size, &cur_edge);
1386 err = handle_qpt_reg_umr(dev, qp, wr, &ctrl, &seg,
1397 err = set_data_inl_seg(qp, wr, &seg, &size, &cur_edge);
1405 handle_post_send_edge(&qp->sq, &seg, size,
1411 (struct mlx5_wqe_data_seg *)seg,
1414 seg += sizeof(struct mlx5_wqe_data_seg);
1419 finish_wqe(qp, ctrl, seg, size, cur_edge, idx, wr->wr_id, nreq,