18c2ecf20Sopenharmony_ci/* 28c2ecf20Sopenharmony_ci * Copyright (c) 2012 - 2018 Intel Corporation. All rights reserved. 38c2ecf20Sopenharmony_ci * Copyright (c) 2006 - 2012 QLogic Corporation. All rights reserved. 48c2ecf20Sopenharmony_ci * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. 58c2ecf20Sopenharmony_ci * 68c2ecf20Sopenharmony_ci * This software is available to you under a choice of one of two 78c2ecf20Sopenharmony_ci * licenses. You may choose to be licensed under the terms of the GNU 88c2ecf20Sopenharmony_ci * General Public License (GPL) Version 2, available from the file 98c2ecf20Sopenharmony_ci * COPYING in the main directory of this source tree, or the 108c2ecf20Sopenharmony_ci * OpenIB.org BSD license below: 118c2ecf20Sopenharmony_ci * 128c2ecf20Sopenharmony_ci * Redistribution and use in source and binary forms, with or 138c2ecf20Sopenharmony_ci * without modification, are permitted provided that the following 148c2ecf20Sopenharmony_ci * conditions are met: 158c2ecf20Sopenharmony_ci * 168c2ecf20Sopenharmony_ci * - Redistributions of source code must retain the above 178c2ecf20Sopenharmony_ci * copyright notice, this list of conditions and the following 188c2ecf20Sopenharmony_ci * disclaimer. 198c2ecf20Sopenharmony_ci * 208c2ecf20Sopenharmony_ci * - Redistributions in binary form must reproduce the above 218c2ecf20Sopenharmony_ci * copyright notice, this list of conditions and the following 228c2ecf20Sopenharmony_ci * disclaimer in the documentation and/or other materials 238c2ecf20Sopenharmony_ci * provided with the distribution. 248c2ecf20Sopenharmony_ci * 258c2ecf20Sopenharmony_ci * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 268c2ecf20Sopenharmony_ci * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 278c2ecf20Sopenharmony_ci * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 288c2ecf20Sopenharmony_ci * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 298c2ecf20Sopenharmony_ci * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 308c2ecf20Sopenharmony_ci * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 318c2ecf20Sopenharmony_ci * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 328c2ecf20Sopenharmony_ci * SOFTWARE. 338c2ecf20Sopenharmony_ci */ 348c2ecf20Sopenharmony_ci 358c2ecf20Sopenharmony_ci#include <rdma/ib_mad.h> 368c2ecf20Sopenharmony_ci#include <rdma/ib_user_verbs.h> 378c2ecf20Sopenharmony_ci#include <linux/io.h> 388c2ecf20Sopenharmony_ci#include <linux/module.h> 398c2ecf20Sopenharmony_ci#include <linux/utsname.h> 408c2ecf20Sopenharmony_ci#include <linux/rculist.h> 418c2ecf20Sopenharmony_ci#include <linux/mm.h> 428c2ecf20Sopenharmony_ci#include <linux/vmalloc.h> 438c2ecf20Sopenharmony_ci#include <rdma/rdma_vt.h> 448c2ecf20Sopenharmony_ci 458c2ecf20Sopenharmony_ci#include "qib.h" 468c2ecf20Sopenharmony_ci#include "qib_common.h" 478c2ecf20Sopenharmony_ci 488c2ecf20Sopenharmony_cistatic unsigned int ib_qib_qp_table_size = 256; 498c2ecf20Sopenharmony_cimodule_param_named(qp_table_size, ib_qib_qp_table_size, uint, S_IRUGO); 508c2ecf20Sopenharmony_ciMODULE_PARM_DESC(qp_table_size, "QP table size"); 518c2ecf20Sopenharmony_ci 528c2ecf20Sopenharmony_cistatic unsigned int qib_lkey_table_size = 16; 538c2ecf20Sopenharmony_cimodule_param_named(lkey_table_size, qib_lkey_table_size, uint, 548c2ecf20Sopenharmony_ci S_IRUGO); 558c2ecf20Sopenharmony_ciMODULE_PARM_DESC(lkey_table_size, 568c2ecf20Sopenharmony_ci "LKEY table size in bits (2^n, 1 <= n <= 23)"); 578c2ecf20Sopenharmony_ci 588c2ecf20Sopenharmony_cistatic unsigned int ib_qib_max_pds = 0xFFFF; 598c2ecf20Sopenharmony_cimodule_param_named(max_pds, ib_qib_max_pds, uint, S_IRUGO); 608c2ecf20Sopenharmony_ciMODULE_PARM_DESC(max_pds, 618c2ecf20Sopenharmony_ci "Maximum number of protection domains to support"); 628c2ecf20Sopenharmony_ci 638c2ecf20Sopenharmony_cistatic unsigned int ib_qib_max_ahs = 0xFFFF; 648c2ecf20Sopenharmony_cimodule_param_named(max_ahs, ib_qib_max_ahs, uint, S_IRUGO); 658c2ecf20Sopenharmony_ciMODULE_PARM_DESC(max_ahs, "Maximum number of address handles to support"); 668c2ecf20Sopenharmony_ci 678c2ecf20Sopenharmony_ciunsigned int ib_qib_max_cqes = 0x2FFFF; 688c2ecf20Sopenharmony_cimodule_param_named(max_cqes, ib_qib_max_cqes, uint, S_IRUGO); 698c2ecf20Sopenharmony_ciMODULE_PARM_DESC(max_cqes, 708c2ecf20Sopenharmony_ci "Maximum number of completion queue entries to support"); 718c2ecf20Sopenharmony_ci 728c2ecf20Sopenharmony_ciunsigned int ib_qib_max_cqs = 0x1FFFF; 738c2ecf20Sopenharmony_cimodule_param_named(max_cqs, ib_qib_max_cqs, uint, S_IRUGO); 748c2ecf20Sopenharmony_ciMODULE_PARM_DESC(max_cqs, "Maximum number of completion queues to support"); 758c2ecf20Sopenharmony_ci 768c2ecf20Sopenharmony_ciunsigned int ib_qib_max_qp_wrs = 0x3FFF; 778c2ecf20Sopenharmony_cimodule_param_named(max_qp_wrs, ib_qib_max_qp_wrs, uint, S_IRUGO); 788c2ecf20Sopenharmony_ciMODULE_PARM_DESC(max_qp_wrs, "Maximum number of QP WRs to support"); 798c2ecf20Sopenharmony_ci 808c2ecf20Sopenharmony_ciunsigned int ib_qib_max_qps = 16384; 818c2ecf20Sopenharmony_cimodule_param_named(max_qps, ib_qib_max_qps, uint, S_IRUGO); 828c2ecf20Sopenharmony_ciMODULE_PARM_DESC(max_qps, "Maximum number of QPs to support"); 838c2ecf20Sopenharmony_ci 848c2ecf20Sopenharmony_ciunsigned int ib_qib_max_sges = 0x60; 858c2ecf20Sopenharmony_cimodule_param_named(max_sges, ib_qib_max_sges, uint, S_IRUGO); 868c2ecf20Sopenharmony_ciMODULE_PARM_DESC(max_sges, "Maximum number of SGEs to support"); 878c2ecf20Sopenharmony_ci 888c2ecf20Sopenharmony_ciunsigned int ib_qib_max_mcast_grps = 16384; 898c2ecf20Sopenharmony_cimodule_param_named(max_mcast_grps, ib_qib_max_mcast_grps, uint, S_IRUGO); 908c2ecf20Sopenharmony_ciMODULE_PARM_DESC(max_mcast_grps, 918c2ecf20Sopenharmony_ci "Maximum number of multicast groups to support"); 928c2ecf20Sopenharmony_ci 938c2ecf20Sopenharmony_ciunsigned int ib_qib_max_mcast_qp_attached = 16; 948c2ecf20Sopenharmony_cimodule_param_named(max_mcast_qp_attached, ib_qib_max_mcast_qp_attached, 958c2ecf20Sopenharmony_ci uint, S_IRUGO); 968c2ecf20Sopenharmony_ciMODULE_PARM_DESC(max_mcast_qp_attached, 978c2ecf20Sopenharmony_ci "Maximum number of attached QPs to support"); 988c2ecf20Sopenharmony_ci 998c2ecf20Sopenharmony_ciunsigned int ib_qib_max_srqs = 1024; 1008c2ecf20Sopenharmony_cimodule_param_named(max_srqs, ib_qib_max_srqs, uint, S_IRUGO); 1018c2ecf20Sopenharmony_ciMODULE_PARM_DESC(max_srqs, "Maximum number of SRQs to support"); 1028c2ecf20Sopenharmony_ci 1038c2ecf20Sopenharmony_ciunsigned int ib_qib_max_srq_sges = 128; 1048c2ecf20Sopenharmony_cimodule_param_named(max_srq_sges, ib_qib_max_srq_sges, uint, S_IRUGO); 1058c2ecf20Sopenharmony_ciMODULE_PARM_DESC(max_srq_sges, "Maximum number of SRQ SGEs to support"); 1068c2ecf20Sopenharmony_ci 1078c2ecf20Sopenharmony_ciunsigned int ib_qib_max_srq_wrs = 0x1FFFF; 1088c2ecf20Sopenharmony_cimodule_param_named(max_srq_wrs, ib_qib_max_srq_wrs, uint, S_IRUGO); 1098c2ecf20Sopenharmony_ciMODULE_PARM_DESC(max_srq_wrs, "Maximum number of SRQ WRs support"); 1108c2ecf20Sopenharmony_ci 1118c2ecf20Sopenharmony_cistatic unsigned int ib_qib_disable_sma; 1128c2ecf20Sopenharmony_cimodule_param_named(disable_sma, ib_qib_disable_sma, uint, S_IWUSR | S_IRUGO); 1138c2ecf20Sopenharmony_ciMODULE_PARM_DESC(disable_sma, "Disable the SMA"); 1148c2ecf20Sopenharmony_ci 1158c2ecf20Sopenharmony_ci/* 1168c2ecf20Sopenharmony_ci * Translate ib_wr_opcode into ib_wc_opcode. 1178c2ecf20Sopenharmony_ci */ 1188c2ecf20Sopenharmony_ciconst enum ib_wc_opcode ib_qib_wc_opcode[] = { 1198c2ecf20Sopenharmony_ci [IB_WR_RDMA_WRITE] = IB_WC_RDMA_WRITE, 1208c2ecf20Sopenharmony_ci [IB_WR_RDMA_WRITE_WITH_IMM] = IB_WC_RDMA_WRITE, 1218c2ecf20Sopenharmony_ci [IB_WR_SEND] = IB_WC_SEND, 1228c2ecf20Sopenharmony_ci [IB_WR_SEND_WITH_IMM] = IB_WC_SEND, 1238c2ecf20Sopenharmony_ci [IB_WR_RDMA_READ] = IB_WC_RDMA_READ, 1248c2ecf20Sopenharmony_ci [IB_WR_ATOMIC_CMP_AND_SWP] = IB_WC_COMP_SWAP, 1258c2ecf20Sopenharmony_ci [IB_WR_ATOMIC_FETCH_AND_ADD] = IB_WC_FETCH_ADD 1268c2ecf20Sopenharmony_ci}; 1278c2ecf20Sopenharmony_ci 1288c2ecf20Sopenharmony_ci/* 1298c2ecf20Sopenharmony_ci * System image GUID. 1308c2ecf20Sopenharmony_ci */ 1318c2ecf20Sopenharmony_ci__be64 ib_qib_sys_image_guid; 1328c2ecf20Sopenharmony_ci 1338c2ecf20Sopenharmony_ci/* 1348c2ecf20Sopenharmony_ci * Count the number of DMA descriptors needed to send length bytes of data. 1358c2ecf20Sopenharmony_ci * Don't modify the qib_sge_state to get the count. 1368c2ecf20Sopenharmony_ci * Return zero if any of the segments is not aligned. 1378c2ecf20Sopenharmony_ci */ 1388c2ecf20Sopenharmony_cistatic u32 qib_count_sge(struct rvt_sge_state *ss, u32 length) 1398c2ecf20Sopenharmony_ci{ 1408c2ecf20Sopenharmony_ci struct rvt_sge *sg_list = ss->sg_list; 1418c2ecf20Sopenharmony_ci struct rvt_sge sge = ss->sge; 1428c2ecf20Sopenharmony_ci u8 num_sge = ss->num_sge; 1438c2ecf20Sopenharmony_ci u32 ndesc = 1; /* count the header */ 1448c2ecf20Sopenharmony_ci 1458c2ecf20Sopenharmony_ci while (length) { 1468c2ecf20Sopenharmony_ci u32 len = rvt_get_sge_length(&sge, length); 1478c2ecf20Sopenharmony_ci 1488c2ecf20Sopenharmony_ci if (((long) sge.vaddr & (sizeof(u32) - 1)) || 1498c2ecf20Sopenharmony_ci (len != length && (len & (sizeof(u32) - 1)))) { 1508c2ecf20Sopenharmony_ci ndesc = 0; 1518c2ecf20Sopenharmony_ci break; 1528c2ecf20Sopenharmony_ci } 1538c2ecf20Sopenharmony_ci ndesc++; 1548c2ecf20Sopenharmony_ci sge.vaddr += len; 1558c2ecf20Sopenharmony_ci sge.length -= len; 1568c2ecf20Sopenharmony_ci sge.sge_length -= len; 1578c2ecf20Sopenharmony_ci if (sge.sge_length == 0) { 1588c2ecf20Sopenharmony_ci if (--num_sge) 1598c2ecf20Sopenharmony_ci sge = *sg_list++; 1608c2ecf20Sopenharmony_ci } else if (sge.length == 0 && sge.mr->lkey) { 1618c2ecf20Sopenharmony_ci if (++sge.n >= RVT_SEGSZ) { 1628c2ecf20Sopenharmony_ci if (++sge.m >= sge.mr->mapsz) 1638c2ecf20Sopenharmony_ci break; 1648c2ecf20Sopenharmony_ci sge.n = 0; 1658c2ecf20Sopenharmony_ci } 1668c2ecf20Sopenharmony_ci sge.vaddr = 1678c2ecf20Sopenharmony_ci sge.mr->map[sge.m]->segs[sge.n].vaddr; 1688c2ecf20Sopenharmony_ci sge.length = 1698c2ecf20Sopenharmony_ci sge.mr->map[sge.m]->segs[sge.n].length; 1708c2ecf20Sopenharmony_ci } 1718c2ecf20Sopenharmony_ci length -= len; 1728c2ecf20Sopenharmony_ci } 1738c2ecf20Sopenharmony_ci return ndesc; 1748c2ecf20Sopenharmony_ci} 1758c2ecf20Sopenharmony_ci 1768c2ecf20Sopenharmony_ci/* 1778c2ecf20Sopenharmony_ci * Copy from the SGEs to the data buffer. 1788c2ecf20Sopenharmony_ci */ 1798c2ecf20Sopenharmony_cistatic void qib_copy_from_sge(void *data, struct rvt_sge_state *ss, u32 length) 1808c2ecf20Sopenharmony_ci{ 1818c2ecf20Sopenharmony_ci struct rvt_sge *sge = &ss->sge; 1828c2ecf20Sopenharmony_ci 1838c2ecf20Sopenharmony_ci while (length) { 1848c2ecf20Sopenharmony_ci u32 len = rvt_get_sge_length(sge, length); 1858c2ecf20Sopenharmony_ci 1868c2ecf20Sopenharmony_ci memcpy(data, sge->vaddr, len); 1878c2ecf20Sopenharmony_ci sge->vaddr += len; 1888c2ecf20Sopenharmony_ci sge->length -= len; 1898c2ecf20Sopenharmony_ci sge->sge_length -= len; 1908c2ecf20Sopenharmony_ci if (sge->sge_length == 0) { 1918c2ecf20Sopenharmony_ci if (--ss->num_sge) 1928c2ecf20Sopenharmony_ci *sge = *ss->sg_list++; 1938c2ecf20Sopenharmony_ci } else if (sge->length == 0 && sge->mr->lkey) { 1948c2ecf20Sopenharmony_ci if (++sge->n >= RVT_SEGSZ) { 1958c2ecf20Sopenharmony_ci if (++sge->m >= sge->mr->mapsz) 1968c2ecf20Sopenharmony_ci break; 1978c2ecf20Sopenharmony_ci sge->n = 0; 1988c2ecf20Sopenharmony_ci } 1998c2ecf20Sopenharmony_ci sge->vaddr = 2008c2ecf20Sopenharmony_ci sge->mr->map[sge->m]->segs[sge->n].vaddr; 2018c2ecf20Sopenharmony_ci sge->length = 2028c2ecf20Sopenharmony_ci sge->mr->map[sge->m]->segs[sge->n].length; 2038c2ecf20Sopenharmony_ci } 2048c2ecf20Sopenharmony_ci data += len; 2058c2ecf20Sopenharmony_ci length -= len; 2068c2ecf20Sopenharmony_ci } 2078c2ecf20Sopenharmony_ci} 2088c2ecf20Sopenharmony_ci 2098c2ecf20Sopenharmony_ci/** 2108c2ecf20Sopenharmony_ci * qib_qp_rcv - processing an incoming packet on a QP 2118c2ecf20Sopenharmony_ci * @rcd: the context pointer 2128c2ecf20Sopenharmony_ci * @hdr: the packet header 2138c2ecf20Sopenharmony_ci * @has_grh: true if the packet has a GRH 2148c2ecf20Sopenharmony_ci * @data: the packet data 2158c2ecf20Sopenharmony_ci * @tlen: the packet length 2168c2ecf20Sopenharmony_ci * @qp: the QP the packet came on 2178c2ecf20Sopenharmony_ci * 2188c2ecf20Sopenharmony_ci * This is called from qib_ib_rcv() to process an incoming packet 2198c2ecf20Sopenharmony_ci * for the given QP. 2208c2ecf20Sopenharmony_ci * Called at interrupt level. 2218c2ecf20Sopenharmony_ci */ 2228c2ecf20Sopenharmony_cistatic void qib_qp_rcv(struct qib_ctxtdata *rcd, struct ib_header *hdr, 2238c2ecf20Sopenharmony_ci int has_grh, void *data, u32 tlen, struct rvt_qp *qp) 2248c2ecf20Sopenharmony_ci{ 2258c2ecf20Sopenharmony_ci struct qib_ibport *ibp = &rcd->ppd->ibport_data; 2268c2ecf20Sopenharmony_ci 2278c2ecf20Sopenharmony_ci spin_lock(&qp->r_lock); 2288c2ecf20Sopenharmony_ci 2298c2ecf20Sopenharmony_ci /* Check for valid receive state. */ 2308c2ecf20Sopenharmony_ci if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) { 2318c2ecf20Sopenharmony_ci ibp->rvp.n_pkt_drops++; 2328c2ecf20Sopenharmony_ci goto unlock; 2338c2ecf20Sopenharmony_ci } 2348c2ecf20Sopenharmony_ci 2358c2ecf20Sopenharmony_ci switch (qp->ibqp.qp_type) { 2368c2ecf20Sopenharmony_ci case IB_QPT_SMI: 2378c2ecf20Sopenharmony_ci case IB_QPT_GSI: 2388c2ecf20Sopenharmony_ci if (ib_qib_disable_sma) 2398c2ecf20Sopenharmony_ci break; 2408c2ecf20Sopenharmony_ci fallthrough; 2418c2ecf20Sopenharmony_ci case IB_QPT_UD: 2428c2ecf20Sopenharmony_ci qib_ud_rcv(ibp, hdr, has_grh, data, tlen, qp); 2438c2ecf20Sopenharmony_ci break; 2448c2ecf20Sopenharmony_ci 2458c2ecf20Sopenharmony_ci case IB_QPT_RC: 2468c2ecf20Sopenharmony_ci qib_rc_rcv(rcd, hdr, has_grh, data, tlen, qp); 2478c2ecf20Sopenharmony_ci break; 2488c2ecf20Sopenharmony_ci 2498c2ecf20Sopenharmony_ci case IB_QPT_UC: 2508c2ecf20Sopenharmony_ci qib_uc_rcv(ibp, hdr, has_grh, data, tlen, qp); 2518c2ecf20Sopenharmony_ci break; 2528c2ecf20Sopenharmony_ci 2538c2ecf20Sopenharmony_ci default: 2548c2ecf20Sopenharmony_ci break; 2558c2ecf20Sopenharmony_ci } 2568c2ecf20Sopenharmony_ci 2578c2ecf20Sopenharmony_ciunlock: 2588c2ecf20Sopenharmony_ci spin_unlock(&qp->r_lock); 2598c2ecf20Sopenharmony_ci} 2608c2ecf20Sopenharmony_ci 2618c2ecf20Sopenharmony_ci/** 2628c2ecf20Sopenharmony_ci * qib_ib_rcv - process an incoming packet 2638c2ecf20Sopenharmony_ci * @rcd: the context pointer 2648c2ecf20Sopenharmony_ci * @rhdr: the header of the packet 2658c2ecf20Sopenharmony_ci * @data: the packet payload 2668c2ecf20Sopenharmony_ci * @tlen: the packet length 2678c2ecf20Sopenharmony_ci * 2688c2ecf20Sopenharmony_ci * This is called from qib_kreceive() to process an incoming packet at 2698c2ecf20Sopenharmony_ci * interrupt level. Tlen is the length of the header + data + CRC in bytes. 2708c2ecf20Sopenharmony_ci */ 2718c2ecf20Sopenharmony_civoid qib_ib_rcv(struct qib_ctxtdata *rcd, void *rhdr, void *data, u32 tlen) 2728c2ecf20Sopenharmony_ci{ 2738c2ecf20Sopenharmony_ci struct qib_pportdata *ppd = rcd->ppd; 2748c2ecf20Sopenharmony_ci struct qib_ibport *ibp = &ppd->ibport_data; 2758c2ecf20Sopenharmony_ci struct ib_header *hdr = rhdr; 2768c2ecf20Sopenharmony_ci struct qib_devdata *dd = ppd->dd; 2778c2ecf20Sopenharmony_ci struct rvt_dev_info *rdi = &dd->verbs_dev.rdi; 2788c2ecf20Sopenharmony_ci struct ib_other_headers *ohdr; 2798c2ecf20Sopenharmony_ci struct rvt_qp *qp; 2808c2ecf20Sopenharmony_ci u32 qp_num; 2818c2ecf20Sopenharmony_ci int lnh; 2828c2ecf20Sopenharmony_ci u8 opcode; 2838c2ecf20Sopenharmony_ci u16 lid; 2848c2ecf20Sopenharmony_ci 2858c2ecf20Sopenharmony_ci /* 24 == LRH+BTH+CRC */ 2868c2ecf20Sopenharmony_ci if (unlikely(tlen < 24)) 2878c2ecf20Sopenharmony_ci goto drop; 2888c2ecf20Sopenharmony_ci 2898c2ecf20Sopenharmony_ci /* Check for a valid destination LID (see ch. 7.11.1). */ 2908c2ecf20Sopenharmony_ci lid = be16_to_cpu(hdr->lrh[1]); 2918c2ecf20Sopenharmony_ci if (lid < be16_to_cpu(IB_MULTICAST_LID_BASE)) { 2928c2ecf20Sopenharmony_ci lid &= ~((1 << ppd->lmc) - 1); 2938c2ecf20Sopenharmony_ci if (unlikely(lid != ppd->lid)) 2948c2ecf20Sopenharmony_ci goto drop; 2958c2ecf20Sopenharmony_ci } 2968c2ecf20Sopenharmony_ci 2978c2ecf20Sopenharmony_ci /* Check for GRH */ 2988c2ecf20Sopenharmony_ci lnh = be16_to_cpu(hdr->lrh[0]) & 3; 2998c2ecf20Sopenharmony_ci if (lnh == QIB_LRH_BTH) 3008c2ecf20Sopenharmony_ci ohdr = &hdr->u.oth; 3018c2ecf20Sopenharmony_ci else if (lnh == QIB_LRH_GRH) { 3028c2ecf20Sopenharmony_ci u32 vtf; 3038c2ecf20Sopenharmony_ci 3048c2ecf20Sopenharmony_ci ohdr = &hdr->u.l.oth; 3058c2ecf20Sopenharmony_ci if (hdr->u.l.grh.next_hdr != IB_GRH_NEXT_HDR) 3068c2ecf20Sopenharmony_ci goto drop; 3078c2ecf20Sopenharmony_ci vtf = be32_to_cpu(hdr->u.l.grh.version_tclass_flow); 3088c2ecf20Sopenharmony_ci if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION) 3098c2ecf20Sopenharmony_ci goto drop; 3108c2ecf20Sopenharmony_ci } else 3118c2ecf20Sopenharmony_ci goto drop; 3128c2ecf20Sopenharmony_ci 3138c2ecf20Sopenharmony_ci opcode = (be32_to_cpu(ohdr->bth[0]) >> 24) & 0x7f; 3148c2ecf20Sopenharmony_ci#ifdef CONFIG_DEBUG_FS 3158c2ecf20Sopenharmony_ci rcd->opstats->stats[opcode].n_bytes += tlen; 3168c2ecf20Sopenharmony_ci rcd->opstats->stats[opcode].n_packets++; 3178c2ecf20Sopenharmony_ci#endif 3188c2ecf20Sopenharmony_ci 3198c2ecf20Sopenharmony_ci /* Get the destination QP number. */ 3208c2ecf20Sopenharmony_ci qp_num = be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK; 3218c2ecf20Sopenharmony_ci if (qp_num == QIB_MULTICAST_QPN) { 3228c2ecf20Sopenharmony_ci struct rvt_mcast *mcast; 3238c2ecf20Sopenharmony_ci struct rvt_mcast_qp *p; 3248c2ecf20Sopenharmony_ci 3258c2ecf20Sopenharmony_ci if (lnh != QIB_LRH_GRH) 3268c2ecf20Sopenharmony_ci goto drop; 3278c2ecf20Sopenharmony_ci mcast = rvt_mcast_find(&ibp->rvp, &hdr->u.l.grh.dgid, lid); 3288c2ecf20Sopenharmony_ci if (mcast == NULL) 3298c2ecf20Sopenharmony_ci goto drop; 3308c2ecf20Sopenharmony_ci this_cpu_inc(ibp->pmastats->n_multicast_rcv); 3318c2ecf20Sopenharmony_ci rcu_read_lock(); 3328c2ecf20Sopenharmony_ci list_for_each_entry_rcu(p, &mcast->qp_list, list) 3338c2ecf20Sopenharmony_ci qib_qp_rcv(rcd, hdr, 1, data, tlen, p->qp); 3348c2ecf20Sopenharmony_ci rcu_read_unlock(); 3358c2ecf20Sopenharmony_ci /* 3368c2ecf20Sopenharmony_ci * Notify rvt_multicast_detach() if it is waiting for us 3378c2ecf20Sopenharmony_ci * to finish. 3388c2ecf20Sopenharmony_ci */ 3398c2ecf20Sopenharmony_ci if (atomic_dec_return(&mcast->refcount) <= 1) 3408c2ecf20Sopenharmony_ci wake_up(&mcast->wait); 3418c2ecf20Sopenharmony_ci } else { 3428c2ecf20Sopenharmony_ci rcu_read_lock(); 3438c2ecf20Sopenharmony_ci qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num); 3448c2ecf20Sopenharmony_ci if (!qp) { 3458c2ecf20Sopenharmony_ci rcu_read_unlock(); 3468c2ecf20Sopenharmony_ci goto drop; 3478c2ecf20Sopenharmony_ci } 3488c2ecf20Sopenharmony_ci this_cpu_inc(ibp->pmastats->n_unicast_rcv); 3498c2ecf20Sopenharmony_ci qib_qp_rcv(rcd, hdr, lnh == QIB_LRH_GRH, data, tlen, qp); 3508c2ecf20Sopenharmony_ci rcu_read_unlock(); 3518c2ecf20Sopenharmony_ci } 3528c2ecf20Sopenharmony_ci return; 3538c2ecf20Sopenharmony_ci 3548c2ecf20Sopenharmony_cidrop: 3558c2ecf20Sopenharmony_ci ibp->rvp.n_pkt_drops++; 3568c2ecf20Sopenharmony_ci} 3578c2ecf20Sopenharmony_ci 3588c2ecf20Sopenharmony_ci/* 3598c2ecf20Sopenharmony_ci * This is called from a timer to check for QPs 3608c2ecf20Sopenharmony_ci * which need kernel memory in order to send a packet. 3618c2ecf20Sopenharmony_ci */ 3628c2ecf20Sopenharmony_cistatic void mem_timer(struct timer_list *t) 3638c2ecf20Sopenharmony_ci{ 3648c2ecf20Sopenharmony_ci struct qib_ibdev *dev = from_timer(dev, t, mem_timer); 3658c2ecf20Sopenharmony_ci struct list_head *list = &dev->memwait; 3668c2ecf20Sopenharmony_ci struct rvt_qp *qp = NULL; 3678c2ecf20Sopenharmony_ci struct qib_qp_priv *priv = NULL; 3688c2ecf20Sopenharmony_ci unsigned long flags; 3698c2ecf20Sopenharmony_ci 3708c2ecf20Sopenharmony_ci spin_lock_irqsave(&dev->rdi.pending_lock, flags); 3718c2ecf20Sopenharmony_ci if (!list_empty(list)) { 3728c2ecf20Sopenharmony_ci priv = list_entry(list->next, struct qib_qp_priv, iowait); 3738c2ecf20Sopenharmony_ci qp = priv->owner; 3748c2ecf20Sopenharmony_ci list_del_init(&priv->iowait); 3758c2ecf20Sopenharmony_ci rvt_get_qp(qp); 3768c2ecf20Sopenharmony_ci if (!list_empty(list)) 3778c2ecf20Sopenharmony_ci mod_timer(&dev->mem_timer, jiffies + 1); 3788c2ecf20Sopenharmony_ci } 3798c2ecf20Sopenharmony_ci spin_unlock_irqrestore(&dev->rdi.pending_lock, flags); 3808c2ecf20Sopenharmony_ci 3818c2ecf20Sopenharmony_ci if (qp) { 3828c2ecf20Sopenharmony_ci spin_lock_irqsave(&qp->s_lock, flags); 3838c2ecf20Sopenharmony_ci if (qp->s_flags & RVT_S_WAIT_KMEM) { 3848c2ecf20Sopenharmony_ci qp->s_flags &= ~RVT_S_WAIT_KMEM; 3858c2ecf20Sopenharmony_ci qib_schedule_send(qp); 3868c2ecf20Sopenharmony_ci } 3878c2ecf20Sopenharmony_ci spin_unlock_irqrestore(&qp->s_lock, flags); 3888c2ecf20Sopenharmony_ci rvt_put_qp(qp); 3898c2ecf20Sopenharmony_ci } 3908c2ecf20Sopenharmony_ci} 3918c2ecf20Sopenharmony_ci 3928c2ecf20Sopenharmony_ci#ifdef __LITTLE_ENDIAN 3938c2ecf20Sopenharmony_cistatic inline u32 get_upper_bits(u32 data, u32 shift) 3948c2ecf20Sopenharmony_ci{ 3958c2ecf20Sopenharmony_ci return data >> shift; 3968c2ecf20Sopenharmony_ci} 3978c2ecf20Sopenharmony_ci 3988c2ecf20Sopenharmony_cistatic inline u32 set_upper_bits(u32 data, u32 shift) 3998c2ecf20Sopenharmony_ci{ 4008c2ecf20Sopenharmony_ci return data << shift; 4018c2ecf20Sopenharmony_ci} 4028c2ecf20Sopenharmony_ci 4038c2ecf20Sopenharmony_cistatic inline u32 clear_upper_bytes(u32 data, u32 n, u32 off) 4048c2ecf20Sopenharmony_ci{ 4058c2ecf20Sopenharmony_ci data <<= ((sizeof(u32) - n) * BITS_PER_BYTE); 4068c2ecf20Sopenharmony_ci data >>= ((sizeof(u32) - n - off) * BITS_PER_BYTE); 4078c2ecf20Sopenharmony_ci return data; 4088c2ecf20Sopenharmony_ci} 4098c2ecf20Sopenharmony_ci#else 4108c2ecf20Sopenharmony_cistatic inline u32 get_upper_bits(u32 data, u32 shift) 4118c2ecf20Sopenharmony_ci{ 4128c2ecf20Sopenharmony_ci return data << shift; 4138c2ecf20Sopenharmony_ci} 4148c2ecf20Sopenharmony_ci 4158c2ecf20Sopenharmony_cistatic inline u32 set_upper_bits(u32 data, u32 shift) 4168c2ecf20Sopenharmony_ci{ 4178c2ecf20Sopenharmony_ci return data >> shift; 4188c2ecf20Sopenharmony_ci} 4198c2ecf20Sopenharmony_ci 4208c2ecf20Sopenharmony_cistatic inline u32 clear_upper_bytes(u32 data, u32 n, u32 off) 4218c2ecf20Sopenharmony_ci{ 4228c2ecf20Sopenharmony_ci data >>= ((sizeof(u32) - n) * BITS_PER_BYTE); 4238c2ecf20Sopenharmony_ci data <<= ((sizeof(u32) - n - off) * BITS_PER_BYTE); 4248c2ecf20Sopenharmony_ci return data; 4258c2ecf20Sopenharmony_ci} 4268c2ecf20Sopenharmony_ci#endif 4278c2ecf20Sopenharmony_ci 4288c2ecf20Sopenharmony_cistatic void copy_io(u32 __iomem *piobuf, struct rvt_sge_state *ss, 4298c2ecf20Sopenharmony_ci u32 length, unsigned flush_wc) 4308c2ecf20Sopenharmony_ci{ 4318c2ecf20Sopenharmony_ci u32 extra = 0; 4328c2ecf20Sopenharmony_ci u32 data = 0; 4338c2ecf20Sopenharmony_ci u32 last; 4348c2ecf20Sopenharmony_ci 4358c2ecf20Sopenharmony_ci while (1) { 4368c2ecf20Sopenharmony_ci u32 len = rvt_get_sge_length(&ss->sge, length); 4378c2ecf20Sopenharmony_ci u32 off; 4388c2ecf20Sopenharmony_ci 4398c2ecf20Sopenharmony_ci /* If the source address is not aligned, try to align it. */ 4408c2ecf20Sopenharmony_ci off = (unsigned long)ss->sge.vaddr & (sizeof(u32) - 1); 4418c2ecf20Sopenharmony_ci if (off) { 4428c2ecf20Sopenharmony_ci u32 *addr = (u32 *)((unsigned long)ss->sge.vaddr & 4438c2ecf20Sopenharmony_ci ~(sizeof(u32) - 1)); 4448c2ecf20Sopenharmony_ci u32 v = get_upper_bits(*addr, off * BITS_PER_BYTE); 4458c2ecf20Sopenharmony_ci u32 y; 4468c2ecf20Sopenharmony_ci 4478c2ecf20Sopenharmony_ci y = sizeof(u32) - off; 4488c2ecf20Sopenharmony_ci if (len > y) 4498c2ecf20Sopenharmony_ci len = y; 4508c2ecf20Sopenharmony_ci if (len + extra >= sizeof(u32)) { 4518c2ecf20Sopenharmony_ci data |= set_upper_bits(v, extra * 4528c2ecf20Sopenharmony_ci BITS_PER_BYTE); 4538c2ecf20Sopenharmony_ci len = sizeof(u32) - extra; 4548c2ecf20Sopenharmony_ci if (len == length) { 4558c2ecf20Sopenharmony_ci last = data; 4568c2ecf20Sopenharmony_ci break; 4578c2ecf20Sopenharmony_ci } 4588c2ecf20Sopenharmony_ci __raw_writel(data, piobuf); 4598c2ecf20Sopenharmony_ci piobuf++; 4608c2ecf20Sopenharmony_ci extra = 0; 4618c2ecf20Sopenharmony_ci data = 0; 4628c2ecf20Sopenharmony_ci } else { 4638c2ecf20Sopenharmony_ci /* Clear unused upper bytes */ 4648c2ecf20Sopenharmony_ci data |= clear_upper_bytes(v, len, extra); 4658c2ecf20Sopenharmony_ci if (len == length) { 4668c2ecf20Sopenharmony_ci last = data; 4678c2ecf20Sopenharmony_ci break; 4688c2ecf20Sopenharmony_ci } 4698c2ecf20Sopenharmony_ci extra += len; 4708c2ecf20Sopenharmony_ci } 4718c2ecf20Sopenharmony_ci } else if (extra) { 4728c2ecf20Sopenharmony_ci /* Source address is aligned. */ 4738c2ecf20Sopenharmony_ci u32 *addr = (u32 *) ss->sge.vaddr; 4748c2ecf20Sopenharmony_ci int shift = extra * BITS_PER_BYTE; 4758c2ecf20Sopenharmony_ci int ushift = 32 - shift; 4768c2ecf20Sopenharmony_ci u32 l = len; 4778c2ecf20Sopenharmony_ci 4788c2ecf20Sopenharmony_ci while (l >= sizeof(u32)) { 4798c2ecf20Sopenharmony_ci u32 v = *addr; 4808c2ecf20Sopenharmony_ci 4818c2ecf20Sopenharmony_ci data |= set_upper_bits(v, shift); 4828c2ecf20Sopenharmony_ci __raw_writel(data, piobuf); 4838c2ecf20Sopenharmony_ci data = get_upper_bits(v, ushift); 4848c2ecf20Sopenharmony_ci piobuf++; 4858c2ecf20Sopenharmony_ci addr++; 4868c2ecf20Sopenharmony_ci l -= sizeof(u32); 4878c2ecf20Sopenharmony_ci } 4888c2ecf20Sopenharmony_ci /* 4898c2ecf20Sopenharmony_ci * We still have 'extra' number of bytes leftover. 4908c2ecf20Sopenharmony_ci */ 4918c2ecf20Sopenharmony_ci if (l) { 4928c2ecf20Sopenharmony_ci u32 v = *addr; 4938c2ecf20Sopenharmony_ci 4948c2ecf20Sopenharmony_ci if (l + extra >= sizeof(u32)) { 4958c2ecf20Sopenharmony_ci data |= set_upper_bits(v, shift); 4968c2ecf20Sopenharmony_ci len -= l + extra - sizeof(u32); 4978c2ecf20Sopenharmony_ci if (len == length) { 4988c2ecf20Sopenharmony_ci last = data; 4998c2ecf20Sopenharmony_ci break; 5008c2ecf20Sopenharmony_ci } 5018c2ecf20Sopenharmony_ci __raw_writel(data, piobuf); 5028c2ecf20Sopenharmony_ci piobuf++; 5038c2ecf20Sopenharmony_ci extra = 0; 5048c2ecf20Sopenharmony_ci data = 0; 5058c2ecf20Sopenharmony_ci } else { 5068c2ecf20Sopenharmony_ci /* Clear unused upper bytes */ 5078c2ecf20Sopenharmony_ci data |= clear_upper_bytes(v, l, extra); 5088c2ecf20Sopenharmony_ci if (len == length) { 5098c2ecf20Sopenharmony_ci last = data; 5108c2ecf20Sopenharmony_ci break; 5118c2ecf20Sopenharmony_ci } 5128c2ecf20Sopenharmony_ci extra += l; 5138c2ecf20Sopenharmony_ci } 5148c2ecf20Sopenharmony_ci } else if (len == length) { 5158c2ecf20Sopenharmony_ci last = data; 5168c2ecf20Sopenharmony_ci break; 5178c2ecf20Sopenharmony_ci } 5188c2ecf20Sopenharmony_ci } else if (len == length) { 5198c2ecf20Sopenharmony_ci u32 w; 5208c2ecf20Sopenharmony_ci 5218c2ecf20Sopenharmony_ci /* 5228c2ecf20Sopenharmony_ci * Need to round up for the last dword in the 5238c2ecf20Sopenharmony_ci * packet. 5248c2ecf20Sopenharmony_ci */ 5258c2ecf20Sopenharmony_ci w = (len + 3) >> 2; 5268c2ecf20Sopenharmony_ci qib_pio_copy(piobuf, ss->sge.vaddr, w - 1); 5278c2ecf20Sopenharmony_ci piobuf += w - 1; 5288c2ecf20Sopenharmony_ci last = ((u32 *) ss->sge.vaddr)[w - 1]; 5298c2ecf20Sopenharmony_ci break; 5308c2ecf20Sopenharmony_ci } else { 5318c2ecf20Sopenharmony_ci u32 w = len >> 2; 5328c2ecf20Sopenharmony_ci 5338c2ecf20Sopenharmony_ci qib_pio_copy(piobuf, ss->sge.vaddr, w); 5348c2ecf20Sopenharmony_ci piobuf += w; 5358c2ecf20Sopenharmony_ci 5368c2ecf20Sopenharmony_ci extra = len & (sizeof(u32) - 1); 5378c2ecf20Sopenharmony_ci if (extra) { 5388c2ecf20Sopenharmony_ci u32 v = ((u32 *) ss->sge.vaddr)[w]; 5398c2ecf20Sopenharmony_ci 5408c2ecf20Sopenharmony_ci /* Clear unused upper bytes */ 5418c2ecf20Sopenharmony_ci data = clear_upper_bytes(v, extra, 0); 5428c2ecf20Sopenharmony_ci } 5438c2ecf20Sopenharmony_ci } 5448c2ecf20Sopenharmony_ci rvt_update_sge(ss, len, false); 5458c2ecf20Sopenharmony_ci length -= len; 5468c2ecf20Sopenharmony_ci } 5478c2ecf20Sopenharmony_ci /* Update address before sending packet. */ 5488c2ecf20Sopenharmony_ci rvt_update_sge(ss, length, false); 5498c2ecf20Sopenharmony_ci if (flush_wc) { 5508c2ecf20Sopenharmony_ci /* must flush early everything before trigger word */ 5518c2ecf20Sopenharmony_ci qib_flush_wc(); 5528c2ecf20Sopenharmony_ci __raw_writel(last, piobuf); 5538c2ecf20Sopenharmony_ci /* be sure trigger word is written */ 5548c2ecf20Sopenharmony_ci qib_flush_wc(); 5558c2ecf20Sopenharmony_ci } else 5568c2ecf20Sopenharmony_ci __raw_writel(last, piobuf); 5578c2ecf20Sopenharmony_ci} 5588c2ecf20Sopenharmony_ci 5598c2ecf20Sopenharmony_cistatic noinline struct qib_verbs_txreq *__get_txreq(struct qib_ibdev *dev, 5608c2ecf20Sopenharmony_ci struct rvt_qp *qp) 5618c2ecf20Sopenharmony_ci{ 5628c2ecf20Sopenharmony_ci struct qib_qp_priv *priv = qp->priv; 5638c2ecf20Sopenharmony_ci struct qib_verbs_txreq *tx; 5648c2ecf20Sopenharmony_ci unsigned long flags; 5658c2ecf20Sopenharmony_ci 5668c2ecf20Sopenharmony_ci spin_lock_irqsave(&qp->s_lock, flags); 5678c2ecf20Sopenharmony_ci spin_lock(&dev->rdi.pending_lock); 5688c2ecf20Sopenharmony_ci 5698c2ecf20Sopenharmony_ci if (!list_empty(&dev->txreq_free)) { 5708c2ecf20Sopenharmony_ci struct list_head *l = dev->txreq_free.next; 5718c2ecf20Sopenharmony_ci 5728c2ecf20Sopenharmony_ci list_del(l); 5738c2ecf20Sopenharmony_ci spin_unlock(&dev->rdi.pending_lock); 5748c2ecf20Sopenharmony_ci spin_unlock_irqrestore(&qp->s_lock, flags); 5758c2ecf20Sopenharmony_ci tx = list_entry(l, struct qib_verbs_txreq, txreq.list); 5768c2ecf20Sopenharmony_ci } else { 5778c2ecf20Sopenharmony_ci if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK && 5788c2ecf20Sopenharmony_ci list_empty(&priv->iowait)) { 5798c2ecf20Sopenharmony_ci dev->n_txwait++; 5808c2ecf20Sopenharmony_ci qp->s_flags |= RVT_S_WAIT_TX; 5818c2ecf20Sopenharmony_ci list_add_tail(&priv->iowait, &dev->txwait); 5828c2ecf20Sopenharmony_ci } 5838c2ecf20Sopenharmony_ci qp->s_flags &= ~RVT_S_BUSY; 5848c2ecf20Sopenharmony_ci spin_unlock(&dev->rdi.pending_lock); 5858c2ecf20Sopenharmony_ci spin_unlock_irqrestore(&qp->s_lock, flags); 5868c2ecf20Sopenharmony_ci tx = ERR_PTR(-EBUSY); 5878c2ecf20Sopenharmony_ci } 5888c2ecf20Sopenharmony_ci return tx; 5898c2ecf20Sopenharmony_ci} 5908c2ecf20Sopenharmony_ci 5918c2ecf20Sopenharmony_cistatic inline struct qib_verbs_txreq *get_txreq(struct qib_ibdev *dev, 5928c2ecf20Sopenharmony_ci struct rvt_qp *qp) 5938c2ecf20Sopenharmony_ci{ 5948c2ecf20Sopenharmony_ci struct qib_verbs_txreq *tx; 5958c2ecf20Sopenharmony_ci unsigned long flags; 5968c2ecf20Sopenharmony_ci 5978c2ecf20Sopenharmony_ci spin_lock_irqsave(&dev->rdi.pending_lock, flags); 5988c2ecf20Sopenharmony_ci /* assume the list non empty */ 5998c2ecf20Sopenharmony_ci if (likely(!list_empty(&dev->txreq_free))) { 6008c2ecf20Sopenharmony_ci struct list_head *l = dev->txreq_free.next; 6018c2ecf20Sopenharmony_ci 6028c2ecf20Sopenharmony_ci list_del(l); 6038c2ecf20Sopenharmony_ci spin_unlock_irqrestore(&dev->rdi.pending_lock, flags); 6048c2ecf20Sopenharmony_ci tx = list_entry(l, struct qib_verbs_txreq, txreq.list); 6058c2ecf20Sopenharmony_ci } else { 6068c2ecf20Sopenharmony_ci /* call slow path to get the extra lock */ 6078c2ecf20Sopenharmony_ci spin_unlock_irqrestore(&dev->rdi.pending_lock, flags); 6088c2ecf20Sopenharmony_ci tx = __get_txreq(dev, qp); 6098c2ecf20Sopenharmony_ci } 6108c2ecf20Sopenharmony_ci return tx; 6118c2ecf20Sopenharmony_ci} 6128c2ecf20Sopenharmony_ci 6138c2ecf20Sopenharmony_civoid qib_put_txreq(struct qib_verbs_txreq *tx) 6148c2ecf20Sopenharmony_ci{ 6158c2ecf20Sopenharmony_ci struct qib_ibdev *dev; 6168c2ecf20Sopenharmony_ci struct rvt_qp *qp; 6178c2ecf20Sopenharmony_ci struct qib_qp_priv *priv; 6188c2ecf20Sopenharmony_ci unsigned long flags; 6198c2ecf20Sopenharmony_ci 6208c2ecf20Sopenharmony_ci qp = tx->qp; 6218c2ecf20Sopenharmony_ci dev = to_idev(qp->ibqp.device); 6228c2ecf20Sopenharmony_ci 6238c2ecf20Sopenharmony_ci if (tx->mr) { 6248c2ecf20Sopenharmony_ci rvt_put_mr(tx->mr); 6258c2ecf20Sopenharmony_ci tx->mr = NULL; 6268c2ecf20Sopenharmony_ci } 6278c2ecf20Sopenharmony_ci if (tx->txreq.flags & QIB_SDMA_TXREQ_F_FREEBUF) { 6288c2ecf20Sopenharmony_ci tx->txreq.flags &= ~QIB_SDMA_TXREQ_F_FREEBUF; 6298c2ecf20Sopenharmony_ci dma_unmap_single(&dd_from_dev(dev)->pcidev->dev, 6308c2ecf20Sopenharmony_ci tx->txreq.addr, tx->hdr_dwords << 2, 6318c2ecf20Sopenharmony_ci DMA_TO_DEVICE); 6328c2ecf20Sopenharmony_ci kfree(tx->align_buf); 6338c2ecf20Sopenharmony_ci } 6348c2ecf20Sopenharmony_ci 6358c2ecf20Sopenharmony_ci spin_lock_irqsave(&dev->rdi.pending_lock, flags); 6368c2ecf20Sopenharmony_ci 6378c2ecf20Sopenharmony_ci /* Put struct back on free list */ 6388c2ecf20Sopenharmony_ci list_add(&tx->txreq.list, &dev->txreq_free); 6398c2ecf20Sopenharmony_ci 6408c2ecf20Sopenharmony_ci if (!list_empty(&dev->txwait)) { 6418c2ecf20Sopenharmony_ci /* Wake up first QP wanting a free struct */ 6428c2ecf20Sopenharmony_ci priv = list_entry(dev->txwait.next, struct qib_qp_priv, 6438c2ecf20Sopenharmony_ci iowait); 6448c2ecf20Sopenharmony_ci qp = priv->owner; 6458c2ecf20Sopenharmony_ci list_del_init(&priv->iowait); 6468c2ecf20Sopenharmony_ci rvt_get_qp(qp); 6478c2ecf20Sopenharmony_ci spin_unlock_irqrestore(&dev->rdi.pending_lock, flags); 6488c2ecf20Sopenharmony_ci 6498c2ecf20Sopenharmony_ci spin_lock_irqsave(&qp->s_lock, flags); 6508c2ecf20Sopenharmony_ci if (qp->s_flags & RVT_S_WAIT_TX) { 6518c2ecf20Sopenharmony_ci qp->s_flags &= ~RVT_S_WAIT_TX; 6528c2ecf20Sopenharmony_ci qib_schedule_send(qp); 6538c2ecf20Sopenharmony_ci } 6548c2ecf20Sopenharmony_ci spin_unlock_irqrestore(&qp->s_lock, flags); 6558c2ecf20Sopenharmony_ci 6568c2ecf20Sopenharmony_ci rvt_put_qp(qp); 6578c2ecf20Sopenharmony_ci } else 6588c2ecf20Sopenharmony_ci spin_unlock_irqrestore(&dev->rdi.pending_lock, flags); 6598c2ecf20Sopenharmony_ci} 6608c2ecf20Sopenharmony_ci 6618c2ecf20Sopenharmony_ci/* 6628c2ecf20Sopenharmony_ci * This is called when there are send DMA descriptors that might be 6638c2ecf20Sopenharmony_ci * available. 6648c2ecf20Sopenharmony_ci * 6658c2ecf20Sopenharmony_ci * This is called with ppd->sdma_lock held. 6668c2ecf20Sopenharmony_ci */ 6678c2ecf20Sopenharmony_civoid qib_verbs_sdma_desc_avail(struct qib_pportdata *ppd, unsigned avail) 6688c2ecf20Sopenharmony_ci{ 6698c2ecf20Sopenharmony_ci struct rvt_qp *qp; 6708c2ecf20Sopenharmony_ci struct qib_qp_priv *qpp, *nqpp; 6718c2ecf20Sopenharmony_ci struct rvt_qp *qps[20]; 6728c2ecf20Sopenharmony_ci struct qib_ibdev *dev; 6738c2ecf20Sopenharmony_ci unsigned i, n; 6748c2ecf20Sopenharmony_ci 6758c2ecf20Sopenharmony_ci n = 0; 6768c2ecf20Sopenharmony_ci dev = &ppd->dd->verbs_dev; 6778c2ecf20Sopenharmony_ci spin_lock(&dev->rdi.pending_lock); 6788c2ecf20Sopenharmony_ci 6798c2ecf20Sopenharmony_ci /* Search wait list for first QP wanting DMA descriptors. */ 6808c2ecf20Sopenharmony_ci list_for_each_entry_safe(qpp, nqpp, &dev->dmawait, iowait) { 6818c2ecf20Sopenharmony_ci qp = qpp->owner; 6828c2ecf20Sopenharmony_ci if (qp->port_num != ppd->port) 6838c2ecf20Sopenharmony_ci continue; 6848c2ecf20Sopenharmony_ci if (n == ARRAY_SIZE(qps)) 6858c2ecf20Sopenharmony_ci break; 6868c2ecf20Sopenharmony_ci if (qpp->s_tx->txreq.sg_count > avail) 6878c2ecf20Sopenharmony_ci break; 6888c2ecf20Sopenharmony_ci avail -= qpp->s_tx->txreq.sg_count; 6898c2ecf20Sopenharmony_ci list_del_init(&qpp->iowait); 6908c2ecf20Sopenharmony_ci rvt_get_qp(qp); 6918c2ecf20Sopenharmony_ci qps[n++] = qp; 6928c2ecf20Sopenharmony_ci } 6938c2ecf20Sopenharmony_ci 6948c2ecf20Sopenharmony_ci spin_unlock(&dev->rdi.pending_lock); 6958c2ecf20Sopenharmony_ci 6968c2ecf20Sopenharmony_ci for (i = 0; i < n; i++) { 6978c2ecf20Sopenharmony_ci qp = qps[i]; 6988c2ecf20Sopenharmony_ci spin_lock(&qp->s_lock); 6998c2ecf20Sopenharmony_ci if (qp->s_flags & RVT_S_WAIT_DMA_DESC) { 7008c2ecf20Sopenharmony_ci qp->s_flags &= ~RVT_S_WAIT_DMA_DESC; 7018c2ecf20Sopenharmony_ci qib_schedule_send(qp); 7028c2ecf20Sopenharmony_ci } 7038c2ecf20Sopenharmony_ci spin_unlock(&qp->s_lock); 7048c2ecf20Sopenharmony_ci rvt_put_qp(qp); 7058c2ecf20Sopenharmony_ci } 7068c2ecf20Sopenharmony_ci} 7078c2ecf20Sopenharmony_ci 7088c2ecf20Sopenharmony_ci/* 7098c2ecf20Sopenharmony_ci * This is called with ppd->sdma_lock held. 7108c2ecf20Sopenharmony_ci */ 7118c2ecf20Sopenharmony_cistatic void sdma_complete(struct qib_sdma_txreq *cookie, int status) 7128c2ecf20Sopenharmony_ci{ 7138c2ecf20Sopenharmony_ci struct qib_verbs_txreq *tx = 7148c2ecf20Sopenharmony_ci container_of(cookie, struct qib_verbs_txreq, txreq); 7158c2ecf20Sopenharmony_ci struct rvt_qp *qp = tx->qp; 7168c2ecf20Sopenharmony_ci struct qib_qp_priv *priv = qp->priv; 7178c2ecf20Sopenharmony_ci 7188c2ecf20Sopenharmony_ci spin_lock(&qp->s_lock); 7198c2ecf20Sopenharmony_ci if (tx->wqe) 7208c2ecf20Sopenharmony_ci rvt_send_complete(qp, tx->wqe, IB_WC_SUCCESS); 7218c2ecf20Sopenharmony_ci else if (qp->ibqp.qp_type == IB_QPT_RC) { 7228c2ecf20Sopenharmony_ci struct ib_header *hdr; 7238c2ecf20Sopenharmony_ci 7248c2ecf20Sopenharmony_ci if (tx->txreq.flags & QIB_SDMA_TXREQ_F_FREEBUF) 7258c2ecf20Sopenharmony_ci hdr = &tx->align_buf->hdr; 7268c2ecf20Sopenharmony_ci else { 7278c2ecf20Sopenharmony_ci struct qib_ibdev *dev = to_idev(qp->ibqp.device); 7288c2ecf20Sopenharmony_ci 7298c2ecf20Sopenharmony_ci hdr = &dev->pio_hdrs[tx->hdr_inx].hdr; 7308c2ecf20Sopenharmony_ci } 7318c2ecf20Sopenharmony_ci qib_rc_send_complete(qp, hdr); 7328c2ecf20Sopenharmony_ci } 7338c2ecf20Sopenharmony_ci if (atomic_dec_and_test(&priv->s_dma_busy)) { 7348c2ecf20Sopenharmony_ci if (qp->state == IB_QPS_RESET) 7358c2ecf20Sopenharmony_ci wake_up(&priv->wait_dma); 7368c2ecf20Sopenharmony_ci else if (qp->s_flags & RVT_S_WAIT_DMA) { 7378c2ecf20Sopenharmony_ci qp->s_flags &= ~RVT_S_WAIT_DMA; 7388c2ecf20Sopenharmony_ci qib_schedule_send(qp); 7398c2ecf20Sopenharmony_ci } 7408c2ecf20Sopenharmony_ci } 7418c2ecf20Sopenharmony_ci spin_unlock(&qp->s_lock); 7428c2ecf20Sopenharmony_ci 7438c2ecf20Sopenharmony_ci qib_put_txreq(tx); 7448c2ecf20Sopenharmony_ci} 7458c2ecf20Sopenharmony_ci 7468c2ecf20Sopenharmony_cistatic int wait_kmem(struct qib_ibdev *dev, struct rvt_qp *qp) 7478c2ecf20Sopenharmony_ci{ 7488c2ecf20Sopenharmony_ci struct qib_qp_priv *priv = qp->priv; 7498c2ecf20Sopenharmony_ci unsigned long flags; 7508c2ecf20Sopenharmony_ci int ret = 0; 7518c2ecf20Sopenharmony_ci 7528c2ecf20Sopenharmony_ci spin_lock_irqsave(&qp->s_lock, flags); 7538c2ecf20Sopenharmony_ci if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) { 7548c2ecf20Sopenharmony_ci spin_lock(&dev->rdi.pending_lock); 7558c2ecf20Sopenharmony_ci if (list_empty(&priv->iowait)) { 7568c2ecf20Sopenharmony_ci if (list_empty(&dev->memwait)) 7578c2ecf20Sopenharmony_ci mod_timer(&dev->mem_timer, jiffies + 1); 7588c2ecf20Sopenharmony_ci qp->s_flags |= RVT_S_WAIT_KMEM; 7598c2ecf20Sopenharmony_ci list_add_tail(&priv->iowait, &dev->memwait); 7608c2ecf20Sopenharmony_ci } 7618c2ecf20Sopenharmony_ci spin_unlock(&dev->rdi.pending_lock); 7628c2ecf20Sopenharmony_ci qp->s_flags &= ~RVT_S_BUSY; 7638c2ecf20Sopenharmony_ci ret = -EBUSY; 7648c2ecf20Sopenharmony_ci } 7658c2ecf20Sopenharmony_ci spin_unlock_irqrestore(&qp->s_lock, flags); 7668c2ecf20Sopenharmony_ci 7678c2ecf20Sopenharmony_ci return ret; 7688c2ecf20Sopenharmony_ci} 7698c2ecf20Sopenharmony_ci 7708c2ecf20Sopenharmony_cistatic int qib_verbs_send_dma(struct rvt_qp *qp, struct ib_header *hdr, 7718c2ecf20Sopenharmony_ci u32 hdrwords, struct rvt_sge_state *ss, u32 len, 7728c2ecf20Sopenharmony_ci u32 plen, u32 dwords) 7738c2ecf20Sopenharmony_ci{ 7748c2ecf20Sopenharmony_ci struct qib_qp_priv *priv = qp->priv; 7758c2ecf20Sopenharmony_ci struct qib_ibdev *dev = to_idev(qp->ibqp.device); 7768c2ecf20Sopenharmony_ci struct qib_devdata *dd = dd_from_dev(dev); 7778c2ecf20Sopenharmony_ci struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); 7788c2ecf20Sopenharmony_ci struct qib_pportdata *ppd = ppd_from_ibp(ibp); 7798c2ecf20Sopenharmony_ci struct qib_verbs_txreq *tx; 7808c2ecf20Sopenharmony_ci struct qib_pio_header *phdr; 7818c2ecf20Sopenharmony_ci u32 control; 7828c2ecf20Sopenharmony_ci u32 ndesc; 7838c2ecf20Sopenharmony_ci int ret; 7848c2ecf20Sopenharmony_ci 7858c2ecf20Sopenharmony_ci tx = priv->s_tx; 7868c2ecf20Sopenharmony_ci if (tx) { 7878c2ecf20Sopenharmony_ci priv->s_tx = NULL; 7888c2ecf20Sopenharmony_ci /* resend previously constructed packet */ 7898c2ecf20Sopenharmony_ci ret = qib_sdma_verbs_send(ppd, tx->ss, tx->dwords, tx); 7908c2ecf20Sopenharmony_ci goto bail; 7918c2ecf20Sopenharmony_ci } 7928c2ecf20Sopenharmony_ci 7938c2ecf20Sopenharmony_ci tx = get_txreq(dev, qp); 7948c2ecf20Sopenharmony_ci if (IS_ERR(tx)) 7958c2ecf20Sopenharmony_ci goto bail_tx; 7968c2ecf20Sopenharmony_ci 7978c2ecf20Sopenharmony_ci control = dd->f_setpbc_control(ppd, plen, qp->s_srate, 7988c2ecf20Sopenharmony_ci be16_to_cpu(hdr->lrh[0]) >> 12); 7998c2ecf20Sopenharmony_ci tx->qp = qp; 8008c2ecf20Sopenharmony_ci tx->wqe = qp->s_wqe; 8018c2ecf20Sopenharmony_ci tx->mr = qp->s_rdma_mr; 8028c2ecf20Sopenharmony_ci if (qp->s_rdma_mr) 8038c2ecf20Sopenharmony_ci qp->s_rdma_mr = NULL; 8048c2ecf20Sopenharmony_ci tx->txreq.callback = sdma_complete; 8058c2ecf20Sopenharmony_ci if (dd->flags & QIB_HAS_SDMA_TIMEOUT) 8068c2ecf20Sopenharmony_ci tx->txreq.flags = QIB_SDMA_TXREQ_F_HEADTOHOST; 8078c2ecf20Sopenharmony_ci else 8088c2ecf20Sopenharmony_ci tx->txreq.flags = QIB_SDMA_TXREQ_F_INTREQ; 8098c2ecf20Sopenharmony_ci if (plen + 1 > dd->piosize2kmax_dwords) 8108c2ecf20Sopenharmony_ci tx->txreq.flags |= QIB_SDMA_TXREQ_F_USELARGEBUF; 8118c2ecf20Sopenharmony_ci 8128c2ecf20Sopenharmony_ci if (len) { 8138c2ecf20Sopenharmony_ci /* 8148c2ecf20Sopenharmony_ci * Don't try to DMA if it takes more descriptors than 8158c2ecf20Sopenharmony_ci * the queue holds. 8168c2ecf20Sopenharmony_ci */ 8178c2ecf20Sopenharmony_ci ndesc = qib_count_sge(ss, len); 8188c2ecf20Sopenharmony_ci if (ndesc >= ppd->sdma_descq_cnt) 8198c2ecf20Sopenharmony_ci ndesc = 0; 8208c2ecf20Sopenharmony_ci } else 8218c2ecf20Sopenharmony_ci ndesc = 1; 8228c2ecf20Sopenharmony_ci if (ndesc) { 8238c2ecf20Sopenharmony_ci phdr = &dev->pio_hdrs[tx->hdr_inx]; 8248c2ecf20Sopenharmony_ci phdr->pbc[0] = cpu_to_le32(plen); 8258c2ecf20Sopenharmony_ci phdr->pbc[1] = cpu_to_le32(control); 8268c2ecf20Sopenharmony_ci memcpy(&phdr->hdr, hdr, hdrwords << 2); 8278c2ecf20Sopenharmony_ci tx->txreq.flags |= QIB_SDMA_TXREQ_F_FREEDESC; 8288c2ecf20Sopenharmony_ci tx->txreq.sg_count = ndesc; 8298c2ecf20Sopenharmony_ci tx->txreq.addr = dev->pio_hdrs_phys + 8308c2ecf20Sopenharmony_ci tx->hdr_inx * sizeof(struct qib_pio_header); 8318c2ecf20Sopenharmony_ci tx->hdr_dwords = hdrwords + 2; /* add PBC length */ 8328c2ecf20Sopenharmony_ci ret = qib_sdma_verbs_send(ppd, ss, dwords, tx); 8338c2ecf20Sopenharmony_ci goto bail; 8348c2ecf20Sopenharmony_ci } 8358c2ecf20Sopenharmony_ci 8368c2ecf20Sopenharmony_ci /* Allocate a buffer and copy the header and payload to it. */ 8378c2ecf20Sopenharmony_ci tx->hdr_dwords = plen + 1; 8388c2ecf20Sopenharmony_ci phdr = kmalloc(tx->hdr_dwords << 2, GFP_ATOMIC); 8398c2ecf20Sopenharmony_ci if (!phdr) 8408c2ecf20Sopenharmony_ci goto err_tx; 8418c2ecf20Sopenharmony_ci phdr->pbc[0] = cpu_to_le32(plen); 8428c2ecf20Sopenharmony_ci phdr->pbc[1] = cpu_to_le32(control); 8438c2ecf20Sopenharmony_ci memcpy(&phdr->hdr, hdr, hdrwords << 2); 8448c2ecf20Sopenharmony_ci qib_copy_from_sge((u32 *) &phdr->hdr + hdrwords, ss, len); 8458c2ecf20Sopenharmony_ci 8468c2ecf20Sopenharmony_ci tx->txreq.addr = dma_map_single(&dd->pcidev->dev, phdr, 8478c2ecf20Sopenharmony_ci tx->hdr_dwords << 2, DMA_TO_DEVICE); 8488c2ecf20Sopenharmony_ci if (dma_mapping_error(&dd->pcidev->dev, tx->txreq.addr)) 8498c2ecf20Sopenharmony_ci goto map_err; 8508c2ecf20Sopenharmony_ci tx->align_buf = phdr; 8518c2ecf20Sopenharmony_ci tx->txreq.flags |= QIB_SDMA_TXREQ_F_FREEBUF; 8528c2ecf20Sopenharmony_ci tx->txreq.sg_count = 1; 8538c2ecf20Sopenharmony_ci ret = qib_sdma_verbs_send(ppd, NULL, 0, tx); 8548c2ecf20Sopenharmony_ci goto unaligned; 8558c2ecf20Sopenharmony_ci 8568c2ecf20Sopenharmony_cimap_err: 8578c2ecf20Sopenharmony_ci kfree(phdr); 8588c2ecf20Sopenharmony_cierr_tx: 8598c2ecf20Sopenharmony_ci qib_put_txreq(tx); 8608c2ecf20Sopenharmony_ci ret = wait_kmem(dev, qp); 8618c2ecf20Sopenharmony_ciunaligned: 8628c2ecf20Sopenharmony_ci ibp->rvp.n_unaligned++; 8638c2ecf20Sopenharmony_cibail: 8648c2ecf20Sopenharmony_ci return ret; 8658c2ecf20Sopenharmony_cibail_tx: 8668c2ecf20Sopenharmony_ci ret = PTR_ERR(tx); 8678c2ecf20Sopenharmony_ci goto bail; 8688c2ecf20Sopenharmony_ci} 8698c2ecf20Sopenharmony_ci 8708c2ecf20Sopenharmony_ci/* 8718c2ecf20Sopenharmony_ci * If we are now in the error state, return zero to flush the 8728c2ecf20Sopenharmony_ci * send work request. 8738c2ecf20Sopenharmony_ci */ 8748c2ecf20Sopenharmony_cistatic int no_bufs_available(struct rvt_qp *qp) 8758c2ecf20Sopenharmony_ci{ 8768c2ecf20Sopenharmony_ci struct qib_qp_priv *priv = qp->priv; 8778c2ecf20Sopenharmony_ci struct qib_ibdev *dev = to_idev(qp->ibqp.device); 8788c2ecf20Sopenharmony_ci struct qib_devdata *dd; 8798c2ecf20Sopenharmony_ci unsigned long flags; 8808c2ecf20Sopenharmony_ci int ret = 0; 8818c2ecf20Sopenharmony_ci 8828c2ecf20Sopenharmony_ci /* 8838c2ecf20Sopenharmony_ci * Note that as soon as want_buffer() is called and 8848c2ecf20Sopenharmony_ci * possibly before it returns, qib_ib_piobufavail() 8858c2ecf20Sopenharmony_ci * could be called. Therefore, put QP on the I/O wait list before 8868c2ecf20Sopenharmony_ci * enabling the PIO avail interrupt. 8878c2ecf20Sopenharmony_ci */ 8888c2ecf20Sopenharmony_ci spin_lock_irqsave(&qp->s_lock, flags); 8898c2ecf20Sopenharmony_ci if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) { 8908c2ecf20Sopenharmony_ci spin_lock(&dev->rdi.pending_lock); 8918c2ecf20Sopenharmony_ci if (list_empty(&priv->iowait)) { 8928c2ecf20Sopenharmony_ci dev->n_piowait++; 8938c2ecf20Sopenharmony_ci qp->s_flags |= RVT_S_WAIT_PIO; 8948c2ecf20Sopenharmony_ci list_add_tail(&priv->iowait, &dev->piowait); 8958c2ecf20Sopenharmony_ci dd = dd_from_dev(dev); 8968c2ecf20Sopenharmony_ci dd->f_wantpiobuf_intr(dd, 1); 8978c2ecf20Sopenharmony_ci } 8988c2ecf20Sopenharmony_ci spin_unlock(&dev->rdi.pending_lock); 8998c2ecf20Sopenharmony_ci qp->s_flags &= ~RVT_S_BUSY; 9008c2ecf20Sopenharmony_ci ret = -EBUSY; 9018c2ecf20Sopenharmony_ci } 9028c2ecf20Sopenharmony_ci spin_unlock_irqrestore(&qp->s_lock, flags); 9038c2ecf20Sopenharmony_ci return ret; 9048c2ecf20Sopenharmony_ci} 9058c2ecf20Sopenharmony_ci 9068c2ecf20Sopenharmony_cistatic int qib_verbs_send_pio(struct rvt_qp *qp, struct ib_header *ibhdr, 9078c2ecf20Sopenharmony_ci u32 hdrwords, struct rvt_sge_state *ss, u32 len, 9088c2ecf20Sopenharmony_ci u32 plen, u32 dwords) 9098c2ecf20Sopenharmony_ci{ 9108c2ecf20Sopenharmony_ci struct qib_devdata *dd = dd_from_ibdev(qp->ibqp.device); 9118c2ecf20Sopenharmony_ci struct qib_pportdata *ppd = dd->pport + qp->port_num - 1; 9128c2ecf20Sopenharmony_ci u32 *hdr = (u32 *) ibhdr; 9138c2ecf20Sopenharmony_ci u32 __iomem *piobuf_orig; 9148c2ecf20Sopenharmony_ci u32 __iomem *piobuf; 9158c2ecf20Sopenharmony_ci u64 pbc; 9168c2ecf20Sopenharmony_ci unsigned long flags; 9178c2ecf20Sopenharmony_ci unsigned flush_wc; 9188c2ecf20Sopenharmony_ci u32 control; 9198c2ecf20Sopenharmony_ci u32 pbufn; 9208c2ecf20Sopenharmony_ci 9218c2ecf20Sopenharmony_ci control = dd->f_setpbc_control(ppd, plen, qp->s_srate, 9228c2ecf20Sopenharmony_ci be16_to_cpu(ibhdr->lrh[0]) >> 12); 9238c2ecf20Sopenharmony_ci pbc = ((u64) control << 32) | plen; 9248c2ecf20Sopenharmony_ci piobuf = dd->f_getsendbuf(ppd, pbc, &pbufn); 9258c2ecf20Sopenharmony_ci if (unlikely(piobuf == NULL)) 9268c2ecf20Sopenharmony_ci return no_bufs_available(qp); 9278c2ecf20Sopenharmony_ci 9288c2ecf20Sopenharmony_ci /* 9298c2ecf20Sopenharmony_ci * Write the pbc. 9308c2ecf20Sopenharmony_ci * We have to flush after the PBC for correctness on some cpus 9318c2ecf20Sopenharmony_ci * or WC buffer can be written out of order. 9328c2ecf20Sopenharmony_ci */ 9338c2ecf20Sopenharmony_ci writeq(pbc, piobuf); 9348c2ecf20Sopenharmony_ci piobuf_orig = piobuf; 9358c2ecf20Sopenharmony_ci piobuf += 2; 9368c2ecf20Sopenharmony_ci 9378c2ecf20Sopenharmony_ci flush_wc = dd->flags & QIB_PIO_FLUSH_WC; 9388c2ecf20Sopenharmony_ci if (len == 0) { 9398c2ecf20Sopenharmony_ci /* 9408c2ecf20Sopenharmony_ci * If there is just the header portion, must flush before 9418c2ecf20Sopenharmony_ci * writing last word of header for correctness, and after 9428c2ecf20Sopenharmony_ci * the last header word (trigger word). 9438c2ecf20Sopenharmony_ci */ 9448c2ecf20Sopenharmony_ci if (flush_wc) { 9458c2ecf20Sopenharmony_ci qib_flush_wc(); 9468c2ecf20Sopenharmony_ci qib_pio_copy(piobuf, hdr, hdrwords - 1); 9478c2ecf20Sopenharmony_ci qib_flush_wc(); 9488c2ecf20Sopenharmony_ci __raw_writel(hdr[hdrwords - 1], piobuf + hdrwords - 1); 9498c2ecf20Sopenharmony_ci qib_flush_wc(); 9508c2ecf20Sopenharmony_ci } else 9518c2ecf20Sopenharmony_ci qib_pio_copy(piobuf, hdr, hdrwords); 9528c2ecf20Sopenharmony_ci goto done; 9538c2ecf20Sopenharmony_ci } 9548c2ecf20Sopenharmony_ci 9558c2ecf20Sopenharmony_ci if (flush_wc) 9568c2ecf20Sopenharmony_ci qib_flush_wc(); 9578c2ecf20Sopenharmony_ci qib_pio_copy(piobuf, hdr, hdrwords); 9588c2ecf20Sopenharmony_ci piobuf += hdrwords; 9598c2ecf20Sopenharmony_ci 9608c2ecf20Sopenharmony_ci /* The common case is aligned and contained in one segment. */ 9618c2ecf20Sopenharmony_ci if (likely(ss->num_sge == 1 && len <= ss->sge.length && 9628c2ecf20Sopenharmony_ci !((unsigned long)ss->sge.vaddr & (sizeof(u32) - 1)))) { 9638c2ecf20Sopenharmony_ci u32 *addr = (u32 *) ss->sge.vaddr; 9648c2ecf20Sopenharmony_ci 9658c2ecf20Sopenharmony_ci /* Update address before sending packet. */ 9668c2ecf20Sopenharmony_ci rvt_update_sge(ss, len, false); 9678c2ecf20Sopenharmony_ci if (flush_wc) { 9688c2ecf20Sopenharmony_ci qib_pio_copy(piobuf, addr, dwords - 1); 9698c2ecf20Sopenharmony_ci /* must flush early everything before trigger word */ 9708c2ecf20Sopenharmony_ci qib_flush_wc(); 9718c2ecf20Sopenharmony_ci __raw_writel(addr[dwords - 1], piobuf + dwords - 1); 9728c2ecf20Sopenharmony_ci /* be sure trigger word is written */ 9738c2ecf20Sopenharmony_ci qib_flush_wc(); 9748c2ecf20Sopenharmony_ci } else 9758c2ecf20Sopenharmony_ci qib_pio_copy(piobuf, addr, dwords); 9768c2ecf20Sopenharmony_ci goto done; 9778c2ecf20Sopenharmony_ci } 9788c2ecf20Sopenharmony_ci copy_io(piobuf, ss, len, flush_wc); 9798c2ecf20Sopenharmony_cidone: 9808c2ecf20Sopenharmony_ci if (dd->flags & QIB_USE_SPCL_TRIG) { 9818c2ecf20Sopenharmony_ci u32 spcl_off = (pbufn >= dd->piobcnt2k) ? 2047 : 1023; 9828c2ecf20Sopenharmony_ci 9838c2ecf20Sopenharmony_ci qib_flush_wc(); 9848c2ecf20Sopenharmony_ci __raw_writel(0xaebecede, piobuf_orig + spcl_off); 9858c2ecf20Sopenharmony_ci } 9868c2ecf20Sopenharmony_ci qib_sendbuf_done(dd, pbufn); 9878c2ecf20Sopenharmony_ci if (qp->s_rdma_mr) { 9888c2ecf20Sopenharmony_ci rvt_put_mr(qp->s_rdma_mr); 9898c2ecf20Sopenharmony_ci qp->s_rdma_mr = NULL; 9908c2ecf20Sopenharmony_ci } 9918c2ecf20Sopenharmony_ci if (qp->s_wqe) { 9928c2ecf20Sopenharmony_ci spin_lock_irqsave(&qp->s_lock, flags); 9938c2ecf20Sopenharmony_ci rvt_send_complete(qp, qp->s_wqe, IB_WC_SUCCESS); 9948c2ecf20Sopenharmony_ci spin_unlock_irqrestore(&qp->s_lock, flags); 9958c2ecf20Sopenharmony_ci } else if (qp->ibqp.qp_type == IB_QPT_RC) { 9968c2ecf20Sopenharmony_ci spin_lock_irqsave(&qp->s_lock, flags); 9978c2ecf20Sopenharmony_ci qib_rc_send_complete(qp, ibhdr); 9988c2ecf20Sopenharmony_ci spin_unlock_irqrestore(&qp->s_lock, flags); 9998c2ecf20Sopenharmony_ci } 10008c2ecf20Sopenharmony_ci return 0; 10018c2ecf20Sopenharmony_ci} 10028c2ecf20Sopenharmony_ci 10038c2ecf20Sopenharmony_ci/** 10048c2ecf20Sopenharmony_ci * qib_verbs_send - send a packet 10058c2ecf20Sopenharmony_ci * @qp: the QP to send on 10068c2ecf20Sopenharmony_ci * @hdr: the packet header 10078c2ecf20Sopenharmony_ci * @hdrwords: the number of 32-bit words in the header 10088c2ecf20Sopenharmony_ci * @ss: the SGE to send 10098c2ecf20Sopenharmony_ci * @len: the length of the packet in bytes 10108c2ecf20Sopenharmony_ci * 10118c2ecf20Sopenharmony_ci * Return zero if packet is sent or queued OK. 10128c2ecf20Sopenharmony_ci * Return non-zero and clear qp->s_flags RVT_S_BUSY otherwise. 10138c2ecf20Sopenharmony_ci */ 10148c2ecf20Sopenharmony_ciint qib_verbs_send(struct rvt_qp *qp, struct ib_header *hdr, 10158c2ecf20Sopenharmony_ci u32 hdrwords, struct rvt_sge_state *ss, u32 len) 10168c2ecf20Sopenharmony_ci{ 10178c2ecf20Sopenharmony_ci struct qib_devdata *dd = dd_from_ibdev(qp->ibqp.device); 10188c2ecf20Sopenharmony_ci u32 plen; 10198c2ecf20Sopenharmony_ci int ret; 10208c2ecf20Sopenharmony_ci u32 dwords = (len + 3) >> 2; 10218c2ecf20Sopenharmony_ci 10228c2ecf20Sopenharmony_ci /* 10238c2ecf20Sopenharmony_ci * Calculate the send buffer trigger address. 10248c2ecf20Sopenharmony_ci * The +1 counts for the pbc control dword following the pbc length. 10258c2ecf20Sopenharmony_ci */ 10268c2ecf20Sopenharmony_ci plen = hdrwords + dwords + 1; 10278c2ecf20Sopenharmony_ci 10288c2ecf20Sopenharmony_ci /* 10298c2ecf20Sopenharmony_ci * VL15 packets (IB_QPT_SMI) will always use PIO, so we 10308c2ecf20Sopenharmony_ci * can defer SDMA restart until link goes ACTIVE without 10318c2ecf20Sopenharmony_ci * worrying about just how we got there. 10328c2ecf20Sopenharmony_ci */ 10338c2ecf20Sopenharmony_ci if (qp->ibqp.qp_type == IB_QPT_SMI || 10348c2ecf20Sopenharmony_ci !(dd->flags & QIB_HAS_SEND_DMA)) 10358c2ecf20Sopenharmony_ci ret = qib_verbs_send_pio(qp, hdr, hdrwords, ss, len, 10368c2ecf20Sopenharmony_ci plen, dwords); 10378c2ecf20Sopenharmony_ci else 10388c2ecf20Sopenharmony_ci ret = qib_verbs_send_dma(qp, hdr, hdrwords, ss, len, 10398c2ecf20Sopenharmony_ci plen, dwords); 10408c2ecf20Sopenharmony_ci 10418c2ecf20Sopenharmony_ci return ret; 10428c2ecf20Sopenharmony_ci} 10438c2ecf20Sopenharmony_ci 10448c2ecf20Sopenharmony_ciint qib_snapshot_counters(struct qib_pportdata *ppd, u64 *swords, 10458c2ecf20Sopenharmony_ci u64 *rwords, u64 *spkts, u64 *rpkts, 10468c2ecf20Sopenharmony_ci u64 *xmit_wait) 10478c2ecf20Sopenharmony_ci{ 10488c2ecf20Sopenharmony_ci int ret; 10498c2ecf20Sopenharmony_ci struct qib_devdata *dd = ppd->dd; 10508c2ecf20Sopenharmony_ci 10518c2ecf20Sopenharmony_ci if (!(dd->flags & QIB_PRESENT)) { 10528c2ecf20Sopenharmony_ci /* no hardware, freeze, etc. */ 10538c2ecf20Sopenharmony_ci ret = -EINVAL; 10548c2ecf20Sopenharmony_ci goto bail; 10558c2ecf20Sopenharmony_ci } 10568c2ecf20Sopenharmony_ci *swords = dd->f_portcntr(ppd, QIBPORTCNTR_WORDSEND); 10578c2ecf20Sopenharmony_ci *rwords = dd->f_portcntr(ppd, QIBPORTCNTR_WORDRCV); 10588c2ecf20Sopenharmony_ci *spkts = dd->f_portcntr(ppd, QIBPORTCNTR_PKTSEND); 10598c2ecf20Sopenharmony_ci *rpkts = dd->f_portcntr(ppd, QIBPORTCNTR_PKTRCV); 10608c2ecf20Sopenharmony_ci *xmit_wait = dd->f_portcntr(ppd, QIBPORTCNTR_SENDSTALL); 10618c2ecf20Sopenharmony_ci 10628c2ecf20Sopenharmony_ci ret = 0; 10638c2ecf20Sopenharmony_ci 10648c2ecf20Sopenharmony_cibail: 10658c2ecf20Sopenharmony_ci return ret; 10668c2ecf20Sopenharmony_ci} 10678c2ecf20Sopenharmony_ci 10688c2ecf20Sopenharmony_ci/** 10698c2ecf20Sopenharmony_ci * qib_get_counters - get various chip counters 10708c2ecf20Sopenharmony_ci * @dd: the qlogic_ib device 10718c2ecf20Sopenharmony_ci * @cntrs: counters are placed here 10728c2ecf20Sopenharmony_ci * 10738c2ecf20Sopenharmony_ci * Return the counters needed by recv_pma_get_portcounters(). 10748c2ecf20Sopenharmony_ci */ 10758c2ecf20Sopenharmony_ciint qib_get_counters(struct qib_pportdata *ppd, 10768c2ecf20Sopenharmony_ci struct qib_verbs_counters *cntrs) 10778c2ecf20Sopenharmony_ci{ 10788c2ecf20Sopenharmony_ci int ret; 10798c2ecf20Sopenharmony_ci 10808c2ecf20Sopenharmony_ci if (!(ppd->dd->flags & QIB_PRESENT)) { 10818c2ecf20Sopenharmony_ci /* no hardware, freeze, etc. */ 10828c2ecf20Sopenharmony_ci ret = -EINVAL; 10838c2ecf20Sopenharmony_ci goto bail; 10848c2ecf20Sopenharmony_ci } 10858c2ecf20Sopenharmony_ci cntrs->symbol_error_counter = 10868c2ecf20Sopenharmony_ci ppd->dd->f_portcntr(ppd, QIBPORTCNTR_IBSYMBOLERR); 10878c2ecf20Sopenharmony_ci cntrs->link_error_recovery_counter = 10888c2ecf20Sopenharmony_ci ppd->dd->f_portcntr(ppd, QIBPORTCNTR_IBLINKERRRECOV); 10898c2ecf20Sopenharmony_ci /* 10908c2ecf20Sopenharmony_ci * The link downed counter counts when the other side downs the 10918c2ecf20Sopenharmony_ci * connection. We add in the number of times we downed the link 10928c2ecf20Sopenharmony_ci * due to local link integrity errors to compensate. 10938c2ecf20Sopenharmony_ci */ 10948c2ecf20Sopenharmony_ci cntrs->link_downed_counter = 10958c2ecf20Sopenharmony_ci ppd->dd->f_portcntr(ppd, QIBPORTCNTR_IBLINKDOWN); 10968c2ecf20Sopenharmony_ci cntrs->port_rcv_errors = 10978c2ecf20Sopenharmony_ci ppd->dd->f_portcntr(ppd, QIBPORTCNTR_RXDROPPKT) + 10988c2ecf20Sopenharmony_ci ppd->dd->f_portcntr(ppd, QIBPORTCNTR_RCVOVFL) + 10998c2ecf20Sopenharmony_ci ppd->dd->f_portcntr(ppd, QIBPORTCNTR_ERR_RLEN) + 11008c2ecf20Sopenharmony_ci ppd->dd->f_portcntr(ppd, QIBPORTCNTR_INVALIDRLEN) + 11018c2ecf20Sopenharmony_ci ppd->dd->f_portcntr(ppd, QIBPORTCNTR_ERRLINK) + 11028c2ecf20Sopenharmony_ci ppd->dd->f_portcntr(ppd, QIBPORTCNTR_ERRICRC) + 11038c2ecf20Sopenharmony_ci ppd->dd->f_portcntr(ppd, QIBPORTCNTR_ERRVCRC) + 11048c2ecf20Sopenharmony_ci ppd->dd->f_portcntr(ppd, QIBPORTCNTR_ERRLPCRC) + 11058c2ecf20Sopenharmony_ci ppd->dd->f_portcntr(ppd, QIBPORTCNTR_BADFORMAT); 11068c2ecf20Sopenharmony_ci cntrs->port_rcv_errors += 11078c2ecf20Sopenharmony_ci ppd->dd->f_portcntr(ppd, QIBPORTCNTR_RXLOCALPHYERR); 11088c2ecf20Sopenharmony_ci cntrs->port_rcv_errors += 11098c2ecf20Sopenharmony_ci ppd->dd->f_portcntr(ppd, QIBPORTCNTR_RXVLERR); 11108c2ecf20Sopenharmony_ci cntrs->port_rcv_remphys_errors = 11118c2ecf20Sopenharmony_ci ppd->dd->f_portcntr(ppd, QIBPORTCNTR_RCVEBP); 11128c2ecf20Sopenharmony_ci cntrs->port_xmit_discards = 11138c2ecf20Sopenharmony_ci ppd->dd->f_portcntr(ppd, QIBPORTCNTR_UNSUPVL); 11148c2ecf20Sopenharmony_ci cntrs->port_xmit_data = ppd->dd->f_portcntr(ppd, 11158c2ecf20Sopenharmony_ci QIBPORTCNTR_WORDSEND); 11168c2ecf20Sopenharmony_ci cntrs->port_rcv_data = ppd->dd->f_portcntr(ppd, 11178c2ecf20Sopenharmony_ci QIBPORTCNTR_WORDRCV); 11188c2ecf20Sopenharmony_ci cntrs->port_xmit_packets = ppd->dd->f_portcntr(ppd, 11198c2ecf20Sopenharmony_ci QIBPORTCNTR_PKTSEND); 11208c2ecf20Sopenharmony_ci cntrs->port_rcv_packets = ppd->dd->f_portcntr(ppd, 11218c2ecf20Sopenharmony_ci QIBPORTCNTR_PKTRCV); 11228c2ecf20Sopenharmony_ci cntrs->local_link_integrity_errors = 11238c2ecf20Sopenharmony_ci ppd->dd->f_portcntr(ppd, QIBPORTCNTR_LLI); 11248c2ecf20Sopenharmony_ci cntrs->excessive_buffer_overrun_errors = 11258c2ecf20Sopenharmony_ci ppd->dd->f_portcntr(ppd, QIBPORTCNTR_EXCESSBUFOVFL); 11268c2ecf20Sopenharmony_ci cntrs->vl15_dropped = 11278c2ecf20Sopenharmony_ci ppd->dd->f_portcntr(ppd, QIBPORTCNTR_VL15PKTDROP); 11288c2ecf20Sopenharmony_ci 11298c2ecf20Sopenharmony_ci ret = 0; 11308c2ecf20Sopenharmony_ci 11318c2ecf20Sopenharmony_cibail: 11328c2ecf20Sopenharmony_ci return ret; 11338c2ecf20Sopenharmony_ci} 11348c2ecf20Sopenharmony_ci 11358c2ecf20Sopenharmony_ci/** 11368c2ecf20Sopenharmony_ci * qib_ib_piobufavail - callback when a PIO buffer is available 11378c2ecf20Sopenharmony_ci * @dd: the device pointer 11388c2ecf20Sopenharmony_ci * 11398c2ecf20Sopenharmony_ci * This is called from qib_intr() at interrupt level when a PIO buffer is 11408c2ecf20Sopenharmony_ci * available after qib_verbs_send() returned an error that no buffers were 11418c2ecf20Sopenharmony_ci * available. Disable the interrupt if there are no more QPs waiting. 11428c2ecf20Sopenharmony_ci */ 11438c2ecf20Sopenharmony_civoid qib_ib_piobufavail(struct qib_devdata *dd) 11448c2ecf20Sopenharmony_ci{ 11458c2ecf20Sopenharmony_ci struct qib_ibdev *dev = &dd->verbs_dev; 11468c2ecf20Sopenharmony_ci struct list_head *list; 11478c2ecf20Sopenharmony_ci struct rvt_qp *qps[5]; 11488c2ecf20Sopenharmony_ci struct rvt_qp *qp; 11498c2ecf20Sopenharmony_ci unsigned long flags; 11508c2ecf20Sopenharmony_ci unsigned i, n; 11518c2ecf20Sopenharmony_ci struct qib_qp_priv *priv; 11528c2ecf20Sopenharmony_ci 11538c2ecf20Sopenharmony_ci list = &dev->piowait; 11548c2ecf20Sopenharmony_ci n = 0; 11558c2ecf20Sopenharmony_ci 11568c2ecf20Sopenharmony_ci /* 11578c2ecf20Sopenharmony_ci * Note: checking that the piowait list is empty and clearing 11588c2ecf20Sopenharmony_ci * the buffer available interrupt needs to be atomic or we 11598c2ecf20Sopenharmony_ci * could end up with QPs on the wait list with the interrupt 11608c2ecf20Sopenharmony_ci * disabled. 11618c2ecf20Sopenharmony_ci */ 11628c2ecf20Sopenharmony_ci spin_lock_irqsave(&dev->rdi.pending_lock, flags); 11638c2ecf20Sopenharmony_ci while (!list_empty(list)) { 11648c2ecf20Sopenharmony_ci if (n == ARRAY_SIZE(qps)) 11658c2ecf20Sopenharmony_ci goto full; 11668c2ecf20Sopenharmony_ci priv = list_entry(list->next, struct qib_qp_priv, iowait); 11678c2ecf20Sopenharmony_ci qp = priv->owner; 11688c2ecf20Sopenharmony_ci list_del_init(&priv->iowait); 11698c2ecf20Sopenharmony_ci rvt_get_qp(qp); 11708c2ecf20Sopenharmony_ci qps[n++] = qp; 11718c2ecf20Sopenharmony_ci } 11728c2ecf20Sopenharmony_ci dd->f_wantpiobuf_intr(dd, 0); 11738c2ecf20Sopenharmony_cifull: 11748c2ecf20Sopenharmony_ci spin_unlock_irqrestore(&dev->rdi.pending_lock, flags); 11758c2ecf20Sopenharmony_ci 11768c2ecf20Sopenharmony_ci for (i = 0; i < n; i++) { 11778c2ecf20Sopenharmony_ci qp = qps[i]; 11788c2ecf20Sopenharmony_ci 11798c2ecf20Sopenharmony_ci spin_lock_irqsave(&qp->s_lock, flags); 11808c2ecf20Sopenharmony_ci if (qp->s_flags & RVT_S_WAIT_PIO) { 11818c2ecf20Sopenharmony_ci qp->s_flags &= ~RVT_S_WAIT_PIO; 11828c2ecf20Sopenharmony_ci qib_schedule_send(qp); 11838c2ecf20Sopenharmony_ci } 11848c2ecf20Sopenharmony_ci spin_unlock_irqrestore(&qp->s_lock, flags); 11858c2ecf20Sopenharmony_ci 11868c2ecf20Sopenharmony_ci /* Notify qib_destroy_qp() if it is waiting. */ 11878c2ecf20Sopenharmony_ci rvt_put_qp(qp); 11888c2ecf20Sopenharmony_ci } 11898c2ecf20Sopenharmony_ci} 11908c2ecf20Sopenharmony_ci 11918c2ecf20Sopenharmony_cistatic int qib_query_port(struct rvt_dev_info *rdi, u8 port_num, 11928c2ecf20Sopenharmony_ci struct ib_port_attr *props) 11938c2ecf20Sopenharmony_ci{ 11948c2ecf20Sopenharmony_ci struct qib_ibdev *ibdev = container_of(rdi, struct qib_ibdev, rdi); 11958c2ecf20Sopenharmony_ci struct qib_devdata *dd = dd_from_dev(ibdev); 11968c2ecf20Sopenharmony_ci struct qib_pportdata *ppd = &dd->pport[port_num - 1]; 11978c2ecf20Sopenharmony_ci enum ib_mtu mtu; 11988c2ecf20Sopenharmony_ci u16 lid = ppd->lid; 11998c2ecf20Sopenharmony_ci 12008c2ecf20Sopenharmony_ci /* props being zeroed by the caller, avoid zeroing it here */ 12018c2ecf20Sopenharmony_ci props->lid = lid ? lid : be16_to_cpu(IB_LID_PERMISSIVE); 12028c2ecf20Sopenharmony_ci props->lmc = ppd->lmc; 12038c2ecf20Sopenharmony_ci props->state = dd->f_iblink_state(ppd->lastibcstat); 12048c2ecf20Sopenharmony_ci props->phys_state = dd->f_ibphys_portstate(ppd->lastibcstat); 12058c2ecf20Sopenharmony_ci props->gid_tbl_len = QIB_GUIDS_PER_PORT; 12068c2ecf20Sopenharmony_ci props->active_width = ppd->link_width_active; 12078c2ecf20Sopenharmony_ci /* See rate_show() */ 12088c2ecf20Sopenharmony_ci props->active_speed = ppd->link_speed_active; 12098c2ecf20Sopenharmony_ci props->max_vl_num = qib_num_vls(ppd->vls_supported); 12108c2ecf20Sopenharmony_ci 12118c2ecf20Sopenharmony_ci props->max_mtu = qib_ibmtu ? qib_ibmtu : IB_MTU_4096; 12128c2ecf20Sopenharmony_ci switch (ppd->ibmtu) { 12138c2ecf20Sopenharmony_ci case 4096: 12148c2ecf20Sopenharmony_ci mtu = IB_MTU_4096; 12158c2ecf20Sopenharmony_ci break; 12168c2ecf20Sopenharmony_ci case 2048: 12178c2ecf20Sopenharmony_ci mtu = IB_MTU_2048; 12188c2ecf20Sopenharmony_ci break; 12198c2ecf20Sopenharmony_ci case 1024: 12208c2ecf20Sopenharmony_ci mtu = IB_MTU_1024; 12218c2ecf20Sopenharmony_ci break; 12228c2ecf20Sopenharmony_ci case 512: 12238c2ecf20Sopenharmony_ci mtu = IB_MTU_512; 12248c2ecf20Sopenharmony_ci break; 12258c2ecf20Sopenharmony_ci case 256: 12268c2ecf20Sopenharmony_ci mtu = IB_MTU_256; 12278c2ecf20Sopenharmony_ci break; 12288c2ecf20Sopenharmony_ci default: 12298c2ecf20Sopenharmony_ci mtu = IB_MTU_2048; 12308c2ecf20Sopenharmony_ci } 12318c2ecf20Sopenharmony_ci props->active_mtu = mtu; 12328c2ecf20Sopenharmony_ci 12338c2ecf20Sopenharmony_ci return 0; 12348c2ecf20Sopenharmony_ci} 12358c2ecf20Sopenharmony_ci 12368c2ecf20Sopenharmony_cistatic int qib_modify_device(struct ib_device *device, 12378c2ecf20Sopenharmony_ci int device_modify_mask, 12388c2ecf20Sopenharmony_ci struct ib_device_modify *device_modify) 12398c2ecf20Sopenharmony_ci{ 12408c2ecf20Sopenharmony_ci struct qib_devdata *dd = dd_from_ibdev(device); 12418c2ecf20Sopenharmony_ci unsigned i; 12428c2ecf20Sopenharmony_ci int ret; 12438c2ecf20Sopenharmony_ci 12448c2ecf20Sopenharmony_ci if (device_modify_mask & ~(IB_DEVICE_MODIFY_SYS_IMAGE_GUID | 12458c2ecf20Sopenharmony_ci IB_DEVICE_MODIFY_NODE_DESC)) { 12468c2ecf20Sopenharmony_ci ret = -EOPNOTSUPP; 12478c2ecf20Sopenharmony_ci goto bail; 12488c2ecf20Sopenharmony_ci } 12498c2ecf20Sopenharmony_ci 12508c2ecf20Sopenharmony_ci if (device_modify_mask & IB_DEVICE_MODIFY_NODE_DESC) { 12518c2ecf20Sopenharmony_ci memcpy(device->node_desc, device_modify->node_desc, 12528c2ecf20Sopenharmony_ci IB_DEVICE_NODE_DESC_MAX); 12538c2ecf20Sopenharmony_ci for (i = 0; i < dd->num_pports; i++) { 12548c2ecf20Sopenharmony_ci struct qib_ibport *ibp = &dd->pport[i].ibport_data; 12558c2ecf20Sopenharmony_ci 12568c2ecf20Sopenharmony_ci qib_node_desc_chg(ibp); 12578c2ecf20Sopenharmony_ci } 12588c2ecf20Sopenharmony_ci } 12598c2ecf20Sopenharmony_ci 12608c2ecf20Sopenharmony_ci if (device_modify_mask & IB_DEVICE_MODIFY_SYS_IMAGE_GUID) { 12618c2ecf20Sopenharmony_ci ib_qib_sys_image_guid = 12628c2ecf20Sopenharmony_ci cpu_to_be64(device_modify->sys_image_guid); 12638c2ecf20Sopenharmony_ci for (i = 0; i < dd->num_pports; i++) { 12648c2ecf20Sopenharmony_ci struct qib_ibport *ibp = &dd->pport[i].ibport_data; 12658c2ecf20Sopenharmony_ci 12668c2ecf20Sopenharmony_ci qib_sys_guid_chg(ibp); 12678c2ecf20Sopenharmony_ci } 12688c2ecf20Sopenharmony_ci } 12698c2ecf20Sopenharmony_ci 12708c2ecf20Sopenharmony_ci ret = 0; 12718c2ecf20Sopenharmony_ci 12728c2ecf20Sopenharmony_cibail: 12738c2ecf20Sopenharmony_ci return ret; 12748c2ecf20Sopenharmony_ci} 12758c2ecf20Sopenharmony_ci 12768c2ecf20Sopenharmony_cistatic int qib_shut_down_port(struct rvt_dev_info *rdi, u8 port_num) 12778c2ecf20Sopenharmony_ci{ 12788c2ecf20Sopenharmony_ci struct qib_ibdev *ibdev = container_of(rdi, struct qib_ibdev, rdi); 12798c2ecf20Sopenharmony_ci struct qib_devdata *dd = dd_from_dev(ibdev); 12808c2ecf20Sopenharmony_ci struct qib_pportdata *ppd = &dd->pport[port_num - 1]; 12818c2ecf20Sopenharmony_ci 12828c2ecf20Sopenharmony_ci qib_set_linkstate(ppd, QIB_IB_LINKDOWN); 12838c2ecf20Sopenharmony_ci 12848c2ecf20Sopenharmony_ci return 0; 12858c2ecf20Sopenharmony_ci} 12868c2ecf20Sopenharmony_ci 12878c2ecf20Sopenharmony_cistatic int qib_get_guid_be(struct rvt_dev_info *rdi, struct rvt_ibport *rvp, 12888c2ecf20Sopenharmony_ci int guid_index, __be64 *guid) 12898c2ecf20Sopenharmony_ci{ 12908c2ecf20Sopenharmony_ci struct qib_ibport *ibp = container_of(rvp, struct qib_ibport, rvp); 12918c2ecf20Sopenharmony_ci struct qib_pportdata *ppd = ppd_from_ibp(ibp); 12928c2ecf20Sopenharmony_ci 12938c2ecf20Sopenharmony_ci if (guid_index == 0) 12948c2ecf20Sopenharmony_ci *guid = ppd->guid; 12958c2ecf20Sopenharmony_ci else if (guid_index < QIB_GUIDS_PER_PORT) 12968c2ecf20Sopenharmony_ci *guid = ibp->guids[guid_index - 1]; 12978c2ecf20Sopenharmony_ci else 12988c2ecf20Sopenharmony_ci return -EINVAL; 12998c2ecf20Sopenharmony_ci 13008c2ecf20Sopenharmony_ci return 0; 13018c2ecf20Sopenharmony_ci} 13028c2ecf20Sopenharmony_ci 13038c2ecf20Sopenharmony_ciint qib_check_ah(struct ib_device *ibdev, struct rdma_ah_attr *ah_attr) 13048c2ecf20Sopenharmony_ci{ 13058c2ecf20Sopenharmony_ci if (rdma_ah_get_sl(ah_attr) > 15) 13068c2ecf20Sopenharmony_ci return -EINVAL; 13078c2ecf20Sopenharmony_ci 13088c2ecf20Sopenharmony_ci if (rdma_ah_get_dlid(ah_attr) == 0) 13098c2ecf20Sopenharmony_ci return -EINVAL; 13108c2ecf20Sopenharmony_ci if (rdma_ah_get_dlid(ah_attr) >= 13118c2ecf20Sopenharmony_ci be16_to_cpu(IB_MULTICAST_LID_BASE) && 13128c2ecf20Sopenharmony_ci rdma_ah_get_dlid(ah_attr) != 13138c2ecf20Sopenharmony_ci be16_to_cpu(IB_LID_PERMISSIVE) && 13148c2ecf20Sopenharmony_ci !(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) 13158c2ecf20Sopenharmony_ci return -EINVAL; 13168c2ecf20Sopenharmony_ci 13178c2ecf20Sopenharmony_ci return 0; 13188c2ecf20Sopenharmony_ci} 13198c2ecf20Sopenharmony_ci 13208c2ecf20Sopenharmony_cistatic void qib_notify_new_ah(struct ib_device *ibdev, 13218c2ecf20Sopenharmony_ci struct rdma_ah_attr *ah_attr, 13228c2ecf20Sopenharmony_ci struct rvt_ah *ah) 13238c2ecf20Sopenharmony_ci{ 13248c2ecf20Sopenharmony_ci struct qib_ibport *ibp; 13258c2ecf20Sopenharmony_ci struct qib_pportdata *ppd; 13268c2ecf20Sopenharmony_ci 13278c2ecf20Sopenharmony_ci /* 13288c2ecf20Sopenharmony_ci * Do not trust reading anything from rvt_ah at this point as it is not 13298c2ecf20Sopenharmony_ci * done being setup. We can however modify things which we need to set. 13308c2ecf20Sopenharmony_ci */ 13318c2ecf20Sopenharmony_ci 13328c2ecf20Sopenharmony_ci ibp = to_iport(ibdev, rdma_ah_get_port_num(ah_attr)); 13338c2ecf20Sopenharmony_ci ppd = ppd_from_ibp(ibp); 13348c2ecf20Sopenharmony_ci ah->vl = ibp->sl_to_vl[rdma_ah_get_sl(&ah->attr)]; 13358c2ecf20Sopenharmony_ci ah->log_pmtu = ilog2(ppd->ibmtu); 13368c2ecf20Sopenharmony_ci} 13378c2ecf20Sopenharmony_ci 13388c2ecf20Sopenharmony_cistruct ib_ah *qib_create_qp0_ah(struct qib_ibport *ibp, u16 dlid) 13398c2ecf20Sopenharmony_ci{ 13408c2ecf20Sopenharmony_ci struct rdma_ah_attr attr; 13418c2ecf20Sopenharmony_ci struct ib_ah *ah = ERR_PTR(-EINVAL); 13428c2ecf20Sopenharmony_ci struct rvt_qp *qp0; 13438c2ecf20Sopenharmony_ci struct qib_pportdata *ppd = ppd_from_ibp(ibp); 13448c2ecf20Sopenharmony_ci struct qib_devdata *dd = dd_from_ppd(ppd); 13458c2ecf20Sopenharmony_ci u8 port_num = ppd->port; 13468c2ecf20Sopenharmony_ci 13478c2ecf20Sopenharmony_ci memset(&attr, 0, sizeof(attr)); 13488c2ecf20Sopenharmony_ci attr.type = rdma_ah_find_type(&dd->verbs_dev.rdi.ibdev, port_num); 13498c2ecf20Sopenharmony_ci rdma_ah_set_dlid(&attr, dlid); 13508c2ecf20Sopenharmony_ci rdma_ah_set_port_num(&attr, port_num); 13518c2ecf20Sopenharmony_ci rcu_read_lock(); 13528c2ecf20Sopenharmony_ci qp0 = rcu_dereference(ibp->rvp.qp[0]); 13538c2ecf20Sopenharmony_ci if (qp0) 13548c2ecf20Sopenharmony_ci ah = rdma_create_ah(qp0->ibqp.pd, &attr, 0); 13558c2ecf20Sopenharmony_ci rcu_read_unlock(); 13568c2ecf20Sopenharmony_ci return ah; 13578c2ecf20Sopenharmony_ci} 13588c2ecf20Sopenharmony_ci 13598c2ecf20Sopenharmony_ci/** 13608c2ecf20Sopenharmony_ci * qib_get_npkeys - return the size of the PKEY table for context 0 13618c2ecf20Sopenharmony_ci * @dd: the qlogic_ib device 13628c2ecf20Sopenharmony_ci */ 13638c2ecf20Sopenharmony_ciunsigned qib_get_npkeys(struct qib_devdata *dd) 13648c2ecf20Sopenharmony_ci{ 13658c2ecf20Sopenharmony_ci return ARRAY_SIZE(dd->rcd[0]->pkeys); 13668c2ecf20Sopenharmony_ci} 13678c2ecf20Sopenharmony_ci 13688c2ecf20Sopenharmony_ci/* 13698c2ecf20Sopenharmony_ci * Return the indexed PKEY from the port PKEY table. 13708c2ecf20Sopenharmony_ci * No need to validate rcd[ctxt]; the port is setup if we are here. 13718c2ecf20Sopenharmony_ci */ 13728c2ecf20Sopenharmony_ciunsigned qib_get_pkey(struct qib_ibport *ibp, unsigned index) 13738c2ecf20Sopenharmony_ci{ 13748c2ecf20Sopenharmony_ci struct qib_pportdata *ppd = ppd_from_ibp(ibp); 13758c2ecf20Sopenharmony_ci struct qib_devdata *dd = ppd->dd; 13768c2ecf20Sopenharmony_ci unsigned ctxt = ppd->hw_pidx; 13778c2ecf20Sopenharmony_ci unsigned ret; 13788c2ecf20Sopenharmony_ci 13798c2ecf20Sopenharmony_ci /* dd->rcd null if mini_init or some init failures */ 13808c2ecf20Sopenharmony_ci if (!dd->rcd || index >= ARRAY_SIZE(dd->rcd[ctxt]->pkeys)) 13818c2ecf20Sopenharmony_ci ret = 0; 13828c2ecf20Sopenharmony_ci else 13838c2ecf20Sopenharmony_ci ret = dd->rcd[ctxt]->pkeys[index]; 13848c2ecf20Sopenharmony_ci 13858c2ecf20Sopenharmony_ci return ret; 13868c2ecf20Sopenharmony_ci} 13878c2ecf20Sopenharmony_ci 13888c2ecf20Sopenharmony_cistatic void init_ibport(struct qib_pportdata *ppd) 13898c2ecf20Sopenharmony_ci{ 13908c2ecf20Sopenharmony_ci struct qib_verbs_counters cntrs; 13918c2ecf20Sopenharmony_ci struct qib_ibport *ibp = &ppd->ibport_data; 13928c2ecf20Sopenharmony_ci 13938c2ecf20Sopenharmony_ci spin_lock_init(&ibp->rvp.lock); 13948c2ecf20Sopenharmony_ci /* Set the prefix to the default value (see ch. 4.1.1) */ 13958c2ecf20Sopenharmony_ci ibp->rvp.gid_prefix = IB_DEFAULT_GID_PREFIX; 13968c2ecf20Sopenharmony_ci ibp->rvp.sm_lid = be16_to_cpu(IB_LID_PERMISSIVE); 13978c2ecf20Sopenharmony_ci ibp->rvp.port_cap_flags = IB_PORT_SYS_IMAGE_GUID_SUP | 13988c2ecf20Sopenharmony_ci IB_PORT_CLIENT_REG_SUP | IB_PORT_SL_MAP_SUP | 13998c2ecf20Sopenharmony_ci IB_PORT_TRAP_SUP | IB_PORT_AUTO_MIGR_SUP | 14008c2ecf20Sopenharmony_ci IB_PORT_DR_NOTICE_SUP | IB_PORT_CAP_MASK_NOTICE_SUP | 14018c2ecf20Sopenharmony_ci IB_PORT_OTHER_LOCAL_CHANGES_SUP; 14028c2ecf20Sopenharmony_ci if (ppd->dd->flags & QIB_HAS_LINK_LATENCY) 14038c2ecf20Sopenharmony_ci ibp->rvp.port_cap_flags |= IB_PORT_LINK_LATENCY_SUP; 14048c2ecf20Sopenharmony_ci ibp->rvp.pma_counter_select[0] = IB_PMA_PORT_XMIT_DATA; 14058c2ecf20Sopenharmony_ci ibp->rvp.pma_counter_select[1] = IB_PMA_PORT_RCV_DATA; 14068c2ecf20Sopenharmony_ci ibp->rvp.pma_counter_select[2] = IB_PMA_PORT_XMIT_PKTS; 14078c2ecf20Sopenharmony_ci ibp->rvp.pma_counter_select[3] = IB_PMA_PORT_RCV_PKTS; 14088c2ecf20Sopenharmony_ci ibp->rvp.pma_counter_select[4] = IB_PMA_PORT_XMIT_WAIT; 14098c2ecf20Sopenharmony_ci 14108c2ecf20Sopenharmony_ci /* Snapshot current HW counters to "clear" them. */ 14118c2ecf20Sopenharmony_ci qib_get_counters(ppd, &cntrs); 14128c2ecf20Sopenharmony_ci ibp->z_symbol_error_counter = cntrs.symbol_error_counter; 14138c2ecf20Sopenharmony_ci ibp->z_link_error_recovery_counter = 14148c2ecf20Sopenharmony_ci cntrs.link_error_recovery_counter; 14158c2ecf20Sopenharmony_ci ibp->z_link_downed_counter = cntrs.link_downed_counter; 14168c2ecf20Sopenharmony_ci ibp->z_port_rcv_errors = cntrs.port_rcv_errors; 14178c2ecf20Sopenharmony_ci ibp->z_port_rcv_remphys_errors = cntrs.port_rcv_remphys_errors; 14188c2ecf20Sopenharmony_ci ibp->z_port_xmit_discards = cntrs.port_xmit_discards; 14198c2ecf20Sopenharmony_ci ibp->z_port_xmit_data = cntrs.port_xmit_data; 14208c2ecf20Sopenharmony_ci ibp->z_port_rcv_data = cntrs.port_rcv_data; 14218c2ecf20Sopenharmony_ci ibp->z_port_xmit_packets = cntrs.port_xmit_packets; 14228c2ecf20Sopenharmony_ci ibp->z_port_rcv_packets = cntrs.port_rcv_packets; 14238c2ecf20Sopenharmony_ci ibp->z_local_link_integrity_errors = 14248c2ecf20Sopenharmony_ci cntrs.local_link_integrity_errors; 14258c2ecf20Sopenharmony_ci ibp->z_excessive_buffer_overrun_errors = 14268c2ecf20Sopenharmony_ci cntrs.excessive_buffer_overrun_errors; 14278c2ecf20Sopenharmony_ci ibp->z_vl15_dropped = cntrs.vl15_dropped; 14288c2ecf20Sopenharmony_ci RCU_INIT_POINTER(ibp->rvp.qp[0], NULL); 14298c2ecf20Sopenharmony_ci RCU_INIT_POINTER(ibp->rvp.qp[1], NULL); 14308c2ecf20Sopenharmony_ci} 14318c2ecf20Sopenharmony_ci 14328c2ecf20Sopenharmony_ci/** 14338c2ecf20Sopenharmony_ci * qib_fill_device_attr - Fill in rvt dev info device attributes. 14348c2ecf20Sopenharmony_ci * @dd: the device data structure 14358c2ecf20Sopenharmony_ci */ 14368c2ecf20Sopenharmony_cistatic void qib_fill_device_attr(struct qib_devdata *dd) 14378c2ecf20Sopenharmony_ci{ 14388c2ecf20Sopenharmony_ci struct rvt_dev_info *rdi = &dd->verbs_dev.rdi; 14398c2ecf20Sopenharmony_ci 14408c2ecf20Sopenharmony_ci memset(&rdi->dparms.props, 0, sizeof(rdi->dparms.props)); 14418c2ecf20Sopenharmony_ci 14428c2ecf20Sopenharmony_ci rdi->dparms.props.max_pd = ib_qib_max_pds; 14438c2ecf20Sopenharmony_ci rdi->dparms.props.max_ah = ib_qib_max_ahs; 14448c2ecf20Sopenharmony_ci rdi->dparms.props.device_cap_flags = IB_DEVICE_BAD_PKEY_CNTR | 14458c2ecf20Sopenharmony_ci IB_DEVICE_BAD_QKEY_CNTR | IB_DEVICE_SHUTDOWN_PORT | 14468c2ecf20Sopenharmony_ci IB_DEVICE_SYS_IMAGE_GUID | IB_DEVICE_RC_RNR_NAK_GEN | 14478c2ecf20Sopenharmony_ci IB_DEVICE_PORT_ACTIVE_EVENT | IB_DEVICE_SRQ_RESIZE; 14488c2ecf20Sopenharmony_ci rdi->dparms.props.page_size_cap = PAGE_SIZE; 14498c2ecf20Sopenharmony_ci rdi->dparms.props.vendor_id = 14508c2ecf20Sopenharmony_ci QIB_SRC_OUI_1 << 16 | QIB_SRC_OUI_2 << 8 | QIB_SRC_OUI_3; 14518c2ecf20Sopenharmony_ci rdi->dparms.props.vendor_part_id = dd->deviceid; 14528c2ecf20Sopenharmony_ci rdi->dparms.props.hw_ver = dd->minrev; 14538c2ecf20Sopenharmony_ci rdi->dparms.props.sys_image_guid = ib_qib_sys_image_guid; 14548c2ecf20Sopenharmony_ci rdi->dparms.props.max_mr_size = ~0ULL; 14558c2ecf20Sopenharmony_ci rdi->dparms.props.max_qp = ib_qib_max_qps; 14568c2ecf20Sopenharmony_ci rdi->dparms.props.max_qp_wr = ib_qib_max_qp_wrs; 14578c2ecf20Sopenharmony_ci rdi->dparms.props.max_send_sge = ib_qib_max_sges; 14588c2ecf20Sopenharmony_ci rdi->dparms.props.max_recv_sge = ib_qib_max_sges; 14598c2ecf20Sopenharmony_ci rdi->dparms.props.max_sge_rd = ib_qib_max_sges; 14608c2ecf20Sopenharmony_ci rdi->dparms.props.max_cq = ib_qib_max_cqs; 14618c2ecf20Sopenharmony_ci rdi->dparms.props.max_cqe = ib_qib_max_cqes; 14628c2ecf20Sopenharmony_ci rdi->dparms.props.max_ah = ib_qib_max_ahs; 14638c2ecf20Sopenharmony_ci rdi->dparms.props.max_qp_rd_atom = QIB_MAX_RDMA_ATOMIC; 14648c2ecf20Sopenharmony_ci rdi->dparms.props.max_qp_init_rd_atom = 255; 14658c2ecf20Sopenharmony_ci rdi->dparms.props.max_srq = ib_qib_max_srqs; 14668c2ecf20Sopenharmony_ci rdi->dparms.props.max_srq_wr = ib_qib_max_srq_wrs; 14678c2ecf20Sopenharmony_ci rdi->dparms.props.max_srq_sge = ib_qib_max_srq_sges; 14688c2ecf20Sopenharmony_ci rdi->dparms.props.atomic_cap = IB_ATOMIC_GLOB; 14698c2ecf20Sopenharmony_ci rdi->dparms.props.max_pkeys = qib_get_npkeys(dd); 14708c2ecf20Sopenharmony_ci rdi->dparms.props.max_mcast_grp = ib_qib_max_mcast_grps; 14718c2ecf20Sopenharmony_ci rdi->dparms.props.max_mcast_qp_attach = ib_qib_max_mcast_qp_attached; 14728c2ecf20Sopenharmony_ci rdi->dparms.props.max_total_mcast_qp_attach = 14738c2ecf20Sopenharmony_ci rdi->dparms.props.max_mcast_qp_attach * 14748c2ecf20Sopenharmony_ci rdi->dparms.props.max_mcast_grp; 14758c2ecf20Sopenharmony_ci /* post send table */ 14768c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.post_parms = qib_post_parms; 14778c2ecf20Sopenharmony_ci 14788c2ecf20Sopenharmony_ci /* opcode translation table */ 14798c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.wc_opcode = ib_qib_wc_opcode; 14808c2ecf20Sopenharmony_ci} 14818c2ecf20Sopenharmony_ci 14828c2ecf20Sopenharmony_cistatic const struct ib_device_ops qib_dev_ops = { 14838c2ecf20Sopenharmony_ci .owner = THIS_MODULE, 14848c2ecf20Sopenharmony_ci .driver_id = RDMA_DRIVER_QIB, 14858c2ecf20Sopenharmony_ci 14868c2ecf20Sopenharmony_ci .init_port = qib_create_port_files, 14878c2ecf20Sopenharmony_ci .modify_device = qib_modify_device, 14888c2ecf20Sopenharmony_ci .process_mad = qib_process_mad, 14898c2ecf20Sopenharmony_ci}; 14908c2ecf20Sopenharmony_ci 14918c2ecf20Sopenharmony_ci/** 14928c2ecf20Sopenharmony_ci * qib_register_ib_device - register our device with the infiniband core 14938c2ecf20Sopenharmony_ci * @dd: the device data structure 14948c2ecf20Sopenharmony_ci * Return the allocated qib_ibdev pointer or NULL on error. 14958c2ecf20Sopenharmony_ci */ 14968c2ecf20Sopenharmony_ciint qib_register_ib_device(struct qib_devdata *dd) 14978c2ecf20Sopenharmony_ci{ 14988c2ecf20Sopenharmony_ci struct qib_ibdev *dev = &dd->verbs_dev; 14998c2ecf20Sopenharmony_ci struct ib_device *ibdev = &dev->rdi.ibdev; 15008c2ecf20Sopenharmony_ci struct qib_pportdata *ppd = dd->pport; 15018c2ecf20Sopenharmony_ci unsigned i, ctxt; 15028c2ecf20Sopenharmony_ci int ret; 15038c2ecf20Sopenharmony_ci 15048c2ecf20Sopenharmony_ci for (i = 0; i < dd->num_pports; i++) 15058c2ecf20Sopenharmony_ci init_ibport(ppd + i); 15068c2ecf20Sopenharmony_ci 15078c2ecf20Sopenharmony_ci /* Only need to initialize non-zero fields. */ 15088c2ecf20Sopenharmony_ci timer_setup(&dev->mem_timer, mem_timer, 0); 15098c2ecf20Sopenharmony_ci 15108c2ecf20Sopenharmony_ci INIT_LIST_HEAD(&dev->piowait); 15118c2ecf20Sopenharmony_ci INIT_LIST_HEAD(&dev->dmawait); 15128c2ecf20Sopenharmony_ci INIT_LIST_HEAD(&dev->txwait); 15138c2ecf20Sopenharmony_ci INIT_LIST_HEAD(&dev->memwait); 15148c2ecf20Sopenharmony_ci INIT_LIST_HEAD(&dev->txreq_free); 15158c2ecf20Sopenharmony_ci 15168c2ecf20Sopenharmony_ci if (ppd->sdma_descq_cnt) { 15178c2ecf20Sopenharmony_ci dev->pio_hdrs = dma_alloc_coherent(&dd->pcidev->dev, 15188c2ecf20Sopenharmony_ci ppd->sdma_descq_cnt * 15198c2ecf20Sopenharmony_ci sizeof(struct qib_pio_header), 15208c2ecf20Sopenharmony_ci &dev->pio_hdrs_phys, 15218c2ecf20Sopenharmony_ci GFP_KERNEL); 15228c2ecf20Sopenharmony_ci if (!dev->pio_hdrs) { 15238c2ecf20Sopenharmony_ci ret = -ENOMEM; 15248c2ecf20Sopenharmony_ci goto err_hdrs; 15258c2ecf20Sopenharmony_ci } 15268c2ecf20Sopenharmony_ci } 15278c2ecf20Sopenharmony_ci 15288c2ecf20Sopenharmony_ci for (i = 0; i < ppd->sdma_descq_cnt; i++) { 15298c2ecf20Sopenharmony_ci struct qib_verbs_txreq *tx; 15308c2ecf20Sopenharmony_ci 15318c2ecf20Sopenharmony_ci tx = kzalloc(sizeof(*tx), GFP_KERNEL); 15328c2ecf20Sopenharmony_ci if (!tx) { 15338c2ecf20Sopenharmony_ci ret = -ENOMEM; 15348c2ecf20Sopenharmony_ci goto err_tx; 15358c2ecf20Sopenharmony_ci } 15368c2ecf20Sopenharmony_ci tx->hdr_inx = i; 15378c2ecf20Sopenharmony_ci list_add(&tx->txreq.list, &dev->txreq_free); 15388c2ecf20Sopenharmony_ci } 15398c2ecf20Sopenharmony_ci 15408c2ecf20Sopenharmony_ci /* 15418c2ecf20Sopenharmony_ci * The system image GUID is supposed to be the same for all 15428c2ecf20Sopenharmony_ci * IB HCAs in a single system but since there can be other 15438c2ecf20Sopenharmony_ci * device types in the system, we can't be sure this is unique. 15448c2ecf20Sopenharmony_ci */ 15458c2ecf20Sopenharmony_ci if (!ib_qib_sys_image_guid) 15468c2ecf20Sopenharmony_ci ib_qib_sys_image_guid = ppd->guid; 15478c2ecf20Sopenharmony_ci 15488c2ecf20Sopenharmony_ci ibdev->node_guid = ppd->guid; 15498c2ecf20Sopenharmony_ci ibdev->phys_port_cnt = dd->num_pports; 15508c2ecf20Sopenharmony_ci ibdev->dev.parent = &dd->pcidev->dev; 15518c2ecf20Sopenharmony_ci 15528c2ecf20Sopenharmony_ci snprintf(ibdev->node_desc, sizeof(ibdev->node_desc), 15538c2ecf20Sopenharmony_ci "Intel Infiniband HCA %s", init_utsname()->nodename); 15548c2ecf20Sopenharmony_ci 15558c2ecf20Sopenharmony_ci /* 15568c2ecf20Sopenharmony_ci * Fill in rvt info object. 15578c2ecf20Sopenharmony_ci */ 15588c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.get_pci_dev = qib_get_pci_dev; 15598c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.check_ah = qib_check_ah; 15608c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.setup_wqe = qib_check_send_wqe; 15618c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.notify_new_ah = qib_notify_new_ah; 15628c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.alloc_qpn = qib_alloc_qpn; 15638c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.qp_priv_alloc = qib_qp_priv_alloc; 15648c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.qp_priv_free = qib_qp_priv_free; 15658c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.free_all_qps = qib_free_all_qps; 15668c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.notify_qp_reset = qib_notify_qp_reset; 15678c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.do_send = qib_do_send; 15688c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.schedule_send = qib_schedule_send; 15698c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.quiesce_qp = qib_quiesce_qp; 15708c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.stop_send_queue = qib_stop_send_queue; 15718c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.flush_qp_waiters = qib_flush_qp_waiters; 15728c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.notify_error_qp = qib_notify_error_qp; 15738c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.notify_restart_rc = qib_restart_rc; 15748c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.mtu_to_path_mtu = qib_mtu_to_path_mtu; 15758c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.mtu_from_qp = qib_mtu_from_qp; 15768c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.get_pmtu_from_attr = qib_get_pmtu_from_attr; 15778c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.schedule_send_no_lock = _qib_schedule_send; 15788c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.query_port_state = qib_query_port; 15798c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.shut_down_port = qib_shut_down_port; 15808c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.cap_mask_chg = qib_cap_mask_chg; 15818c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.notify_create_mad_agent = 15828c2ecf20Sopenharmony_ci qib_notify_create_mad_agent; 15838c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.notify_free_mad_agent = 15848c2ecf20Sopenharmony_ci qib_notify_free_mad_agent; 15858c2ecf20Sopenharmony_ci 15868c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.dparms.max_rdma_atomic = QIB_MAX_RDMA_ATOMIC; 15878c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.driver_f.get_guid_be = qib_get_guid_be; 15888c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.dparms.lkey_table_size = qib_lkey_table_size; 15898c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.dparms.qp_table_size = ib_qib_qp_table_size; 15908c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.dparms.qpn_start = 1; 15918c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.dparms.qpn_res_start = QIB_KD_QP; 15928c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.dparms.qpn_res_end = QIB_KD_QP; /* Reserve one QP */ 15938c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.dparms.qpn_inc = 1; 15948c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.dparms.qos_shift = 1; 15958c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.dparms.psn_mask = QIB_PSN_MASK; 15968c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.dparms.psn_shift = QIB_PSN_SHIFT; 15978c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.dparms.psn_modify_mask = QIB_PSN_MASK; 15988c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.dparms.nports = dd->num_pports; 15998c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.dparms.npkeys = qib_get_npkeys(dd); 16008c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.dparms.node = dd->assigned_node_id; 16018c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.dparms.core_cap_flags = RDMA_CORE_PORT_IBA_IB; 16028c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.dparms.max_mad_size = IB_MGMT_MAD_SIZE; 16038c2ecf20Sopenharmony_ci dd->verbs_dev.rdi.dparms.sge_copy_mode = RVT_SGE_COPY_MEMCPY; 16048c2ecf20Sopenharmony_ci 16058c2ecf20Sopenharmony_ci qib_fill_device_attr(dd); 16068c2ecf20Sopenharmony_ci 16078c2ecf20Sopenharmony_ci ppd = dd->pport; 16088c2ecf20Sopenharmony_ci for (i = 0; i < dd->num_pports; i++, ppd++) { 16098c2ecf20Sopenharmony_ci ctxt = ppd->hw_pidx; 16108c2ecf20Sopenharmony_ci rvt_init_port(&dd->verbs_dev.rdi, 16118c2ecf20Sopenharmony_ci &ppd->ibport_data.rvp, 16128c2ecf20Sopenharmony_ci i, 16138c2ecf20Sopenharmony_ci dd->rcd[ctxt]->pkeys); 16148c2ecf20Sopenharmony_ci } 16158c2ecf20Sopenharmony_ci rdma_set_device_sysfs_group(&dd->verbs_dev.rdi.ibdev, &qib_attr_group); 16168c2ecf20Sopenharmony_ci 16178c2ecf20Sopenharmony_ci ib_set_device_ops(ibdev, &qib_dev_ops); 16188c2ecf20Sopenharmony_ci ret = rvt_register_device(&dd->verbs_dev.rdi); 16198c2ecf20Sopenharmony_ci if (ret) 16208c2ecf20Sopenharmony_ci goto err_tx; 16218c2ecf20Sopenharmony_ci 16228c2ecf20Sopenharmony_ci return ret; 16238c2ecf20Sopenharmony_ci 16248c2ecf20Sopenharmony_cierr_tx: 16258c2ecf20Sopenharmony_ci while (!list_empty(&dev->txreq_free)) { 16268c2ecf20Sopenharmony_ci struct list_head *l = dev->txreq_free.next; 16278c2ecf20Sopenharmony_ci struct qib_verbs_txreq *tx; 16288c2ecf20Sopenharmony_ci 16298c2ecf20Sopenharmony_ci list_del(l); 16308c2ecf20Sopenharmony_ci tx = list_entry(l, struct qib_verbs_txreq, txreq.list); 16318c2ecf20Sopenharmony_ci kfree(tx); 16328c2ecf20Sopenharmony_ci } 16338c2ecf20Sopenharmony_ci if (ppd->sdma_descq_cnt) 16348c2ecf20Sopenharmony_ci dma_free_coherent(&dd->pcidev->dev, 16358c2ecf20Sopenharmony_ci ppd->sdma_descq_cnt * 16368c2ecf20Sopenharmony_ci sizeof(struct qib_pio_header), 16378c2ecf20Sopenharmony_ci dev->pio_hdrs, dev->pio_hdrs_phys); 16388c2ecf20Sopenharmony_cierr_hdrs: 16398c2ecf20Sopenharmony_ci qib_dev_err(dd, "cannot register verbs: %d!\n", -ret); 16408c2ecf20Sopenharmony_ci return ret; 16418c2ecf20Sopenharmony_ci} 16428c2ecf20Sopenharmony_ci 16438c2ecf20Sopenharmony_civoid qib_unregister_ib_device(struct qib_devdata *dd) 16448c2ecf20Sopenharmony_ci{ 16458c2ecf20Sopenharmony_ci struct qib_ibdev *dev = &dd->verbs_dev; 16468c2ecf20Sopenharmony_ci 16478c2ecf20Sopenharmony_ci qib_verbs_unregister_sysfs(dd); 16488c2ecf20Sopenharmony_ci 16498c2ecf20Sopenharmony_ci rvt_unregister_device(&dd->verbs_dev.rdi); 16508c2ecf20Sopenharmony_ci 16518c2ecf20Sopenharmony_ci if (!list_empty(&dev->piowait)) 16528c2ecf20Sopenharmony_ci qib_dev_err(dd, "piowait list not empty!\n"); 16538c2ecf20Sopenharmony_ci if (!list_empty(&dev->dmawait)) 16548c2ecf20Sopenharmony_ci qib_dev_err(dd, "dmawait list not empty!\n"); 16558c2ecf20Sopenharmony_ci if (!list_empty(&dev->txwait)) 16568c2ecf20Sopenharmony_ci qib_dev_err(dd, "txwait list not empty!\n"); 16578c2ecf20Sopenharmony_ci if (!list_empty(&dev->memwait)) 16588c2ecf20Sopenharmony_ci qib_dev_err(dd, "memwait list not empty!\n"); 16598c2ecf20Sopenharmony_ci 16608c2ecf20Sopenharmony_ci del_timer_sync(&dev->mem_timer); 16618c2ecf20Sopenharmony_ci while (!list_empty(&dev->txreq_free)) { 16628c2ecf20Sopenharmony_ci struct list_head *l = dev->txreq_free.next; 16638c2ecf20Sopenharmony_ci struct qib_verbs_txreq *tx; 16648c2ecf20Sopenharmony_ci 16658c2ecf20Sopenharmony_ci list_del(l); 16668c2ecf20Sopenharmony_ci tx = list_entry(l, struct qib_verbs_txreq, txreq.list); 16678c2ecf20Sopenharmony_ci kfree(tx); 16688c2ecf20Sopenharmony_ci } 16698c2ecf20Sopenharmony_ci if (dd->pport->sdma_descq_cnt) 16708c2ecf20Sopenharmony_ci dma_free_coherent(&dd->pcidev->dev, 16718c2ecf20Sopenharmony_ci dd->pport->sdma_descq_cnt * 16728c2ecf20Sopenharmony_ci sizeof(struct qib_pio_header), 16738c2ecf20Sopenharmony_ci dev->pio_hdrs, dev->pio_hdrs_phys); 16748c2ecf20Sopenharmony_ci} 16758c2ecf20Sopenharmony_ci 16768c2ecf20Sopenharmony_ci/** 16778c2ecf20Sopenharmony_ci * _qib_schedule_send - schedule progress 16788c2ecf20Sopenharmony_ci * @qp - the qp 16798c2ecf20Sopenharmony_ci * 16808c2ecf20Sopenharmony_ci * This schedules progress w/o regard to the s_flags. 16818c2ecf20Sopenharmony_ci * 16828c2ecf20Sopenharmony_ci * It is only used in post send, which doesn't hold 16838c2ecf20Sopenharmony_ci * the s_lock. 16848c2ecf20Sopenharmony_ci */ 16858c2ecf20Sopenharmony_cibool _qib_schedule_send(struct rvt_qp *qp) 16868c2ecf20Sopenharmony_ci{ 16878c2ecf20Sopenharmony_ci struct qib_ibport *ibp = 16888c2ecf20Sopenharmony_ci to_iport(qp->ibqp.device, qp->port_num); 16898c2ecf20Sopenharmony_ci struct qib_pportdata *ppd = ppd_from_ibp(ibp); 16908c2ecf20Sopenharmony_ci struct qib_qp_priv *priv = qp->priv; 16918c2ecf20Sopenharmony_ci 16928c2ecf20Sopenharmony_ci return queue_work(ppd->qib_wq, &priv->s_work); 16938c2ecf20Sopenharmony_ci} 16948c2ecf20Sopenharmony_ci 16958c2ecf20Sopenharmony_ci/** 16968c2ecf20Sopenharmony_ci * qib_schedule_send - schedule progress 16978c2ecf20Sopenharmony_ci * @qp - the qp 16988c2ecf20Sopenharmony_ci * 16998c2ecf20Sopenharmony_ci * This schedules qp progress. The s_lock 17008c2ecf20Sopenharmony_ci * should be held. 17018c2ecf20Sopenharmony_ci */ 17028c2ecf20Sopenharmony_cibool qib_schedule_send(struct rvt_qp *qp) 17038c2ecf20Sopenharmony_ci{ 17048c2ecf20Sopenharmony_ci if (qib_send_ok(qp)) 17058c2ecf20Sopenharmony_ci return _qib_schedule_send(qp); 17068c2ecf20Sopenharmony_ci return false; 17078c2ecf20Sopenharmony_ci} 1708