1/******************************************************************* 2 * This file is part of the Emulex Linux Device Driver for * 3 * Fibre Channel Host Bus Adapters. * 4 * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * 5 * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * 6 * Copyright (C) 2004-2016 Emulex. All rights reserved. * 7 * EMULEX and SLI are trademarks of Emulex. * 8 * www.broadcom.com * 9 * Portions Copyright (C) 2004-2005 Christoph Hellwig * 10 * * 11 * This program is free software; you can redistribute it and/or * 12 * modify it under the terms of version 2 of the GNU General * 13 * Public License as published by the Free Software Foundation. * 14 * This program is distributed in the hope that it will be useful. * 15 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND * 16 * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, * 17 * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE * 18 * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD * 19 * TO BE LEGALLY INVALID. See the GNU General Public License for * 20 * more details, a copy of which can be found in the file COPYING * 21 * included with this package. * 22 ********************************************************************/ 23#include <linux/pci.h> 24#include <linux/slab.h> 25#include <linux/interrupt.h> 26#include <linux/delay.h> 27#include <asm/unaligned.h> 28#include <linux/crc-t10dif.h> 29#include <net/checksum.h> 30 31#include <scsi/scsi.h> 32#include <scsi/scsi_device.h> 33#include <scsi/scsi_eh.h> 34#include <scsi/scsi_host.h> 35#include <scsi/scsi_tcq.h> 36#include <scsi/scsi_transport_fc.h> 37#include <scsi/fc/fc_fs.h> 38 39#include "lpfc_version.h" 40#include "lpfc_hw4.h" 41#include "lpfc_hw.h" 42#include "lpfc_sli.h" 43#include "lpfc_sli4.h" 44#include "lpfc_nl.h" 45#include "lpfc_disc.h" 46#include "lpfc.h" 47#include "lpfc_nvme.h" 48#include "lpfc_scsi.h" 49#include "lpfc_logmsg.h" 50#include "lpfc_crtn.h" 51#include "lpfc_vport.h" 52#include "lpfc_debugfs.h" 53 54/* NVME initiator-based functions */ 55 56static struct lpfc_io_buf * 57lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, 58 int idx, int expedite); 59 60static void 61lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_io_buf *); 62 63static struct nvme_fc_port_template lpfc_nvme_template; 64 65static union lpfc_wqe128 lpfc_iread_cmd_template; 66static union lpfc_wqe128 lpfc_iwrite_cmd_template; 67static union lpfc_wqe128 lpfc_icmnd_cmd_template; 68 69/* Setup WQE templates for NVME IOs */ 70void 71lpfc_nvme_cmd_template(void) 72{ 73 union lpfc_wqe128 *wqe; 74 75 /* IREAD template */ 76 wqe = &lpfc_iread_cmd_template; 77 memset(wqe, 0, sizeof(union lpfc_wqe128)); 78 79 /* Word 0, 1, 2 - BDE is variable */ 80 81 /* Word 3 - cmd_buff_len, payload_offset_len is zero */ 82 83 /* Word 4 - total_xfer_len is variable */ 84 85 /* Word 5 - is zero */ 86 87 /* Word 6 - ctxt_tag, xri_tag is variable */ 88 89 /* Word 7 */ 90 bf_set(wqe_cmnd, &wqe->fcp_iread.wqe_com, CMD_FCP_IREAD64_WQE); 91 bf_set(wqe_pu, &wqe->fcp_iread.wqe_com, PARM_READ_CHECK); 92 bf_set(wqe_class, &wqe->fcp_iread.wqe_com, CLASS3); 93 bf_set(wqe_ct, &wqe->fcp_iread.wqe_com, SLI4_CT_RPI); 94 95 /* Word 8 - abort_tag is variable */ 96 97 /* Word 9 - reqtag is variable */ 98 99 /* Word 10 - dbde, wqes is variable */ 100 bf_set(wqe_qosd, &wqe->fcp_iread.wqe_com, 0); 101 bf_set(wqe_nvme, &wqe->fcp_iread.wqe_com, 1); 102 bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, LPFC_WQE_IOD_READ); 103 bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com, LPFC_WQE_LENLOC_WORD4); 104 bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 0); 105 bf_set(wqe_wqes, &wqe->fcp_iread.wqe_com, 1); 106 107 /* Word 11 - pbde is variable */ 108 bf_set(wqe_cmd_type, &wqe->fcp_iread.wqe_com, NVME_READ_CMD); 109 bf_set(wqe_cqid, &wqe->fcp_iread.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 110 bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 1); 111 112 /* Word 12 - is zero */ 113 114 /* Word 13, 14, 15 - PBDE is variable */ 115 116 /* IWRITE template */ 117 wqe = &lpfc_iwrite_cmd_template; 118 memset(wqe, 0, sizeof(union lpfc_wqe128)); 119 120 /* Word 0, 1, 2 - BDE is variable */ 121 122 /* Word 3 - cmd_buff_len, payload_offset_len is zero */ 123 124 /* Word 4 - total_xfer_len is variable */ 125 126 /* Word 5 - initial_xfer_len is variable */ 127 128 /* Word 6 - ctxt_tag, xri_tag is variable */ 129 130 /* Word 7 */ 131 bf_set(wqe_cmnd, &wqe->fcp_iwrite.wqe_com, CMD_FCP_IWRITE64_WQE); 132 bf_set(wqe_pu, &wqe->fcp_iwrite.wqe_com, PARM_READ_CHECK); 133 bf_set(wqe_class, &wqe->fcp_iwrite.wqe_com, CLASS3); 134 bf_set(wqe_ct, &wqe->fcp_iwrite.wqe_com, SLI4_CT_RPI); 135 136 /* Word 8 - abort_tag is variable */ 137 138 /* Word 9 - reqtag is variable */ 139 140 /* Word 10 - dbde, wqes is variable */ 141 bf_set(wqe_qosd, &wqe->fcp_iwrite.wqe_com, 0); 142 bf_set(wqe_nvme, &wqe->fcp_iwrite.wqe_com, 1); 143 bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_IOD_WRITE); 144 bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_LENLOC_WORD4); 145 bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 0); 146 bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1); 147 148 /* Word 11 - pbde is variable */ 149 bf_set(wqe_cmd_type, &wqe->fcp_iwrite.wqe_com, NVME_WRITE_CMD); 150 bf_set(wqe_cqid, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 151 bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 1); 152 153 /* Word 12 - is zero */ 154 155 /* Word 13, 14, 15 - PBDE is variable */ 156 157 /* ICMND template */ 158 wqe = &lpfc_icmnd_cmd_template; 159 memset(wqe, 0, sizeof(union lpfc_wqe128)); 160 161 /* Word 0, 1, 2 - BDE is variable */ 162 163 /* Word 3 - payload_offset_len is variable */ 164 165 /* Word 4, 5 - is zero */ 166 167 /* Word 6 - ctxt_tag, xri_tag is variable */ 168 169 /* Word 7 */ 170 bf_set(wqe_cmnd, &wqe->fcp_icmd.wqe_com, CMD_FCP_ICMND64_WQE); 171 bf_set(wqe_pu, &wqe->fcp_icmd.wqe_com, 0); 172 bf_set(wqe_class, &wqe->fcp_icmd.wqe_com, CLASS3); 173 bf_set(wqe_ct, &wqe->fcp_icmd.wqe_com, SLI4_CT_RPI); 174 175 /* Word 8 - abort_tag is variable */ 176 177 /* Word 9 - reqtag is variable */ 178 179 /* Word 10 - dbde, wqes is variable */ 180 bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1); 181 bf_set(wqe_nvme, &wqe->fcp_icmd.wqe_com, 1); 182 bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_NONE); 183 bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com, LPFC_WQE_LENLOC_NONE); 184 bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 0); 185 bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1); 186 187 /* Word 11 */ 188 bf_set(wqe_cmd_type, &wqe->fcp_icmd.wqe_com, FCP_COMMAND); 189 bf_set(wqe_cqid, &wqe->fcp_icmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 190 bf_set(wqe_pbde, &wqe->fcp_icmd.wqe_com, 0); 191 192 /* Word 12, 13, 14, 15 - is zero */ 193} 194 195/** 196 * lpfc_nvme_prep_abort_wqe - set up 'abort' work queue entry. 197 * @pwqeq: Pointer to command iocb. 198 * @xritag: Tag that uniqely identifies the local exchange resource. 199 * @opt: Option bits - 200 * bit 0 = inhibit sending abts on the link 201 * 202 * This function is called with hbalock held. 203 **/ 204void 205lpfc_nvme_prep_abort_wqe(struct lpfc_iocbq *pwqeq, u16 xritag, u8 opt) 206{ 207 union lpfc_wqe128 *wqe = &pwqeq->wqe; 208 209 /* WQEs are reused. Clear stale data and set key fields to 210 * zero like ia, iaab, iaar, xri_tag, and ctxt_tag. 211 */ 212 memset(wqe, 0, sizeof(*wqe)); 213 214 if (opt & INHIBIT_ABORT) 215 bf_set(abort_cmd_ia, &wqe->abort_cmd, 1); 216 /* Abort specified xri tag, with the mask deliberately zeroed */ 217 bf_set(abort_cmd_criteria, &wqe->abort_cmd, T_XRI_TAG); 218 219 bf_set(wqe_cmnd, &wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX); 220 221 /* Abort the IO associated with this outstanding exchange ID. */ 222 wqe->abort_cmd.wqe_com.abort_tag = xritag; 223 224 /* iotag for the wqe completion. */ 225 bf_set(wqe_reqtag, &wqe->abort_cmd.wqe_com, pwqeq->iotag); 226 227 bf_set(wqe_qosd, &wqe->abort_cmd.wqe_com, 1); 228 bf_set(wqe_lenloc, &wqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE); 229 230 bf_set(wqe_cmd_type, &wqe->abort_cmd.wqe_com, OTHER_COMMAND); 231 bf_set(wqe_wqec, &wqe->abort_cmd.wqe_com, 1); 232 bf_set(wqe_cqid, &wqe->abort_cmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 233} 234 235/** 236 * lpfc_nvme_create_queue - 237 * @pnvme_lport: Transport localport that LS is to be issued from 238 * @lpfc_pnvme: Pointer to the driver's nvme instance data 239 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors. 240 * @qsize: Size of the queue in bytes 241 * @handle: An opaque driver handle used in follow-up calls. 242 * 243 * Driver registers this routine to preallocate and initialize any 244 * internal data structures to bind the @qidx to its internal IO queues. 245 * A hardware queue maps (qidx) to a specific driver MSI-X vector/EQ/CQ/WQ. 246 * 247 * Return value : 248 * 0 - Success 249 * -EINVAL - Unsupported input value. 250 * -ENOMEM - Could not alloc necessary memory 251 **/ 252static int 253lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport, 254 unsigned int qidx, u16 qsize, 255 void **handle) 256{ 257 struct lpfc_nvme_lport *lport; 258 struct lpfc_vport *vport; 259 struct lpfc_nvme_qhandle *qhandle; 260 char *str; 261 262 if (!pnvme_lport->private) 263 return -ENOMEM; 264 265 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 266 vport = lport->vport; 267 qhandle = kzalloc(sizeof(struct lpfc_nvme_qhandle), GFP_KERNEL); 268 if (qhandle == NULL) 269 return -ENOMEM; 270 271 qhandle->cpu_id = raw_smp_processor_id(); 272 qhandle->qidx = qidx; 273 /* 274 * NVME qidx == 0 is the admin queue, so both admin queue 275 * and first IO queue will use MSI-X vector and associated 276 * EQ/CQ/WQ at index 0. After that they are sequentially assigned. 277 */ 278 if (qidx) { 279 str = "IO "; /* IO queue */ 280 qhandle->index = ((qidx - 1) % 281 lpfc_nvme_template.max_hw_queues); 282 } else { 283 str = "ADM"; /* Admin queue */ 284 qhandle->index = qidx; 285 } 286 287 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 288 "6073 Binding %s HdwQueue %d (cpu %d) to " 289 "hdw_queue %d qhandle x%px\n", str, 290 qidx, qhandle->cpu_id, qhandle->index, qhandle); 291 *handle = (void *)qhandle; 292 return 0; 293} 294 295/** 296 * lpfc_nvme_delete_queue - 297 * @pnvme_lport: Transport localport that LS is to be issued from 298 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors. 299 * @handle: An opaque driver handle from lpfc_nvme_create_queue 300 * 301 * Driver registers this routine to free 302 * any internal data structures to bind the @qidx to its internal 303 * IO queues. 304 * 305 * Return value : 306 * 0 - Success 307 * TODO: What are the failure codes. 308 **/ 309static void 310lpfc_nvme_delete_queue(struct nvme_fc_local_port *pnvme_lport, 311 unsigned int qidx, 312 void *handle) 313{ 314 struct lpfc_nvme_lport *lport; 315 struct lpfc_vport *vport; 316 317 if (!pnvme_lport->private) 318 return; 319 320 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 321 vport = lport->vport; 322 323 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 324 "6001 ENTER. lpfc_pnvme x%px, qidx x%x qhandle x%px\n", 325 lport, qidx, handle); 326 kfree(handle); 327} 328 329static void 330lpfc_nvme_localport_delete(struct nvme_fc_local_port *localport) 331{ 332 struct lpfc_nvme_lport *lport = localport->private; 333 334 lpfc_printf_vlog(lport->vport, KERN_INFO, LOG_NVME, 335 "6173 localport x%px delete complete\n", 336 lport); 337 338 /* release any threads waiting for the unreg to complete */ 339 if (lport->vport->localport) 340 complete(lport->lport_unreg_cmp); 341} 342 343/* lpfc_nvme_remoteport_delete 344 * 345 * @remoteport: Pointer to an nvme transport remoteport instance. 346 * 347 * This is a template downcall. NVME transport calls this function 348 * when it has completed the unregistration of a previously 349 * registered remoteport. 350 * 351 * Return value : 352 * None 353 */ 354static void 355lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport) 356{ 357 struct lpfc_nvme_rport *rport = remoteport->private; 358 struct lpfc_vport *vport; 359 struct lpfc_nodelist *ndlp; 360 361 ndlp = rport->ndlp; 362 if (!ndlp) 363 goto rport_err; 364 365 vport = ndlp->vport; 366 if (!vport) 367 goto rport_err; 368 369 /* Remove this rport from the lport's list - memory is owned by the 370 * transport. Remove the ndlp reference for the NVME transport before 371 * calling state machine to remove the node. 372 */ 373 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 374 "6146 remoteport delete of remoteport x%px\n", 375 remoteport); 376 spin_lock_irq(&vport->phba->hbalock); 377 378 /* The register rebind might have occurred before the delete 379 * downcall. Guard against this race. 380 */ 381 if (ndlp->upcall_flags & NLP_WAIT_FOR_UNREG) { 382 ndlp->nrport = NULL; 383 ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG; 384 spin_unlock_irq(&vport->phba->hbalock); 385 386 /* Remove original register reference. The host transport 387 * won't reference this rport/remoteport any further. 388 */ 389 lpfc_nlp_put(ndlp); 390 } else { 391 spin_unlock_irq(&vport->phba->hbalock); 392 } 393 394 rport_err: 395 return; 396} 397 398/** 399 * lpfc_nvme_handle_lsreq - Process an unsolicited NVME LS request 400 * @phba: pointer to lpfc hba data structure. 401 * @axchg: pointer to exchange context for the NVME LS request 402 * 403 * This routine is used for processing an asychronously received NVME LS 404 * request. Any remaining validation is done and the LS is then forwarded 405 * to the nvme-fc transport via nvme_fc_rcv_ls_req(). 406 * 407 * The calling sequence should be: nvme_fc_rcv_ls_req() -> (processing) 408 * -> lpfc_nvme_xmt_ls_rsp/cmp -> req->done. 409 * __lpfc_nvme_xmt_ls_rsp_cmp should free the allocated axchg. 410 * 411 * Returns 0 if LS was handled and delivered to the transport 412 * Returns 1 if LS failed to be handled and should be dropped 413 */ 414int 415lpfc_nvme_handle_lsreq(struct lpfc_hba *phba, 416 struct lpfc_async_xchg_ctx *axchg) 417{ 418#if (IS_ENABLED(CONFIG_NVME_FC)) 419 struct lpfc_vport *vport; 420 struct lpfc_nvme_rport *lpfc_rport; 421 struct nvme_fc_remote_port *remoteport; 422 struct lpfc_nvme_lport *lport; 423 uint32_t *payload = axchg->payload; 424 int rc; 425 426 vport = axchg->ndlp->vport; 427 lpfc_rport = axchg->ndlp->nrport; 428 if (!lpfc_rport) 429 return -EINVAL; 430 431 remoteport = lpfc_rport->remoteport; 432 if (!vport->localport) 433 return -EINVAL; 434 435 lport = vport->localport->private; 436 if (!lport) 437 return -EINVAL; 438 439 rc = nvme_fc_rcv_ls_req(remoteport, &axchg->ls_rsp, axchg->payload, 440 axchg->size); 441 442 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, 443 "6205 NVME Unsol rcv: sz %d rc %d: %08x %08x %08x " 444 "%08x %08x %08x\n", 445 axchg->size, rc, 446 *payload, *(payload+1), *(payload+2), 447 *(payload+3), *(payload+4), *(payload+5)); 448 449 if (!rc) 450 return 0; 451#endif 452 return 1; 453} 454 455/** 456 * __lpfc_nvme_ls_req_cmp - Generic completion handler for a NVME 457 * LS request. 458 * @phba: Pointer to HBA context object 459 * @vport: The local port that issued the LS 460 * @cmdwqe: Pointer to driver command WQE object. 461 * @wcqe: Pointer to driver response CQE object. 462 * 463 * This function is the generic completion handler for NVME LS requests. 464 * The function updates any states and statistics, calls the transport 465 * ls_req done() routine, then tears down the command and buffers used 466 * for the LS request. 467 **/ 468void 469__lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_vport *vport, 470 struct lpfc_iocbq *cmdwqe, 471 struct lpfc_wcqe_complete *wcqe) 472{ 473 struct nvmefc_ls_req *pnvme_lsreq; 474 struct lpfc_dmabuf *buf_ptr; 475 struct lpfc_nodelist *ndlp; 476 uint32_t status; 477 478 pnvme_lsreq = (struct nvmefc_ls_req *)cmdwqe->context2; 479 ndlp = (struct lpfc_nodelist *)cmdwqe->context1; 480 status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK; 481 482 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 483 "6047 NVMEx LS REQ %px cmpl DID %x Xri: %x " 484 "status %x reason x%x cmd:x%px lsreg:x%px bmp:x%px " 485 "ndlp:x%px\n", 486 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0, 487 cmdwqe->sli4_xritag, status, 488 (wcqe->parameter & 0xffff), 489 cmdwqe, pnvme_lsreq, cmdwqe->context3, ndlp); 490 491 lpfc_nvmeio_data(phba, "NVMEx LS CMPL: xri x%x stat x%x parm x%x\n", 492 cmdwqe->sli4_xritag, status, wcqe->parameter); 493 494 if (cmdwqe->context3) { 495 buf_ptr = (struct lpfc_dmabuf *)cmdwqe->context3; 496 lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys); 497 kfree(buf_ptr); 498 cmdwqe->context3 = NULL; 499 } 500 if (pnvme_lsreq->done) 501 pnvme_lsreq->done(pnvme_lsreq, status); 502 else 503 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 504 "6046 NVMEx cmpl without done call back? " 505 "Data %px DID %x Xri: %x status %x\n", 506 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0, 507 cmdwqe->sli4_xritag, status); 508 if (ndlp) { 509 lpfc_nlp_put(ndlp); 510 cmdwqe->context1 = NULL; 511 } 512 lpfc_sli_release_iocbq(phba, cmdwqe); 513} 514 515static void 516lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 517 struct lpfc_wcqe_complete *wcqe) 518{ 519 struct lpfc_vport *vport = cmdwqe->vport; 520 struct lpfc_nvme_lport *lport; 521 uint32_t status; 522 523 status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK; 524 525 if (vport->localport) { 526 lport = (struct lpfc_nvme_lport *)vport->localport->private; 527 if (lport) { 528 atomic_inc(&lport->fc4NvmeLsCmpls); 529 if (status) { 530 if (bf_get(lpfc_wcqe_c_xb, wcqe)) 531 atomic_inc(&lport->cmpl_ls_xb); 532 atomic_inc(&lport->cmpl_ls_err); 533 } 534 } 535 } 536 537 __lpfc_nvme_ls_req_cmp(phba, vport, cmdwqe, wcqe); 538} 539 540static int 541lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp, 542 struct lpfc_dmabuf *inp, 543 struct nvmefc_ls_req *pnvme_lsreq, 544 void (*cmpl)(struct lpfc_hba *, struct lpfc_iocbq *, 545 struct lpfc_wcqe_complete *), 546 struct lpfc_nodelist *ndlp, uint32_t num_entry, 547 uint32_t tmo, uint8_t retry) 548{ 549 struct lpfc_hba *phba = vport->phba; 550 union lpfc_wqe128 *wqe; 551 struct lpfc_iocbq *genwqe; 552 struct ulp_bde64 *bpl; 553 struct ulp_bde64 bde; 554 int i, rc, xmit_len, first_len; 555 556 /* Allocate buffer for command WQE */ 557 genwqe = lpfc_sli_get_iocbq(phba); 558 if (genwqe == NULL) 559 return 1; 560 561 wqe = &genwqe->wqe; 562 /* Initialize only 64 bytes */ 563 memset(wqe, 0, sizeof(union lpfc_wqe)); 564 565 genwqe->context3 = (uint8_t *)bmp; 566 genwqe->iocb_flag |= LPFC_IO_NVME_LS; 567 568 /* Save for completion so we can release these resources */ 569 genwqe->context1 = lpfc_nlp_get(ndlp); 570 genwqe->context2 = (uint8_t *)pnvme_lsreq; 571 /* Fill in payload, bp points to frame payload */ 572 573 if (!tmo) 574 /* FC spec states we need 3 * ratov for CT requests */ 575 tmo = (3 * phba->fc_ratov); 576 577 /* For this command calculate the xmit length of the request bde. */ 578 xmit_len = 0; 579 first_len = 0; 580 bpl = (struct ulp_bde64 *)bmp->virt; 581 for (i = 0; i < num_entry; i++) { 582 bde.tus.w = bpl[i].tus.w; 583 if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64) 584 break; 585 xmit_len += bde.tus.f.bdeSize; 586 if (i == 0) 587 first_len = xmit_len; 588 } 589 590 genwqe->rsvd2 = num_entry; 591 genwqe->hba_wqidx = 0; 592 593 /* Words 0 - 2 */ 594 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 595 wqe->generic.bde.tus.f.bdeSize = first_len; 596 wqe->generic.bde.addrLow = bpl[0].addrLow; 597 wqe->generic.bde.addrHigh = bpl[0].addrHigh; 598 599 /* Word 3 */ 600 wqe->gen_req.request_payload_len = first_len; 601 602 /* Word 4 */ 603 604 /* Word 5 */ 605 bf_set(wqe_dfctl, &wqe->gen_req.wge_ctl, 0); 606 bf_set(wqe_si, &wqe->gen_req.wge_ctl, 1); 607 bf_set(wqe_la, &wqe->gen_req.wge_ctl, 1); 608 bf_set(wqe_rctl, &wqe->gen_req.wge_ctl, FC_RCTL_ELS4_REQ); 609 bf_set(wqe_type, &wqe->gen_req.wge_ctl, FC_TYPE_NVME); 610 611 /* Word 6 */ 612 bf_set(wqe_ctxt_tag, &wqe->gen_req.wqe_com, 613 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); 614 bf_set(wqe_xri_tag, &wqe->gen_req.wqe_com, genwqe->sli4_xritag); 615 616 /* Word 7 */ 617 bf_set(wqe_tmo, &wqe->gen_req.wqe_com, (vport->phba->fc_ratov-1)); 618 bf_set(wqe_class, &wqe->gen_req.wqe_com, CLASS3); 619 bf_set(wqe_cmnd, &wqe->gen_req.wqe_com, CMD_GEN_REQUEST64_WQE); 620 bf_set(wqe_ct, &wqe->gen_req.wqe_com, SLI4_CT_RPI); 621 622 /* Word 8 */ 623 wqe->gen_req.wqe_com.abort_tag = genwqe->iotag; 624 625 /* Word 9 */ 626 bf_set(wqe_reqtag, &wqe->gen_req.wqe_com, genwqe->iotag); 627 628 /* Word 10 */ 629 bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1); 630 bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ); 631 bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1); 632 bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE); 633 bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0); 634 635 /* Word 11 */ 636 bf_set(wqe_cqid, &wqe->gen_req.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 637 bf_set(wqe_cmd_type, &wqe->gen_req.wqe_com, OTHER_COMMAND); 638 639 640 /* Issue GEN REQ WQE for NPORT <did> */ 641 genwqe->wqe_cmpl = cmpl; 642 genwqe->iocb_cmpl = NULL; 643 genwqe->drvrTimeout = tmo + LPFC_DRVR_TIMEOUT; 644 genwqe->vport = vport; 645 genwqe->retry = retry; 646 647 lpfc_nvmeio_data(phba, "NVME LS XMIT: xri x%x iotag x%x to x%06x\n", 648 genwqe->sli4_xritag, genwqe->iotag, ndlp->nlp_DID); 649 650 rc = lpfc_sli4_issue_wqe(phba, &phba->sli4_hba.hdwq[0], genwqe); 651 if (rc) { 652 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 653 "6045 Issue GEN REQ WQE to NPORT x%x " 654 "Data: x%x x%x rc x%x\n", 655 ndlp->nlp_DID, genwqe->iotag, 656 vport->port_state, rc); 657 lpfc_sli_release_iocbq(phba, genwqe); 658 return 1; 659 } 660 661 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_ELS, 662 "6050 Issue GEN REQ WQE to NPORT x%x " 663 "Data: oxid: x%x state: x%x wq:x%px lsreq:x%px " 664 "bmp:x%px xmit:%d 1st:%d\n", 665 ndlp->nlp_DID, genwqe->sli4_xritag, 666 vport->port_state, 667 genwqe, pnvme_lsreq, bmp, xmit_len, first_len); 668 return 0; 669} 670 671 672/** 673 * __lpfc_nvme_ls_req - Generic service routine to issue an NVME LS request 674 * @vport: The local port issuing the LS 675 * @ndlp: The remote port to send the LS to 676 * @pnvme_lsreq: Pointer to LS request structure from the transport 677 * @gen_req_cmp: Completion call-back 678 * 679 * Routine validates the ndlp, builds buffers and sends a GEN_REQUEST 680 * WQE to perform the LS operation. 681 * 682 * Return value : 683 * 0 - Success 684 * non-zero: various error codes, in form of -Exxx 685 **/ 686int 687__lpfc_nvme_ls_req(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, 688 struct nvmefc_ls_req *pnvme_lsreq, 689 void (*gen_req_cmp)(struct lpfc_hba *phba, 690 struct lpfc_iocbq *cmdwqe, 691 struct lpfc_wcqe_complete *wcqe)) 692{ 693 struct lpfc_dmabuf *bmp; 694 struct ulp_bde64 *bpl; 695 int ret; 696 uint16_t ntype, nstate; 697 698 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { 699 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 700 "6051 NVMEx LS REQ: Bad NDLP x%px, Failing " 701 "LS Req\n", 702 ndlp); 703 return -ENODEV; 704 } 705 706 ntype = ndlp->nlp_type; 707 nstate = ndlp->nlp_state; 708 if ((ntype & NLP_NVME_TARGET && nstate != NLP_STE_MAPPED_NODE) || 709 (ntype & NLP_NVME_INITIATOR && nstate != NLP_STE_UNMAPPED_NODE)) { 710 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 711 "6088 NVMEx LS REQ: Fail DID x%06x not " 712 "ready for IO. Type x%x, State x%x\n", 713 ndlp->nlp_DID, ntype, nstate); 714 return -ENODEV; 715 } 716 717 if (!vport->phba->sli4_hba.nvmels_wq) 718 return -ENOMEM; 719 720 /* 721 * there are two dma buf in the request, actually there is one and 722 * the second one is just the start address + cmd size. 723 * Before calling lpfc_nvme_gen_req these buffers need to be wrapped 724 * in a lpfc_dmabuf struct. When freeing we just free the wrapper 725 * because the nvem layer owns the data bufs. 726 * We do not have to break these packets open, we don't care what is 727 * in them. And we do not have to look at the resonse data, we only 728 * care that we got a response. All of the caring is going to happen 729 * in the nvme-fc layer. 730 */ 731 732 bmp = kmalloc(sizeof(*bmp), GFP_KERNEL); 733 if (!bmp) { 734 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 735 "6044 NVMEx LS REQ: Could not alloc LS buf " 736 "for DID %x\n", 737 ndlp->nlp_DID); 738 return -ENOMEM; 739 } 740 741 bmp->virt = lpfc_mbuf_alloc(vport->phba, MEM_PRI, &(bmp->phys)); 742 if (!bmp->virt) { 743 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 744 "6042 NVMEx LS REQ: Could not alloc mbuf " 745 "for DID %x\n", 746 ndlp->nlp_DID); 747 kfree(bmp); 748 return -ENOMEM; 749 } 750 751 INIT_LIST_HEAD(&bmp->list); 752 753 bpl = (struct ulp_bde64 *)bmp->virt; 754 bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rqstdma)); 755 bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rqstdma)); 756 bpl->tus.f.bdeFlags = 0; 757 bpl->tus.f.bdeSize = pnvme_lsreq->rqstlen; 758 bpl->tus.w = le32_to_cpu(bpl->tus.w); 759 bpl++; 760 761 bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rspdma)); 762 bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rspdma)); 763 bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I; 764 bpl->tus.f.bdeSize = pnvme_lsreq->rsplen; 765 bpl->tus.w = le32_to_cpu(bpl->tus.w); 766 767 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 768 "6149 NVMEx LS REQ: Issue to DID 0x%06x lsreq x%px, " 769 "rqstlen:%d rsplen:%d %pad %pad\n", 770 ndlp->nlp_DID, pnvme_lsreq, pnvme_lsreq->rqstlen, 771 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma, 772 &pnvme_lsreq->rspdma); 773 774 ret = lpfc_nvme_gen_req(vport, bmp, pnvme_lsreq->rqstaddr, 775 pnvme_lsreq, gen_req_cmp, ndlp, 2, 776 LPFC_NVME_LS_TIMEOUT, 0); 777 if (ret != WQE_SUCCESS) { 778 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 779 "6052 NVMEx REQ: EXIT. issue ls wqe failed " 780 "lsreq x%px Status %x DID %x\n", 781 pnvme_lsreq, ret, ndlp->nlp_DID); 782 lpfc_mbuf_free(vport->phba, bmp->virt, bmp->phys); 783 kfree(bmp); 784 return -EIO; 785 } 786 787 return 0; 788} 789 790/** 791 * lpfc_nvme_ls_req - Issue an NVME Link Service request 792 * @pnvme_lport: Transport localport that LS is to be issued from. 793 * @nvme_rport: Transport remoteport that LS is to be sent to. 794 * @pnvme_lsreq: the transport nvme_ls_req structure for the LS 795 * 796 * Driver registers this routine to handle any link service request 797 * from the nvme_fc transport to a remote nvme-aware port. 798 * 799 * Return value : 800 * 0 - Success 801 * non-zero: various error codes, in form of -Exxx 802 **/ 803static int 804lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport, 805 struct nvme_fc_remote_port *pnvme_rport, 806 struct nvmefc_ls_req *pnvme_lsreq) 807{ 808 struct lpfc_nvme_lport *lport; 809 struct lpfc_nvme_rport *rport; 810 struct lpfc_vport *vport; 811 int ret; 812 813 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 814 rport = (struct lpfc_nvme_rport *)pnvme_rport->private; 815 if (unlikely(!lport) || unlikely(!rport)) 816 return -EINVAL; 817 818 vport = lport->vport; 819 if (vport->load_flag & FC_UNLOADING) 820 return -ENODEV; 821 822 atomic_inc(&lport->fc4NvmeLsRequests); 823 824 ret = __lpfc_nvme_ls_req(vport, rport->ndlp, pnvme_lsreq, 825 lpfc_nvme_ls_req_cmp); 826 if (ret) 827 atomic_inc(&lport->xmt_ls_err); 828 829 return ret; 830} 831 832/** 833 * __lpfc_nvme_ls_abort - Generic service routine to abort a prior 834 * NVME LS request 835 * @vport: The local port that issued the LS 836 * @ndlp: The remote port the LS was sent to 837 * @pnvme_lsreq: Pointer to LS request structure from the transport 838 * 839 * The driver validates the ndlp, looks for the LS, and aborts the 840 * LS if found. 841 * 842 * Returns: 843 * 0 : if LS found and aborted 844 * non-zero: various error conditions in form -Exxx 845 **/ 846int 847__lpfc_nvme_ls_abort(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, 848 struct nvmefc_ls_req *pnvme_lsreq) 849{ 850 struct lpfc_hba *phba = vport->phba; 851 struct lpfc_sli_ring *pring; 852 struct lpfc_iocbq *wqe, *next_wqe; 853 bool foundit = false; 854 855 if (!ndlp) { 856 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 857 "6049 NVMEx LS REQ Abort: Bad NDLP x%px DID " 858 "x%06x, Failing LS Req\n", 859 ndlp, ndlp ? ndlp->nlp_DID : 0); 860 return -EINVAL; 861 } 862 863 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS, 864 "6040 NVMEx LS REQ Abort: Issue LS_ABORT for lsreq " 865 "x%p rqstlen:%d rsplen:%d %pad %pad\n", 866 pnvme_lsreq, pnvme_lsreq->rqstlen, 867 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma, 868 &pnvme_lsreq->rspdma); 869 870 /* 871 * Lock the ELS ring txcmplq and look for the wqe that matches 872 * this ELS. If found, issue an abort on the wqe. 873 */ 874 pring = phba->sli4_hba.nvmels_wq->pring; 875 spin_lock_irq(&phba->hbalock); 876 spin_lock(&pring->ring_lock); 877 list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) { 878 if (wqe->context2 == pnvme_lsreq) { 879 wqe->iocb_flag |= LPFC_DRIVER_ABORTED; 880 foundit = true; 881 break; 882 } 883 } 884 spin_unlock(&pring->ring_lock); 885 886 if (foundit) 887 lpfc_sli_issue_abort_iotag(phba, pring, wqe); 888 spin_unlock_irq(&phba->hbalock); 889 890 if (foundit) 891 return 0; 892 893 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS, 894 "6213 NVMEx LS REQ Abort: Unable to locate req x%p\n", 895 pnvme_lsreq); 896 return -EINVAL; 897} 898 899static int 900lpfc_nvme_xmt_ls_rsp(struct nvme_fc_local_port *localport, 901 struct nvme_fc_remote_port *remoteport, 902 struct nvmefc_ls_rsp *ls_rsp) 903{ 904 struct lpfc_async_xchg_ctx *axchg = 905 container_of(ls_rsp, struct lpfc_async_xchg_ctx, ls_rsp); 906 struct lpfc_nvme_lport *lport; 907 int rc; 908 909 if (axchg->phba->pport->load_flag & FC_UNLOADING) 910 return -ENODEV; 911 912 lport = (struct lpfc_nvme_lport *)localport->private; 913 914 rc = __lpfc_nvme_xmt_ls_rsp(axchg, ls_rsp, __lpfc_nvme_xmt_ls_rsp_cmp); 915 916 if (rc) { 917 /* 918 * unless the failure is due to having already sent 919 * the response, an abort will be generated for the 920 * exchange if the rsp can't be sent. 921 */ 922 if (rc != -EALREADY) 923 atomic_inc(&lport->xmt_ls_abort); 924 return rc; 925 } 926 927 return 0; 928} 929 930/** 931 * lpfc_nvme_ls_abort - Abort a prior NVME LS request 932 * @pnvme_lport: Transport localport that LS is to be issued from. 933 * @pnvme_rport: Transport remoteport that LS is to be sent to. 934 * @pnvme_lsreq: the transport nvme_ls_req structure for the LS 935 * 936 * Driver registers this routine to abort a NVME LS request that is 937 * in progress (from the transports perspective). 938 **/ 939static void 940lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport, 941 struct nvme_fc_remote_port *pnvme_rport, 942 struct nvmefc_ls_req *pnvme_lsreq) 943{ 944 struct lpfc_nvme_lport *lport; 945 struct lpfc_vport *vport; 946 struct lpfc_hba *phba; 947 struct lpfc_nodelist *ndlp; 948 int ret; 949 950 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 951 if (unlikely(!lport)) 952 return; 953 vport = lport->vport; 954 phba = vport->phba; 955 956 if (vport->load_flag & FC_UNLOADING) 957 return; 958 959 ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id); 960 961 ret = __lpfc_nvme_ls_abort(vport, ndlp, pnvme_lsreq); 962 if (!ret) 963 atomic_inc(&lport->xmt_ls_abort); 964} 965 966/* Fix up the existing sgls for NVME IO. */ 967static inline void 968lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport, 969 struct lpfc_io_buf *lpfc_ncmd, 970 struct nvmefc_fcp_req *nCmd) 971{ 972 struct lpfc_hba *phba = vport->phba; 973 struct sli4_sge *sgl; 974 union lpfc_wqe128 *wqe; 975 uint32_t *wptr, *dptr; 976 977 /* 978 * Get a local pointer to the built-in wqe and correct 979 * the cmd size to match NVME's 96 bytes and fix 980 * the dma address. 981 */ 982 983 wqe = &lpfc_ncmd->cur_iocbq.wqe; 984 985 /* 986 * Adjust the FCP_CMD and FCP_RSP DMA data and sge_len to 987 * match NVME. NVME sends 96 bytes. Also, use the 988 * nvme commands command and response dma addresses 989 * rather than the virtual memory to ease the restore 990 * operation. 991 */ 992 sgl = lpfc_ncmd->dma_sgl; 993 sgl->sge_len = cpu_to_le32(nCmd->cmdlen); 994 if (phba->cfg_nvme_embed_cmd) { 995 sgl->addr_hi = 0; 996 sgl->addr_lo = 0; 997 998 /* Word 0-2 - NVME CMND IU (embedded payload) */ 999 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_IMMED; 1000 wqe->generic.bde.tus.f.bdeSize = 56; 1001 wqe->generic.bde.addrHigh = 0; 1002 wqe->generic.bde.addrLow = 64; /* Word 16 */ 1003 1004 /* Word 10 - dbde is 0, wqes is 1 in template */ 1005 1006 /* 1007 * Embed the payload in the last half of the WQE 1008 * WQE words 16-30 get the NVME CMD IU payload 1009 * 1010 * WQE words 16-19 get payload Words 1-4 1011 * WQE words 20-21 get payload Words 6-7 1012 * WQE words 22-29 get payload Words 16-23 1013 */ 1014 wptr = &wqe->words[16]; /* WQE ptr */ 1015 dptr = (uint32_t *)nCmd->cmdaddr; /* payload ptr */ 1016 dptr++; /* Skip Word 0 in payload */ 1017 1018 *wptr++ = *dptr++; /* Word 1 */ 1019 *wptr++ = *dptr++; /* Word 2 */ 1020 *wptr++ = *dptr++; /* Word 3 */ 1021 *wptr++ = *dptr++; /* Word 4 */ 1022 dptr++; /* Skip Word 5 in payload */ 1023 *wptr++ = *dptr++; /* Word 6 */ 1024 *wptr++ = *dptr++; /* Word 7 */ 1025 dptr += 8; /* Skip Words 8-15 in payload */ 1026 *wptr++ = *dptr++; /* Word 16 */ 1027 *wptr++ = *dptr++; /* Word 17 */ 1028 *wptr++ = *dptr++; /* Word 18 */ 1029 *wptr++ = *dptr++; /* Word 19 */ 1030 *wptr++ = *dptr++; /* Word 20 */ 1031 *wptr++ = *dptr++; /* Word 21 */ 1032 *wptr++ = *dptr++; /* Word 22 */ 1033 *wptr = *dptr; /* Word 23 */ 1034 } else { 1035 sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->cmddma)); 1036 sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->cmddma)); 1037 1038 /* Word 0-2 - NVME CMND IU Inline BDE */ 1039 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 1040 wqe->generic.bde.tus.f.bdeSize = nCmd->cmdlen; 1041 wqe->generic.bde.addrHigh = sgl->addr_hi; 1042 wqe->generic.bde.addrLow = sgl->addr_lo; 1043 1044 /* Word 10 */ 1045 bf_set(wqe_dbde, &wqe->generic.wqe_com, 1); 1046 bf_set(wqe_wqes, &wqe->generic.wqe_com, 0); 1047 } 1048 1049 sgl++; 1050 1051 /* Setup the physical region for the FCP RSP */ 1052 sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->rspdma)); 1053 sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->rspdma)); 1054 sgl->word2 = le32_to_cpu(sgl->word2); 1055 if (nCmd->sg_cnt) 1056 bf_set(lpfc_sli4_sge_last, sgl, 0); 1057 else 1058 bf_set(lpfc_sli4_sge_last, sgl, 1); 1059 sgl->word2 = cpu_to_le32(sgl->word2); 1060 sgl->sge_len = cpu_to_le32(nCmd->rsplen); 1061} 1062 1063 1064/* 1065 * lpfc_nvme_io_cmd_wqe_cmpl - Complete an NVME-over-FCP IO 1066 * 1067 * Driver registers this routine as it io request handler. This 1068 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 1069 * data structure to the rport indicated in @lpfc_nvme_rport. 1070 * 1071 * Return value : 1072 * 0 - Success 1073 * TODO: What are the failure codes. 1074 **/ 1075static void 1076lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn, 1077 struct lpfc_wcqe_complete *wcqe) 1078{ 1079 struct lpfc_io_buf *lpfc_ncmd = 1080 (struct lpfc_io_buf *)pwqeIn->context1; 1081 struct lpfc_vport *vport = pwqeIn->vport; 1082 struct nvmefc_fcp_req *nCmd; 1083 struct nvme_fc_ersp_iu *ep; 1084 struct nvme_fc_cmd_iu *cp; 1085 struct lpfc_nodelist *ndlp; 1086 struct lpfc_nvme_fcpreq_priv *freqpriv; 1087 struct lpfc_nvme_lport *lport; 1088 uint32_t code, status, idx; 1089 uint16_t cid, sqhd, data; 1090 uint32_t *ptr; 1091#ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1092 int cpu; 1093#endif 1094 1095 /* Sanity check on return of outstanding command */ 1096 if (!lpfc_ncmd) { 1097 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 1098 "6071 Null lpfc_ncmd pointer. No " 1099 "release, skip completion\n"); 1100 return; 1101 } 1102 1103 /* Guard against abort handler being called at same time */ 1104 spin_lock(&lpfc_ncmd->buf_lock); 1105 1106 if (!lpfc_ncmd->nvmeCmd) { 1107 spin_unlock(&lpfc_ncmd->buf_lock); 1108 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 1109 "6066 Missing cmpl ptrs: lpfc_ncmd x%px, " 1110 "nvmeCmd x%px\n", 1111 lpfc_ncmd, lpfc_ncmd->nvmeCmd); 1112 1113 /* Release the lpfc_ncmd regardless of the missing elements. */ 1114 lpfc_release_nvme_buf(phba, lpfc_ncmd); 1115 return; 1116 } 1117 nCmd = lpfc_ncmd->nvmeCmd; 1118 status = bf_get(lpfc_wcqe_c_status, wcqe); 1119 1120 idx = lpfc_ncmd->cur_iocbq.hba_wqidx; 1121 phba->sli4_hba.hdwq[idx].nvme_cstat.io_cmpls++; 1122 1123 if (unlikely(status && vport->localport)) { 1124 lport = (struct lpfc_nvme_lport *)vport->localport->private; 1125 if (lport) { 1126 if (bf_get(lpfc_wcqe_c_xb, wcqe)) 1127 atomic_inc(&lport->cmpl_fcp_xb); 1128 atomic_inc(&lport->cmpl_fcp_err); 1129 } 1130 } 1131 1132 lpfc_nvmeio_data(phba, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n", 1133 lpfc_ncmd->cur_iocbq.sli4_xritag, 1134 status, wcqe->parameter); 1135 /* 1136 * Catch race where our node has transitioned, but the 1137 * transport is still transitioning. 1138 */ 1139 ndlp = lpfc_ncmd->ndlp; 1140 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { 1141 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 1142 "6062 Ignoring NVME cmpl. No ndlp\n"); 1143 goto out_err; 1144 } 1145 1146 code = bf_get(lpfc_wcqe_c_code, wcqe); 1147 if (code == CQE_CODE_NVME_ERSP) { 1148 /* For this type of CQE, we need to rebuild the rsp */ 1149 ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr; 1150 1151 /* 1152 * Get Command Id from cmd to plug into response. This 1153 * code is not needed in the next NVME Transport drop. 1154 */ 1155 cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr; 1156 cid = cp->sqe.common.command_id; 1157 1158 /* 1159 * RSN is in CQE word 2 1160 * SQHD is in CQE Word 3 bits 15:0 1161 * Cmd Specific info is in CQE Word 1 1162 * and in CQE Word 0 bits 15:0 1163 */ 1164 sqhd = bf_get(lpfc_wcqe_c_sqhead, wcqe); 1165 1166 /* Now lets build the NVME ERSP IU */ 1167 ep->iu_len = cpu_to_be16(8); 1168 ep->rsn = wcqe->parameter; 1169 ep->xfrd_len = cpu_to_be32(nCmd->payload_length); 1170 ep->rsvd12 = 0; 1171 ptr = (uint32_t *)&ep->cqe.result.u64; 1172 *ptr++ = wcqe->total_data_placed; 1173 data = bf_get(lpfc_wcqe_c_ersp0, wcqe); 1174 *ptr = (uint32_t)data; 1175 ep->cqe.sq_head = sqhd; 1176 ep->cqe.sq_id = nCmd->sqid; 1177 ep->cqe.command_id = cid; 1178 ep->cqe.status = 0; 1179 1180 lpfc_ncmd->status = IOSTAT_SUCCESS; 1181 lpfc_ncmd->result = 0; 1182 nCmd->rcv_rsplen = LPFC_NVME_ERSP_LEN; 1183 nCmd->transferred_length = nCmd->payload_length; 1184 } else { 1185 lpfc_ncmd->status = (status & LPFC_IOCB_STATUS_MASK); 1186 lpfc_ncmd->result = (wcqe->parameter & IOERR_PARAM_MASK); 1187 1188 /* For NVME, the only failure path that results in an 1189 * IO error is when the adapter rejects it. All other 1190 * conditions are a success case and resolved by the 1191 * transport. 1192 * IOSTAT_FCP_RSP_ERROR means: 1193 * 1. Length of data received doesn't match total 1194 * transfer length in WQE 1195 * 2. If the RSP payload does NOT match these cases: 1196 * a. RSP length 12/24 bytes and all zeros 1197 * b. NVME ERSP 1198 */ 1199 switch (lpfc_ncmd->status) { 1200 case IOSTAT_SUCCESS: 1201 nCmd->transferred_length = wcqe->total_data_placed; 1202 nCmd->rcv_rsplen = 0; 1203 nCmd->status = 0; 1204 break; 1205 case IOSTAT_FCP_RSP_ERROR: 1206 nCmd->transferred_length = wcqe->total_data_placed; 1207 nCmd->rcv_rsplen = wcqe->parameter; 1208 nCmd->status = 0; 1209 /* Sanity check */ 1210 if (nCmd->rcv_rsplen == LPFC_NVME_ERSP_LEN) 1211 break; 1212 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 1213 "6081 NVME Completion Protocol Error: " 1214 "xri %x status x%x result x%x " 1215 "placed x%x\n", 1216 lpfc_ncmd->cur_iocbq.sli4_xritag, 1217 lpfc_ncmd->status, lpfc_ncmd->result, 1218 wcqe->total_data_placed); 1219 break; 1220 case IOSTAT_LOCAL_REJECT: 1221 /* Let fall through to set command final state. */ 1222 if (lpfc_ncmd->result == IOERR_ABORT_REQUESTED) 1223 lpfc_printf_vlog(vport, KERN_INFO, 1224 LOG_NVME_IOERR, 1225 "6032 Delay Aborted cmd x%px " 1226 "nvme cmd x%px, xri x%x, " 1227 "xb %d\n", 1228 lpfc_ncmd, nCmd, 1229 lpfc_ncmd->cur_iocbq.sli4_xritag, 1230 bf_get(lpfc_wcqe_c_xb, wcqe)); 1231 fallthrough; 1232 default: 1233out_err: 1234 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1235 "6072 NVME Completion Error: xri %x " 1236 "status x%x result x%x [x%x] " 1237 "placed x%x\n", 1238 lpfc_ncmd->cur_iocbq.sli4_xritag, 1239 lpfc_ncmd->status, lpfc_ncmd->result, 1240 wcqe->parameter, 1241 wcqe->total_data_placed); 1242 nCmd->transferred_length = 0; 1243 nCmd->rcv_rsplen = 0; 1244 nCmd->status = NVME_SC_INTERNAL; 1245 } 1246 } 1247 1248 /* pick up SLI4 exhange busy condition */ 1249 if (bf_get(lpfc_wcqe_c_xb, wcqe)) 1250 lpfc_ncmd->flags |= LPFC_SBUF_XBUSY; 1251 else 1252 lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY; 1253 1254 /* Update stats and complete the IO. There is 1255 * no need for dma unprep because the nvme_transport 1256 * owns the dma address. 1257 */ 1258#ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1259 if (lpfc_ncmd->ts_cmd_start) { 1260 lpfc_ncmd->ts_isr_cmpl = pwqeIn->isr_timestamp; 1261 lpfc_ncmd->ts_data_io = ktime_get_ns(); 1262 phba->ktime_last_cmd = lpfc_ncmd->ts_data_io; 1263 lpfc_io_ktime(phba, lpfc_ncmd); 1264 } 1265 if (unlikely(phba->hdwqstat_on & LPFC_CHECK_NVME_IO)) { 1266 cpu = raw_smp_processor_id(); 1267 this_cpu_inc(phba->sli4_hba.c_stat->cmpl_io); 1268 if (lpfc_ncmd->cpu != cpu) 1269 lpfc_printf_vlog(vport, 1270 KERN_INFO, LOG_NVME_IOERR, 1271 "6701 CPU Check cmpl: " 1272 "cpu %d expect %d\n", 1273 cpu, lpfc_ncmd->cpu); 1274 } 1275#endif 1276 1277 /* NVME targets need completion held off until the abort exchange 1278 * completes unless the NVME Rport is getting unregistered. 1279 */ 1280 1281 if (!(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) { 1282 freqpriv = nCmd->private; 1283 freqpriv->nvme_buf = NULL; 1284 lpfc_ncmd->nvmeCmd = NULL; 1285 spin_unlock(&lpfc_ncmd->buf_lock); 1286 nCmd->done(nCmd); 1287 } else 1288 spin_unlock(&lpfc_ncmd->buf_lock); 1289 1290 /* Call release with XB=1 to queue the IO into the abort list. */ 1291 lpfc_release_nvme_buf(phba, lpfc_ncmd); 1292} 1293 1294 1295/** 1296 * lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO 1297 * @vport: pointer to a host virtual N_Port data structure 1298 * @lpfcn_cmd: Pointer to lpfc scsi command 1299 * @pnode: pointer to a node-list data structure 1300 * @cstat: pointer to the control status structure 1301 * 1302 * Driver registers this routine as it io request handler. This 1303 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 1304 * data structure to the rport indicated in @lpfc_nvme_rport. 1305 * 1306 * Return value : 1307 * 0 - Success 1308 * TODO: What are the failure codes. 1309 **/ 1310static int 1311lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport, 1312 struct lpfc_io_buf *lpfc_ncmd, 1313 struct lpfc_nodelist *pnode, 1314 struct lpfc_fc4_ctrl_stat *cstat) 1315{ 1316 struct lpfc_hba *phba = vport->phba; 1317 struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd; 1318 struct nvme_common_command *sqe; 1319 struct lpfc_iocbq *pwqeq = &lpfc_ncmd->cur_iocbq; 1320 union lpfc_wqe128 *wqe = &pwqeq->wqe; 1321 uint32_t req_len; 1322 1323 if (!NLP_CHK_NODE_ACT(pnode)) 1324 return -EINVAL; 1325 1326 /* 1327 * There are three possibilities here - use scatter-gather segment, use 1328 * the single mapping, or neither. 1329 */ 1330 if (nCmd->sg_cnt) { 1331 if (nCmd->io_dir == NVMEFC_FCP_WRITE) { 1332 /* From the iwrite template, initialize words 7 - 11 */ 1333 memcpy(&wqe->words[7], 1334 &lpfc_iwrite_cmd_template.words[7], 1335 sizeof(uint32_t) * 5); 1336 1337 /* Word 4 */ 1338 wqe->fcp_iwrite.total_xfer_len = nCmd->payload_length; 1339 1340 /* Word 5 */ 1341 if ((phba->cfg_nvme_enable_fb) && 1342 (pnode->nlp_flag & NLP_FIRSTBURST)) { 1343 req_len = lpfc_ncmd->nvmeCmd->payload_length; 1344 if (req_len < pnode->nvme_fb_size) 1345 wqe->fcp_iwrite.initial_xfer_len = 1346 req_len; 1347 else 1348 wqe->fcp_iwrite.initial_xfer_len = 1349 pnode->nvme_fb_size; 1350 } else { 1351 wqe->fcp_iwrite.initial_xfer_len = 0; 1352 } 1353 cstat->output_requests++; 1354 } else { 1355 /* From the iread template, initialize words 7 - 11 */ 1356 memcpy(&wqe->words[7], 1357 &lpfc_iread_cmd_template.words[7], 1358 sizeof(uint32_t) * 5); 1359 1360 /* Word 4 */ 1361 wqe->fcp_iread.total_xfer_len = nCmd->payload_length; 1362 1363 /* Word 5 */ 1364 wqe->fcp_iread.rsrvd5 = 0; 1365 1366 cstat->input_requests++; 1367 } 1368 } else { 1369 /* From the icmnd template, initialize words 4 - 11 */ 1370 memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4], 1371 sizeof(uint32_t) * 8); 1372 cstat->control_requests++; 1373 } 1374 1375 if (pnode->nlp_nvme_info & NLP_NVME_NSLER) { 1376 bf_set(wqe_erp, &wqe->generic.wqe_com, 1); 1377 sqe = &((struct nvme_fc_cmd_iu *) 1378 nCmd->cmdaddr)->sqe.common; 1379 if (sqe->opcode == nvme_admin_async_event) 1380 bf_set(wqe_ffrq, &wqe->generic.wqe_com, 1); 1381 } 1382 1383 /* 1384 * Finish initializing those WQE fields that are independent 1385 * of the nvme_cmnd request_buffer 1386 */ 1387 1388 /* Word 3 */ 1389 bf_set(payload_offset_len, &wqe->fcp_icmd, 1390 (nCmd->rsplen + nCmd->cmdlen)); 1391 1392 /* Word 6 */ 1393 bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com, 1394 phba->sli4_hba.rpi_ids[pnode->nlp_rpi]); 1395 bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag); 1396 1397 /* Word 8 */ 1398 wqe->generic.wqe_com.abort_tag = pwqeq->iotag; 1399 1400 /* Word 9 */ 1401 bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag); 1402 1403 /* Words 13 14 15 are for PBDE support */ 1404 1405 pwqeq->vport = vport; 1406 return 0; 1407} 1408 1409 1410/** 1411 * lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO 1412 * @vport: pointer to a host virtual N_Port data structure 1413 * @lpfcn_cmd: Pointer to lpfc scsi command 1414 * 1415 * Driver registers this routine as it io request handler. This 1416 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 1417 * data structure to the rport indicated in @lpfc_nvme_rport. 1418 * 1419 * Return value : 1420 * 0 - Success 1421 * TODO: What are the failure codes. 1422 **/ 1423static int 1424lpfc_nvme_prep_io_dma(struct lpfc_vport *vport, 1425 struct lpfc_io_buf *lpfc_ncmd) 1426{ 1427 struct lpfc_hba *phba = vport->phba; 1428 struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd; 1429 union lpfc_wqe128 *wqe = &lpfc_ncmd->cur_iocbq.wqe; 1430 struct sli4_sge *sgl = lpfc_ncmd->dma_sgl; 1431 struct sli4_hybrid_sgl *sgl_xtra = NULL; 1432 struct scatterlist *data_sg; 1433 struct sli4_sge *first_data_sgl; 1434 struct ulp_bde64 *bde; 1435 dma_addr_t physaddr = 0; 1436 uint32_t num_bde = 0; 1437 uint32_t dma_len = 0; 1438 uint32_t dma_offset = 0; 1439 int nseg, i, j; 1440 bool lsp_just_set = false; 1441 1442 /* Fix up the command and response DMA stuff. */ 1443 lpfc_nvme_adj_fcp_sgls(vport, lpfc_ncmd, nCmd); 1444 1445 /* 1446 * There are three possibilities here - use scatter-gather segment, use 1447 * the single mapping, or neither. 1448 */ 1449 if (nCmd->sg_cnt) { 1450 /* 1451 * Jump over the cmd and rsp SGEs. The fix routine 1452 * has already adjusted for this. 1453 */ 1454 sgl += 2; 1455 1456 first_data_sgl = sgl; 1457 lpfc_ncmd->seg_cnt = nCmd->sg_cnt; 1458 if (lpfc_ncmd->seg_cnt > lpfc_nvme_template.max_sgl_segments) { 1459 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, 1460 "6058 Too many sg segments from " 1461 "NVME Transport. Max %d, " 1462 "nvmeIO sg_cnt %d\n", 1463 phba->cfg_nvme_seg_cnt + 1, 1464 lpfc_ncmd->seg_cnt); 1465 lpfc_ncmd->seg_cnt = 0; 1466 return 1; 1467 } 1468 1469 /* 1470 * The driver established a maximum scatter-gather segment count 1471 * during probe that limits the number of sg elements in any 1472 * single nvme command. Just run through the seg_cnt and format 1473 * the sge's. 1474 */ 1475 nseg = nCmd->sg_cnt; 1476 data_sg = nCmd->first_sgl; 1477 1478 /* for tracking the segment boundaries */ 1479 j = 2; 1480 for (i = 0; i < nseg; i++) { 1481 if (data_sg == NULL) { 1482 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, 1483 "6059 dptr err %d, nseg %d\n", 1484 i, nseg); 1485 lpfc_ncmd->seg_cnt = 0; 1486 return 1; 1487 } 1488 1489 sgl->word2 = 0; 1490 if ((num_bde + 1) == nseg) { 1491 bf_set(lpfc_sli4_sge_last, sgl, 1); 1492 bf_set(lpfc_sli4_sge_type, sgl, 1493 LPFC_SGE_TYPE_DATA); 1494 } else { 1495 bf_set(lpfc_sli4_sge_last, sgl, 0); 1496 1497 /* expand the segment */ 1498 if (!lsp_just_set && 1499 !((j + 1) % phba->border_sge_num) && 1500 ((nseg - 1) != i)) { 1501 /* set LSP type */ 1502 bf_set(lpfc_sli4_sge_type, sgl, 1503 LPFC_SGE_TYPE_LSP); 1504 1505 sgl_xtra = lpfc_get_sgl_per_hdwq( 1506 phba, lpfc_ncmd); 1507 1508 if (unlikely(!sgl_xtra)) { 1509 lpfc_ncmd->seg_cnt = 0; 1510 return 1; 1511 } 1512 sgl->addr_lo = cpu_to_le32(putPaddrLow( 1513 sgl_xtra->dma_phys_sgl)); 1514 sgl->addr_hi = cpu_to_le32(putPaddrHigh( 1515 sgl_xtra->dma_phys_sgl)); 1516 1517 } else { 1518 bf_set(lpfc_sli4_sge_type, sgl, 1519 LPFC_SGE_TYPE_DATA); 1520 } 1521 } 1522 1523 if (!(bf_get(lpfc_sli4_sge_type, sgl) & 1524 LPFC_SGE_TYPE_LSP)) { 1525 if ((nseg - 1) == i) 1526 bf_set(lpfc_sli4_sge_last, sgl, 1); 1527 1528 physaddr = data_sg->dma_address; 1529 dma_len = data_sg->length; 1530 sgl->addr_lo = cpu_to_le32( 1531 putPaddrLow(physaddr)); 1532 sgl->addr_hi = cpu_to_le32( 1533 putPaddrHigh(physaddr)); 1534 1535 bf_set(lpfc_sli4_sge_offset, sgl, dma_offset); 1536 sgl->word2 = cpu_to_le32(sgl->word2); 1537 sgl->sge_len = cpu_to_le32(dma_len); 1538 1539 dma_offset += dma_len; 1540 data_sg = sg_next(data_sg); 1541 1542 sgl++; 1543 1544 lsp_just_set = false; 1545 } else { 1546 sgl->word2 = cpu_to_le32(sgl->word2); 1547 1548 sgl->sge_len = cpu_to_le32( 1549 phba->cfg_sg_dma_buf_size); 1550 1551 sgl = (struct sli4_sge *)sgl_xtra->dma_sgl; 1552 i = i - 1; 1553 1554 lsp_just_set = true; 1555 } 1556 1557 j++; 1558 } 1559 if (phba->cfg_enable_pbde) { 1560 /* Use PBDE support for first SGL only, offset == 0 */ 1561 /* Words 13-15 */ 1562 bde = (struct ulp_bde64 *) 1563 &wqe->words[13]; 1564 bde->addrLow = first_data_sgl->addr_lo; 1565 bde->addrHigh = first_data_sgl->addr_hi; 1566 bde->tus.f.bdeSize = 1567 le32_to_cpu(first_data_sgl->sge_len); 1568 bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64; 1569 bde->tus.w = cpu_to_le32(bde->tus.w); 1570 /* wqe_pbde is 1 in template */ 1571 } else { 1572 memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3)); 1573 bf_set(wqe_pbde, &wqe->generic.wqe_com, 0); 1574 } 1575 1576 } else { 1577 lpfc_ncmd->seg_cnt = 0; 1578 1579 /* For this clause to be valid, the payload_length 1580 * and sg_cnt must zero. 1581 */ 1582 if (nCmd->payload_length != 0) { 1583 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, 1584 "6063 NVME DMA Prep Err: sg_cnt %d " 1585 "payload_length x%x\n", 1586 nCmd->sg_cnt, nCmd->payload_length); 1587 return 1; 1588 } 1589 } 1590 return 0; 1591} 1592 1593/** 1594 * lpfc_nvme_fcp_io_submit - Issue an NVME-over-FCP IO 1595 * @lpfc_pnvme: Pointer to the driver's nvme instance data 1596 * @lpfc_nvme_lport: Pointer to the driver's local port data 1597 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 1598 * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 1599 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 1600 * 1601 * Driver registers this routine as it io request handler. This 1602 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 1603 * data structure to the rport 1604 indicated in @lpfc_nvme_rport. 1605 * 1606 * Return value : 1607 * 0 - Success 1608 * TODO: What are the failure codes. 1609 **/ 1610static int 1611lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, 1612 struct nvme_fc_remote_port *pnvme_rport, 1613 void *hw_queue_handle, 1614 struct nvmefc_fcp_req *pnvme_fcreq) 1615{ 1616 int ret = 0; 1617 int expedite = 0; 1618 int idx, cpu; 1619 struct lpfc_nvme_lport *lport; 1620 struct lpfc_fc4_ctrl_stat *cstat; 1621 struct lpfc_vport *vport; 1622 struct lpfc_hba *phba; 1623 struct lpfc_nodelist *ndlp; 1624 struct lpfc_io_buf *lpfc_ncmd; 1625 struct lpfc_nvme_rport *rport; 1626 struct lpfc_nvme_qhandle *lpfc_queue_info; 1627 struct lpfc_nvme_fcpreq_priv *freqpriv; 1628 struct nvme_common_command *sqe; 1629#ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1630 uint64_t start = 0; 1631#endif 1632 1633 /* Validate pointers. LLDD fault handling with transport does 1634 * have timing races. 1635 */ 1636 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 1637 if (unlikely(!lport)) { 1638 ret = -EINVAL; 1639 goto out_fail; 1640 } 1641 1642 vport = lport->vport; 1643 1644 if (unlikely(!hw_queue_handle)) { 1645 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1646 "6117 Fail IO, NULL hw_queue_handle\n"); 1647 atomic_inc(&lport->xmt_fcp_err); 1648 ret = -EBUSY; 1649 goto out_fail; 1650 } 1651 1652 phba = vport->phba; 1653 1654 if (unlikely(vport->load_flag & FC_UNLOADING)) { 1655 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1656 "6124 Fail IO, Driver unload\n"); 1657 atomic_inc(&lport->xmt_fcp_err); 1658 ret = -ENODEV; 1659 goto out_fail; 1660 } 1661 1662 freqpriv = pnvme_fcreq->private; 1663 if (unlikely(!freqpriv)) { 1664 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1665 "6158 Fail IO, NULL request data\n"); 1666 atomic_inc(&lport->xmt_fcp_err); 1667 ret = -EINVAL; 1668 goto out_fail; 1669 } 1670 1671#ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1672 if (phba->ktime_on) 1673 start = ktime_get_ns(); 1674#endif 1675 rport = (struct lpfc_nvme_rport *)pnvme_rport->private; 1676 lpfc_queue_info = (struct lpfc_nvme_qhandle *)hw_queue_handle; 1677 1678 /* 1679 * Catch race where our node has transitioned, but the 1680 * transport is still transitioning. 1681 */ 1682 ndlp = rport->ndlp; 1683 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { 1684 lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR, 1685 "6053 Busy IO, ndlp not ready: rport x%px " 1686 "ndlp x%px, DID x%06x\n", 1687 rport, ndlp, pnvme_rport->port_id); 1688 atomic_inc(&lport->xmt_fcp_err); 1689 ret = -EBUSY; 1690 goto out_fail; 1691 } 1692 1693 /* The remote node has to be a mapped target or it's an error. */ 1694 if ((ndlp->nlp_type & NLP_NVME_TARGET) && 1695 (ndlp->nlp_state != NLP_STE_MAPPED_NODE)) { 1696 lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR, 1697 "6036 Fail IO, DID x%06x not ready for " 1698 "IO. State x%x, Type x%x Flg x%x\n", 1699 pnvme_rport->port_id, 1700 ndlp->nlp_state, ndlp->nlp_type, 1701 ndlp->upcall_flags); 1702 atomic_inc(&lport->xmt_fcp_bad_ndlp); 1703 ret = -EBUSY; 1704 goto out_fail; 1705 1706 } 1707 1708 /* Currently only NVME Keep alive commands should be expedited 1709 * if the driver runs out of a resource. These should only be 1710 * issued on the admin queue, qidx 0 1711 */ 1712 if (!lpfc_queue_info->qidx && !pnvme_fcreq->sg_cnt) { 1713 sqe = &((struct nvme_fc_cmd_iu *) 1714 pnvme_fcreq->cmdaddr)->sqe.common; 1715 if (sqe->opcode == nvme_admin_keep_alive) 1716 expedite = 1; 1717 } 1718 1719 /* The node is shared with FCP IO, make sure the IO pending count does 1720 * not exceed the programmed depth. 1721 */ 1722 if (lpfc_ndlp_check_qdepth(phba, ndlp)) { 1723 if ((atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) && 1724 !expedite) { 1725 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1726 "6174 Fail IO, ndlp qdepth exceeded: " 1727 "idx %d DID %x pend %d qdepth %d\n", 1728 lpfc_queue_info->index, ndlp->nlp_DID, 1729 atomic_read(&ndlp->cmd_pending), 1730 ndlp->cmd_qdepth); 1731 atomic_inc(&lport->xmt_fcp_qdepth); 1732 ret = -EBUSY; 1733 goto out_fail; 1734 } 1735 } 1736 1737 /* Lookup Hardware Queue index based on fcp_io_sched module parameter */ 1738 if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) { 1739 idx = lpfc_queue_info->index; 1740 } else { 1741 cpu = raw_smp_processor_id(); 1742 idx = phba->sli4_hba.cpu_map[cpu].hdwq; 1743 } 1744 1745 lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, idx, expedite); 1746 if (lpfc_ncmd == NULL) { 1747 atomic_inc(&lport->xmt_fcp_noxri); 1748 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1749 "6065 Fail IO, driver buffer pool is empty: " 1750 "idx %d DID %x\n", 1751 lpfc_queue_info->index, ndlp->nlp_DID); 1752 ret = -EBUSY; 1753 goto out_fail; 1754 } 1755#ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1756 if (start) { 1757 lpfc_ncmd->ts_cmd_start = start; 1758 lpfc_ncmd->ts_last_cmd = phba->ktime_last_cmd; 1759 } else { 1760 lpfc_ncmd->ts_cmd_start = 0; 1761 } 1762#endif 1763 1764 /* 1765 * Store the data needed by the driver to issue, abort, and complete 1766 * an IO. 1767 * Do not let the IO hang out forever. There is no midlayer issuing 1768 * an abort so inform the FW of the maximum IO pending time. 1769 */ 1770 freqpriv->nvme_buf = lpfc_ncmd; 1771 lpfc_ncmd->nvmeCmd = pnvme_fcreq; 1772 lpfc_ncmd->ndlp = ndlp; 1773 lpfc_ncmd->qidx = lpfc_queue_info->qidx; 1774 1775 /* 1776 * Issue the IO on the WQ indicated by index in the hw_queue_handle. 1777 * This identfier was create in our hardware queue create callback 1778 * routine. The driver now is dependent on the IO queue steering from 1779 * the transport. We are trusting the upper NVME layers know which 1780 * index to use and that they have affinitized a CPU to this hardware 1781 * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ. 1782 */ 1783 lpfc_ncmd->cur_iocbq.hba_wqidx = idx; 1784 cstat = &phba->sli4_hba.hdwq[idx].nvme_cstat; 1785 1786 lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp, cstat); 1787 ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd); 1788 if (ret) { 1789 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1790 "6175 Fail IO, Prep DMA: " 1791 "idx %d DID %x\n", 1792 lpfc_queue_info->index, ndlp->nlp_DID); 1793 atomic_inc(&lport->xmt_fcp_err); 1794 ret = -ENOMEM; 1795 goto out_free_nvme_buf; 1796 } 1797 1798 lpfc_nvmeio_data(phba, "NVME FCP XMIT: xri x%x idx %d to %06x\n", 1799 lpfc_ncmd->cur_iocbq.sli4_xritag, 1800 lpfc_queue_info->index, ndlp->nlp_DID); 1801 1802 ret = lpfc_sli4_issue_wqe(phba, lpfc_ncmd->hdwq, &lpfc_ncmd->cur_iocbq); 1803 if (ret) { 1804 atomic_inc(&lport->xmt_fcp_wqerr); 1805 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1806 "6113 Fail IO, Could not issue WQE err %x " 1807 "sid: x%x did: x%x oxid: x%x\n", 1808 ret, vport->fc_myDID, ndlp->nlp_DID, 1809 lpfc_ncmd->cur_iocbq.sli4_xritag); 1810 goto out_free_nvme_buf; 1811 } 1812 1813 if (phba->cfg_xri_rebalancing) 1814 lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_ncmd->hdwq_no); 1815 1816#ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1817 if (lpfc_ncmd->ts_cmd_start) 1818 lpfc_ncmd->ts_cmd_wqput = ktime_get_ns(); 1819 1820 if (phba->hdwqstat_on & LPFC_CHECK_NVME_IO) { 1821 cpu = raw_smp_processor_id(); 1822 this_cpu_inc(phba->sli4_hba.c_stat->xmt_io); 1823 lpfc_ncmd->cpu = cpu; 1824 if (idx != cpu) 1825 lpfc_printf_vlog(vport, 1826 KERN_INFO, LOG_NVME_IOERR, 1827 "6702 CPU Check cmd: " 1828 "cpu %d wq %d\n", 1829 lpfc_ncmd->cpu, 1830 lpfc_queue_info->index); 1831 } 1832#endif 1833 return 0; 1834 1835 out_free_nvme_buf: 1836 if (lpfc_ncmd->nvmeCmd->sg_cnt) { 1837 if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE) 1838 cstat->output_requests--; 1839 else 1840 cstat->input_requests--; 1841 } else 1842 cstat->control_requests--; 1843 lpfc_release_nvme_buf(phba, lpfc_ncmd); 1844 out_fail: 1845 return ret; 1846} 1847 1848/** 1849 * lpfc_nvme_abort_fcreq_cmpl - Complete an NVME FCP abort request. 1850 * @phba: Pointer to HBA context object 1851 * @cmdiocb: Pointer to command iocb object. 1852 * @rspiocb: Pointer to response iocb object. 1853 * 1854 * This is the callback function for any NVME FCP IO that was aborted. 1855 * 1856 * Return value: 1857 * None 1858 **/ 1859void 1860lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, 1861 struct lpfc_wcqe_complete *abts_cmpl) 1862{ 1863 lpfc_printf_log(phba, KERN_INFO, LOG_NVME, 1864 "6145 ABORT_XRI_CN completing on rpi x%x " 1865 "original iotag x%x, abort cmd iotag x%x " 1866 "req_tag x%x, status x%x, hwstatus x%x\n", 1867 cmdiocb->iocb.un.acxri.abortContextTag, 1868 cmdiocb->iocb.un.acxri.abortIoTag, 1869 cmdiocb->iotag, 1870 bf_get(lpfc_wcqe_c_request_tag, abts_cmpl), 1871 bf_get(lpfc_wcqe_c_status, abts_cmpl), 1872 bf_get(lpfc_wcqe_c_hw_status, abts_cmpl)); 1873 lpfc_sli_release_iocbq(phba, cmdiocb); 1874} 1875 1876/** 1877 * lpfc_nvme_fcp_abort - Issue an NVME-over-FCP ABTS 1878 * @lpfc_pnvme: Pointer to the driver's nvme instance data 1879 * @lpfc_nvme_lport: Pointer to the driver's local port data 1880 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 1881 * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 1882 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 1883 * 1884 * Driver registers this routine as its nvme request io abort handler. This 1885 * routine issues an fcp Abort WQE with data from the @lpfc_nvme_fcpreq 1886 * data structure to the rport indicated in @lpfc_nvme_rport. This routine 1887 * is executed asynchronously - one the target is validated as "MAPPED" and 1888 * ready for IO, the driver issues the abort request and returns. 1889 * 1890 * Return value: 1891 * None 1892 **/ 1893static void 1894lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport, 1895 struct nvme_fc_remote_port *pnvme_rport, 1896 void *hw_queue_handle, 1897 struct nvmefc_fcp_req *pnvme_fcreq) 1898{ 1899 struct lpfc_nvme_lport *lport; 1900 struct lpfc_vport *vport; 1901 struct lpfc_hba *phba; 1902 struct lpfc_io_buf *lpfc_nbuf; 1903 struct lpfc_iocbq *abts_buf; 1904 struct lpfc_iocbq *nvmereq_wqe; 1905 struct lpfc_nvme_fcpreq_priv *freqpriv; 1906 unsigned long flags; 1907 int ret_val; 1908 1909 /* Validate pointers. LLDD fault handling with transport does 1910 * have timing races. 1911 */ 1912 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 1913 if (unlikely(!lport)) 1914 return; 1915 1916 vport = lport->vport; 1917 1918 if (unlikely(!hw_queue_handle)) { 1919 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 1920 "6129 Fail Abort, HW Queue Handle NULL.\n"); 1921 return; 1922 } 1923 1924 phba = vport->phba; 1925 freqpriv = pnvme_fcreq->private; 1926 1927 if (unlikely(!freqpriv)) 1928 return; 1929 if (vport->load_flag & FC_UNLOADING) 1930 return; 1931 1932 /* Announce entry to new IO submit field. */ 1933 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 1934 "6002 Abort Request to rport DID x%06x " 1935 "for nvme_fc_req x%px\n", 1936 pnvme_rport->port_id, 1937 pnvme_fcreq); 1938 1939 /* If the hba is getting reset, this flag is set. It is 1940 * cleared when the reset is complete and rings reestablished. 1941 */ 1942 spin_lock_irqsave(&phba->hbalock, flags); 1943 /* driver queued commands are in process of being flushed */ 1944 if (phba->hba_flag & HBA_IOQ_FLUSH) { 1945 spin_unlock_irqrestore(&phba->hbalock, flags); 1946 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 1947 "6139 Driver in reset cleanup - flushing " 1948 "NVME Req now. hba_flag x%x\n", 1949 phba->hba_flag); 1950 return; 1951 } 1952 1953 lpfc_nbuf = freqpriv->nvme_buf; 1954 if (!lpfc_nbuf) { 1955 spin_unlock_irqrestore(&phba->hbalock, flags); 1956 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 1957 "6140 NVME IO req has no matching lpfc nvme " 1958 "io buffer. Skipping abort req.\n"); 1959 return; 1960 } else if (!lpfc_nbuf->nvmeCmd) { 1961 spin_unlock_irqrestore(&phba->hbalock, flags); 1962 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 1963 "6141 lpfc NVME IO req has no nvme_fcreq " 1964 "io buffer. Skipping abort req.\n"); 1965 return; 1966 } 1967 nvmereq_wqe = &lpfc_nbuf->cur_iocbq; 1968 1969 /* Guard against IO completion being called at same time */ 1970 spin_lock(&lpfc_nbuf->buf_lock); 1971 1972 /* 1973 * The lpfc_nbuf and the mapped nvme_fcreq in the driver's 1974 * state must match the nvme_fcreq passed by the nvme 1975 * transport. If they don't match, it is likely the driver 1976 * has already completed the NVME IO and the nvme transport 1977 * has not seen it yet. 1978 */ 1979 if (lpfc_nbuf->nvmeCmd != pnvme_fcreq) { 1980 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 1981 "6143 NVME req mismatch: " 1982 "lpfc_nbuf x%px nvmeCmd x%px, " 1983 "pnvme_fcreq x%px. Skipping Abort xri x%x\n", 1984 lpfc_nbuf, lpfc_nbuf->nvmeCmd, 1985 pnvme_fcreq, nvmereq_wqe->sli4_xritag); 1986 goto out_unlock; 1987 } 1988 1989 /* Don't abort IOs no longer on the pending queue. */ 1990 if (!(nvmereq_wqe->iocb_flag & LPFC_IO_ON_TXCMPLQ)) { 1991 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 1992 "6142 NVME IO req x%px not queued - skipping " 1993 "abort req xri x%x\n", 1994 pnvme_fcreq, nvmereq_wqe->sli4_xritag); 1995 goto out_unlock; 1996 } 1997 1998 atomic_inc(&lport->xmt_fcp_abort); 1999 lpfc_nvmeio_data(phba, "NVME FCP ABORT: xri x%x idx %d to %06x\n", 2000 nvmereq_wqe->sli4_xritag, 2001 nvmereq_wqe->hba_wqidx, pnvme_rport->port_id); 2002 2003 /* Outstanding abort is in progress */ 2004 if (nvmereq_wqe->iocb_flag & LPFC_DRIVER_ABORTED) { 2005 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 2006 "6144 Outstanding NVME I/O Abort Request " 2007 "still pending on nvme_fcreq x%px, " 2008 "lpfc_ncmd %px xri x%x\n", 2009 pnvme_fcreq, lpfc_nbuf, 2010 nvmereq_wqe->sli4_xritag); 2011 goto out_unlock; 2012 } 2013 2014 abts_buf = __lpfc_sli_get_iocbq(phba); 2015 if (!abts_buf) { 2016 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 2017 "6136 No available abort wqes. Skipping " 2018 "Abts req for nvme_fcreq x%px xri x%x\n", 2019 pnvme_fcreq, nvmereq_wqe->sli4_xritag); 2020 goto out_unlock; 2021 } 2022 2023 /* Ready - mark outstanding as aborted by driver. */ 2024 nvmereq_wqe->iocb_flag |= LPFC_DRIVER_ABORTED; 2025 2026 lpfc_nvme_prep_abort_wqe(abts_buf, nvmereq_wqe->sli4_xritag, 0); 2027 2028 /* ABTS WQE must go to the same WQ as the WQE to be aborted */ 2029 abts_buf->iocb_flag |= LPFC_IO_NVME; 2030 abts_buf->hba_wqidx = nvmereq_wqe->hba_wqidx; 2031 abts_buf->vport = vport; 2032 abts_buf->wqe_cmpl = lpfc_nvme_abort_fcreq_cmpl; 2033 ret_val = lpfc_sli4_issue_wqe(phba, lpfc_nbuf->hdwq, abts_buf); 2034 spin_unlock(&lpfc_nbuf->buf_lock); 2035 spin_unlock_irqrestore(&phba->hbalock, flags); 2036 if (ret_val) { 2037 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 2038 "6137 Failed abts issue_wqe with status x%x " 2039 "for nvme_fcreq x%px.\n", 2040 ret_val, pnvme_fcreq); 2041 lpfc_sli_release_iocbq(phba, abts_buf); 2042 return; 2043 } 2044 2045 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 2046 "6138 Transport Abort NVME Request Issued for " 2047 "ox_id x%x on reqtag x%x\n", 2048 nvmereq_wqe->sli4_xritag, 2049 abts_buf->iotag); 2050 return; 2051 2052out_unlock: 2053 spin_unlock(&lpfc_nbuf->buf_lock); 2054 spin_unlock_irqrestore(&phba->hbalock, flags); 2055 return; 2056} 2057 2058/* Declare and initialization an instance of the FC NVME template. */ 2059static struct nvme_fc_port_template lpfc_nvme_template = { 2060 /* initiator-based functions */ 2061 .localport_delete = lpfc_nvme_localport_delete, 2062 .remoteport_delete = lpfc_nvme_remoteport_delete, 2063 .create_queue = lpfc_nvme_create_queue, 2064 .delete_queue = lpfc_nvme_delete_queue, 2065 .ls_req = lpfc_nvme_ls_req, 2066 .fcp_io = lpfc_nvme_fcp_io_submit, 2067 .ls_abort = lpfc_nvme_ls_abort, 2068 .fcp_abort = lpfc_nvme_fcp_abort, 2069 .xmt_ls_rsp = lpfc_nvme_xmt_ls_rsp, 2070 2071 .max_hw_queues = 1, 2072 .max_sgl_segments = LPFC_NVME_DEFAULT_SEGS, 2073 .max_dif_sgl_segments = LPFC_NVME_DEFAULT_SEGS, 2074 .dma_boundary = 0xFFFFFFFF, 2075 2076 /* Sizes of additional private data for data structures. 2077 * No use for the last two sizes at this time. 2078 */ 2079 .local_priv_sz = sizeof(struct lpfc_nvme_lport), 2080 .remote_priv_sz = sizeof(struct lpfc_nvme_rport), 2081 .lsrqst_priv_sz = 0, 2082 .fcprqst_priv_sz = sizeof(struct lpfc_nvme_fcpreq_priv), 2083}; 2084 2085/** 2086 * lpfc_get_nvme_buf - Get a nvme buffer from io_buf_list of the HBA 2087 * @phba: The HBA for which this call is being executed. 2088 * 2089 * This routine removes a nvme buffer from head of @hdwq io_buf_list 2090 * and returns to caller. 2091 * 2092 * Return codes: 2093 * NULL - Error 2094 * Pointer to lpfc_nvme_buf - Success 2095 **/ 2096static struct lpfc_io_buf * 2097lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, 2098 int idx, int expedite) 2099{ 2100 struct lpfc_io_buf *lpfc_ncmd; 2101 struct lpfc_sli4_hdw_queue *qp; 2102 struct sli4_sge *sgl; 2103 struct lpfc_iocbq *pwqeq; 2104 union lpfc_wqe128 *wqe; 2105 2106 lpfc_ncmd = lpfc_get_io_buf(phba, NULL, idx, expedite); 2107 2108 if (lpfc_ncmd) { 2109 pwqeq = &(lpfc_ncmd->cur_iocbq); 2110 wqe = &pwqeq->wqe; 2111 2112 /* Setup key fields in buffer that may have been changed 2113 * if other protocols used this buffer. 2114 */ 2115 pwqeq->iocb_flag = LPFC_IO_NVME; 2116 pwqeq->wqe_cmpl = lpfc_nvme_io_cmd_wqe_cmpl; 2117 lpfc_ncmd->start_time = jiffies; 2118 lpfc_ncmd->flags = 0; 2119 2120 /* Rsp SGE will be filled in when we rcv an IO 2121 * from the NVME Layer to be sent. 2122 * The cmd is going to be embedded so we need a SKIP SGE. 2123 */ 2124 sgl = lpfc_ncmd->dma_sgl; 2125 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP); 2126 bf_set(lpfc_sli4_sge_last, sgl, 0); 2127 sgl->word2 = cpu_to_le32(sgl->word2); 2128 /* Fill in word 3 / sgl_len during cmd submission */ 2129 2130 /* Initialize 64 bytes only */ 2131 memset(wqe, 0, sizeof(union lpfc_wqe)); 2132 2133 if (lpfc_ndlp_check_qdepth(phba, ndlp)) { 2134 atomic_inc(&ndlp->cmd_pending); 2135 lpfc_ncmd->flags |= LPFC_SBUF_BUMP_QDEPTH; 2136 } 2137 2138 } else { 2139 qp = &phba->sli4_hba.hdwq[idx]; 2140 qp->empty_io_bufs++; 2141 } 2142 2143 return lpfc_ncmd; 2144} 2145 2146/** 2147 * lpfc_release_nvme_buf: Return a nvme buffer back to hba nvme buf list. 2148 * @phba: The Hba for which this call is being executed. 2149 * @lpfc_ncmd: The nvme buffer which is being released. 2150 * 2151 * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba 2152 * lpfc_io_buf_list list. For SLI4 XRI's are tied to the nvme buffer 2153 * and cannot be reused for at least RA_TOV amount of time if it was 2154 * aborted. 2155 **/ 2156static void 2157lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd) 2158{ 2159 struct lpfc_sli4_hdw_queue *qp; 2160 unsigned long iflag = 0; 2161 2162 if ((lpfc_ncmd->flags & LPFC_SBUF_BUMP_QDEPTH) && lpfc_ncmd->ndlp) 2163 atomic_dec(&lpfc_ncmd->ndlp->cmd_pending); 2164 2165 lpfc_ncmd->ndlp = NULL; 2166 lpfc_ncmd->flags &= ~LPFC_SBUF_BUMP_QDEPTH; 2167 2168 qp = lpfc_ncmd->hdwq; 2169 if (unlikely(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) { 2170 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2171 "6310 XB release deferred for " 2172 "ox_id x%x on reqtag x%x\n", 2173 lpfc_ncmd->cur_iocbq.sli4_xritag, 2174 lpfc_ncmd->cur_iocbq.iotag); 2175 2176 spin_lock_irqsave(&qp->abts_io_buf_list_lock, iflag); 2177 list_add_tail(&lpfc_ncmd->list, 2178 &qp->lpfc_abts_io_buf_list); 2179 qp->abts_nvme_io_bufs++; 2180 spin_unlock_irqrestore(&qp->abts_io_buf_list_lock, iflag); 2181 } else 2182 lpfc_release_io_buf(phba, (struct lpfc_io_buf *)lpfc_ncmd, qp); 2183} 2184 2185/** 2186 * lpfc_nvme_create_localport - Create/Bind an nvme localport instance. 2187 * @pvport - the lpfc_vport instance requesting a localport. 2188 * 2189 * This routine is invoked to create an nvme localport instance to bind 2190 * to the nvme_fc_transport. It is called once during driver load 2191 * like lpfc_create_shost after all other services are initialized. 2192 * It requires a vport, vpi, and wwns at call time. Other localport 2193 * parameters are modified as the driver's FCID and the Fabric WWN 2194 * are established. 2195 * 2196 * Return codes 2197 * 0 - successful 2198 * -ENOMEM - no heap memory available 2199 * other values - from nvme registration upcall 2200 **/ 2201int 2202lpfc_nvme_create_localport(struct lpfc_vport *vport) 2203{ 2204 int ret = 0; 2205 struct lpfc_hba *phba = vport->phba; 2206 struct nvme_fc_port_info nfcp_info; 2207 struct nvme_fc_local_port *localport; 2208 struct lpfc_nvme_lport *lport; 2209 2210 /* Initialize this localport instance. The vport wwn usage ensures 2211 * that NPIV is accounted for. 2212 */ 2213 memset(&nfcp_info, 0, sizeof(struct nvme_fc_port_info)); 2214 nfcp_info.port_role = FC_PORT_ROLE_NVME_INITIATOR; 2215 nfcp_info.node_name = wwn_to_u64(vport->fc_nodename.u.wwn); 2216 nfcp_info.port_name = wwn_to_u64(vport->fc_portname.u.wwn); 2217 2218 /* We need to tell the transport layer + 1 because it takes page 2219 * alignment into account. When space for the SGL is allocated we 2220 * allocate + 3, one for cmd, one for rsp and one for this alignment 2221 */ 2222 lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1; 2223 2224 /* Advertise how many hw queues we support based on cfg_hdw_queue, 2225 * which will not exceed cpu count. 2226 */ 2227 lpfc_nvme_template.max_hw_queues = phba->cfg_hdw_queue; 2228 2229 if (!IS_ENABLED(CONFIG_NVME_FC)) 2230 return ret; 2231 2232 /* localport is allocated from the stack, but the registration 2233 * call allocates heap memory as well as the private area. 2234 */ 2235 2236 ret = nvme_fc_register_localport(&nfcp_info, &lpfc_nvme_template, 2237 &vport->phba->pcidev->dev, &localport); 2238 if (!ret) { 2239 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_NVME_DISC, 2240 "6005 Successfully registered local " 2241 "NVME port num %d, localP x%px, private " 2242 "x%px, sg_seg %d\n", 2243 localport->port_num, localport, 2244 localport->private, 2245 lpfc_nvme_template.max_sgl_segments); 2246 2247 /* Private is our lport size declared in the template. */ 2248 lport = (struct lpfc_nvme_lport *)localport->private; 2249 vport->localport = localport; 2250 lport->vport = vport; 2251 vport->nvmei_support = 1; 2252 2253 atomic_set(&lport->xmt_fcp_noxri, 0); 2254 atomic_set(&lport->xmt_fcp_bad_ndlp, 0); 2255 atomic_set(&lport->xmt_fcp_qdepth, 0); 2256 atomic_set(&lport->xmt_fcp_err, 0); 2257 atomic_set(&lport->xmt_fcp_wqerr, 0); 2258 atomic_set(&lport->xmt_fcp_abort, 0); 2259 atomic_set(&lport->xmt_ls_abort, 0); 2260 atomic_set(&lport->xmt_ls_err, 0); 2261 atomic_set(&lport->cmpl_fcp_xb, 0); 2262 atomic_set(&lport->cmpl_fcp_err, 0); 2263 atomic_set(&lport->cmpl_ls_xb, 0); 2264 atomic_set(&lport->cmpl_ls_err, 0); 2265 2266 atomic_set(&lport->fc4NvmeLsRequests, 0); 2267 atomic_set(&lport->fc4NvmeLsCmpls, 0); 2268 } 2269 2270 return ret; 2271} 2272 2273#if (IS_ENABLED(CONFIG_NVME_FC)) 2274/* lpfc_nvme_lport_unreg_wait - Wait for the host to complete an lport unreg. 2275 * 2276 * The driver has to wait for the host nvme transport to callback 2277 * indicating the localport has successfully unregistered all 2278 * resources. Since this is an uninterruptible wait, loop every ten 2279 * seconds and print a message indicating no progress. 2280 * 2281 * An uninterruptible wait is used because of the risk of transport-to- 2282 * driver state mismatch. 2283 */ 2284static void 2285lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport, 2286 struct lpfc_nvme_lport *lport, 2287 struct completion *lport_unreg_cmp) 2288{ 2289 u32 wait_tmo; 2290 int ret, i, pending = 0; 2291 struct lpfc_sli_ring *pring; 2292 struct lpfc_hba *phba = vport->phba; 2293 struct lpfc_sli4_hdw_queue *qp; 2294 int abts_scsi, abts_nvme; 2295 2296 /* Host transport has to clean up and confirm requiring an indefinite 2297 * wait. Print a message if a 10 second wait expires and renew the 2298 * wait. This is unexpected. 2299 */ 2300 wait_tmo = msecs_to_jiffies(LPFC_NVME_WAIT_TMO * 1000); 2301 while (true) { 2302 ret = wait_for_completion_timeout(lport_unreg_cmp, wait_tmo); 2303 if (unlikely(!ret)) { 2304 pending = 0; 2305 abts_scsi = 0; 2306 abts_nvme = 0; 2307 for (i = 0; i < phba->cfg_hdw_queue; i++) { 2308 qp = &phba->sli4_hba.hdwq[i]; 2309 pring = qp->io_wq->pring; 2310 if (!pring) 2311 continue; 2312 pending += pring->txcmplq_cnt; 2313 abts_scsi += qp->abts_scsi_io_bufs; 2314 abts_nvme += qp->abts_nvme_io_bufs; 2315 } 2316 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 2317 "6176 Lport x%px Localport x%px wait " 2318 "timed out. Pending %d [%d:%d]. " 2319 "Renewing.\n", 2320 lport, vport->localport, pending, 2321 abts_scsi, abts_nvme); 2322 continue; 2323 } 2324 break; 2325 } 2326 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 2327 "6177 Lport x%px Localport x%px Complete Success\n", 2328 lport, vport->localport); 2329} 2330#endif 2331 2332/** 2333 * lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport. 2334 * @pnvme: pointer to lpfc nvme data structure. 2335 * 2336 * This routine is invoked to destroy all lports bound to the phba. 2337 * The lport memory was allocated by the nvme fc transport and is 2338 * released there. This routine ensures all rports bound to the 2339 * lport have been disconnected. 2340 * 2341 **/ 2342void 2343lpfc_nvme_destroy_localport(struct lpfc_vport *vport) 2344{ 2345#if (IS_ENABLED(CONFIG_NVME_FC)) 2346 struct nvme_fc_local_port *localport; 2347 struct lpfc_nvme_lport *lport; 2348 int ret; 2349 DECLARE_COMPLETION_ONSTACK(lport_unreg_cmp); 2350 2351 if (vport->nvmei_support == 0) 2352 return; 2353 2354 localport = vport->localport; 2355 lport = (struct lpfc_nvme_lport *)localport->private; 2356 2357 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 2358 "6011 Destroying NVME localport x%px\n", 2359 localport); 2360 2361 /* lport's rport list is clear. Unregister 2362 * lport and release resources. 2363 */ 2364 lport->lport_unreg_cmp = &lport_unreg_cmp; 2365 ret = nvme_fc_unregister_localport(localport); 2366 2367 /* Wait for completion. This either blocks 2368 * indefinitely or succeeds 2369 */ 2370 lpfc_nvme_lport_unreg_wait(vport, lport, &lport_unreg_cmp); 2371 vport->localport = NULL; 2372 2373 /* Regardless of the unregister upcall response, clear 2374 * nvmei_support. All rports are unregistered and the 2375 * driver will clean up. 2376 */ 2377 vport->nvmei_support = 0; 2378 if (ret == 0) { 2379 lpfc_printf_vlog(vport, 2380 KERN_INFO, LOG_NVME_DISC, 2381 "6009 Unregistered lport Success\n"); 2382 } else { 2383 lpfc_printf_vlog(vport, 2384 KERN_INFO, LOG_NVME_DISC, 2385 "6010 Unregistered lport " 2386 "Failed, status x%x\n", 2387 ret); 2388 } 2389#endif 2390} 2391 2392void 2393lpfc_nvme_update_localport(struct lpfc_vport *vport) 2394{ 2395#if (IS_ENABLED(CONFIG_NVME_FC)) 2396 struct nvme_fc_local_port *localport; 2397 struct lpfc_nvme_lport *lport; 2398 2399 localport = vport->localport; 2400 if (!localport) { 2401 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME, 2402 "6710 Update NVME fail. No localport\n"); 2403 return; 2404 } 2405 lport = (struct lpfc_nvme_lport *)localport->private; 2406 if (!lport) { 2407 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME, 2408 "6171 Update NVME fail. localP x%px, No lport\n", 2409 localport); 2410 return; 2411 } 2412 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 2413 "6012 Update NVME lport x%px did x%x\n", 2414 localport, vport->fc_myDID); 2415 2416 localport->port_id = vport->fc_myDID; 2417 if (localport->port_id == 0) 2418 localport->port_role = FC_PORT_ROLE_NVME_DISCOVERY; 2419 else 2420 localport->port_role = FC_PORT_ROLE_NVME_INITIATOR; 2421 2422 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2423 "6030 bound lport x%px to DID x%06x\n", 2424 lport, localport->port_id); 2425#endif 2426} 2427 2428int 2429lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) 2430{ 2431#if (IS_ENABLED(CONFIG_NVME_FC)) 2432 int ret = 0; 2433 struct nvme_fc_local_port *localport; 2434 struct lpfc_nvme_lport *lport; 2435 struct lpfc_nvme_rport *rport; 2436 struct lpfc_nvme_rport *oldrport; 2437 struct nvme_fc_remote_port *remote_port; 2438 struct nvme_fc_port_info rpinfo; 2439 struct lpfc_nodelist *prev_ndlp = NULL; 2440 struct fc_rport *srport = ndlp->rport; 2441 2442 lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NVME_DISC, 2443 "6006 Register NVME PORT. DID x%06x nlptype x%x\n", 2444 ndlp->nlp_DID, ndlp->nlp_type); 2445 2446 localport = vport->localport; 2447 if (!localport) 2448 return 0; 2449 2450 lport = (struct lpfc_nvme_lport *)localport->private; 2451 2452 /* NVME rports are not preserved across devloss. 2453 * Just register this instance. Note, rpinfo->dev_loss_tmo 2454 * is left 0 to indicate accept transport defaults. The 2455 * driver communicates port role capabilities consistent 2456 * with the PRLI response data. 2457 */ 2458 memset(&rpinfo, 0, sizeof(struct nvme_fc_port_info)); 2459 rpinfo.port_id = ndlp->nlp_DID; 2460 if (ndlp->nlp_type & NLP_NVME_TARGET) 2461 rpinfo.port_role |= FC_PORT_ROLE_NVME_TARGET; 2462 if (ndlp->nlp_type & NLP_NVME_INITIATOR) 2463 rpinfo.port_role |= FC_PORT_ROLE_NVME_INITIATOR; 2464 2465 if (ndlp->nlp_type & NLP_NVME_DISCOVERY) 2466 rpinfo.port_role |= FC_PORT_ROLE_NVME_DISCOVERY; 2467 2468 rpinfo.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn); 2469 rpinfo.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn); 2470 if (srport) 2471 rpinfo.dev_loss_tmo = srport->dev_loss_tmo; 2472 else 2473 rpinfo.dev_loss_tmo = vport->cfg_devloss_tmo; 2474 2475 spin_lock_irq(&vport->phba->hbalock); 2476 oldrport = lpfc_ndlp_get_nrport(ndlp); 2477 if (oldrport) { 2478 prev_ndlp = oldrport->ndlp; 2479 spin_unlock_irq(&vport->phba->hbalock); 2480 } else { 2481 spin_unlock_irq(&vport->phba->hbalock); 2482 lpfc_nlp_get(ndlp); 2483 } 2484 2485 ret = nvme_fc_register_remoteport(localport, &rpinfo, &remote_port); 2486 if (!ret) { 2487 /* If the ndlp already has an nrport, this is just 2488 * a resume of the existing rport. Else this is a 2489 * new rport. 2490 */ 2491 /* Guard against an unregister/reregister 2492 * race that leaves the WAIT flag set. 2493 */ 2494 spin_lock_irq(&vport->phba->hbalock); 2495 ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG; 2496 spin_unlock_irq(&vport->phba->hbalock); 2497 rport = remote_port->private; 2498 if (oldrport) { 2499 2500 /* Sever the ndlp<->rport association 2501 * before dropping the ndlp ref from 2502 * register. 2503 */ 2504 spin_lock_irq(&vport->phba->hbalock); 2505 ndlp->nrport = NULL; 2506 ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG; 2507 spin_unlock_irq(&vport->phba->hbalock); 2508 rport->ndlp = NULL; 2509 rport->remoteport = NULL; 2510 2511 /* Reference only removed if previous NDLP is no longer 2512 * active. It might be just a swap and removing the 2513 * reference would cause a premature cleanup. 2514 */ 2515 if (prev_ndlp && prev_ndlp != ndlp) { 2516 if ((!NLP_CHK_NODE_ACT(prev_ndlp)) || 2517 (!prev_ndlp->nrport)) 2518 lpfc_nlp_put(prev_ndlp); 2519 } 2520 } 2521 2522 /* Clean bind the rport to the ndlp. */ 2523 rport->remoteport = remote_port; 2524 rport->lport = lport; 2525 rport->ndlp = ndlp; 2526 spin_lock_irq(&vport->phba->hbalock); 2527 ndlp->nrport = rport; 2528 spin_unlock_irq(&vport->phba->hbalock); 2529 lpfc_printf_vlog(vport, KERN_INFO, 2530 LOG_NVME_DISC | LOG_NODE, 2531 "6022 Bind lport x%px to remoteport x%px " 2532 "rport x%px WWNN 0x%llx, " 2533 "Rport WWPN 0x%llx DID " 2534 "x%06x Role x%x, ndlp %p prev_ndlp x%px\n", 2535 lport, remote_port, rport, 2536 rpinfo.node_name, rpinfo.port_name, 2537 rpinfo.port_id, rpinfo.port_role, 2538 ndlp, prev_ndlp); 2539 } else { 2540 lpfc_printf_vlog(vport, KERN_ERR, 2541 LOG_TRACE_EVENT, 2542 "6031 RemotePort Registration failed " 2543 "err: %d, DID x%06x\n", 2544 ret, ndlp->nlp_DID); 2545 } 2546 2547 return ret; 2548#else 2549 return 0; 2550#endif 2551} 2552 2553/** 2554 * lpfc_nvme_rescan_port - Check to see if we should rescan this remoteport 2555 * 2556 * If the ndlp represents an NVME Target, that we are logged into, 2557 * ping the NVME FC Transport layer to initiate a device rescan 2558 * on this remote NPort. 2559 */ 2560void 2561lpfc_nvme_rescan_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) 2562{ 2563#if (IS_ENABLED(CONFIG_NVME_FC)) 2564 struct lpfc_nvme_rport *nrport; 2565 struct nvme_fc_remote_port *remoteport = NULL; 2566 2567 spin_lock_irq(&vport->phba->hbalock); 2568 nrport = lpfc_ndlp_get_nrport(ndlp); 2569 if (nrport) 2570 remoteport = nrport->remoteport; 2571 spin_unlock_irq(&vport->phba->hbalock); 2572 2573 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2574 "6170 Rescan NPort DID x%06x type x%x " 2575 "state x%x nrport x%px remoteport x%px\n", 2576 ndlp->nlp_DID, ndlp->nlp_type, ndlp->nlp_state, 2577 nrport, remoteport); 2578 2579 if (!nrport || !remoteport) 2580 goto rescan_exit; 2581 2582 /* Only rescan if we are an NVME target in the MAPPED state */ 2583 if (remoteport->port_role & FC_PORT_ROLE_NVME_DISCOVERY && 2584 ndlp->nlp_state == NLP_STE_MAPPED_NODE) { 2585 nvme_fc_rescan_remoteport(remoteport); 2586 2587 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 2588 "6172 NVME rescanned DID x%06x " 2589 "port_state x%x\n", 2590 ndlp->nlp_DID, remoteport->port_state); 2591 } 2592 return; 2593 rescan_exit: 2594 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2595 "6169 Skip NVME Rport Rescan, NVME remoteport " 2596 "unregistered\n"); 2597#endif 2598} 2599 2600/* lpfc_nvme_unregister_port - unbind the DID and port_role from this rport. 2601 * 2602 * There is no notion of Devloss or rport recovery from the current 2603 * nvme_transport perspective. Loss of an rport just means IO cannot 2604 * be sent and recovery is completely up to the initator. 2605 * For now, the driver just unbinds the DID and port_role so that 2606 * no further IO can be issued. Changes are planned for later. 2607 * 2608 * Notes - the ndlp reference count is not decremented here since 2609 * since there is no nvme_transport api for devloss. Node ref count 2610 * is only adjusted in driver unload. 2611 */ 2612void 2613lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) 2614{ 2615#if (IS_ENABLED(CONFIG_NVME_FC)) 2616 int ret; 2617 struct nvme_fc_local_port *localport; 2618 struct lpfc_nvme_lport *lport; 2619 struct lpfc_nvme_rport *rport; 2620 struct nvme_fc_remote_port *remoteport = NULL; 2621 2622 localport = vport->localport; 2623 2624 /* This is fundamental error. The localport is always 2625 * available until driver unload. Just exit. 2626 */ 2627 if (!localport) 2628 return; 2629 2630 lport = (struct lpfc_nvme_lport *)localport->private; 2631 if (!lport) 2632 goto input_err; 2633 2634 spin_lock_irq(&vport->phba->hbalock); 2635 rport = lpfc_ndlp_get_nrport(ndlp); 2636 if (rport) 2637 remoteport = rport->remoteport; 2638 spin_unlock_irq(&vport->phba->hbalock); 2639 if (!remoteport) 2640 goto input_err; 2641 2642 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2643 "6033 Unreg nvme remoteport x%px, portname x%llx, " 2644 "port_id x%06x, portstate x%x port type x%x\n", 2645 remoteport, remoteport->port_name, 2646 remoteport->port_id, remoteport->port_state, 2647 ndlp->nlp_type); 2648 2649 /* Sanity check ndlp type. Only call for NVME ports. Don't 2650 * clear any rport state until the transport calls back. 2651 */ 2652 2653 if (ndlp->nlp_type & NLP_NVME_TARGET) { 2654 /* No concern about the role change on the nvme remoteport. 2655 * The transport will update it. 2656 */ 2657 ndlp->upcall_flags |= NLP_WAIT_FOR_UNREG; 2658 2659 /* Don't let the host nvme transport keep sending keep-alives 2660 * on this remoteport. Vport is unloading, no recovery. The 2661 * return values is ignored. The upcall is a courtesy to the 2662 * transport. 2663 */ 2664 if (vport->load_flag & FC_UNLOADING) 2665 (void)nvme_fc_set_remoteport_devloss(remoteport, 0); 2666 2667 ret = nvme_fc_unregister_remoteport(remoteport); 2668 if (ret != 0) { 2669 lpfc_nlp_put(ndlp); 2670 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 2671 "6167 NVME unregister failed %d " 2672 "port_state x%x\n", 2673 ret, remoteport->port_state); 2674 } 2675 } 2676 return; 2677 2678 input_err: 2679#endif 2680 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 2681 "6168 State error: lport x%px, rport x%px FCID x%06x\n", 2682 vport->localport, ndlp->rport, ndlp->nlp_DID); 2683} 2684 2685/** 2686 * lpfc_sli4_nvme_xri_aborted - Fast-path process of NVME xri abort 2687 * @phba: pointer to lpfc hba data structure. 2688 * @axri: pointer to the fcp xri abort wcqe structure. 2689 * @lpfc_ncmd: The nvme job structure for the request being aborted. 2690 * 2691 * This routine is invoked by the worker thread to process a SLI4 fast-path 2692 * NVME aborted xri. Aborted NVME IO commands are completed to the transport 2693 * here. 2694 **/ 2695void 2696lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba, 2697 struct sli4_wcqe_xri_aborted *axri, 2698 struct lpfc_io_buf *lpfc_ncmd) 2699{ 2700 uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri); 2701 struct nvmefc_fcp_req *nvme_cmd = NULL; 2702 struct lpfc_nodelist *ndlp = lpfc_ncmd->ndlp; 2703 2704 2705 if (ndlp) 2706 lpfc_sli4_abts_err_handler(phba, ndlp, axri); 2707 2708 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2709 "6311 nvme_cmd %p xri x%x tag x%x abort complete and " 2710 "xri released\n", 2711 lpfc_ncmd->nvmeCmd, xri, 2712 lpfc_ncmd->cur_iocbq.iotag); 2713 2714 /* Aborted NVME commands are required to not complete 2715 * before the abort exchange command fully completes. 2716 * Once completed, it is available via the put list. 2717 */ 2718 if (lpfc_ncmd->nvmeCmd) { 2719 nvme_cmd = lpfc_ncmd->nvmeCmd; 2720 nvme_cmd->done(nvme_cmd); 2721 lpfc_ncmd->nvmeCmd = NULL; 2722 } 2723 lpfc_release_nvme_buf(phba, lpfc_ncmd); 2724} 2725 2726/** 2727 * lpfc_nvme_wait_for_io_drain - Wait for all NVME wqes to complete 2728 * @phba: Pointer to HBA context object. 2729 * 2730 * This function flushes all wqes in the nvme rings and frees all resources 2731 * in the txcmplq. This function does not issue abort wqes for the IO 2732 * commands in txcmplq, they will just be returned with 2733 * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI 2734 * slot has been permanently disabled. 2735 **/ 2736void 2737lpfc_nvme_wait_for_io_drain(struct lpfc_hba *phba) 2738{ 2739 struct lpfc_sli_ring *pring; 2740 u32 i, wait_cnt = 0; 2741 2742 if (phba->sli_rev < LPFC_SLI_REV4 || !phba->sli4_hba.hdwq) 2743 return; 2744 2745 /* Cycle through all IO rings and make sure all outstanding 2746 * WQEs have been removed from the txcmplqs. 2747 */ 2748 for (i = 0; i < phba->cfg_hdw_queue; i++) { 2749 if (!phba->sli4_hba.hdwq[i].io_wq) 2750 continue; 2751 pring = phba->sli4_hba.hdwq[i].io_wq->pring; 2752 2753 if (!pring) 2754 continue; 2755 2756 /* Retrieve everything on the txcmplq */ 2757 while (!list_empty(&pring->txcmplq)) { 2758 msleep(LPFC_XRI_EXCH_BUSY_WAIT_T1); 2759 wait_cnt++; 2760 2761 /* The sleep is 10mS. Every ten seconds, 2762 * dump a message. Something is wrong. 2763 */ 2764 if ((wait_cnt % 1000) == 0) { 2765 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, 2766 "6178 NVME IO not empty, " 2767 "cnt %d\n", wait_cnt); 2768 } 2769 } 2770 } 2771} 2772 2773void 2774lpfc_nvme_cancel_iocb(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn) 2775{ 2776#if (IS_ENABLED(CONFIG_NVME_FC)) 2777 struct lpfc_io_buf *lpfc_ncmd; 2778 struct nvmefc_fcp_req *nCmd; 2779 struct lpfc_nvme_fcpreq_priv *freqpriv; 2780 2781 if (!pwqeIn->context1) { 2782 lpfc_sli_release_iocbq(phba, pwqeIn); 2783 return; 2784 } 2785 /* For abort iocb just return, IO iocb will do a done call */ 2786 if (bf_get(wqe_cmnd, &pwqeIn->wqe.gen_req.wqe_com) == 2787 CMD_ABORT_XRI_CX) { 2788 lpfc_sli_release_iocbq(phba, pwqeIn); 2789 return; 2790 } 2791 lpfc_ncmd = (struct lpfc_io_buf *)pwqeIn->context1; 2792 2793 spin_lock(&lpfc_ncmd->buf_lock); 2794 if (!lpfc_ncmd->nvmeCmd) { 2795 spin_unlock(&lpfc_ncmd->buf_lock); 2796 lpfc_release_nvme_buf(phba, lpfc_ncmd); 2797 return; 2798 } 2799 2800 nCmd = lpfc_ncmd->nvmeCmd; 2801 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, 2802 "6194 NVME Cancel xri %x\n", 2803 lpfc_ncmd->cur_iocbq.sli4_xritag); 2804 2805 nCmd->transferred_length = 0; 2806 nCmd->rcv_rsplen = 0; 2807 nCmd->status = NVME_SC_INTERNAL; 2808 freqpriv = nCmd->private; 2809 freqpriv->nvme_buf = NULL; 2810 lpfc_ncmd->nvmeCmd = NULL; 2811 2812 spin_unlock(&lpfc_ncmd->buf_lock); 2813 nCmd->done(nCmd); 2814 2815 /* Call release with XB=1 to queue the IO into the abort list. */ 2816 lpfc_release_nvme_buf(phba, lpfc_ncmd); 2817#endif 2818} 2819