1// SPDX-License-Identifier: GPL-2.0 2/* 3 * NVMe over Fabrics TCP host. 4 * Copyright (c) 2018 Lightbits Labs. All rights reserved. 5 */ 6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7#include <linux/module.h> 8#include <linux/init.h> 9#include <linux/slab.h> 10#include <linux/err.h> 11#include <linux/nvme-tcp.h> 12#include <net/sock.h> 13#include <net/tcp.h> 14#include <linux/blk-mq.h> 15#include <crypto/hash.h> 16#include <net/busy_poll.h> 17 18#include "nvme.h" 19#include "fabrics.h" 20 21struct nvme_tcp_queue; 22 23/* Define the socket priority to use for connections were it is desirable 24 * that the NIC consider performing optimized packet processing or filtering. 25 * A non-zero value being sufficient to indicate general consideration of any 26 * possible optimization. Making it a module param allows for alternative 27 * values that may be unique for some NIC implementations. 28 */ 29static int so_priority; 30module_param(so_priority, int, 0644); 31MODULE_PARM_DESC(so_priority, "nvme tcp socket optimize priority"); 32 33#ifdef CONFIG_DEBUG_LOCK_ALLOC 34/* lockdep can detect a circular dependency of the form 35 * sk_lock -> mmap_lock (page fault) -> fs locks -> sk_lock 36 * because dependencies are tracked for both nvme-tcp and user contexts. Using 37 * a separate class prevents lockdep from conflating nvme-tcp socket use with 38 * user-space socket API use. 39 */ 40static struct lock_class_key nvme_tcp_sk_key[2]; 41static struct lock_class_key nvme_tcp_slock_key[2]; 42 43static void nvme_tcp_reclassify_socket(struct socket *sock) 44{ 45 struct sock *sk = sock->sk; 46 47 if (WARN_ON_ONCE(!sock_allow_reclassification(sk))) 48 return; 49 50 switch (sk->sk_family) { 51 case AF_INET: 52 sock_lock_init_class_and_name(sk, "slock-AF_INET-NVME", 53 &nvme_tcp_slock_key[0], 54 "sk_lock-AF_INET-NVME", 55 &nvme_tcp_sk_key[0]); 56 break; 57 case AF_INET6: 58 sock_lock_init_class_and_name(sk, "slock-AF_INET6-NVME", 59 &nvme_tcp_slock_key[1], 60 "sk_lock-AF_INET6-NVME", 61 &nvme_tcp_sk_key[1]); 62 break; 63 default: 64 WARN_ON_ONCE(1); 65 } 66} 67#else 68static void nvme_tcp_reclassify_socket(struct socket *sock) { } 69#endif 70 71enum nvme_tcp_send_state { 72 NVME_TCP_SEND_CMD_PDU = 0, 73 NVME_TCP_SEND_H2C_PDU, 74 NVME_TCP_SEND_DATA, 75 NVME_TCP_SEND_DDGST, 76}; 77 78struct nvme_tcp_request { 79 struct nvme_request req; 80 void *pdu; 81 struct nvme_tcp_queue *queue; 82 u32 data_len; 83 u32 pdu_len; 84 u32 pdu_sent; 85 u16 ttag; 86 struct list_head entry; 87 struct llist_node lentry; 88 __le32 ddgst; 89 90 struct bio *curr_bio; 91 struct iov_iter iter; 92 93 /* send state */ 94 size_t offset; 95 size_t data_sent; 96 enum nvme_tcp_send_state state; 97}; 98 99enum nvme_tcp_queue_flags { 100 NVME_TCP_Q_ALLOCATED = 0, 101 NVME_TCP_Q_LIVE = 1, 102 NVME_TCP_Q_POLLING = 2, 103}; 104 105enum nvme_tcp_recv_state { 106 NVME_TCP_RECV_PDU = 0, 107 NVME_TCP_RECV_DATA, 108 NVME_TCP_RECV_DDGST, 109}; 110 111struct nvme_tcp_ctrl; 112struct nvme_tcp_queue { 113 struct socket *sock; 114 struct work_struct io_work; 115 int io_cpu; 116 117 struct mutex queue_lock; 118 struct mutex send_mutex; 119 struct llist_head req_list; 120 struct list_head send_list; 121 122 /* recv state */ 123 void *pdu; 124 int pdu_remaining; 125 int pdu_offset; 126 size_t data_remaining; 127 size_t ddgst_remaining; 128 unsigned int nr_cqe; 129 130 /* send state */ 131 struct nvme_tcp_request *request; 132 133 int queue_size; 134 size_t cmnd_capsule_len; 135 struct nvme_tcp_ctrl *ctrl; 136 unsigned long flags; 137 bool rd_enabled; 138 139 bool hdr_digest; 140 bool data_digest; 141 struct ahash_request *rcv_hash; 142 struct ahash_request *snd_hash; 143 __le32 exp_ddgst; 144 __le32 recv_ddgst; 145 146 struct page_frag_cache pf_cache; 147 148 void (*state_change)(struct sock *); 149 void (*data_ready)(struct sock *); 150 void (*write_space)(struct sock *); 151}; 152 153struct nvme_tcp_ctrl { 154 /* read only in the hot path */ 155 struct nvme_tcp_queue *queues; 156 struct blk_mq_tag_set tag_set; 157 158 /* other member variables */ 159 struct list_head list; 160 struct blk_mq_tag_set admin_tag_set; 161 struct sockaddr_storage addr; 162 struct sockaddr_storage src_addr; 163 struct nvme_ctrl ctrl; 164 165 struct work_struct err_work; 166 struct delayed_work connect_work; 167 struct nvme_tcp_request async_req; 168 u32 io_queues[HCTX_MAX_TYPES]; 169}; 170 171static LIST_HEAD(nvme_tcp_ctrl_list); 172static DEFINE_MUTEX(nvme_tcp_ctrl_mutex); 173static struct workqueue_struct *nvme_tcp_wq; 174static const struct blk_mq_ops nvme_tcp_mq_ops; 175static const struct blk_mq_ops nvme_tcp_admin_mq_ops; 176static int nvme_tcp_try_send(struct nvme_tcp_queue *queue); 177 178static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl) 179{ 180 return container_of(ctrl, struct nvme_tcp_ctrl, ctrl); 181} 182 183static inline int nvme_tcp_queue_id(struct nvme_tcp_queue *queue) 184{ 185 return queue - queue->ctrl->queues; 186} 187 188static inline struct blk_mq_tags *nvme_tcp_tagset(struct nvme_tcp_queue *queue) 189{ 190 u32 queue_idx = nvme_tcp_queue_id(queue); 191 192 if (queue_idx == 0) 193 return queue->ctrl->admin_tag_set.tags[queue_idx]; 194 return queue->ctrl->tag_set.tags[queue_idx - 1]; 195} 196 197static inline u8 nvme_tcp_hdgst_len(struct nvme_tcp_queue *queue) 198{ 199 return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0; 200} 201 202static inline u8 nvme_tcp_ddgst_len(struct nvme_tcp_queue *queue) 203{ 204 return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0; 205} 206 207static inline size_t nvme_tcp_inline_data_size(struct nvme_tcp_queue *queue) 208{ 209 return queue->cmnd_capsule_len - sizeof(struct nvme_command); 210} 211 212static inline bool nvme_tcp_async_req(struct nvme_tcp_request *req) 213{ 214 return req == &req->queue->ctrl->async_req; 215} 216 217static inline bool nvme_tcp_has_inline_data(struct nvme_tcp_request *req) 218{ 219 struct request *rq; 220 221 if (unlikely(nvme_tcp_async_req(req))) 222 return false; /* async events don't have a request */ 223 224 rq = blk_mq_rq_from_pdu(req); 225 226 return rq_data_dir(rq) == WRITE && req->data_len && 227 req->data_len <= nvme_tcp_inline_data_size(req->queue); 228} 229 230static inline struct page *nvme_tcp_req_cur_page(struct nvme_tcp_request *req) 231{ 232 return req->iter.bvec->bv_page; 233} 234 235static inline size_t nvme_tcp_req_cur_offset(struct nvme_tcp_request *req) 236{ 237 return req->iter.bvec->bv_offset + req->iter.iov_offset; 238} 239 240static inline size_t nvme_tcp_req_cur_length(struct nvme_tcp_request *req) 241{ 242 return min_t(size_t, iov_iter_single_seg_count(&req->iter), 243 req->pdu_len - req->pdu_sent); 244} 245 246static inline size_t nvme_tcp_req_offset(struct nvme_tcp_request *req) 247{ 248 return req->iter.iov_offset; 249} 250 251static inline size_t nvme_tcp_pdu_data_left(struct nvme_tcp_request *req) 252{ 253 return rq_data_dir(blk_mq_rq_from_pdu(req)) == WRITE ? 254 req->pdu_len - req->pdu_sent : 0; 255} 256 257static inline size_t nvme_tcp_pdu_last_send(struct nvme_tcp_request *req, 258 int len) 259{ 260 return nvme_tcp_pdu_data_left(req) <= len; 261} 262 263static void nvme_tcp_init_iter(struct nvme_tcp_request *req, 264 unsigned int dir) 265{ 266 struct request *rq = blk_mq_rq_from_pdu(req); 267 struct bio_vec *vec; 268 unsigned int size; 269 int nsegs; 270 size_t offset; 271 272 if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) { 273 vec = &rq->special_vec; 274 nsegs = 1; 275 size = blk_rq_payload_bytes(rq); 276 offset = 0; 277 } else { 278 struct bio *bio = req->curr_bio; 279 280 vec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter); 281 nsegs = bio_segments(bio); 282 size = bio->bi_iter.bi_size; 283 offset = bio->bi_iter.bi_bvec_done; 284 } 285 286 iov_iter_bvec(&req->iter, dir, vec, nsegs, size); 287 req->iter.iov_offset = offset; 288} 289 290static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req, 291 int len) 292{ 293 req->data_sent += len; 294 req->pdu_sent += len; 295 iov_iter_advance(&req->iter, len); 296 if (!iov_iter_count(&req->iter) && 297 req->data_sent < req->data_len) { 298 req->curr_bio = req->curr_bio->bi_next; 299 nvme_tcp_init_iter(req, WRITE); 300 } 301} 302 303static inline void nvme_tcp_send_all(struct nvme_tcp_queue *queue) 304{ 305 int ret; 306 307 /* drain the send queue as much as we can... */ 308 do { 309 ret = nvme_tcp_try_send(queue); 310 } while (ret > 0); 311} 312 313static inline bool nvme_tcp_queue_more(struct nvme_tcp_queue *queue) 314{ 315 return !list_empty(&queue->send_list) || 316 !llist_empty(&queue->req_list); 317} 318 319static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req, 320 bool sync, bool last) 321{ 322 struct nvme_tcp_queue *queue = req->queue; 323 bool empty; 324 325 empty = llist_add(&req->lentry, &queue->req_list) && 326 list_empty(&queue->send_list) && !queue->request; 327 328 /* 329 * if we're the first on the send_list and we can try to send 330 * directly, otherwise queue io_work. Also, only do that if we 331 * are on the same cpu, so we don't introduce contention. 332 */ 333 if (queue->io_cpu == raw_smp_processor_id() && 334 sync && empty && mutex_trylock(&queue->send_mutex)) { 335 nvme_tcp_send_all(queue); 336 mutex_unlock(&queue->send_mutex); 337 } 338 339 if (last && nvme_tcp_queue_more(queue)) 340 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 341} 342 343static void nvme_tcp_process_req_list(struct nvme_tcp_queue *queue) 344{ 345 struct nvme_tcp_request *req; 346 struct llist_node *node; 347 348 for (node = llist_del_all(&queue->req_list); node; node = node->next) { 349 req = llist_entry(node, struct nvme_tcp_request, lentry); 350 list_add(&req->entry, &queue->send_list); 351 } 352} 353 354static inline struct nvme_tcp_request * 355nvme_tcp_fetch_request(struct nvme_tcp_queue *queue) 356{ 357 struct nvme_tcp_request *req; 358 359 req = list_first_entry_or_null(&queue->send_list, 360 struct nvme_tcp_request, entry); 361 if (!req) { 362 nvme_tcp_process_req_list(queue); 363 req = list_first_entry_or_null(&queue->send_list, 364 struct nvme_tcp_request, entry); 365 if (unlikely(!req)) 366 return NULL; 367 } 368 369 list_del(&req->entry); 370 return req; 371} 372 373static inline void nvme_tcp_ddgst_final(struct ahash_request *hash, 374 __le32 *dgst) 375{ 376 ahash_request_set_crypt(hash, NULL, (u8 *)dgst, 0); 377 crypto_ahash_final(hash); 378} 379 380static inline void nvme_tcp_ddgst_update(struct ahash_request *hash, 381 struct page *page, off_t off, size_t len) 382{ 383 struct scatterlist sg; 384 385 sg_init_marker(&sg, 1); 386 sg_set_page(&sg, page, len, off); 387 ahash_request_set_crypt(hash, &sg, NULL, len); 388 crypto_ahash_update(hash); 389} 390 391static inline void nvme_tcp_hdgst(struct ahash_request *hash, 392 void *pdu, size_t len) 393{ 394 struct scatterlist sg; 395 396 sg_init_one(&sg, pdu, len); 397 ahash_request_set_crypt(hash, &sg, pdu + len, len); 398 crypto_ahash_digest(hash); 399} 400 401static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue, 402 void *pdu, size_t pdu_len) 403{ 404 struct nvme_tcp_hdr *hdr = pdu; 405 __le32 recv_digest; 406 __le32 exp_digest; 407 408 if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) { 409 dev_err(queue->ctrl->ctrl.device, 410 "queue %d: header digest flag is cleared\n", 411 nvme_tcp_queue_id(queue)); 412 return -EPROTO; 413 } 414 415 recv_digest = *(__le32 *)(pdu + hdr->hlen); 416 nvme_tcp_hdgst(queue->rcv_hash, pdu, pdu_len); 417 exp_digest = *(__le32 *)(pdu + hdr->hlen); 418 if (recv_digest != exp_digest) { 419 dev_err(queue->ctrl->ctrl.device, 420 "header digest error: recv %#x expected %#x\n", 421 le32_to_cpu(recv_digest), le32_to_cpu(exp_digest)); 422 return -EIO; 423 } 424 425 return 0; 426} 427 428static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu) 429{ 430 struct nvme_tcp_hdr *hdr = pdu; 431 u8 digest_len = nvme_tcp_hdgst_len(queue); 432 u32 len; 433 434 len = le32_to_cpu(hdr->plen) - hdr->hlen - 435 ((hdr->flags & NVME_TCP_F_HDGST) ? digest_len : 0); 436 437 if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) { 438 dev_err(queue->ctrl->ctrl.device, 439 "queue %d: data digest flag is cleared\n", 440 nvme_tcp_queue_id(queue)); 441 return -EPROTO; 442 } 443 crypto_ahash_init(queue->rcv_hash); 444 445 return 0; 446} 447 448static void nvme_tcp_exit_request(struct blk_mq_tag_set *set, 449 struct request *rq, unsigned int hctx_idx) 450{ 451 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 452 453 page_frag_free(req->pdu); 454} 455 456static int nvme_tcp_init_request(struct blk_mq_tag_set *set, 457 struct request *rq, unsigned int hctx_idx, 458 unsigned int numa_node) 459{ 460 struct nvme_tcp_ctrl *ctrl = set->driver_data; 461 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 462 int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0; 463 struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx]; 464 u8 hdgst = nvme_tcp_hdgst_len(queue); 465 466 req->pdu = page_frag_alloc(&queue->pf_cache, 467 sizeof(struct nvme_tcp_cmd_pdu) + hdgst, 468 GFP_KERNEL | __GFP_ZERO); 469 if (!req->pdu) 470 return -ENOMEM; 471 472 req->queue = queue; 473 nvme_req(rq)->ctrl = &ctrl->ctrl; 474 475 return 0; 476} 477 478static int nvme_tcp_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, 479 unsigned int hctx_idx) 480{ 481 struct nvme_tcp_ctrl *ctrl = data; 482 struct nvme_tcp_queue *queue = &ctrl->queues[hctx_idx + 1]; 483 484 hctx->driver_data = queue; 485 return 0; 486} 487 488static int nvme_tcp_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data, 489 unsigned int hctx_idx) 490{ 491 struct nvme_tcp_ctrl *ctrl = data; 492 struct nvme_tcp_queue *queue = &ctrl->queues[0]; 493 494 hctx->driver_data = queue; 495 return 0; 496} 497 498static enum nvme_tcp_recv_state 499nvme_tcp_recv_state(struct nvme_tcp_queue *queue) 500{ 501 return (queue->pdu_remaining) ? NVME_TCP_RECV_PDU : 502 (queue->ddgst_remaining) ? NVME_TCP_RECV_DDGST : 503 NVME_TCP_RECV_DATA; 504} 505 506static void nvme_tcp_init_recv_ctx(struct nvme_tcp_queue *queue) 507{ 508 queue->pdu_remaining = sizeof(struct nvme_tcp_rsp_pdu) + 509 nvme_tcp_hdgst_len(queue); 510 queue->pdu_offset = 0; 511 queue->data_remaining = -1; 512 queue->ddgst_remaining = 0; 513} 514 515static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl) 516{ 517 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) 518 return; 519 520 dev_warn(ctrl->device, "starting error recovery\n"); 521 queue_work(nvme_reset_wq, &to_tcp_ctrl(ctrl)->err_work); 522} 523 524static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue, 525 struct nvme_completion *cqe) 526{ 527 struct request *rq; 528 529 rq = nvme_find_rq(nvme_tcp_tagset(queue), cqe->command_id); 530 if (!rq) { 531 dev_err(queue->ctrl->ctrl.device, 532 "got bad cqe.command_id %#x on queue %d\n", 533 cqe->command_id, nvme_tcp_queue_id(queue)); 534 nvme_tcp_error_recovery(&queue->ctrl->ctrl); 535 return -EINVAL; 536 } 537 538 if (!nvme_try_complete_req(rq, cqe->status, cqe->result)) 539 nvme_complete_rq(rq); 540 queue->nr_cqe++; 541 542 return 0; 543} 544 545static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue, 546 struct nvme_tcp_data_pdu *pdu) 547{ 548 struct request *rq; 549 550 rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id); 551 if (!rq) { 552 dev_err(queue->ctrl->ctrl.device, 553 "got bad c2hdata.command_id %#x on queue %d\n", 554 pdu->command_id, nvme_tcp_queue_id(queue)); 555 return -ENOENT; 556 } 557 558 if (!blk_rq_payload_bytes(rq)) { 559 dev_err(queue->ctrl->ctrl.device, 560 "queue %d tag %#x unexpected data\n", 561 nvme_tcp_queue_id(queue), rq->tag); 562 return -EIO; 563 } 564 565 queue->data_remaining = le32_to_cpu(pdu->data_length); 566 567 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS && 568 unlikely(!(pdu->hdr.flags & NVME_TCP_F_DATA_LAST))) { 569 dev_err(queue->ctrl->ctrl.device, 570 "queue %d tag %#x SUCCESS set but not last PDU\n", 571 nvme_tcp_queue_id(queue), rq->tag); 572 nvme_tcp_error_recovery(&queue->ctrl->ctrl); 573 return -EPROTO; 574 } 575 576 return 0; 577} 578 579static int nvme_tcp_handle_comp(struct nvme_tcp_queue *queue, 580 struct nvme_tcp_rsp_pdu *pdu) 581{ 582 struct nvme_completion *cqe = &pdu->cqe; 583 int ret = 0; 584 585 /* 586 * AEN requests are special as they don't time out and can 587 * survive any kind of queue freeze and often don't respond to 588 * aborts. We don't even bother to allocate a struct request 589 * for them but rather special case them here. 590 */ 591 if (unlikely(nvme_is_aen_req(nvme_tcp_queue_id(queue), 592 cqe->command_id))) 593 nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status, 594 &cqe->result); 595 else 596 ret = nvme_tcp_process_nvme_cqe(queue, cqe); 597 598 return ret; 599} 600 601static int nvme_tcp_setup_h2c_data_pdu(struct nvme_tcp_request *req, 602 struct nvme_tcp_r2t_pdu *pdu) 603{ 604 struct nvme_tcp_data_pdu *data = req->pdu; 605 struct nvme_tcp_queue *queue = req->queue; 606 struct request *rq = blk_mq_rq_from_pdu(req); 607 u8 hdgst = nvme_tcp_hdgst_len(queue); 608 u8 ddgst = nvme_tcp_ddgst_len(queue); 609 610 req->pdu_len = le32_to_cpu(pdu->r2t_length); 611 req->pdu_sent = 0; 612 613 if (unlikely(!req->pdu_len)) { 614 dev_err(queue->ctrl->ctrl.device, 615 "req %d r2t len is %u, probably a bug...\n", 616 rq->tag, req->pdu_len); 617 return -EPROTO; 618 } 619 620 if (unlikely(req->data_sent + req->pdu_len > req->data_len)) { 621 dev_err(queue->ctrl->ctrl.device, 622 "req %d r2t len %u exceeded data len %u (%zu sent)\n", 623 rq->tag, req->pdu_len, req->data_len, 624 req->data_sent); 625 return -EPROTO; 626 } 627 628 if (unlikely(le32_to_cpu(pdu->r2t_offset) < req->data_sent)) { 629 dev_err(queue->ctrl->ctrl.device, 630 "req %d unexpected r2t offset %u (expected %zu)\n", 631 rq->tag, le32_to_cpu(pdu->r2t_offset), 632 req->data_sent); 633 return -EPROTO; 634 } 635 636 memset(data, 0, sizeof(*data)); 637 data->hdr.type = nvme_tcp_h2c_data; 638 data->hdr.flags = NVME_TCP_F_DATA_LAST; 639 if (queue->hdr_digest) 640 data->hdr.flags |= NVME_TCP_F_HDGST; 641 if (queue->data_digest) 642 data->hdr.flags |= NVME_TCP_F_DDGST; 643 data->hdr.hlen = sizeof(*data); 644 data->hdr.pdo = data->hdr.hlen + hdgst; 645 data->hdr.plen = 646 cpu_to_le32(data->hdr.hlen + hdgst + req->pdu_len + ddgst); 647 data->ttag = pdu->ttag; 648 data->command_id = nvme_cid(rq); 649 data->data_offset = pdu->r2t_offset; 650 data->data_length = cpu_to_le32(req->pdu_len); 651 return 0; 652} 653 654static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue, 655 struct nvme_tcp_r2t_pdu *pdu) 656{ 657 struct nvme_tcp_request *req; 658 struct request *rq; 659 int ret; 660 661 rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id); 662 if (!rq) { 663 dev_err(queue->ctrl->ctrl.device, 664 "got bad r2t.command_id %#x on queue %d\n", 665 pdu->command_id, nvme_tcp_queue_id(queue)); 666 return -ENOENT; 667 } 668 req = blk_mq_rq_to_pdu(rq); 669 670 ret = nvme_tcp_setup_h2c_data_pdu(req, pdu); 671 if (unlikely(ret)) 672 return ret; 673 674 req->state = NVME_TCP_SEND_H2C_PDU; 675 req->offset = 0; 676 677 nvme_tcp_queue_request(req, false, true); 678 679 return 0; 680} 681 682static int nvme_tcp_recv_pdu(struct nvme_tcp_queue *queue, struct sk_buff *skb, 683 unsigned int *offset, size_t *len) 684{ 685 struct nvme_tcp_hdr *hdr; 686 char *pdu = queue->pdu; 687 size_t rcv_len = min_t(size_t, *len, queue->pdu_remaining); 688 int ret; 689 690 ret = skb_copy_bits(skb, *offset, 691 &pdu[queue->pdu_offset], rcv_len); 692 if (unlikely(ret)) 693 return ret; 694 695 queue->pdu_remaining -= rcv_len; 696 queue->pdu_offset += rcv_len; 697 *offset += rcv_len; 698 *len -= rcv_len; 699 if (queue->pdu_remaining) 700 return 0; 701 702 hdr = queue->pdu; 703 if (queue->hdr_digest) { 704 ret = nvme_tcp_verify_hdgst(queue, queue->pdu, hdr->hlen); 705 if (unlikely(ret)) 706 return ret; 707 } 708 709 710 if (queue->data_digest) { 711 ret = nvme_tcp_check_ddgst(queue, queue->pdu); 712 if (unlikely(ret)) 713 return ret; 714 } 715 716 switch (hdr->type) { 717 case nvme_tcp_c2h_data: 718 return nvme_tcp_handle_c2h_data(queue, (void *)queue->pdu); 719 case nvme_tcp_rsp: 720 nvme_tcp_init_recv_ctx(queue); 721 return nvme_tcp_handle_comp(queue, (void *)queue->pdu); 722 case nvme_tcp_r2t: 723 nvme_tcp_init_recv_ctx(queue); 724 return nvme_tcp_handle_r2t(queue, (void *)queue->pdu); 725 default: 726 dev_err(queue->ctrl->ctrl.device, 727 "unsupported pdu type (%d)\n", hdr->type); 728 return -EINVAL; 729 } 730} 731 732static inline void nvme_tcp_end_request(struct request *rq, u16 status) 733{ 734 union nvme_result res = {}; 735 736 if (!nvme_try_complete_req(rq, cpu_to_le16(status << 1), res)) 737 nvme_complete_rq(rq); 738} 739 740static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb, 741 unsigned int *offset, size_t *len) 742{ 743 struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu; 744 struct request *rq = 745 nvme_cid_to_rq(nvme_tcp_tagset(queue), pdu->command_id); 746 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 747 748 while (true) { 749 int recv_len, ret; 750 751 recv_len = min_t(size_t, *len, queue->data_remaining); 752 if (!recv_len) 753 break; 754 755 if (!iov_iter_count(&req->iter)) { 756 req->curr_bio = req->curr_bio->bi_next; 757 758 /* 759 * If we don`t have any bios it means that controller 760 * sent more data than we requested, hence error 761 */ 762 if (!req->curr_bio) { 763 dev_err(queue->ctrl->ctrl.device, 764 "queue %d no space in request %#x", 765 nvme_tcp_queue_id(queue), rq->tag); 766 nvme_tcp_init_recv_ctx(queue); 767 return -EIO; 768 } 769 nvme_tcp_init_iter(req, READ); 770 } 771 772 /* we can read only from what is left in this bio */ 773 recv_len = min_t(size_t, recv_len, 774 iov_iter_count(&req->iter)); 775 776 if (queue->data_digest) 777 ret = skb_copy_and_hash_datagram_iter(skb, *offset, 778 &req->iter, recv_len, queue->rcv_hash); 779 else 780 ret = skb_copy_datagram_iter(skb, *offset, 781 &req->iter, recv_len); 782 if (ret) { 783 dev_err(queue->ctrl->ctrl.device, 784 "queue %d failed to copy request %#x data", 785 nvme_tcp_queue_id(queue), rq->tag); 786 return ret; 787 } 788 789 *len -= recv_len; 790 *offset += recv_len; 791 queue->data_remaining -= recv_len; 792 } 793 794 if (!queue->data_remaining) { 795 if (queue->data_digest) { 796 nvme_tcp_ddgst_final(queue->rcv_hash, &queue->exp_ddgst); 797 queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH; 798 } else { 799 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) { 800 nvme_tcp_end_request(rq, NVME_SC_SUCCESS); 801 queue->nr_cqe++; 802 } 803 nvme_tcp_init_recv_ctx(queue); 804 } 805 } 806 807 return 0; 808} 809 810static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue, 811 struct sk_buff *skb, unsigned int *offset, size_t *len) 812{ 813 struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu; 814 char *ddgst = (char *)&queue->recv_ddgst; 815 size_t recv_len = min_t(size_t, *len, queue->ddgst_remaining); 816 off_t off = NVME_TCP_DIGEST_LENGTH - queue->ddgst_remaining; 817 int ret; 818 819 ret = skb_copy_bits(skb, *offset, &ddgst[off], recv_len); 820 if (unlikely(ret)) 821 return ret; 822 823 queue->ddgst_remaining -= recv_len; 824 *offset += recv_len; 825 *len -= recv_len; 826 if (queue->ddgst_remaining) 827 return 0; 828 829 if (queue->recv_ddgst != queue->exp_ddgst) { 830 dev_err(queue->ctrl->ctrl.device, 831 "data digest error: recv %#x expected %#x\n", 832 le32_to_cpu(queue->recv_ddgst), 833 le32_to_cpu(queue->exp_ddgst)); 834 return -EIO; 835 } 836 837 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) { 838 struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue), 839 pdu->command_id); 840 841 nvme_tcp_end_request(rq, NVME_SC_SUCCESS); 842 queue->nr_cqe++; 843 } 844 845 nvme_tcp_init_recv_ctx(queue); 846 return 0; 847} 848 849static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb, 850 unsigned int offset, size_t len) 851{ 852 struct nvme_tcp_queue *queue = desc->arg.data; 853 size_t consumed = len; 854 int result; 855 856 while (len) { 857 switch (nvme_tcp_recv_state(queue)) { 858 case NVME_TCP_RECV_PDU: 859 result = nvme_tcp_recv_pdu(queue, skb, &offset, &len); 860 break; 861 case NVME_TCP_RECV_DATA: 862 result = nvme_tcp_recv_data(queue, skb, &offset, &len); 863 break; 864 case NVME_TCP_RECV_DDGST: 865 result = nvme_tcp_recv_ddgst(queue, skb, &offset, &len); 866 break; 867 default: 868 result = -EFAULT; 869 } 870 if (result) { 871 dev_err(queue->ctrl->ctrl.device, 872 "receive failed: %d\n", result); 873 queue->rd_enabled = false; 874 nvme_tcp_error_recovery(&queue->ctrl->ctrl); 875 return result; 876 } 877 } 878 879 return consumed; 880} 881 882static void nvme_tcp_data_ready(struct sock *sk) 883{ 884 struct nvme_tcp_queue *queue; 885 886 read_lock_bh(&sk->sk_callback_lock); 887 queue = sk->sk_user_data; 888 if (likely(queue && queue->rd_enabled) && 889 !test_bit(NVME_TCP_Q_POLLING, &queue->flags)) 890 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 891 read_unlock_bh(&sk->sk_callback_lock); 892} 893 894static void nvme_tcp_write_space(struct sock *sk) 895{ 896 struct nvme_tcp_queue *queue; 897 898 read_lock_bh(&sk->sk_callback_lock); 899 queue = sk->sk_user_data; 900 if (likely(queue && sk_stream_is_writeable(sk))) { 901 clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 902 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 903 } 904 read_unlock_bh(&sk->sk_callback_lock); 905} 906 907static void nvme_tcp_state_change(struct sock *sk) 908{ 909 struct nvme_tcp_queue *queue; 910 911 read_lock_bh(&sk->sk_callback_lock); 912 queue = sk->sk_user_data; 913 if (!queue) 914 goto done; 915 916 switch (sk->sk_state) { 917 case TCP_CLOSE: 918 case TCP_CLOSE_WAIT: 919 case TCP_LAST_ACK: 920 case TCP_FIN_WAIT1: 921 case TCP_FIN_WAIT2: 922 nvme_tcp_error_recovery(&queue->ctrl->ctrl); 923 break; 924 default: 925 dev_info(queue->ctrl->ctrl.device, 926 "queue %d socket state %d\n", 927 nvme_tcp_queue_id(queue), sk->sk_state); 928 } 929 930 queue->state_change(sk); 931done: 932 read_unlock_bh(&sk->sk_callback_lock); 933} 934 935static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue) 936{ 937 queue->request = NULL; 938} 939 940static void nvme_tcp_fail_request(struct nvme_tcp_request *req) 941{ 942 if (nvme_tcp_async_req(req)) { 943 union nvme_result res = {}; 944 945 nvme_complete_async_event(&req->queue->ctrl->ctrl, 946 cpu_to_le16(NVME_SC_HOST_PATH_ERROR), &res); 947 } else { 948 nvme_tcp_end_request(blk_mq_rq_from_pdu(req), 949 NVME_SC_HOST_PATH_ERROR); 950 } 951} 952 953static int nvme_tcp_try_send_data(struct nvme_tcp_request *req) 954{ 955 struct nvme_tcp_queue *queue = req->queue; 956 int req_data_len = req->data_len; 957 958 while (true) { 959 struct page *page = nvme_tcp_req_cur_page(req); 960 size_t offset = nvme_tcp_req_cur_offset(req); 961 size_t len = nvme_tcp_req_cur_length(req); 962 bool last = nvme_tcp_pdu_last_send(req, len); 963 int req_data_sent = req->data_sent; 964 int ret, flags = MSG_DONTWAIT; 965 966 if (last && !queue->data_digest && !nvme_tcp_queue_more(queue)) 967 flags |= MSG_EOR; 968 else 969 flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST; 970 971 if (sendpage_ok(page)) { 972 ret = kernel_sendpage(queue->sock, page, offset, len, 973 flags); 974 } else { 975 ret = sock_no_sendpage(queue->sock, page, offset, len, 976 flags); 977 } 978 if (ret <= 0) 979 return ret; 980 981 if (queue->data_digest) 982 nvme_tcp_ddgst_update(queue->snd_hash, page, 983 offset, ret); 984 985 /* 986 * update the request iterator except for the last payload send 987 * in the request where we don't want to modify it as we may 988 * compete with the RX path completing the request. 989 */ 990 if (req_data_sent + ret < req_data_len) 991 nvme_tcp_advance_req(req, ret); 992 993 /* fully successful last send in current PDU */ 994 if (last && ret == len) { 995 if (queue->data_digest) { 996 nvme_tcp_ddgst_final(queue->snd_hash, 997 &req->ddgst); 998 req->state = NVME_TCP_SEND_DDGST; 999 req->offset = 0; 1000 } else { 1001 nvme_tcp_done_send_req(queue); 1002 } 1003 return 1; 1004 } 1005 } 1006 return -EAGAIN; 1007} 1008 1009static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req) 1010{ 1011 struct nvme_tcp_queue *queue = req->queue; 1012 struct nvme_tcp_cmd_pdu *pdu = req->pdu; 1013 bool inline_data = nvme_tcp_has_inline_data(req); 1014 u8 hdgst = nvme_tcp_hdgst_len(queue); 1015 int len = sizeof(*pdu) + hdgst - req->offset; 1016 int flags = MSG_DONTWAIT; 1017 int ret; 1018 1019 if (inline_data || nvme_tcp_queue_more(queue)) 1020 flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST; 1021 else 1022 flags |= MSG_EOR; 1023 1024 if (queue->hdr_digest && !req->offset) 1025 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu)); 1026 1027 ret = kernel_sendpage(queue->sock, virt_to_page(pdu), 1028 offset_in_page(pdu) + req->offset, len, flags); 1029 if (unlikely(ret <= 0)) 1030 return ret; 1031 1032 len -= ret; 1033 if (!len) { 1034 if (inline_data) { 1035 req->state = NVME_TCP_SEND_DATA; 1036 if (queue->data_digest) 1037 crypto_ahash_init(queue->snd_hash); 1038 nvme_tcp_init_iter(req, WRITE); 1039 } else { 1040 nvme_tcp_done_send_req(queue); 1041 } 1042 return 1; 1043 } 1044 req->offset += ret; 1045 1046 return -EAGAIN; 1047} 1048 1049static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req) 1050{ 1051 struct nvme_tcp_queue *queue = req->queue; 1052 struct nvme_tcp_data_pdu *pdu = req->pdu; 1053 u8 hdgst = nvme_tcp_hdgst_len(queue); 1054 int len = sizeof(*pdu) - req->offset + hdgst; 1055 int ret; 1056 1057 if (queue->hdr_digest && !req->offset) 1058 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu)); 1059 1060 ret = kernel_sendpage(queue->sock, virt_to_page(pdu), 1061 offset_in_page(pdu) + req->offset, len, 1062 MSG_DONTWAIT | MSG_MORE | MSG_SENDPAGE_NOTLAST); 1063 if (unlikely(ret <= 0)) 1064 return ret; 1065 1066 len -= ret; 1067 if (!len) { 1068 req->state = NVME_TCP_SEND_DATA; 1069 if (queue->data_digest) 1070 crypto_ahash_init(queue->snd_hash); 1071 if (!req->data_sent) 1072 nvme_tcp_init_iter(req, WRITE); 1073 return 1; 1074 } 1075 req->offset += ret; 1076 1077 return -EAGAIN; 1078} 1079 1080static int nvme_tcp_try_send_ddgst(struct nvme_tcp_request *req) 1081{ 1082 struct nvme_tcp_queue *queue = req->queue; 1083 size_t offset = req->offset; 1084 int ret; 1085 struct msghdr msg = { .msg_flags = MSG_DONTWAIT }; 1086 struct kvec iov = { 1087 .iov_base = (u8 *)&req->ddgst + req->offset, 1088 .iov_len = NVME_TCP_DIGEST_LENGTH - req->offset 1089 }; 1090 1091 if (nvme_tcp_queue_more(queue)) 1092 msg.msg_flags |= MSG_MORE; 1093 else 1094 msg.msg_flags |= MSG_EOR; 1095 1096 ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); 1097 if (unlikely(ret <= 0)) 1098 return ret; 1099 1100 if (offset + ret == NVME_TCP_DIGEST_LENGTH) { 1101 nvme_tcp_done_send_req(queue); 1102 return 1; 1103 } 1104 1105 req->offset += ret; 1106 return -EAGAIN; 1107} 1108 1109static int nvme_tcp_try_send(struct nvme_tcp_queue *queue) 1110{ 1111 struct nvme_tcp_request *req; 1112 int ret = 1; 1113 1114 if (!queue->request) { 1115 queue->request = nvme_tcp_fetch_request(queue); 1116 if (!queue->request) 1117 return 0; 1118 } 1119 req = queue->request; 1120 1121 if (req->state == NVME_TCP_SEND_CMD_PDU) { 1122 ret = nvme_tcp_try_send_cmd_pdu(req); 1123 if (ret <= 0) 1124 goto done; 1125 if (!nvme_tcp_has_inline_data(req)) 1126 return ret; 1127 } 1128 1129 if (req->state == NVME_TCP_SEND_H2C_PDU) { 1130 ret = nvme_tcp_try_send_data_pdu(req); 1131 if (ret <= 0) 1132 goto done; 1133 } 1134 1135 if (req->state == NVME_TCP_SEND_DATA) { 1136 ret = nvme_tcp_try_send_data(req); 1137 if (ret <= 0) 1138 goto done; 1139 } 1140 1141 if (req->state == NVME_TCP_SEND_DDGST) 1142 ret = nvme_tcp_try_send_ddgst(req); 1143done: 1144 if (ret == -EAGAIN) { 1145 ret = 0; 1146 } else if (ret < 0) { 1147 dev_err(queue->ctrl->ctrl.device, 1148 "failed to send request %d\n", ret); 1149 nvme_tcp_fail_request(queue->request); 1150 nvme_tcp_done_send_req(queue); 1151 } 1152 return ret; 1153} 1154 1155static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue) 1156{ 1157 struct socket *sock = queue->sock; 1158 struct sock *sk = sock->sk; 1159 read_descriptor_t rd_desc; 1160 int consumed; 1161 1162 rd_desc.arg.data = queue; 1163 rd_desc.count = 1; 1164 lock_sock(sk); 1165 queue->nr_cqe = 0; 1166 consumed = sock->ops->read_sock(sk, &rd_desc, nvme_tcp_recv_skb); 1167 release_sock(sk); 1168 return consumed; 1169} 1170 1171static void nvme_tcp_io_work(struct work_struct *w) 1172{ 1173 struct nvme_tcp_queue *queue = 1174 container_of(w, struct nvme_tcp_queue, io_work); 1175 unsigned long deadline = jiffies + msecs_to_jiffies(1); 1176 1177 do { 1178 bool pending = false; 1179 int result; 1180 1181 if (mutex_trylock(&queue->send_mutex)) { 1182 result = nvme_tcp_try_send(queue); 1183 mutex_unlock(&queue->send_mutex); 1184 if (result > 0) 1185 pending = true; 1186 else if (unlikely(result < 0)) 1187 break; 1188 } 1189 1190 result = nvme_tcp_try_recv(queue); 1191 if (result > 0) 1192 pending = true; 1193 else if (unlikely(result < 0)) 1194 return; 1195 1196 if (!pending || !queue->rd_enabled) 1197 return; 1198 1199 } while (!time_after(jiffies, deadline)); /* quota is exhausted */ 1200 1201 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 1202} 1203 1204static void nvme_tcp_free_crypto(struct nvme_tcp_queue *queue) 1205{ 1206 struct crypto_ahash *tfm = crypto_ahash_reqtfm(queue->rcv_hash); 1207 1208 ahash_request_free(queue->rcv_hash); 1209 ahash_request_free(queue->snd_hash); 1210 crypto_free_ahash(tfm); 1211} 1212 1213static int nvme_tcp_alloc_crypto(struct nvme_tcp_queue *queue) 1214{ 1215 struct crypto_ahash *tfm; 1216 1217 tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC); 1218 if (IS_ERR(tfm)) 1219 return PTR_ERR(tfm); 1220 1221 queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL); 1222 if (!queue->snd_hash) 1223 goto free_tfm; 1224 ahash_request_set_callback(queue->snd_hash, 0, NULL, NULL); 1225 1226 queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL); 1227 if (!queue->rcv_hash) 1228 goto free_snd_hash; 1229 ahash_request_set_callback(queue->rcv_hash, 0, NULL, NULL); 1230 1231 return 0; 1232free_snd_hash: 1233 ahash_request_free(queue->snd_hash); 1234free_tfm: 1235 crypto_free_ahash(tfm); 1236 return -ENOMEM; 1237} 1238 1239static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl) 1240{ 1241 struct nvme_tcp_request *async = &ctrl->async_req; 1242 1243 page_frag_free(async->pdu); 1244} 1245 1246static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl) 1247{ 1248 struct nvme_tcp_queue *queue = &ctrl->queues[0]; 1249 struct nvme_tcp_request *async = &ctrl->async_req; 1250 u8 hdgst = nvme_tcp_hdgst_len(queue); 1251 1252 async->pdu = page_frag_alloc(&queue->pf_cache, 1253 sizeof(struct nvme_tcp_cmd_pdu) + hdgst, 1254 GFP_KERNEL | __GFP_ZERO); 1255 if (!async->pdu) 1256 return -ENOMEM; 1257 1258 async->queue = &ctrl->queues[0]; 1259 return 0; 1260} 1261 1262static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid) 1263{ 1264 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1265 struct nvme_tcp_queue *queue = &ctrl->queues[qid]; 1266 1267 if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags)) 1268 return; 1269 1270 if (queue->hdr_digest || queue->data_digest) 1271 nvme_tcp_free_crypto(queue); 1272 1273 sock_release(queue->sock); 1274 kfree(queue->pdu); 1275 mutex_destroy(&queue->queue_lock); 1276} 1277 1278static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue) 1279{ 1280 struct nvme_tcp_icreq_pdu *icreq; 1281 struct nvme_tcp_icresp_pdu *icresp; 1282 struct msghdr msg = {}; 1283 struct kvec iov; 1284 bool ctrl_hdgst, ctrl_ddgst; 1285 int ret; 1286 1287 icreq = kzalloc(sizeof(*icreq), GFP_KERNEL); 1288 if (!icreq) 1289 return -ENOMEM; 1290 1291 icresp = kzalloc(sizeof(*icresp), GFP_KERNEL); 1292 if (!icresp) { 1293 ret = -ENOMEM; 1294 goto free_icreq; 1295 } 1296 1297 icreq->hdr.type = nvme_tcp_icreq; 1298 icreq->hdr.hlen = sizeof(*icreq); 1299 icreq->hdr.pdo = 0; 1300 icreq->hdr.plen = cpu_to_le32(icreq->hdr.hlen); 1301 icreq->pfv = cpu_to_le16(NVME_TCP_PFV_1_0); 1302 icreq->maxr2t = 0; /* single inflight r2t supported */ 1303 icreq->hpda = 0; /* no alignment constraint */ 1304 if (queue->hdr_digest) 1305 icreq->digest |= NVME_TCP_HDR_DIGEST_ENABLE; 1306 if (queue->data_digest) 1307 icreq->digest |= NVME_TCP_DATA_DIGEST_ENABLE; 1308 1309 iov.iov_base = icreq; 1310 iov.iov_len = sizeof(*icreq); 1311 ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); 1312 if (ret < 0) 1313 goto free_icresp; 1314 1315 memset(&msg, 0, sizeof(msg)); 1316 iov.iov_base = icresp; 1317 iov.iov_len = sizeof(*icresp); 1318 ret = kernel_recvmsg(queue->sock, &msg, &iov, 1, 1319 iov.iov_len, msg.msg_flags); 1320 if (ret < 0) 1321 goto free_icresp; 1322 1323 ret = -EINVAL; 1324 if (icresp->hdr.type != nvme_tcp_icresp) { 1325 pr_err("queue %d: bad type returned %d\n", 1326 nvme_tcp_queue_id(queue), icresp->hdr.type); 1327 goto free_icresp; 1328 } 1329 1330 if (le32_to_cpu(icresp->hdr.plen) != sizeof(*icresp)) { 1331 pr_err("queue %d: bad pdu length returned %d\n", 1332 nvme_tcp_queue_id(queue), icresp->hdr.plen); 1333 goto free_icresp; 1334 } 1335 1336 if (icresp->pfv != NVME_TCP_PFV_1_0) { 1337 pr_err("queue %d: bad pfv returned %d\n", 1338 nvme_tcp_queue_id(queue), icresp->pfv); 1339 goto free_icresp; 1340 } 1341 1342 ctrl_ddgst = !!(icresp->digest & NVME_TCP_DATA_DIGEST_ENABLE); 1343 if ((queue->data_digest && !ctrl_ddgst) || 1344 (!queue->data_digest && ctrl_ddgst)) { 1345 pr_err("queue %d: data digest mismatch host: %s ctrl: %s\n", 1346 nvme_tcp_queue_id(queue), 1347 queue->data_digest ? "enabled" : "disabled", 1348 ctrl_ddgst ? "enabled" : "disabled"); 1349 goto free_icresp; 1350 } 1351 1352 ctrl_hdgst = !!(icresp->digest & NVME_TCP_HDR_DIGEST_ENABLE); 1353 if ((queue->hdr_digest && !ctrl_hdgst) || 1354 (!queue->hdr_digest && ctrl_hdgst)) { 1355 pr_err("queue %d: header digest mismatch host: %s ctrl: %s\n", 1356 nvme_tcp_queue_id(queue), 1357 queue->hdr_digest ? "enabled" : "disabled", 1358 ctrl_hdgst ? "enabled" : "disabled"); 1359 goto free_icresp; 1360 } 1361 1362 if (icresp->cpda != 0) { 1363 pr_err("queue %d: unsupported cpda returned %d\n", 1364 nvme_tcp_queue_id(queue), icresp->cpda); 1365 goto free_icresp; 1366 } 1367 1368 ret = 0; 1369free_icresp: 1370 kfree(icresp); 1371free_icreq: 1372 kfree(icreq); 1373 return ret; 1374} 1375 1376static bool nvme_tcp_admin_queue(struct nvme_tcp_queue *queue) 1377{ 1378 return nvme_tcp_queue_id(queue) == 0; 1379} 1380 1381static bool nvme_tcp_default_queue(struct nvme_tcp_queue *queue) 1382{ 1383 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1384 int qid = nvme_tcp_queue_id(queue); 1385 1386 return !nvme_tcp_admin_queue(queue) && 1387 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT]; 1388} 1389 1390static bool nvme_tcp_read_queue(struct nvme_tcp_queue *queue) 1391{ 1392 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1393 int qid = nvme_tcp_queue_id(queue); 1394 1395 return !nvme_tcp_admin_queue(queue) && 1396 !nvme_tcp_default_queue(queue) && 1397 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] + 1398 ctrl->io_queues[HCTX_TYPE_READ]; 1399} 1400 1401static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue) 1402{ 1403 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1404 int qid = nvme_tcp_queue_id(queue); 1405 1406 return !nvme_tcp_admin_queue(queue) && 1407 !nvme_tcp_default_queue(queue) && 1408 !nvme_tcp_read_queue(queue) && 1409 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] + 1410 ctrl->io_queues[HCTX_TYPE_READ] + 1411 ctrl->io_queues[HCTX_TYPE_POLL]; 1412} 1413 1414static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue) 1415{ 1416 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1417 int qid = nvme_tcp_queue_id(queue); 1418 int n = 0; 1419 1420 if (nvme_tcp_default_queue(queue)) 1421 n = qid - 1; 1422 else if (nvme_tcp_read_queue(queue)) 1423 n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] - 1; 1424 else if (nvme_tcp_poll_queue(queue)) 1425 n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] - 1426 ctrl->io_queues[HCTX_TYPE_READ] - 1; 1427 queue->io_cpu = cpumask_next_wrap(n - 1, cpu_online_mask, -1, false); 1428} 1429 1430static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, 1431 int qid, size_t queue_size) 1432{ 1433 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1434 struct nvme_tcp_queue *queue = &ctrl->queues[qid]; 1435 int ret, rcv_pdu_size; 1436 1437 mutex_init(&queue->queue_lock); 1438 queue->ctrl = ctrl; 1439 init_llist_head(&queue->req_list); 1440 INIT_LIST_HEAD(&queue->send_list); 1441 mutex_init(&queue->send_mutex); 1442 INIT_WORK(&queue->io_work, nvme_tcp_io_work); 1443 queue->queue_size = queue_size; 1444 1445 if (qid > 0) 1446 queue->cmnd_capsule_len = nctrl->ioccsz * 16; 1447 else 1448 queue->cmnd_capsule_len = sizeof(struct nvme_command) + 1449 NVME_TCP_ADMIN_CCSZ; 1450 1451 ret = sock_create(ctrl->addr.ss_family, SOCK_STREAM, 1452 IPPROTO_TCP, &queue->sock); 1453 if (ret) { 1454 dev_err(nctrl->device, 1455 "failed to create socket: %d\n", ret); 1456 goto err_destroy_mutex; 1457 } 1458 1459 nvme_tcp_reclassify_socket(queue->sock); 1460 1461 /* Single syn retry */ 1462 tcp_sock_set_syncnt(queue->sock->sk, 1); 1463 1464 /* Set TCP no delay */ 1465 tcp_sock_set_nodelay(queue->sock->sk); 1466 1467 /* 1468 * Cleanup whatever is sitting in the TCP transmit queue on socket 1469 * close. This is done to prevent stale data from being sent should 1470 * the network connection be restored before TCP times out. 1471 */ 1472 sock_no_linger(queue->sock->sk); 1473 1474 if (so_priority > 0) 1475 sock_set_priority(queue->sock->sk, so_priority); 1476 1477 /* Set socket type of service */ 1478 if (nctrl->opts->tos >= 0) 1479 ip_sock_set_tos(queue->sock->sk, nctrl->opts->tos); 1480 1481 /* Set 10 seconds timeout for icresp recvmsg */ 1482 queue->sock->sk->sk_rcvtimeo = 10 * HZ; 1483 1484 queue->sock->sk->sk_allocation = GFP_ATOMIC; 1485 nvme_tcp_set_queue_io_cpu(queue); 1486 queue->request = NULL; 1487 queue->data_remaining = 0; 1488 queue->ddgst_remaining = 0; 1489 queue->pdu_remaining = 0; 1490 queue->pdu_offset = 0; 1491 sk_set_memalloc(queue->sock->sk); 1492 1493 if (nctrl->opts->mask & NVMF_OPT_HOST_TRADDR) { 1494 ret = kernel_bind(queue->sock, (struct sockaddr *)&ctrl->src_addr, 1495 sizeof(ctrl->src_addr)); 1496 if (ret) { 1497 dev_err(nctrl->device, 1498 "failed to bind queue %d socket %d\n", 1499 qid, ret); 1500 goto err_sock; 1501 } 1502 } 1503 1504 queue->hdr_digest = nctrl->opts->hdr_digest; 1505 queue->data_digest = nctrl->opts->data_digest; 1506 if (queue->hdr_digest || queue->data_digest) { 1507 ret = nvme_tcp_alloc_crypto(queue); 1508 if (ret) { 1509 dev_err(nctrl->device, 1510 "failed to allocate queue %d crypto\n", qid); 1511 goto err_sock; 1512 } 1513 } 1514 1515 rcv_pdu_size = sizeof(struct nvme_tcp_rsp_pdu) + 1516 nvme_tcp_hdgst_len(queue); 1517 queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL); 1518 if (!queue->pdu) { 1519 ret = -ENOMEM; 1520 goto err_crypto; 1521 } 1522 1523 dev_dbg(nctrl->device, "connecting queue %d\n", 1524 nvme_tcp_queue_id(queue)); 1525 1526 ret = kernel_connect(queue->sock, (struct sockaddr *)&ctrl->addr, 1527 sizeof(ctrl->addr), 0); 1528 if (ret) { 1529 dev_err(nctrl->device, 1530 "failed to connect socket: %d\n", ret); 1531 goto err_rcv_pdu; 1532 } 1533 1534 ret = nvme_tcp_init_connection(queue); 1535 if (ret) 1536 goto err_init_connect; 1537 1538 set_bit(NVME_TCP_Q_ALLOCATED, &queue->flags); 1539 1540 return 0; 1541 1542err_init_connect: 1543 kernel_sock_shutdown(queue->sock, SHUT_RDWR); 1544err_rcv_pdu: 1545 kfree(queue->pdu); 1546err_crypto: 1547 if (queue->hdr_digest || queue->data_digest) 1548 nvme_tcp_free_crypto(queue); 1549err_sock: 1550 sock_release(queue->sock); 1551 queue->sock = NULL; 1552err_destroy_mutex: 1553 mutex_destroy(&queue->queue_lock); 1554 return ret; 1555} 1556 1557static void nvme_tcp_restore_sock_ops(struct nvme_tcp_queue *queue) 1558{ 1559 struct socket *sock = queue->sock; 1560 1561 write_lock_bh(&sock->sk->sk_callback_lock); 1562 sock->sk->sk_user_data = NULL; 1563 sock->sk->sk_data_ready = queue->data_ready; 1564 sock->sk->sk_state_change = queue->state_change; 1565 sock->sk->sk_write_space = queue->write_space; 1566 write_unlock_bh(&sock->sk->sk_callback_lock); 1567} 1568 1569static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue) 1570{ 1571 kernel_sock_shutdown(queue->sock, SHUT_RDWR); 1572 nvme_tcp_restore_sock_ops(queue); 1573 cancel_work_sync(&queue->io_work); 1574} 1575 1576static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid) 1577{ 1578 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1579 struct nvme_tcp_queue *queue = &ctrl->queues[qid]; 1580 1581 mutex_lock(&queue->queue_lock); 1582 if (test_and_clear_bit(NVME_TCP_Q_LIVE, &queue->flags)) 1583 __nvme_tcp_stop_queue(queue); 1584 mutex_unlock(&queue->queue_lock); 1585} 1586 1587static void nvme_tcp_setup_sock_ops(struct nvme_tcp_queue *queue) 1588{ 1589 write_lock_bh(&queue->sock->sk->sk_callback_lock); 1590 queue->sock->sk->sk_user_data = queue; 1591 queue->state_change = queue->sock->sk->sk_state_change; 1592 queue->data_ready = queue->sock->sk->sk_data_ready; 1593 queue->write_space = queue->sock->sk->sk_write_space; 1594 queue->sock->sk->sk_data_ready = nvme_tcp_data_ready; 1595 queue->sock->sk->sk_state_change = nvme_tcp_state_change; 1596 queue->sock->sk->sk_write_space = nvme_tcp_write_space; 1597#ifdef CONFIG_NET_RX_BUSY_POLL 1598 queue->sock->sk->sk_ll_usec = 1; 1599#endif 1600 write_unlock_bh(&queue->sock->sk->sk_callback_lock); 1601} 1602 1603static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx) 1604{ 1605 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1606 struct nvme_tcp_queue *queue = &ctrl->queues[idx]; 1607 int ret; 1608 1609 queue->rd_enabled = true; 1610 nvme_tcp_init_recv_ctx(queue); 1611 nvme_tcp_setup_sock_ops(queue); 1612 1613 if (idx) 1614 ret = nvmf_connect_io_queue(nctrl, idx, false); 1615 else 1616 ret = nvmf_connect_admin_queue(nctrl); 1617 1618 if (!ret) { 1619 set_bit(NVME_TCP_Q_LIVE, &queue->flags); 1620 } else { 1621 if (test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags)) 1622 __nvme_tcp_stop_queue(queue); 1623 dev_err(nctrl->device, 1624 "failed to connect queue: %d ret=%d\n", idx, ret); 1625 } 1626 return ret; 1627} 1628 1629static struct blk_mq_tag_set *nvme_tcp_alloc_tagset(struct nvme_ctrl *nctrl, 1630 bool admin) 1631{ 1632 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1633 struct blk_mq_tag_set *set; 1634 int ret; 1635 1636 if (admin) { 1637 set = &ctrl->admin_tag_set; 1638 memset(set, 0, sizeof(*set)); 1639 set->ops = &nvme_tcp_admin_mq_ops; 1640 set->queue_depth = NVME_AQ_MQ_TAG_DEPTH; 1641 set->reserved_tags = 2; /* connect + keep-alive */ 1642 set->numa_node = nctrl->numa_node; 1643 set->flags = BLK_MQ_F_BLOCKING; 1644 set->cmd_size = sizeof(struct nvme_tcp_request); 1645 set->driver_data = ctrl; 1646 set->nr_hw_queues = 1; 1647 set->timeout = ADMIN_TIMEOUT; 1648 } else { 1649 set = &ctrl->tag_set; 1650 memset(set, 0, sizeof(*set)); 1651 set->ops = &nvme_tcp_mq_ops; 1652 set->queue_depth = nctrl->sqsize + 1; 1653 set->reserved_tags = 1; /* fabric connect */ 1654 set->numa_node = nctrl->numa_node; 1655 set->flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_BLOCKING; 1656 set->cmd_size = sizeof(struct nvme_tcp_request); 1657 set->driver_data = ctrl; 1658 set->nr_hw_queues = nctrl->queue_count - 1; 1659 set->timeout = NVME_IO_TIMEOUT; 1660 set->nr_maps = nctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2; 1661 } 1662 1663 ret = blk_mq_alloc_tag_set(set); 1664 if (ret) 1665 return ERR_PTR(ret); 1666 1667 return set; 1668} 1669 1670static void nvme_tcp_free_admin_queue(struct nvme_ctrl *ctrl) 1671{ 1672 if (to_tcp_ctrl(ctrl)->async_req.pdu) { 1673 cancel_work_sync(&ctrl->async_event_work); 1674 nvme_tcp_free_async_req(to_tcp_ctrl(ctrl)); 1675 to_tcp_ctrl(ctrl)->async_req.pdu = NULL; 1676 } 1677 1678 nvme_tcp_free_queue(ctrl, 0); 1679} 1680 1681static void nvme_tcp_free_io_queues(struct nvme_ctrl *ctrl) 1682{ 1683 int i; 1684 1685 for (i = 1; i < ctrl->queue_count; i++) 1686 nvme_tcp_free_queue(ctrl, i); 1687} 1688 1689static void nvme_tcp_stop_io_queues(struct nvme_ctrl *ctrl) 1690{ 1691 int i; 1692 1693 for (i = 1; i < ctrl->queue_count; i++) 1694 nvme_tcp_stop_queue(ctrl, i); 1695} 1696 1697static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl) 1698{ 1699 int i, ret = 0; 1700 1701 for (i = 1; i < ctrl->queue_count; i++) { 1702 ret = nvme_tcp_start_queue(ctrl, i); 1703 if (ret) 1704 goto out_stop_queues; 1705 } 1706 1707 return 0; 1708 1709out_stop_queues: 1710 for (i--; i >= 1; i--) 1711 nvme_tcp_stop_queue(ctrl, i); 1712 return ret; 1713} 1714 1715static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl) 1716{ 1717 int ret; 1718 1719 ret = nvme_tcp_alloc_queue(ctrl, 0, NVME_AQ_DEPTH); 1720 if (ret) 1721 return ret; 1722 1723 ret = nvme_tcp_alloc_async_req(to_tcp_ctrl(ctrl)); 1724 if (ret) 1725 goto out_free_queue; 1726 1727 return 0; 1728 1729out_free_queue: 1730 nvme_tcp_free_queue(ctrl, 0); 1731 return ret; 1732} 1733 1734static int __nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl) 1735{ 1736 int i, ret; 1737 1738 for (i = 1; i < ctrl->queue_count; i++) { 1739 ret = nvme_tcp_alloc_queue(ctrl, i, 1740 ctrl->sqsize + 1); 1741 if (ret) 1742 goto out_free_queues; 1743 } 1744 1745 return 0; 1746 1747out_free_queues: 1748 for (i--; i >= 1; i--) 1749 nvme_tcp_free_queue(ctrl, i); 1750 1751 return ret; 1752} 1753 1754static unsigned int nvme_tcp_nr_io_queues(struct nvme_ctrl *ctrl) 1755{ 1756 unsigned int nr_io_queues; 1757 1758 nr_io_queues = min(ctrl->opts->nr_io_queues, num_online_cpus()); 1759 nr_io_queues += min(ctrl->opts->nr_write_queues, num_online_cpus()); 1760 nr_io_queues += min(ctrl->opts->nr_poll_queues, num_online_cpus()); 1761 1762 return nr_io_queues; 1763} 1764 1765static void nvme_tcp_set_io_queues(struct nvme_ctrl *nctrl, 1766 unsigned int nr_io_queues) 1767{ 1768 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1769 struct nvmf_ctrl_options *opts = nctrl->opts; 1770 1771 if (opts->nr_write_queues && opts->nr_io_queues < nr_io_queues) { 1772 /* 1773 * separate read/write queues 1774 * hand out dedicated default queues only after we have 1775 * sufficient read queues. 1776 */ 1777 ctrl->io_queues[HCTX_TYPE_READ] = opts->nr_io_queues; 1778 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_READ]; 1779 ctrl->io_queues[HCTX_TYPE_DEFAULT] = 1780 min(opts->nr_write_queues, nr_io_queues); 1781 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT]; 1782 } else { 1783 /* 1784 * shared read/write queues 1785 * either no write queues were requested, or we don't have 1786 * sufficient queue count to have dedicated default queues. 1787 */ 1788 ctrl->io_queues[HCTX_TYPE_DEFAULT] = 1789 min(opts->nr_io_queues, nr_io_queues); 1790 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT]; 1791 } 1792 1793 if (opts->nr_poll_queues && nr_io_queues) { 1794 /* map dedicated poll queues only if we have queues left */ 1795 ctrl->io_queues[HCTX_TYPE_POLL] = 1796 min(opts->nr_poll_queues, nr_io_queues); 1797 } 1798} 1799 1800static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl) 1801{ 1802 unsigned int nr_io_queues; 1803 int ret; 1804 1805 nr_io_queues = nvme_tcp_nr_io_queues(ctrl); 1806 ret = nvme_set_queue_count(ctrl, &nr_io_queues); 1807 if (ret) 1808 return ret; 1809 1810 if (nr_io_queues == 0) { 1811 dev_err(ctrl->device, 1812 "unable to set any I/O queues\n"); 1813 return -ENOMEM; 1814 } 1815 1816 ctrl->queue_count = nr_io_queues + 1; 1817 dev_info(ctrl->device, 1818 "creating %d I/O queues.\n", nr_io_queues); 1819 1820 nvme_tcp_set_io_queues(ctrl, nr_io_queues); 1821 1822 return __nvme_tcp_alloc_io_queues(ctrl); 1823} 1824 1825static void nvme_tcp_destroy_io_queues(struct nvme_ctrl *ctrl, bool remove) 1826{ 1827 nvme_tcp_stop_io_queues(ctrl); 1828 if (remove) { 1829 blk_cleanup_queue(ctrl->connect_q); 1830 blk_mq_free_tag_set(ctrl->tagset); 1831 } 1832 nvme_tcp_free_io_queues(ctrl); 1833} 1834 1835static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new) 1836{ 1837 int ret; 1838 1839 ret = nvme_tcp_alloc_io_queues(ctrl); 1840 if (ret) 1841 return ret; 1842 1843 if (new) { 1844 ctrl->tagset = nvme_tcp_alloc_tagset(ctrl, false); 1845 if (IS_ERR(ctrl->tagset)) { 1846 ret = PTR_ERR(ctrl->tagset); 1847 goto out_free_io_queues; 1848 } 1849 1850 ctrl->connect_q = blk_mq_init_queue(ctrl->tagset); 1851 if (IS_ERR(ctrl->connect_q)) { 1852 ret = PTR_ERR(ctrl->connect_q); 1853 goto out_free_tag_set; 1854 } 1855 } 1856 1857 ret = nvme_tcp_start_io_queues(ctrl); 1858 if (ret) 1859 goto out_cleanup_connect_q; 1860 1861 if (!new) { 1862 nvme_start_freeze(ctrl); 1863 nvme_start_queues(ctrl); 1864 if (!nvme_wait_freeze_timeout(ctrl, NVME_IO_TIMEOUT)) { 1865 /* 1866 * If we timed out waiting for freeze we are likely to 1867 * be stuck. Fail the controller initialization just 1868 * to be safe. 1869 */ 1870 ret = -ENODEV; 1871 nvme_unfreeze(ctrl); 1872 goto out_wait_freeze_timed_out; 1873 } 1874 blk_mq_update_nr_hw_queues(ctrl->tagset, 1875 ctrl->queue_count - 1); 1876 nvme_unfreeze(ctrl); 1877 } 1878 1879 return 0; 1880 1881out_wait_freeze_timed_out: 1882 nvme_stop_queues(ctrl); 1883 nvme_sync_io_queues(ctrl); 1884 nvme_tcp_stop_io_queues(ctrl); 1885out_cleanup_connect_q: 1886 nvme_cancel_tagset(ctrl); 1887 if (new) 1888 blk_cleanup_queue(ctrl->connect_q); 1889out_free_tag_set: 1890 if (new) 1891 blk_mq_free_tag_set(ctrl->tagset); 1892out_free_io_queues: 1893 nvme_tcp_free_io_queues(ctrl); 1894 return ret; 1895} 1896 1897static void nvme_tcp_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove) 1898{ 1899 nvme_tcp_stop_queue(ctrl, 0); 1900 if (remove) { 1901 blk_cleanup_queue(ctrl->admin_q); 1902 blk_cleanup_queue(ctrl->fabrics_q); 1903 blk_mq_free_tag_set(ctrl->admin_tagset); 1904 } 1905 nvme_tcp_free_admin_queue(ctrl); 1906} 1907 1908static int nvme_tcp_configure_admin_queue(struct nvme_ctrl *ctrl, bool new) 1909{ 1910 int error; 1911 1912 error = nvme_tcp_alloc_admin_queue(ctrl); 1913 if (error) 1914 return error; 1915 1916 if (new) { 1917 ctrl->admin_tagset = nvme_tcp_alloc_tagset(ctrl, true); 1918 if (IS_ERR(ctrl->admin_tagset)) { 1919 error = PTR_ERR(ctrl->admin_tagset); 1920 goto out_free_queue; 1921 } 1922 1923 ctrl->fabrics_q = blk_mq_init_queue(ctrl->admin_tagset); 1924 if (IS_ERR(ctrl->fabrics_q)) { 1925 error = PTR_ERR(ctrl->fabrics_q); 1926 goto out_free_tagset; 1927 } 1928 1929 ctrl->admin_q = blk_mq_init_queue(ctrl->admin_tagset); 1930 if (IS_ERR(ctrl->admin_q)) { 1931 error = PTR_ERR(ctrl->admin_q); 1932 goto out_cleanup_fabrics_q; 1933 } 1934 } 1935 1936 error = nvme_tcp_start_queue(ctrl, 0); 1937 if (error) 1938 goto out_cleanup_queue; 1939 1940 error = nvme_enable_ctrl(ctrl); 1941 if (error) 1942 goto out_stop_queue; 1943 1944 nvme_start_admin_queue(ctrl); 1945 1946 error = nvme_init_identify(ctrl); 1947 if (error) 1948 goto out_quiesce_queue; 1949 1950 return 0; 1951 1952out_quiesce_queue: 1953 nvme_stop_admin_queue(ctrl); 1954 blk_sync_queue(ctrl->admin_q); 1955out_stop_queue: 1956 nvme_tcp_stop_queue(ctrl, 0); 1957 nvme_cancel_admin_tagset(ctrl); 1958out_cleanup_queue: 1959 if (new) 1960 blk_cleanup_queue(ctrl->admin_q); 1961out_cleanup_fabrics_q: 1962 if (new) 1963 blk_cleanup_queue(ctrl->fabrics_q); 1964out_free_tagset: 1965 if (new) 1966 blk_mq_free_tag_set(ctrl->admin_tagset); 1967out_free_queue: 1968 nvme_tcp_free_admin_queue(ctrl); 1969 return error; 1970} 1971 1972static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl, 1973 bool remove) 1974{ 1975 nvme_stop_admin_queue(ctrl); 1976 blk_sync_queue(ctrl->admin_q); 1977 nvme_tcp_stop_queue(ctrl, 0); 1978 if (ctrl->admin_tagset) { 1979 blk_mq_tagset_busy_iter(ctrl->admin_tagset, 1980 nvme_cancel_request, ctrl); 1981 blk_mq_tagset_wait_completed_request(ctrl->admin_tagset); 1982 } 1983 if (remove) 1984 nvme_start_admin_queue(ctrl); 1985 nvme_tcp_destroy_admin_queue(ctrl, remove); 1986} 1987 1988static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl, 1989 bool remove) 1990{ 1991 if (ctrl->queue_count <= 1) 1992 return; 1993 nvme_stop_admin_queue(ctrl); 1994 nvme_stop_queues(ctrl); 1995 nvme_sync_io_queues(ctrl); 1996 nvme_tcp_stop_io_queues(ctrl); 1997 if (ctrl->tagset) { 1998 blk_mq_tagset_busy_iter(ctrl->tagset, 1999 nvme_cancel_request, ctrl); 2000 blk_mq_tagset_wait_completed_request(ctrl->tagset); 2001 } 2002 if (remove) 2003 nvme_start_queues(ctrl); 2004 nvme_tcp_destroy_io_queues(ctrl, remove); 2005} 2006 2007static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl) 2008{ 2009 /* If we are resetting/deleting then do nothing */ 2010 if (ctrl->state != NVME_CTRL_CONNECTING) { 2011 WARN_ON_ONCE(ctrl->state == NVME_CTRL_NEW || 2012 ctrl->state == NVME_CTRL_LIVE); 2013 return; 2014 } 2015 2016 if (nvmf_should_reconnect(ctrl)) { 2017 dev_info(ctrl->device, "Reconnecting in %d seconds...\n", 2018 ctrl->opts->reconnect_delay); 2019 queue_delayed_work(nvme_wq, &to_tcp_ctrl(ctrl)->connect_work, 2020 ctrl->opts->reconnect_delay * HZ); 2021 } else { 2022 dev_info(ctrl->device, "Removing controller...\n"); 2023 nvme_delete_ctrl(ctrl); 2024 } 2025} 2026 2027static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new) 2028{ 2029 struct nvmf_ctrl_options *opts = ctrl->opts; 2030 int ret; 2031 2032 ret = nvme_tcp_configure_admin_queue(ctrl, new); 2033 if (ret) 2034 return ret; 2035 2036 if (ctrl->icdoff) { 2037 dev_err(ctrl->device, "icdoff is not supported!\n"); 2038 goto destroy_admin; 2039 } 2040 2041 if (opts->queue_size > ctrl->sqsize + 1) 2042 dev_warn(ctrl->device, 2043 "queue_size %zu > ctrl sqsize %u, clamping down\n", 2044 opts->queue_size, ctrl->sqsize + 1); 2045 2046 if (ctrl->sqsize + 1 > ctrl->maxcmd) { 2047 dev_warn(ctrl->device, 2048 "sqsize %u > ctrl maxcmd %u, clamping down\n", 2049 ctrl->sqsize + 1, ctrl->maxcmd); 2050 ctrl->sqsize = ctrl->maxcmd - 1; 2051 } 2052 2053 if (ctrl->queue_count > 1) { 2054 ret = nvme_tcp_configure_io_queues(ctrl, new); 2055 if (ret) 2056 goto destroy_admin; 2057 } 2058 2059 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE)) { 2060 /* 2061 * state change failure is ok if we started ctrl delete, 2062 * unless we're during creation of a new controller to 2063 * avoid races with teardown flow. 2064 */ 2065 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING && 2066 ctrl->state != NVME_CTRL_DELETING_NOIO); 2067 WARN_ON_ONCE(new); 2068 ret = -EINVAL; 2069 goto destroy_io; 2070 } 2071 2072 nvme_start_ctrl(ctrl); 2073 return 0; 2074 2075destroy_io: 2076 if (ctrl->queue_count > 1) { 2077 nvme_stop_queues(ctrl); 2078 nvme_sync_io_queues(ctrl); 2079 nvme_tcp_stop_io_queues(ctrl); 2080 nvme_cancel_tagset(ctrl); 2081 nvme_tcp_destroy_io_queues(ctrl, new); 2082 } 2083destroy_admin: 2084 nvme_stop_admin_queue(ctrl); 2085 blk_sync_queue(ctrl->admin_q); 2086 nvme_tcp_stop_queue(ctrl, 0); 2087 nvme_cancel_admin_tagset(ctrl); 2088 nvme_tcp_destroy_admin_queue(ctrl, new); 2089 return ret; 2090} 2091 2092static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work) 2093{ 2094 struct nvme_tcp_ctrl *tcp_ctrl = container_of(to_delayed_work(work), 2095 struct nvme_tcp_ctrl, connect_work); 2096 struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; 2097 2098 ++ctrl->nr_reconnects; 2099 2100 if (nvme_tcp_setup_ctrl(ctrl, false)) 2101 goto requeue; 2102 2103 dev_info(ctrl->device, "Successfully reconnected (%d attempt)\n", 2104 ctrl->nr_reconnects); 2105 2106 ctrl->nr_reconnects = 0; 2107 2108 return; 2109 2110requeue: 2111 dev_info(ctrl->device, "Failed reconnect attempt %d\n", 2112 ctrl->nr_reconnects); 2113 nvme_tcp_reconnect_or_remove(ctrl); 2114} 2115 2116static void nvme_tcp_error_recovery_work(struct work_struct *work) 2117{ 2118 struct nvme_tcp_ctrl *tcp_ctrl = container_of(work, 2119 struct nvme_tcp_ctrl, err_work); 2120 struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; 2121 2122 nvme_stop_keep_alive(ctrl); 2123 flush_work(&ctrl->async_event_work); 2124 nvme_tcp_teardown_io_queues(ctrl, false); 2125 /* unquiesce to fail fast pending requests */ 2126 nvme_start_queues(ctrl); 2127 nvme_tcp_teardown_admin_queue(ctrl, false); 2128 nvme_start_admin_queue(ctrl); 2129 2130 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) { 2131 /* state change failure is ok if we started ctrl delete */ 2132 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING && 2133 ctrl->state != NVME_CTRL_DELETING_NOIO); 2134 return; 2135 } 2136 2137 nvme_tcp_reconnect_or_remove(ctrl); 2138} 2139 2140static void nvme_tcp_teardown_ctrl(struct nvme_ctrl *ctrl, bool shutdown) 2141{ 2142 nvme_tcp_teardown_io_queues(ctrl, shutdown); 2143 nvme_stop_admin_queue(ctrl); 2144 if (shutdown) 2145 nvme_shutdown_ctrl(ctrl); 2146 else 2147 nvme_disable_ctrl(ctrl); 2148 nvme_tcp_teardown_admin_queue(ctrl, shutdown); 2149} 2150 2151static void nvme_tcp_delete_ctrl(struct nvme_ctrl *ctrl) 2152{ 2153 nvme_tcp_teardown_ctrl(ctrl, true); 2154} 2155 2156static void nvme_reset_ctrl_work(struct work_struct *work) 2157{ 2158 struct nvme_ctrl *ctrl = 2159 container_of(work, struct nvme_ctrl, reset_work); 2160 2161 nvme_stop_ctrl(ctrl); 2162 nvme_tcp_teardown_ctrl(ctrl, false); 2163 2164 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) { 2165 /* state change failure is ok if we started ctrl delete */ 2166 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING && 2167 ctrl->state != NVME_CTRL_DELETING_NOIO); 2168 return; 2169 } 2170 2171 if (nvme_tcp_setup_ctrl(ctrl, false)) 2172 goto out_fail; 2173 2174 return; 2175 2176out_fail: 2177 ++ctrl->nr_reconnects; 2178 nvme_tcp_reconnect_or_remove(ctrl); 2179} 2180 2181static void nvme_tcp_stop_ctrl(struct nvme_ctrl *ctrl) 2182{ 2183 cancel_work_sync(&to_tcp_ctrl(ctrl)->err_work); 2184 cancel_delayed_work_sync(&to_tcp_ctrl(ctrl)->connect_work); 2185} 2186 2187static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl) 2188{ 2189 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 2190 2191 if (list_empty(&ctrl->list)) 2192 goto free_ctrl; 2193 2194 mutex_lock(&nvme_tcp_ctrl_mutex); 2195 list_del(&ctrl->list); 2196 mutex_unlock(&nvme_tcp_ctrl_mutex); 2197 2198 nvmf_free_options(nctrl->opts); 2199free_ctrl: 2200 kfree(ctrl->queues); 2201 kfree(ctrl); 2202} 2203 2204static void nvme_tcp_set_sg_null(struct nvme_command *c) 2205{ 2206 struct nvme_sgl_desc *sg = &c->common.dptr.sgl; 2207 2208 sg->addr = 0; 2209 sg->length = 0; 2210 sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) | 2211 NVME_SGL_FMT_TRANSPORT_A; 2212} 2213 2214static void nvme_tcp_set_sg_inline(struct nvme_tcp_queue *queue, 2215 struct nvme_command *c, u32 data_len) 2216{ 2217 struct nvme_sgl_desc *sg = &c->common.dptr.sgl; 2218 2219 sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff); 2220 sg->length = cpu_to_le32(data_len); 2221 sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET; 2222} 2223 2224static void nvme_tcp_set_sg_host_data(struct nvme_command *c, 2225 u32 data_len) 2226{ 2227 struct nvme_sgl_desc *sg = &c->common.dptr.sgl; 2228 2229 sg->addr = 0; 2230 sg->length = cpu_to_le32(data_len); 2231 sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) | 2232 NVME_SGL_FMT_TRANSPORT_A; 2233} 2234 2235static void nvme_tcp_submit_async_event(struct nvme_ctrl *arg) 2236{ 2237 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(arg); 2238 struct nvme_tcp_queue *queue = &ctrl->queues[0]; 2239 struct nvme_tcp_cmd_pdu *pdu = ctrl->async_req.pdu; 2240 struct nvme_command *cmd = &pdu->cmd; 2241 u8 hdgst = nvme_tcp_hdgst_len(queue); 2242 2243 memset(pdu, 0, sizeof(*pdu)); 2244 pdu->hdr.type = nvme_tcp_cmd; 2245 if (queue->hdr_digest) 2246 pdu->hdr.flags |= NVME_TCP_F_HDGST; 2247 pdu->hdr.hlen = sizeof(*pdu); 2248 pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst); 2249 2250 cmd->common.opcode = nvme_admin_async_event; 2251 cmd->common.command_id = NVME_AQ_BLK_MQ_DEPTH; 2252 cmd->common.flags |= NVME_CMD_SGL_METABUF; 2253 nvme_tcp_set_sg_null(cmd); 2254 2255 ctrl->async_req.state = NVME_TCP_SEND_CMD_PDU; 2256 ctrl->async_req.offset = 0; 2257 ctrl->async_req.curr_bio = NULL; 2258 ctrl->async_req.data_len = 0; 2259 2260 nvme_tcp_queue_request(&ctrl->async_req, true, true); 2261} 2262 2263static void nvme_tcp_complete_timed_out(struct request *rq) 2264{ 2265 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2266 struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl; 2267 2268 nvme_tcp_stop_queue(ctrl, nvme_tcp_queue_id(req->queue)); 2269 if (blk_mq_request_started(rq) && !blk_mq_request_completed(rq)) { 2270 nvme_req(rq)->status = NVME_SC_HOST_ABORTED_CMD; 2271 blk_mq_complete_request(rq); 2272 } 2273} 2274 2275static enum blk_eh_timer_return 2276nvme_tcp_timeout(struct request *rq, bool reserved) 2277{ 2278 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2279 struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl; 2280 struct nvme_tcp_cmd_pdu *pdu = req->pdu; 2281 2282 dev_warn(ctrl->device, 2283 "queue %d: timeout request %#x type %d\n", 2284 nvme_tcp_queue_id(req->queue), rq->tag, pdu->hdr.type); 2285 2286 if (ctrl->state != NVME_CTRL_LIVE) { 2287 /* 2288 * If we are resetting, connecting or deleting we should 2289 * complete immediately because we may block controller 2290 * teardown or setup sequence 2291 * - ctrl disable/shutdown fabrics requests 2292 * - connect requests 2293 * - initialization admin requests 2294 * - I/O requests that entered after unquiescing and 2295 * the controller stopped responding 2296 * 2297 * All other requests should be cancelled by the error 2298 * recovery work, so it's fine that we fail it here. 2299 */ 2300 nvme_tcp_complete_timed_out(rq); 2301 return BLK_EH_DONE; 2302 } 2303 2304 /* 2305 * LIVE state should trigger the normal error recovery which will 2306 * handle completing this request. 2307 */ 2308 nvme_tcp_error_recovery(ctrl); 2309 return BLK_EH_RESET_TIMER; 2310} 2311 2312static blk_status_t nvme_tcp_map_data(struct nvme_tcp_queue *queue, 2313 struct request *rq) 2314{ 2315 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2316 struct nvme_tcp_cmd_pdu *pdu = req->pdu; 2317 struct nvme_command *c = &pdu->cmd; 2318 2319 c->common.flags |= NVME_CMD_SGL_METABUF; 2320 2321 if (!blk_rq_nr_phys_segments(rq)) 2322 nvme_tcp_set_sg_null(c); 2323 else if (rq_data_dir(rq) == WRITE && 2324 req->data_len <= nvme_tcp_inline_data_size(queue)) 2325 nvme_tcp_set_sg_inline(queue, c, req->data_len); 2326 else 2327 nvme_tcp_set_sg_host_data(c, req->data_len); 2328 2329 return 0; 2330} 2331 2332static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns, 2333 struct request *rq) 2334{ 2335 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2336 struct nvme_tcp_cmd_pdu *pdu = req->pdu; 2337 struct nvme_tcp_queue *queue = req->queue; 2338 u8 hdgst = nvme_tcp_hdgst_len(queue), ddgst = 0; 2339 blk_status_t ret; 2340 2341 ret = nvme_setup_cmd(ns, rq, &pdu->cmd); 2342 if (ret) 2343 return ret; 2344 2345 req->state = NVME_TCP_SEND_CMD_PDU; 2346 req->offset = 0; 2347 req->data_sent = 0; 2348 req->pdu_len = 0; 2349 req->pdu_sent = 0; 2350 req->data_len = blk_rq_nr_phys_segments(rq) ? 2351 blk_rq_payload_bytes(rq) : 0; 2352 req->curr_bio = rq->bio; 2353 2354 if (rq_data_dir(rq) == WRITE && 2355 req->data_len <= nvme_tcp_inline_data_size(queue)) 2356 req->pdu_len = req->data_len; 2357 else if (req->curr_bio) 2358 nvme_tcp_init_iter(req, READ); 2359 2360 pdu->hdr.type = nvme_tcp_cmd; 2361 pdu->hdr.flags = 0; 2362 if (queue->hdr_digest) 2363 pdu->hdr.flags |= NVME_TCP_F_HDGST; 2364 if (queue->data_digest && req->pdu_len) { 2365 pdu->hdr.flags |= NVME_TCP_F_DDGST; 2366 ddgst = nvme_tcp_ddgst_len(queue); 2367 } 2368 pdu->hdr.hlen = sizeof(*pdu); 2369 pdu->hdr.pdo = req->pdu_len ? pdu->hdr.hlen + hdgst : 0; 2370 pdu->hdr.plen = 2371 cpu_to_le32(pdu->hdr.hlen + hdgst + req->pdu_len + ddgst); 2372 2373 ret = nvme_tcp_map_data(queue, rq); 2374 if (unlikely(ret)) { 2375 nvme_cleanup_cmd(rq); 2376 dev_err(queue->ctrl->ctrl.device, 2377 "Failed to map data (%d)\n", ret); 2378 return ret; 2379 } 2380 2381 return 0; 2382} 2383 2384static void nvme_tcp_commit_rqs(struct blk_mq_hw_ctx *hctx) 2385{ 2386 struct nvme_tcp_queue *queue = hctx->driver_data; 2387 2388 if (!llist_empty(&queue->req_list)) 2389 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 2390} 2391 2392static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx, 2393 const struct blk_mq_queue_data *bd) 2394{ 2395 struct nvme_ns *ns = hctx->queue->queuedata; 2396 struct nvme_tcp_queue *queue = hctx->driver_data; 2397 struct request *rq = bd->rq; 2398 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2399 bool queue_ready = test_bit(NVME_TCP_Q_LIVE, &queue->flags); 2400 blk_status_t ret; 2401 2402 if (!nvmf_check_ready(&queue->ctrl->ctrl, rq, queue_ready)) 2403 return nvmf_fail_nonready_command(&queue->ctrl->ctrl, rq); 2404 2405 ret = nvme_tcp_setup_cmd_pdu(ns, rq); 2406 if (unlikely(ret)) 2407 return ret; 2408 2409 blk_mq_start_request(rq); 2410 2411 nvme_tcp_queue_request(req, true, bd->last); 2412 2413 return BLK_STS_OK; 2414} 2415 2416static int nvme_tcp_map_queues(struct blk_mq_tag_set *set) 2417{ 2418 struct nvme_tcp_ctrl *ctrl = set->driver_data; 2419 struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; 2420 2421 if (opts->nr_write_queues && ctrl->io_queues[HCTX_TYPE_READ]) { 2422 /* separate read/write queues */ 2423 set->map[HCTX_TYPE_DEFAULT].nr_queues = 2424 ctrl->io_queues[HCTX_TYPE_DEFAULT]; 2425 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0; 2426 set->map[HCTX_TYPE_READ].nr_queues = 2427 ctrl->io_queues[HCTX_TYPE_READ]; 2428 set->map[HCTX_TYPE_READ].queue_offset = 2429 ctrl->io_queues[HCTX_TYPE_DEFAULT]; 2430 } else { 2431 /* shared read/write queues */ 2432 set->map[HCTX_TYPE_DEFAULT].nr_queues = 2433 ctrl->io_queues[HCTX_TYPE_DEFAULT]; 2434 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0; 2435 set->map[HCTX_TYPE_READ].nr_queues = 2436 ctrl->io_queues[HCTX_TYPE_DEFAULT]; 2437 set->map[HCTX_TYPE_READ].queue_offset = 0; 2438 } 2439 blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]); 2440 blk_mq_map_queues(&set->map[HCTX_TYPE_READ]); 2441 2442 if (opts->nr_poll_queues && ctrl->io_queues[HCTX_TYPE_POLL]) { 2443 /* map dedicated poll queues only if we have queues left */ 2444 set->map[HCTX_TYPE_POLL].nr_queues = 2445 ctrl->io_queues[HCTX_TYPE_POLL]; 2446 set->map[HCTX_TYPE_POLL].queue_offset = 2447 ctrl->io_queues[HCTX_TYPE_DEFAULT] + 2448 ctrl->io_queues[HCTX_TYPE_READ]; 2449 blk_mq_map_queues(&set->map[HCTX_TYPE_POLL]); 2450 } 2451 2452 dev_info(ctrl->ctrl.device, 2453 "mapped %d/%d/%d default/read/poll queues.\n", 2454 ctrl->io_queues[HCTX_TYPE_DEFAULT], 2455 ctrl->io_queues[HCTX_TYPE_READ], 2456 ctrl->io_queues[HCTX_TYPE_POLL]); 2457 2458 return 0; 2459} 2460 2461static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx) 2462{ 2463 struct nvme_tcp_queue *queue = hctx->driver_data; 2464 struct sock *sk = queue->sock->sk; 2465 2466 if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags)) 2467 return 0; 2468 2469 set_bit(NVME_TCP_Q_POLLING, &queue->flags); 2470 if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue)) 2471 sk_busy_loop(sk, true); 2472 nvme_tcp_try_recv(queue); 2473 clear_bit(NVME_TCP_Q_POLLING, &queue->flags); 2474 return queue->nr_cqe; 2475} 2476 2477static const struct blk_mq_ops nvme_tcp_mq_ops = { 2478 .queue_rq = nvme_tcp_queue_rq, 2479 .commit_rqs = nvme_tcp_commit_rqs, 2480 .complete = nvme_complete_rq, 2481 .init_request = nvme_tcp_init_request, 2482 .exit_request = nvme_tcp_exit_request, 2483 .init_hctx = nvme_tcp_init_hctx, 2484 .timeout = nvme_tcp_timeout, 2485 .map_queues = nvme_tcp_map_queues, 2486 .poll = nvme_tcp_poll, 2487}; 2488 2489static const struct blk_mq_ops nvme_tcp_admin_mq_ops = { 2490 .queue_rq = nvme_tcp_queue_rq, 2491 .complete = nvme_complete_rq, 2492 .init_request = nvme_tcp_init_request, 2493 .exit_request = nvme_tcp_exit_request, 2494 .init_hctx = nvme_tcp_init_admin_hctx, 2495 .timeout = nvme_tcp_timeout, 2496}; 2497 2498static const struct nvme_ctrl_ops nvme_tcp_ctrl_ops = { 2499 .name = "tcp", 2500 .module = THIS_MODULE, 2501 .flags = NVME_F_FABRICS, 2502 .reg_read32 = nvmf_reg_read32, 2503 .reg_read64 = nvmf_reg_read64, 2504 .reg_write32 = nvmf_reg_write32, 2505 .free_ctrl = nvme_tcp_free_ctrl, 2506 .submit_async_event = nvme_tcp_submit_async_event, 2507 .delete_ctrl = nvme_tcp_delete_ctrl, 2508 .get_address = nvmf_get_address, 2509 .stop_ctrl = nvme_tcp_stop_ctrl, 2510}; 2511 2512static bool 2513nvme_tcp_existing_controller(struct nvmf_ctrl_options *opts) 2514{ 2515 struct nvme_tcp_ctrl *ctrl; 2516 bool found = false; 2517 2518 mutex_lock(&nvme_tcp_ctrl_mutex); 2519 list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) { 2520 found = nvmf_ip_options_match(&ctrl->ctrl, opts); 2521 if (found) 2522 break; 2523 } 2524 mutex_unlock(&nvme_tcp_ctrl_mutex); 2525 2526 return found; 2527} 2528 2529static struct nvme_ctrl *nvme_tcp_create_ctrl(struct device *dev, 2530 struct nvmf_ctrl_options *opts) 2531{ 2532 struct nvme_tcp_ctrl *ctrl; 2533 int ret; 2534 2535 ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL); 2536 if (!ctrl) 2537 return ERR_PTR(-ENOMEM); 2538 2539 INIT_LIST_HEAD(&ctrl->list); 2540 ctrl->ctrl.opts = opts; 2541 ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues + 2542 opts->nr_poll_queues + 1; 2543 ctrl->ctrl.sqsize = opts->queue_size - 1; 2544 ctrl->ctrl.kato = opts->kato; 2545 2546 INIT_DELAYED_WORK(&ctrl->connect_work, 2547 nvme_tcp_reconnect_ctrl_work); 2548 INIT_WORK(&ctrl->err_work, nvme_tcp_error_recovery_work); 2549 INIT_WORK(&ctrl->ctrl.reset_work, nvme_reset_ctrl_work); 2550 2551 if (!(opts->mask & NVMF_OPT_TRSVCID)) { 2552 opts->trsvcid = 2553 kstrdup(__stringify(NVME_TCP_DISC_PORT), GFP_KERNEL); 2554 if (!opts->trsvcid) { 2555 ret = -ENOMEM; 2556 goto out_free_ctrl; 2557 } 2558 opts->mask |= NVMF_OPT_TRSVCID; 2559 } 2560 2561 ret = inet_pton_with_scope(&init_net, AF_UNSPEC, 2562 opts->traddr, opts->trsvcid, &ctrl->addr); 2563 if (ret) { 2564 pr_err("malformed address passed: %s:%s\n", 2565 opts->traddr, opts->trsvcid); 2566 goto out_free_ctrl; 2567 } 2568 2569 if (opts->mask & NVMF_OPT_HOST_TRADDR) { 2570 ret = inet_pton_with_scope(&init_net, AF_UNSPEC, 2571 opts->host_traddr, NULL, &ctrl->src_addr); 2572 if (ret) { 2573 pr_err("malformed src address passed: %s\n", 2574 opts->host_traddr); 2575 goto out_free_ctrl; 2576 } 2577 } 2578 2579 if (!opts->duplicate_connect && nvme_tcp_existing_controller(opts)) { 2580 ret = -EALREADY; 2581 goto out_free_ctrl; 2582 } 2583 2584 ctrl->queues = kcalloc(ctrl->ctrl.queue_count, sizeof(*ctrl->queues), 2585 GFP_KERNEL); 2586 if (!ctrl->queues) { 2587 ret = -ENOMEM; 2588 goto out_free_ctrl; 2589 } 2590 2591 ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_tcp_ctrl_ops, 0); 2592 if (ret) 2593 goto out_kfree_queues; 2594 2595 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) { 2596 WARN_ON_ONCE(1); 2597 ret = -EINTR; 2598 goto out_uninit_ctrl; 2599 } 2600 2601 ret = nvme_tcp_setup_ctrl(&ctrl->ctrl, true); 2602 if (ret) 2603 goto out_uninit_ctrl; 2604 2605 dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp\n", 2606 ctrl->ctrl.opts->subsysnqn, &ctrl->addr); 2607 2608 mutex_lock(&nvme_tcp_ctrl_mutex); 2609 list_add_tail(&ctrl->list, &nvme_tcp_ctrl_list); 2610 mutex_unlock(&nvme_tcp_ctrl_mutex); 2611 2612 return &ctrl->ctrl; 2613 2614out_uninit_ctrl: 2615 nvme_uninit_ctrl(&ctrl->ctrl); 2616 nvme_put_ctrl(&ctrl->ctrl); 2617 if (ret > 0) 2618 ret = -EIO; 2619 return ERR_PTR(ret); 2620out_kfree_queues: 2621 kfree(ctrl->queues); 2622out_free_ctrl: 2623 kfree(ctrl); 2624 return ERR_PTR(ret); 2625} 2626 2627static struct nvmf_transport_ops nvme_tcp_transport = { 2628 .name = "tcp", 2629 .module = THIS_MODULE, 2630 .required_opts = NVMF_OPT_TRADDR, 2631 .allowed_opts = NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY | 2632 NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO | 2633 NVMF_OPT_HDR_DIGEST | NVMF_OPT_DATA_DIGEST | 2634 NVMF_OPT_NR_WRITE_QUEUES | NVMF_OPT_NR_POLL_QUEUES | 2635 NVMF_OPT_TOS, 2636 .create_ctrl = nvme_tcp_create_ctrl, 2637}; 2638 2639static int __init nvme_tcp_init_module(void) 2640{ 2641 nvme_tcp_wq = alloc_workqueue("nvme_tcp_wq", 2642 WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); 2643 if (!nvme_tcp_wq) 2644 return -ENOMEM; 2645 2646 nvmf_register_transport(&nvme_tcp_transport); 2647 return 0; 2648} 2649 2650static void __exit nvme_tcp_cleanup_module(void) 2651{ 2652 struct nvme_tcp_ctrl *ctrl; 2653 2654 nvmf_unregister_transport(&nvme_tcp_transport); 2655 2656 mutex_lock(&nvme_tcp_ctrl_mutex); 2657 list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) 2658 nvme_delete_ctrl(&ctrl->ctrl); 2659 mutex_unlock(&nvme_tcp_ctrl_mutex); 2660 flush_workqueue(nvme_delete_wq); 2661 2662 destroy_workqueue(nvme_tcp_wq); 2663} 2664 2665module_init(nvme_tcp_init_module); 2666module_exit(nvme_tcp_cleanup_module); 2667 2668MODULE_LICENSE("GPL v2"); 2669