162306a36Sopenharmony_ci// SPDX-License-Identifier: GPL-2.0 262306a36Sopenharmony_ci/* 362306a36Sopenharmony_ci * NVMe over Fabrics loopback device. 462306a36Sopenharmony_ci * Copyright (c) 2015-2016 HGST, a Western Digital Company. 562306a36Sopenharmony_ci */ 662306a36Sopenharmony_ci#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 762306a36Sopenharmony_ci#include <linux/scatterlist.h> 862306a36Sopenharmony_ci#include <linux/blk-mq.h> 962306a36Sopenharmony_ci#include <linux/nvme.h> 1062306a36Sopenharmony_ci#include <linux/module.h> 1162306a36Sopenharmony_ci#include <linux/parser.h> 1262306a36Sopenharmony_ci#include "nvmet.h" 1362306a36Sopenharmony_ci#include "../host/nvme.h" 1462306a36Sopenharmony_ci#include "../host/fabrics.h" 1562306a36Sopenharmony_ci 1662306a36Sopenharmony_ci#define NVME_LOOP_MAX_SEGMENTS 256 1762306a36Sopenharmony_ci 1862306a36Sopenharmony_cistruct nvme_loop_iod { 1962306a36Sopenharmony_ci struct nvme_request nvme_req; 2062306a36Sopenharmony_ci struct nvme_command cmd; 2162306a36Sopenharmony_ci struct nvme_completion cqe; 2262306a36Sopenharmony_ci struct nvmet_req req; 2362306a36Sopenharmony_ci struct nvme_loop_queue *queue; 2462306a36Sopenharmony_ci struct work_struct work; 2562306a36Sopenharmony_ci struct sg_table sg_table; 2662306a36Sopenharmony_ci struct scatterlist first_sgl[]; 2762306a36Sopenharmony_ci}; 2862306a36Sopenharmony_ci 2962306a36Sopenharmony_cistruct nvme_loop_ctrl { 3062306a36Sopenharmony_ci struct nvme_loop_queue *queues; 3162306a36Sopenharmony_ci 3262306a36Sopenharmony_ci struct blk_mq_tag_set admin_tag_set; 3362306a36Sopenharmony_ci 3462306a36Sopenharmony_ci struct list_head list; 3562306a36Sopenharmony_ci struct blk_mq_tag_set tag_set; 3662306a36Sopenharmony_ci struct nvme_loop_iod async_event_iod; 3762306a36Sopenharmony_ci struct nvme_ctrl ctrl; 3862306a36Sopenharmony_ci 3962306a36Sopenharmony_ci struct nvmet_port *port; 4062306a36Sopenharmony_ci}; 4162306a36Sopenharmony_ci 4262306a36Sopenharmony_cistatic inline struct nvme_loop_ctrl *to_loop_ctrl(struct nvme_ctrl *ctrl) 4362306a36Sopenharmony_ci{ 4462306a36Sopenharmony_ci return container_of(ctrl, struct nvme_loop_ctrl, ctrl); 4562306a36Sopenharmony_ci} 4662306a36Sopenharmony_ci 4762306a36Sopenharmony_cienum nvme_loop_queue_flags { 4862306a36Sopenharmony_ci NVME_LOOP_Q_LIVE = 0, 4962306a36Sopenharmony_ci}; 5062306a36Sopenharmony_ci 5162306a36Sopenharmony_cistruct nvme_loop_queue { 5262306a36Sopenharmony_ci struct nvmet_cq nvme_cq; 5362306a36Sopenharmony_ci struct nvmet_sq nvme_sq; 5462306a36Sopenharmony_ci struct nvme_loop_ctrl *ctrl; 5562306a36Sopenharmony_ci unsigned long flags; 5662306a36Sopenharmony_ci}; 5762306a36Sopenharmony_ci 5862306a36Sopenharmony_cistatic LIST_HEAD(nvme_loop_ports); 5962306a36Sopenharmony_cistatic DEFINE_MUTEX(nvme_loop_ports_mutex); 6062306a36Sopenharmony_ci 6162306a36Sopenharmony_cistatic LIST_HEAD(nvme_loop_ctrl_list); 6262306a36Sopenharmony_cistatic DEFINE_MUTEX(nvme_loop_ctrl_mutex); 6362306a36Sopenharmony_ci 6462306a36Sopenharmony_cistatic void nvme_loop_queue_response(struct nvmet_req *nvme_req); 6562306a36Sopenharmony_cistatic void nvme_loop_delete_ctrl(struct nvmet_ctrl *ctrl); 6662306a36Sopenharmony_ci 6762306a36Sopenharmony_cistatic const struct nvmet_fabrics_ops nvme_loop_ops; 6862306a36Sopenharmony_ci 6962306a36Sopenharmony_cistatic inline int nvme_loop_queue_idx(struct nvme_loop_queue *queue) 7062306a36Sopenharmony_ci{ 7162306a36Sopenharmony_ci return queue - queue->ctrl->queues; 7262306a36Sopenharmony_ci} 7362306a36Sopenharmony_ci 7462306a36Sopenharmony_cistatic void nvme_loop_complete_rq(struct request *req) 7562306a36Sopenharmony_ci{ 7662306a36Sopenharmony_ci struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req); 7762306a36Sopenharmony_ci 7862306a36Sopenharmony_ci sg_free_table_chained(&iod->sg_table, NVME_INLINE_SG_CNT); 7962306a36Sopenharmony_ci nvme_complete_rq(req); 8062306a36Sopenharmony_ci} 8162306a36Sopenharmony_ci 8262306a36Sopenharmony_cistatic struct blk_mq_tags *nvme_loop_tagset(struct nvme_loop_queue *queue) 8362306a36Sopenharmony_ci{ 8462306a36Sopenharmony_ci u32 queue_idx = nvme_loop_queue_idx(queue); 8562306a36Sopenharmony_ci 8662306a36Sopenharmony_ci if (queue_idx == 0) 8762306a36Sopenharmony_ci return queue->ctrl->admin_tag_set.tags[queue_idx]; 8862306a36Sopenharmony_ci return queue->ctrl->tag_set.tags[queue_idx - 1]; 8962306a36Sopenharmony_ci} 9062306a36Sopenharmony_ci 9162306a36Sopenharmony_cistatic void nvme_loop_queue_response(struct nvmet_req *req) 9262306a36Sopenharmony_ci{ 9362306a36Sopenharmony_ci struct nvme_loop_queue *queue = 9462306a36Sopenharmony_ci container_of(req->sq, struct nvme_loop_queue, nvme_sq); 9562306a36Sopenharmony_ci struct nvme_completion *cqe = req->cqe; 9662306a36Sopenharmony_ci 9762306a36Sopenharmony_ci /* 9862306a36Sopenharmony_ci * AEN requests are special as they don't time out and can 9962306a36Sopenharmony_ci * survive any kind of queue freeze and often don't respond to 10062306a36Sopenharmony_ci * aborts. We don't even bother to allocate a struct request 10162306a36Sopenharmony_ci * for them but rather special case them here. 10262306a36Sopenharmony_ci */ 10362306a36Sopenharmony_ci if (unlikely(nvme_is_aen_req(nvme_loop_queue_idx(queue), 10462306a36Sopenharmony_ci cqe->command_id))) { 10562306a36Sopenharmony_ci nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status, 10662306a36Sopenharmony_ci &cqe->result); 10762306a36Sopenharmony_ci } else { 10862306a36Sopenharmony_ci struct request *rq; 10962306a36Sopenharmony_ci 11062306a36Sopenharmony_ci rq = nvme_find_rq(nvme_loop_tagset(queue), cqe->command_id); 11162306a36Sopenharmony_ci if (!rq) { 11262306a36Sopenharmony_ci dev_err(queue->ctrl->ctrl.device, 11362306a36Sopenharmony_ci "got bad command_id %#x on queue %d\n", 11462306a36Sopenharmony_ci cqe->command_id, nvme_loop_queue_idx(queue)); 11562306a36Sopenharmony_ci return; 11662306a36Sopenharmony_ci } 11762306a36Sopenharmony_ci 11862306a36Sopenharmony_ci if (!nvme_try_complete_req(rq, cqe->status, cqe->result)) 11962306a36Sopenharmony_ci nvme_loop_complete_rq(rq); 12062306a36Sopenharmony_ci } 12162306a36Sopenharmony_ci} 12262306a36Sopenharmony_ci 12362306a36Sopenharmony_cistatic void nvme_loop_execute_work(struct work_struct *work) 12462306a36Sopenharmony_ci{ 12562306a36Sopenharmony_ci struct nvme_loop_iod *iod = 12662306a36Sopenharmony_ci container_of(work, struct nvme_loop_iod, work); 12762306a36Sopenharmony_ci 12862306a36Sopenharmony_ci iod->req.execute(&iod->req); 12962306a36Sopenharmony_ci} 13062306a36Sopenharmony_ci 13162306a36Sopenharmony_cistatic blk_status_t nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx, 13262306a36Sopenharmony_ci const struct blk_mq_queue_data *bd) 13362306a36Sopenharmony_ci{ 13462306a36Sopenharmony_ci struct nvme_ns *ns = hctx->queue->queuedata; 13562306a36Sopenharmony_ci struct nvme_loop_queue *queue = hctx->driver_data; 13662306a36Sopenharmony_ci struct request *req = bd->rq; 13762306a36Sopenharmony_ci struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req); 13862306a36Sopenharmony_ci bool queue_ready = test_bit(NVME_LOOP_Q_LIVE, &queue->flags); 13962306a36Sopenharmony_ci blk_status_t ret; 14062306a36Sopenharmony_ci 14162306a36Sopenharmony_ci if (!nvme_check_ready(&queue->ctrl->ctrl, req, queue_ready)) 14262306a36Sopenharmony_ci return nvme_fail_nonready_command(&queue->ctrl->ctrl, req); 14362306a36Sopenharmony_ci 14462306a36Sopenharmony_ci ret = nvme_setup_cmd(ns, req); 14562306a36Sopenharmony_ci if (ret) 14662306a36Sopenharmony_ci return ret; 14762306a36Sopenharmony_ci 14862306a36Sopenharmony_ci nvme_start_request(req); 14962306a36Sopenharmony_ci iod->cmd.common.flags |= NVME_CMD_SGL_METABUF; 15062306a36Sopenharmony_ci iod->req.port = queue->ctrl->port; 15162306a36Sopenharmony_ci if (!nvmet_req_init(&iod->req, &queue->nvme_cq, 15262306a36Sopenharmony_ci &queue->nvme_sq, &nvme_loop_ops)) 15362306a36Sopenharmony_ci return BLK_STS_OK; 15462306a36Sopenharmony_ci 15562306a36Sopenharmony_ci if (blk_rq_nr_phys_segments(req)) { 15662306a36Sopenharmony_ci iod->sg_table.sgl = iod->first_sgl; 15762306a36Sopenharmony_ci if (sg_alloc_table_chained(&iod->sg_table, 15862306a36Sopenharmony_ci blk_rq_nr_phys_segments(req), 15962306a36Sopenharmony_ci iod->sg_table.sgl, NVME_INLINE_SG_CNT)) { 16062306a36Sopenharmony_ci nvme_cleanup_cmd(req); 16162306a36Sopenharmony_ci return BLK_STS_RESOURCE; 16262306a36Sopenharmony_ci } 16362306a36Sopenharmony_ci 16462306a36Sopenharmony_ci iod->req.sg = iod->sg_table.sgl; 16562306a36Sopenharmony_ci iod->req.sg_cnt = blk_rq_map_sg(req->q, req, iod->sg_table.sgl); 16662306a36Sopenharmony_ci iod->req.transfer_len = blk_rq_payload_bytes(req); 16762306a36Sopenharmony_ci } 16862306a36Sopenharmony_ci 16962306a36Sopenharmony_ci queue_work(nvmet_wq, &iod->work); 17062306a36Sopenharmony_ci return BLK_STS_OK; 17162306a36Sopenharmony_ci} 17262306a36Sopenharmony_ci 17362306a36Sopenharmony_cistatic void nvme_loop_submit_async_event(struct nvme_ctrl *arg) 17462306a36Sopenharmony_ci{ 17562306a36Sopenharmony_ci struct nvme_loop_ctrl *ctrl = to_loop_ctrl(arg); 17662306a36Sopenharmony_ci struct nvme_loop_queue *queue = &ctrl->queues[0]; 17762306a36Sopenharmony_ci struct nvme_loop_iod *iod = &ctrl->async_event_iod; 17862306a36Sopenharmony_ci 17962306a36Sopenharmony_ci memset(&iod->cmd, 0, sizeof(iod->cmd)); 18062306a36Sopenharmony_ci iod->cmd.common.opcode = nvme_admin_async_event; 18162306a36Sopenharmony_ci iod->cmd.common.command_id = NVME_AQ_BLK_MQ_DEPTH; 18262306a36Sopenharmony_ci iod->cmd.common.flags |= NVME_CMD_SGL_METABUF; 18362306a36Sopenharmony_ci 18462306a36Sopenharmony_ci if (!nvmet_req_init(&iod->req, &queue->nvme_cq, &queue->nvme_sq, 18562306a36Sopenharmony_ci &nvme_loop_ops)) { 18662306a36Sopenharmony_ci dev_err(ctrl->ctrl.device, "failed async event work\n"); 18762306a36Sopenharmony_ci return; 18862306a36Sopenharmony_ci } 18962306a36Sopenharmony_ci 19062306a36Sopenharmony_ci queue_work(nvmet_wq, &iod->work); 19162306a36Sopenharmony_ci} 19262306a36Sopenharmony_ci 19362306a36Sopenharmony_cistatic int nvme_loop_init_iod(struct nvme_loop_ctrl *ctrl, 19462306a36Sopenharmony_ci struct nvme_loop_iod *iod, unsigned int queue_idx) 19562306a36Sopenharmony_ci{ 19662306a36Sopenharmony_ci iod->req.cmd = &iod->cmd; 19762306a36Sopenharmony_ci iod->req.cqe = &iod->cqe; 19862306a36Sopenharmony_ci iod->queue = &ctrl->queues[queue_idx]; 19962306a36Sopenharmony_ci INIT_WORK(&iod->work, nvme_loop_execute_work); 20062306a36Sopenharmony_ci return 0; 20162306a36Sopenharmony_ci} 20262306a36Sopenharmony_ci 20362306a36Sopenharmony_cistatic int nvme_loop_init_request(struct blk_mq_tag_set *set, 20462306a36Sopenharmony_ci struct request *req, unsigned int hctx_idx, 20562306a36Sopenharmony_ci unsigned int numa_node) 20662306a36Sopenharmony_ci{ 20762306a36Sopenharmony_ci struct nvme_loop_ctrl *ctrl = to_loop_ctrl(set->driver_data); 20862306a36Sopenharmony_ci struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req); 20962306a36Sopenharmony_ci 21062306a36Sopenharmony_ci nvme_req(req)->ctrl = &ctrl->ctrl; 21162306a36Sopenharmony_ci nvme_req(req)->cmd = &iod->cmd; 21262306a36Sopenharmony_ci return nvme_loop_init_iod(ctrl, blk_mq_rq_to_pdu(req), 21362306a36Sopenharmony_ci (set == &ctrl->tag_set) ? hctx_idx + 1 : 0); 21462306a36Sopenharmony_ci} 21562306a36Sopenharmony_ci 21662306a36Sopenharmony_cistatic struct lock_class_key loop_hctx_fq_lock_key; 21762306a36Sopenharmony_ci 21862306a36Sopenharmony_cistatic int nvme_loop_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, 21962306a36Sopenharmony_ci unsigned int hctx_idx) 22062306a36Sopenharmony_ci{ 22162306a36Sopenharmony_ci struct nvme_loop_ctrl *ctrl = to_loop_ctrl(data); 22262306a36Sopenharmony_ci struct nvme_loop_queue *queue = &ctrl->queues[hctx_idx + 1]; 22362306a36Sopenharmony_ci 22462306a36Sopenharmony_ci BUG_ON(hctx_idx >= ctrl->ctrl.queue_count); 22562306a36Sopenharmony_ci 22662306a36Sopenharmony_ci /* 22762306a36Sopenharmony_ci * flush_end_io() can be called recursively for us, so use our own 22862306a36Sopenharmony_ci * lock class key for avoiding lockdep possible recursive locking, 22962306a36Sopenharmony_ci * then we can remove the dynamically allocated lock class for each 23062306a36Sopenharmony_ci * flush queue, that way may cause horrible boot delay. 23162306a36Sopenharmony_ci */ 23262306a36Sopenharmony_ci blk_mq_hctx_set_fq_lock_class(hctx, &loop_hctx_fq_lock_key); 23362306a36Sopenharmony_ci 23462306a36Sopenharmony_ci hctx->driver_data = queue; 23562306a36Sopenharmony_ci return 0; 23662306a36Sopenharmony_ci} 23762306a36Sopenharmony_ci 23862306a36Sopenharmony_cistatic int nvme_loop_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data, 23962306a36Sopenharmony_ci unsigned int hctx_idx) 24062306a36Sopenharmony_ci{ 24162306a36Sopenharmony_ci struct nvme_loop_ctrl *ctrl = to_loop_ctrl(data); 24262306a36Sopenharmony_ci struct nvme_loop_queue *queue = &ctrl->queues[0]; 24362306a36Sopenharmony_ci 24462306a36Sopenharmony_ci BUG_ON(hctx_idx != 0); 24562306a36Sopenharmony_ci 24662306a36Sopenharmony_ci hctx->driver_data = queue; 24762306a36Sopenharmony_ci return 0; 24862306a36Sopenharmony_ci} 24962306a36Sopenharmony_ci 25062306a36Sopenharmony_cistatic const struct blk_mq_ops nvme_loop_mq_ops = { 25162306a36Sopenharmony_ci .queue_rq = nvme_loop_queue_rq, 25262306a36Sopenharmony_ci .complete = nvme_loop_complete_rq, 25362306a36Sopenharmony_ci .init_request = nvme_loop_init_request, 25462306a36Sopenharmony_ci .init_hctx = nvme_loop_init_hctx, 25562306a36Sopenharmony_ci}; 25662306a36Sopenharmony_ci 25762306a36Sopenharmony_cistatic const struct blk_mq_ops nvme_loop_admin_mq_ops = { 25862306a36Sopenharmony_ci .queue_rq = nvme_loop_queue_rq, 25962306a36Sopenharmony_ci .complete = nvme_loop_complete_rq, 26062306a36Sopenharmony_ci .init_request = nvme_loop_init_request, 26162306a36Sopenharmony_ci .init_hctx = nvme_loop_init_admin_hctx, 26262306a36Sopenharmony_ci}; 26362306a36Sopenharmony_ci 26462306a36Sopenharmony_cistatic void nvme_loop_destroy_admin_queue(struct nvme_loop_ctrl *ctrl) 26562306a36Sopenharmony_ci{ 26662306a36Sopenharmony_ci if (!test_and_clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags)) 26762306a36Sopenharmony_ci return; 26862306a36Sopenharmony_ci nvmet_sq_destroy(&ctrl->queues[0].nvme_sq); 26962306a36Sopenharmony_ci nvme_remove_admin_tag_set(&ctrl->ctrl); 27062306a36Sopenharmony_ci} 27162306a36Sopenharmony_ci 27262306a36Sopenharmony_cistatic void nvme_loop_free_ctrl(struct nvme_ctrl *nctrl) 27362306a36Sopenharmony_ci{ 27462306a36Sopenharmony_ci struct nvme_loop_ctrl *ctrl = to_loop_ctrl(nctrl); 27562306a36Sopenharmony_ci 27662306a36Sopenharmony_ci if (list_empty(&ctrl->list)) 27762306a36Sopenharmony_ci goto free_ctrl; 27862306a36Sopenharmony_ci 27962306a36Sopenharmony_ci mutex_lock(&nvme_loop_ctrl_mutex); 28062306a36Sopenharmony_ci list_del(&ctrl->list); 28162306a36Sopenharmony_ci mutex_unlock(&nvme_loop_ctrl_mutex); 28262306a36Sopenharmony_ci 28362306a36Sopenharmony_ci if (nctrl->tagset) 28462306a36Sopenharmony_ci nvme_remove_io_tag_set(nctrl); 28562306a36Sopenharmony_ci kfree(ctrl->queues); 28662306a36Sopenharmony_ci nvmf_free_options(nctrl->opts); 28762306a36Sopenharmony_cifree_ctrl: 28862306a36Sopenharmony_ci kfree(ctrl); 28962306a36Sopenharmony_ci} 29062306a36Sopenharmony_ci 29162306a36Sopenharmony_cistatic void nvme_loop_destroy_io_queues(struct nvme_loop_ctrl *ctrl) 29262306a36Sopenharmony_ci{ 29362306a36Sopenharmony_ci int i; 29462306a36Sopenharmony_ci 29562306a36Sopenharmony_ci for (i = 1; i < ctrl->ctrl.queue_count; i++) { 29662306a36Sopenharmony_ci clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[i].flags); 29762306a36Sopenharmony_ci nvmet_sq_destroy(&ctrl->queues[i].nvme_sq); 29862306a36Sopenharmony_ci } 29962306a36Sopenharmony_ci ctrl->ctrl.queue_count = 1; 30062306a36Sopenharmony_ci} 30162306a36Sopenharmony_ci 30262306a36Sopenharmony_cistatic int nvme_loop_init_io_queues(struct nvme_loop_ctrl *ctrl) 30362306a36Sopenharmony_ci{ 30462306a36Sopenharmony_ci struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; 30562306a36Sopenharmony_ci unsigned int nr_io_queues; 30662306a36Sopenharmony_ci int ret, i; 30762306a36Sopenharmony_ci 30862306a36Sopenharmony_ci nr_io_queues = min(opts->nr_io_queues, num_online_cpus()); 30962306a36Sopenharmony_ci ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues); 31062306a36Sopenharmony_ci if (ret || !nr_io_queues) 31162306a36Sopenharmony_ci return ret; 31262306a36Sopenharmony_ci 31362306a36Sopenharmony_ci dev_info(ctrl->ctrl.device, "creating %d I/O queues.\n", nr_io_queues); 31462306a36Sopenharmony_ci 31562306a36Sopenharmony_ci for (i = 1; i <= nr_io_queues; i++) { 31662306a36Sopenharmony_ci ctrl->queues[i].ctrl = ctrl; 31762306a36Sopenharmony_ci ret = nvmet_sq_init(&ctrl->queues[i].nvme_sq); 31862306a36Sopenharmony_ci if (ret) 31962306a36Sopenharmony_ci goto out_destroy_queues; 32062306a36Sopenharmony_ci 32162306a36Sopenharmony_ci ctrl->ctrl.queue_count++; 32262306a36Sopenharmony_ci } 32362306a36Sopenharmony_ci 32462306a36Sopenharmony_ci return 0; 32562306a36Sopenharmony_ci 32662306a36Sopenharmony_ciout_destroy_queues: 32762306a36Sopenharmony_ci nvme_loop_destroy_io_queues(ctrl); 32862306a36Sopenharmony_ci return ret; 32962306a36Sopenharmony_ci} 33062306a36Sopenharmony_ci 33162306a36Sopenharmony_cistatic int nvme_loop_connect_io_queues(struct nvme_loop_ctrl *ctrl) 33262306a36Sopenharmony_ci{ 33362306a36Sopenharmony_ci int i, ret; 33462306a36Sopenharmony_ci 33562306a36Sopenharmony_ci for (i = 1; i < ctrl->ctrl.queue_count; i++) { 33662306a36Sopenharmony_ci ret = nvmf_connect_io_queue(&ctrl->ctrl, i); 33762306a36Sopenharmony_ci if (ret) 33862306a36Sopenharmony_ci return ret; 33962306a36Sopenharmony_ci set_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[i].flags); 34062306a36Sopenharmony_ci } 34162306a36Sopenharmony_ci 34262306a36Sopenharmony_ci return 0; 34362306a36Sopenharmony_ci} 34462306a36Sopenharmony_ci 34562306a36Sopenharmony_cistatic int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl) 34662306a36Sopenharmony_ci{ 34762306a36Sopenharmony_ci int error; 34862306a36Sopenharmony_ci 34962306a36Sopenharmony_ci ctrl->queues[0].ctrl = ctrl; 35062306a36Sopenharmony_ci error = nvmet_sq_init(&ctrl->queues[0].nvme_sq); 35162306a36Sopenharmony_ci if (error) 35262306a36Sopenharmony_ci return error; 35362306a36Sopenharmony_ci ctrl->ctrl.queue_count = 1; 35462306a36Sopenharmony_ci 35562306a36Sopenharmony_ci error = nvme_alloc_admin_tag_set(&ctrl->ctrl, &ctrl->admin_tag_set, 35662306a36Sopenharmony_ci &nvme_loop_admin_mq_ops, 35762306a36Sopenharmony_ci sizeof(struct nvme_loop_iod) + 35862306a36Sopenharmony_ci NVME_INLINE_SG_CNT * sizeof(struct scatterlist)); 35962306a36Sopenharmony_ci if (error) 36062306a36Sopenharmony_ci goto out_free_sq; 36162306a36Sopenharmony_ci 36262306a36Sopenharmony_ci /* reset stopped state for the fresh admin queue */ 36362306a36Sopenharmony_ci clear_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->ctrl.flags); 36462306a36Sopenharmony_ci 36562306a36Sopenharmony_ci error = nvmf_connect_admin_queue(&ctrl->ctrl); 36662306a36Sopenharmony_ci if (error) 36762306a36Sopenharmony_ci goto out_cleanup_tagset; 36862306a36Sopenharmony_ci 36962306a36Sopenharmony_ci set_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags); 37062306a36Sopenharmony_ci 37162306a36Sopenharmony_ci error = nvme_enable_ctrl(&ctrl->ctrl); 37262306a36Sopenharmony_ci if (error) 37362306a36Sopenharmony_ci goto out_cleanup_tagset; 37462306a36Sopenharmony_ci 37562306a36Sopenharmony_ci ctrl->ctrl.max_hw_sectors = 37662306a36Sopenharmony_ci (NVME_LOOP_MAX_SEGMENTS - 1) << PAGE_SECTORS_SHIFT; 37762306a36Sopenharmony_ci 37862306a36Sopenharmony_ci nvme_unquiesce_admin_queue(&ctrl->ctrl); 37962306a36Sopenharmony_ci 38062306a36Sopenharmony_ci error = nvme_init_ctrl_finish(&ctrl->ctrl, false); 38162306a36Sopenharmony_ci if (error) 38262306a36Sopenharmony_ci goto out_cleanup_tagset; 38362306a36Sopenharmony_ci 38462306a36Sopenharmony_ci return 0; 38562306a36Sopenharmony_ci 38662306a36Sopenharmony_ciout_cleanup_tagset: 38762306a36Sopenharmony_ci clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags); 38862306a36Sopenharmony_ci nvme_remove_admin_tag_set(&ctrl->ctrl); 38962306a36Sopenharmony_ciout_free_sq: 39062306a36Sopenharmony_ci nvmet_sq_destroy(&ctrl->queues[0].nvme_sq); 39162306a36Sopenharmony_ci return error; 39262306a36Sopenharmony_ci} 39362306a36Sopenharmony_ci 39462306a36Sopenharmony_cistatic void nvme_loop_shutdown_ctrl(struct nvme_loop_ctrl *ctrl) 39562306a36Sopenharmony_ci{ 39662306a36Sopenharmony_ci if (ctrl->ctrl.queue_count > 1) { 39762306a36Sopenharmony_ci nvme_quiesce_io_queues(&ctrl->ctrl); 39862306a36Sopenharmony_ci nvme_cancel_tagset(&ctrl->ctrl); 39962306a36Sopenharmony_ci nvme_loop_destroy_io_queues(ctrl); 40062306a36Sopenharmony_ci } 40162306a36Sopenharmony_ci 40262306a36Sopenharmony_ci nvme_quiesce_admin_queue(&ctrl->ctrl); 40362306a36Sopenharmony_ci if (ctrl->ctrl.state == NVME_CTRL_LIVE) 40462306a36Sopenharmony_ci nvme_disable_ctrl(&ctrl->ctrl, true); 40562306a36Sopenharmony_ci 40662306a36Sopenharmony_ci nvme_cancel_admin_tagset(&ctrl->ctrl); 40762306a36Sopenharmony_ci nvme_loop_destroy_admin_queue(ctrl); 40862306a36Sopenharmony_ci} 40962306a36Sopenharmony_ci 41062306a36Sopenharmony_cistatic void nvme_loop_delete_ctrl_host(struct nvme_ctrl *ctrl) 41162306a36Sopenharmony_ci{ 41262306a36Sopenharmony_ci nvme_loop_shutdown_ctrl(to_loop_ctrl(ctrl)); 41362306a36Sopenharmony_ci} 41462306a36Sopenharmony_ci 41562306a36Sopenharmony_cistatic void nvme_loop_delete_ctrl(struct nvmet_ctrl *nctrl) 41662306a36Sopenharmony_ci{ 41762306a36Sopenharmony_ci struct nvme_loop_ctrl *ctrl; 41862306a36Sopenharmony_ci 41962306a36Sopenharmony_ci mutex_lock(&nvme_loop_ctrl_mutex); 42062306a36Sopenharmony_ci list_for_each_entry(ctrl, &nvme_loop_ctrl_list, list) { 42162306a36Sopenharmony_ci if (ctrl->ctrl.cntlid == nctrl->cntlid) 42262306a36Sopenharmony_ci nvme_delete_ctrl(&ctrl->ctrl); 42362306a36Sopenharmony_ci } 42462306a36Sopenharmony_ci mutex_unlock(&nvme_loop_ctrl_mutex); 42562306a36Sopenharmony_ci} 42662306a36Sopenharmony_ci 42762306a36Sopenharmony_cistatic void nvme_loop_reset_ctrl_work(struct work_struct *work) 42862306a36Sopenharmony_ci{ 42962306a36Sopenharmony_ci struct nvme_loop_ctrl *ctrl = 43062306a36Sopenharmony_ci container_of(work, struct nvme_loop_ctrl, ctrl.reset_work); 43162306a36Sopenharmony_ci int ret; 43262306a36Sopenharmony_ci 43362306a36Sopenharmony_ci nvme_stop_ctrl(&ctrl->ctrl); 43462306a36Sopenharmony_ci nvme_loop_shutdown_ctrl(ctrl); 43562306a36Sopenharmony_ci 43662306a36Sopenharmony_ci if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) { 43762306a36Sopenharmony_ci if (ctrl->ctrl.state != NVME_CTRL_DELETING && 43862306a36Sopenharmony_ci ctrl->ctrl.state != NVME_CTRL_DELETING_NOIO) 43962306a36Sopenharmony_ci /* state change failure for non-deleted ctrl? */ 44062306a36Sopenharmony_ci WARN_ON_ONCE(1); 44162306a36Sopenharmony_ci return; 44262306a36Sopenharmony_ci } 44362306a36Sopenharmony_ci 44462306a36Sopenharmony_ci ret = nvme_loop_configure_admin_queue(ctrl); 44562306a36Sopenharmony_ci if (ret) 44662306a36Sopenharmony_ci goto out_disable; 44762306a36Sopenharmony_ci 44862306a36Sopenharmony_ci ret = nvme_loop_init_io_queues(ctrl); 44962306a36Sopenharmony_ci if (ret) 45062306a36Sopenharmony_ci goto out_destroy_admin; 45162306a36Sopenharmony_ci 45262306a36Sopenharmony_ci ret = nvme_loop_connect_io_queues(ctrl); 45362306a36Sopenharmony_ci if (ret) 45462306a36Sopenharmony_ci goto out_destroy_io; 45562306a36Sopenharmony_ci 45662306a36Sopenharmony_ci blk_mq_update_nr_hw_queues(&ctrl->tag_set, 45762306a36Sopenharmony_ci ctrl->ctrl.queue_count - 1); 45862306a36Sopenharmony_ci 45962306a36Sopenharmony_ci if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE)) 46062306a36Sopenharmony_ci WARN_ON_ONCE(1); 46162306a36Sopenharmony_ci 46262306a36Sopenharmony_ci nvme_start_ctrl(&ctrl->ctrl); 46362306a36Sopenharmony_ci 46462306a36Sopenharmony_ci return; 46562306a36Sopenharmony_ci 46662306a36Sopenharmony_ciout_destroy_io: 46762306a36Sopenharmony_ci nvme_loop_destroy_io_queues(ctrl); 46862306a36Sopenharmony_ciout_destroy_admin: 46962306a36Sopenharmony_ci nvme_loop_destroy_admin_queue(ctrl); 47062306a36Sopenharmony_ciout_disable: 47162306a36Sopenharmony_ci dev_warn(ctrl->ctrl.device, "Removing after reset failure\n"); 47262306a36Sopenharmony_ci nvme_uninit_ctrl(&ctrl->ctrl); 47362306a36Sopenharmony_ci} 47462306a36Sopenharmony_ci 47562306a36Sopenharmony_cistatic const struct nvme_ctrl_ops nvme_loop_ctrl_ops = { 47662306a36Sopenharmony_ci .name = "loop", 47762306a36Sopenharmony_ci .module = THIS_MODULE, 47862306a36Sopenharmony_ci .flags = NVME_F_FABRICS, 47962306a36Sopenharmony_ci .reg_read32 = nvmf_reg_read32, 48062306a36Sopenharmony_ci .reg_read64 = nvmf_reg_read64, 48162306a36Sopenharmony_ci .reg_write32 = nvmf_reg_write32, 48262306a36Sopenharmony_ci .free_ctrl = nvme_loop_free_ctrl, 48362306a36Sopenharmony_ci .submit_async_event = nvme_loop_submit_async_event, 48462306a36Sopenharmony_ci .delete_ctrl = nvme_loop_delete_ctrl_host, 48562306a36Sopenharmony_ci .get_address = nvmf_get_address, 48662306a36Sopenharmony_ci}; 48762306a36Sopenharmony_ci 48862306a36Sopenharmony_cistatic int nvme_loop_create_io_queues(struct nvme_loop_ctrl *ctrl) 48962306a36Sopenharmony_ci{ 49062306a36Sopenharmony_ci int ret; 49162306a36Sopenharmony_ci 49262306a36Sopenharmony_ci ret = nvme_loop_init_io_queues(ctrl); 49362306a36Sopenharmony_ci if (ret) 49462306a36Sopenharmony_ci return ret; 49562306a36Sopenharmony_ci 49662306a36Sopenharmony_ci ret = nvme_alloc_io_tag_set(&ctrl->ctrl, &ctrl->tag_set, 49762306a36Sopenharmony_ci &nvme_loop_mq_ops, 1, 49862306a36Sopenharmony_ci sizeof(struct nvme_loop_iod) + 49962306a36Sopenharmony_ci NVME_INLINE_SG_CNT * sizeof(struct scatterlist)); 50062306a36Sopenharmony_ci if (ret) 50162306a36Sopenharmony_ci goto out_destroy_queues; 50262306a36Sopenharmony_ci 50362306a36Sopenharmony_ci ret = nvme_loop_connect_io_queues(ctrl); 50462306a36Sopenharmony_ci if (ret) 50562306a36Sopenharmony_ci goto out_cleanup_tagset; 50662306a36Sopenharmony_ci 50762306a36Sopenharmony_ci return 0; 50862306a36Sopenharmony_ci 50962306a36Sopenharmony_ciout_cleanup_tagset: 51062306a36Sopenharmony_ci nvme_remove_io_tag_set(&ctrl->ctrl); 51162306a36Sopenharmony_ciout_destroy_queues: 51262306a36Sopenharmony_ci nvme_loop_destroy_io_queues(ctrl); 51362306a36Sopenharmony_ci return ret; 51462306a36Sopenharmony_ci} 51562306a36Sopenharmony_ci 51662306a36Sopenharmony_cistatic struct nvmet_port *nvme_loop_find_port(struct nvme_ctrl *ctrl) 51762306a36Sopenharmony_ci{ 51862306a36Sopenharmony_ci struct nvmet_port *p, *found = NULL; 51962306a36Sopenharmony_ci 52062306a36Sopenharmony_ci mutex_lock(&nvme_loop_ports_mutex); 52162306a36Sopenharmony_ci list_for_each_entry(p, &nvme_loop_ports, entry) { 52262306a36Sopenharmony_ci /* if no transport address is specified use the first port */ 52362306a36Sopenharmony_ci if ((ctrl->opts->mask & NVMF_OPT_TRADDR) && 52462306a36Sopenharmony_ci strcmp(ctrl->opts->traddr, p->disc_addr.traddr)) 52562306a36Sopenharmony_ci continue; 52662306a36Sopenharmony_ci found = p; 52762306a36Sopenharmony_ci break; 52862306a36Sopenharmony_ci } 52962306a36Sopenharmony_ci mutex_unlock(&nvme_loop_ports_mutex); 53062306a36Sopenharmony_ci return found; 53162306a36Sopenharmony_ci} 53262306a36Sopenharmony_ci 53362306a36Sopenharmony_cistatic struct nvme_ctrl *nvme_loop_create_ctrl(struct device *dev, 53462306a36Sopenharmony_ci struct nvmf_ctrl_options *opts) 53562306a36Sopenharmony_ci{ 53662306a36Sopenharmony_ci struct nvme_loop_ctrl *ctrl; 53762306a36Sopenharmony_ci int ret; 53862306a36Sopenharmony_ci 53962306a36Sopenharmony_ci ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL); 54062306a36Sopenharmony_ci if (!ctrl) 54162306a36Sopenharmony_ci return ERR_PTR(-ENOMEM); 54262306a36Sopenharmony_ci ctrl->ctrl.opts = opts; 54362306a36Sopenharmony_ci INIT_LIST_HEAD(&ctrl->list); 54462306a36Sopenharmony_ci 54562306a36Sopenharmony_ci INIT_WORK(&ctrl->ctrl.reset_work, nvme_loop_reset_ctrl_work); 54662306a36Sopenharmony_ci 54762306a36Sopenharmony_ci ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_loop_ctrl_ops, 54862306a36Sopenharmony_ci 0 /* no quirks, we're perfect! */); 54962306a36Sopenharmony_ci if (ret) { 55062306a36Sopenharmony_ci kfree(ctrl); 55162306a36Sopenharmony_ci goto out; 55262306a36Sopenharmony_ci } 55362306a36Sopenharmony_ci 55462306a36Sopenharmony_ci if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) 55562306a36Sopenharmony_ci WARN_ON_ONCE(1); 55662306a36Sopenharmony_ci 55762306a36Sopenharmony_ci ret = -ENOMEM; 55862306a36Sopenharmony_ci 55962306a36Sopenharmony_ci ctrl->ctrl.kato = opts->kato; 56062306a36Sopenharmony_ci ctrl->port = nvme_loop_find_port(&ctrl->ctrl); 56162306a36Sopenharmony_ci 56262306a36Sopenharmony_ci ctrl->queues = kcalloc(opts->nr_io_queues + 1, sizeof(*ctrl->queues), 56362306a36Sopenharmony_ci GFP_KERNEL); 56462306a36Sopenharmony_ci if (!ctrl->queues) 56562306a36Sopenharmony_ci goto out_uninit_ctrl; 56662306a36Sopenharmony_ci 56762306a36Sopenharmony_ci ret = nvme_loop_configure_admin_queue(ctrl); 56862306a36Sopenharmony_ci if (ret) 56962306a36Sopenharmony_ci goto out_free_queues; 57062306a36Sopenharmony_ci 57162306a36Sopenharmony_ci if (opts->queue_size > ctrl->ctrl.maxcmd) { 57262306a36Sopenharmony_ci /* warn if maxcmd is lower than queue_size */ 57362306a36Sopenharmony_ci dev_warn(ctrl->ctrl.device, 57462306a36Sopenharmony_ci "queue_size %zu > ctrl maxcmd %u, clamping down\n", 57562306a36Sopenharmony_ci opts->queue_size, ctrl->ctrl.maxcmd); 57662306a36Sopenharmony_ci opts->queue_size = ctrl->ctrl.maxcmd; 57762306a36Sopenharmony_ci } 57862306a36Sopenharmony_ci ctrl->ctrl.sqsize = opts->queue_size - 1; 57962306a36Sopenharmony_ci 58062306a36Sopenharmony_ci if (opts->nr_io_queues) { 58162306a36Sopenharmony_ci ret = nvme_loop_create_io_queues(ctrl); 58262306a36Sopenharmony_ci if (ret) 58362306a36Sopenharmony_ci goto out_remove_admin_queue; 58462306a36Sopenharmony_ci } 58562306a36Sopenharmony_ci 58662306a36Sopenharmony_ci nvme_loop_init_iod(ctrl, &ctrl->async_event_iod, 0); 58762306a36Sopenharmony_ci 58862306a36Sopenharmony_ci dev_info(ctrl->ctrl.device, 58962306a36Sopenharmony_ci "new ctrl: \"%s\"\n", ctrl->ctrl.opts->subsysnqn); 59062306a36Sopenharmony_ci 59162306a36Sopenharmony_ci if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE)) 59262306a36Sopenharmony_ci WARN_ON_ONCE(1); 59362306a36Sopenharmony_ci 59462306a36Sopenharmony_ci mutex_lock(&nvme_loop_ctrl_mutex); 59562306a36Sopenharmony_ci list_add_tail(&ctrl->list, &nvme_loop_ctrl_list); 59662306a36Sopenharmony_ci mutex_unlock(&nvme_loop_ctrl_mutex); 59762306a36Sopenharmony_ci 59862306a36Sopenharmony_ci nvme_start_ctrl(&ctrl->ctrl); 59962306a36Sopenharmony_ci 60062306a36Sopenharmony_ci return &ctrl->ctrl; 60162306a36Sopenharmony_ci 60262306a36Sopenharmony_ciout_remove_admin_queue: 60362306a36Sopenharmony_ci nvme_loop_destroy_admin_queue(ctrl); 60462306a36Sopenharmony_ciout_free_queues: 60562306a36Sopenharmony_ci kfree(ctrl->queues); 60662306a36Sopenharmony_ciout_uninit_ctrl: 60762306a36Sopenharmony_ci nvme_uninit_ctrl(&ctrl->ctrl); 60862306a36Sopenharmony_ci nvme_put_ctrl(&ctrl->ctrl); 60962306a36Sopenharmony_ciout: 61062306a36Sopenharmony_ci if (ret > 0) 61162306a36Sopenharmony_ci ret = -EIO; 61262306a36Sopenharmony_ci return ERR_PTR(ret); 61362306a36Sopenharmony_ci} 61462306a36Sopenharmony_ci 61562306a36Sopenharmony_cistatic int nvme_loop_add_port(struct nvmet_port *port) 61662306a36Sopenharmony_ci{ 61762306a36Sopenharmony_ci mutex_lock(&nvme_loop_ports_mutex); 61862306a36Sopenharmony_ci list_add_tail(&port->entry, &nvme_loop_ports); 61962306a36Sopenharmony_ci mutex_unlock(&nvme_loop_ports_mutex); 62062306a36Sopenharmony_ci return 0; 62162306a36Sopenharmony_ci} 62262306a36Sopenharmony_ci 62362306a36Sopenharmony_cistatic void nvme_loop_remove_port(struct nvmet_port *port) 62462306a36Sopenharmony_ci{ 62562306a36Sopenharmony_ci mutex_lock(&nvme_loop_ports_mutex); 62662306a36Sopenharmony_ci list_del_init(&port->entry); 62762306a36Sopenharmony_ci mutex_unlock(&nvme_loop_ports_mutex); 62862306a36Sopenharmony_ci 62962306a36Sopenharmony_ci /* 63062306a36Sopenharmony_ci * Ensure any ctrls that are in the process of being 63162306a36Sopenharmony_ci * deleted are in fact deleted before we return 63262306a36Sopenharmony_ci * and free the port. This is to prevent active 63362306a36Sopenharmony_ci * ctrls from using a port after it's freed. 63462306a36Sopenharmony_ci */ 63562306a36Sopenharmony_ci flush_workqueue(nvme_delete_wq); 63662306a36Sopenharmony_ci} 63762306a36Sopenharmony_ci 63862306a36Sopenharmony_cistatic const struct nvmet_fabrics_ops nvme_loop_ops = { 63962306a36Sopenharmony_ci .owner = THIS_MODULE, 64062306a36Sopenharmony_ci .type = NVMF_TRTYPE_LOOP, 64162306a36Sopenharmony_ci .add_port = nvme_loop_add_port, 64262306a36Sopenharmony_ci .remove_port = nvme_loop_remove_port, 64362306a36Sopenharmony_ci .queue_response = nvme_loop_queue_response, 64462306a36Sopenharmony_ci .delete_ctrl = nvme_loop_delete_ctrl, 64562306a36Sopenharmony_ci}; 64662306a36Sopenharmony_ci 64762306a36Sopenharmony_cistatic struct nvmf_transport_ops nvme_loop_transport = { 64862306a36Sopenharmony_ci .name = "loop", 64962306a36Sopenharmony_ci .module = THIS_MODULE, 65062306a36Sopenharmony_ci .create_ctrl = nvme_loop_create_ctrl, 65162306a36Sopenharmony_ci .allowed_opts = NVMF_OPT_TRADDR, 65262306a36Sopenharmony_ci}; 65362306a36Sopenharmony_ci 65462306a36Sopenharmony_cistatic int __init nvme_loop_init_module(void) 65562306a36Sopenharmony_ci{ 65662306a36Sopenharmony_ci int ret; 65762306a36Sopenharmony_ci 65862306a36Sopenharmony_ci ret = nvmet_register_transport(&nvme_loop_ops); 65962306a36Sopenharmony_ci if (ret) 66062306a36Sopenharmony_ci return ret; 66162306a36Sopenharmony_ci 66262306a36Sopenharmony_ci ret = nvmf_register_transport(&nvme_loop_transport); 66362306a36Sopenharmony_ci if (ret) 66462306a36Sopenharmony_ci nvmet_unregister_transport(&nvme_loop_ops); 66562306a36Sopenharmony_ci 66662306a36Sopenharmony_ci return ret; 66762306a36Sopenharmony_ci} 66862306a36Sopenharmony_ci 66962306a36Sopenharmony_cistatic void __exit nvme_loop_cleanup_module(void) 67062306a36Sopenharmony_ci{ 67162306a36Sopenharmony_ci struct nvme_loop_ctrl *ctrl, *next; 67262306a36Sopenharmony_ci 67362306a36Sopenharmony_ci nvmf_unregister_transport(&nvme_loop_transport); 67462306a36Sopenharmony_ci nvmet_unregister_transport(&nvme_loop_ops); 67562306a36Sopenharmony_ci 67662306a36Sopenharmony_ci mutex_lock(&nvme_loop_ctrl_mutex); 67762306a36Sopenharmony_ci list_for_each_entry_safe(ctrl, next, &nvme_loop_ctrl_list, list) 67862306a36Sopenharmony_ci nvme_delete_ctrl(&ctrl->ctrl); 67962306a36Sopenharmony_ci mutex_unlock(&nvme_loop_ctrl_mutex); 68062306a36Sopenharmony_ci 68162306a36Sopenharmony_ci flush_workqueue(nvme_delete_wq); 68262306a36Sopenharmony_ci} 68362306a36Sopenharmony_ci 68462306a36Sopenharmony_cimodule_init(nvme_loop_init_module); 68562306a36Sopenharmony_cimodule_exit(nvme_loop_cleanup_module); 68662306a36Sopenharmony_ci 68762306a36Sopenharmony_ciMODULE_LICENSE("GPL v2"); 68862306a36Sopenharmony_ciMODULE_ALIAS("nvmet-transport-254"); /* 254 == NVMF_TRTYPE_LOOP */ 689