162306a36Sopenharmony_ci// SPDX-License-Identifier: GPL-2.0
262306a36Sopenharmony_ci/*
362306a36Sopenharmony_ci * NVMe over Fabrics loopback device.
462306a36Sopenharmony_ci * Copyright (c) 2015-2016 HGST, a Western Digital Company.
562306a36Sopenharmony_ci */
662306a36Sopenharmony_ci#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
762306a36Sopenharmony_ci#include <linux/scatterlist.h>
862306a36Sopenharmony_ci#include <linux/blk-mq.h>
962306a36Sopenharmony_ci#include <linux/nvme.h>
1062306a36Sopenharmony_ci#include <linux/module.h>
1162306a36Sopenharmony_ci#include <linux/parser.h>
1262306a36Sopenharmony_ci#include "nvmet.h"
1362306a36Sopenharmony_ci#include "../host/nvme.h"
1462306a36Sopenharmony_ci#include "../host/fabrics.h"
1562306a36Sopenharmony_ci
1662306a36Sopenharmony_ci#define NVME_LOOP_MAX_SEGMENTS		256
1762306a36Sopenharmony_ci
1862306a36Sopenharmony_cistruct nvme_loop_iod {
1962306a36Sopenharmony_ci	struct nvme_request	nvme_req;
2062306a36Sopenharmony_ci	struct nvme_command	cmd;
2162306a36Sopenharmony_ci	struct nvme_completion	cqe;
2262306a36Sopenharmony_ci	struct nvmet_req	req;
2362306a36Sopenharmony_ci	struct nvme_loop_queue	*queue;
2462306a36Sopenharmony_ci	struct work_struct	work;
2562306a36Sopenharmony_ci	struct sg_table		sg_table;
2662306a36Sopenharmony_ci	struct scatterlist	first_sgl[];
2762306a36Sopenharmony_ci};
2862306a36Sopenharmony_ci
2962306a36Sopenharmony_cistruct nvme_loop_ctrl {
3062306a36Sopenharmony_ci	struct nvme_loop_queue	*queues;
3162306a36Sopenharmony_ci
3262306a36Sopenharmony_ci	struct blk_mq_tag_set	admin_tag_set;
3362306a36Sopenharmony_ci
3462306a36Sopenharmony_ci	struct list_head	list;
3562306a36Sopenharmony_ci	struct blk_mq_tag_set	tag_set;
3662306a36Sopenharmony_ci	struct nvme_loop_iod	async_event_iod;
3762306a36Sopenharmony_ci	struct nvme_ctrl	ctrl;
3862306a36Sopenharmony_ci
3962306a36Sopenharmony_ci	struct nvmet_port	*port;
4062306a36Sopenharmony_ci};
4162306a36Sopenharmony_ci
4262306a36Sopenharmony_cistatic inline struct nvme_loop_ctrl *to_loop_ctrl(struct nvme_ctrl *ctrl)
4362306a36Sopenharmony_ci{
4462306a36Sopenharmony_ci	return container_of(ctrl, struct nvme_loop_ctrl, ctrl);
4562306a36Sopenharmony_ci}
4662306a36Sopenharmony_ci
4762306a36Sopenharmony_cienum nvme_loop_queue_flags {
4862306a36Sopenharmony_ci	NVME_LOOP_Q_LIVE	= 0,
4962306a36Sopenharmony_ci};
5062306a36Sopenharmony_ci
5162306a36Sopenharmony_cistruct nvme_loop_queue {
5262306a36Sopenharmony_ci	struct nvmet_cq		nvme_cq;
5362306a36Sopenharmony_ci	struct nvmet_sq		nvme_sq;
5462306a36Sopenharmony_ci	struct nvme_loop_ctrl	*ctrl;
5562306a36Sopenharmony_ci	unsigned long		flags;
5662306a36Sopenharmony_ci};
5762306a36Sopenharmony_ci
5862306a36Sopenharmony_cistatic LIST_HEAD(nvme_loop_ports);
5962306a36Sopenharmony_cistatic DEFINE_MUTEX(nvme_loop_ports_mutex);
6062306a36Sopenharmony_ci
6162306a36Sopenharmony_cistatic LIST_HEAD(nvme_loop_ctrl_list);
6262306a36Sopenharmony_cistatic DEFINE_MUTEX(nvme_loop_ctrl_mutex);
6362306a36Sopenharmony_ci
6462306a36Sopenharmony_cistatic void nvme_loop_queue_response(struct nvmet_req *nvme_req);
6562306a36Sopenharmony_cistatic void nvme_loop_delete_ctrl(struct nvmet_ctrl *ctrl);
6662306a36Sopenharmony_ci
6762306a36Sopenharmony_cistatic const struct nvmet_fabrics_ops nvme_loop_ops;
6862306a36Sopenharmony_ci
6962306a36Sopenharmony_cistatic inline int nvme_loop_queue_idx(struct nvme_loop_queue *queue)
7062306a36Sopenharmony_ci{
7162306a36Sopenharmony_ci	return queue - queue->ctrl->queues;
7262306a36Sopenharmony_ci}
7362306a36Sopenharmony_ci
7462306a36Sopenharmony_cistatic void nvme_loop_complete_rq(struct request *req)
7562306a36Sopenharmony_ci{
7662306a36Sopenharmony_ci	struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req);
7762306a36Sopenharmony_ci
7862306a36Sopenharmony_ci	sg_free_table_chained(&iod->sg_table, NVME_INLINE_SG_CNT);
7962306a36Sopenharmony_ci	nvme_complete_rq(req);
8062306a36Sopenharmony_ci}
8162306a36Sopenharmony_ci
8262306a36Sopenharmony_cistatic struct blk_mq_tags *nvme_loop_tagset(struct nvme_loop_queue *queue)
8362306a36Sopenharmony_ci{
8462306a36Sopenharmony_ci	u32 queue_idx = nvme_loop_queue_idx(queue);
8562306a36Sopenharmony_ci
8662306a36Sopenharmony_ci	if (queue_idx == 0)
8762306a36Sopenharmony_ci		return queue->ctrl->admin_tag_set.tags[queue_idx];
8862306a36Sopenharmony_ci	return queue->ctrl->tag_set.tags[queue_idx - 1];
8962306a36Sopenharmony_ci}
9062306a36Sopenharmony_ci
9162306a36Sopenharmony_cistatic void nvme_loop_queue_response(struct nvmet_req *req)
9262306a36Sopenharmony_ci{
9362306a36Sopenharmony_ci	struct nvme_loop_queue *queue =
9462306a36Sopenharmony_ci		container_of(req->sq, struct nvme_loop_queue, nvme_sq);
9562306a36Sopenharmony_ci	struct nvme_completion *cqe = req->cqe;
9662306a36Sopenharmony_ci
9762306a36Sopenharmony_ci	/*
9862306a36Sopenharmony_ci	 * AEN requests are special as they don't time out and can
9962306a36Sopenharmony_ci	 * survive any kind of queue freeze and often don't respond to
10062306a36Sopenharmony_ci	 * aborts.  We don't even bother to allocate a struct request
10162306a36Sopenharmony_ci	 * for them but rather special case them here.
10262306a36Sopenharmony_ci	 */
10362306a36Sopenharmony_ci	if (unlikely(nvme_is_aen_req(nvme_loop_queue_idx(queue),
10462306a36Sopenharmony_ci				     cqe->command_id))) {
10562306a36Sopenharmony_ci		nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,
10662306a36Sopenharmony_ci				&cqe->result);
10762306a36Sopenharmony_ci	} else {
10862306a36Sopenharmony_ci		struct request *rq;
10962306a36Sopenharmony_ci
11062306a36Sopenharmony_ci		rq = nvme_find_rq(nvme_loop_tagset(queue), cqe->command_id);
11162306a36Sopenharmony_ci		if (!rq) {
11262306a36Sopenharmony_ci			dev_err(queue->ctrl->ctrl.device,
11362306a36Sopenharmony_ci				"got bad command_id %#x on queue %d\n",
11462306a36Sopenharmony_ci				cqe->command_id, nvme_loop_queue_idx(queue));
11562306a36Sopenharmony_ci			return;
11662306a36Sopenharmony_ci		}
11762306a36Sopenharmony_ci
11862306a36Sopenharmony_ci		if (!nvme_try_complete_req(rq, cqe->status, cqe->result))
11962306a36Sopenharmony_ci			nvme_loop_complete_rq(rq);
12062306a36Sopenharmony_ci	}
12162306a36Sopenharmony_ci}
12262306a36Sopenharmony_ci
12362306a36Sopenharmony_cistatic void nvme_loop_execute_work(struct work_struct *work)
12462306a36Sopenharmony_ci{
12562306a36Sopenharmony_ci	struct nvme_loop_iod *iod =
12662306a36Sopenharmony_ci		container_of(work, struct nvme_loop_iod, work);
12762306a36Sopenharmony_ci
12862306a36Sopenharmony_ci	iod->req.execute(&iod->req);
12962306a36Sopenharmony_ci}
13062306a36Sopenharmony_ci
13162306a36Sopenharmony_cistatic blk_status_t nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx,
13262306a36Sopenharmony_ci		const struct blk_mq_queue_data *bd)
13362306a36Sopenharmony_ci{
13462306a36Sopenharmony_ci	struct nvme_ns *ns = hctx->queue->queuedata;
13562306a36Sopenharmony_ci	struct nvme_loop_queue *queue = hctx->driver_data;
13662306a36Sopenharmony_ci	struct request *req = bd->rq;
13762306a36Sopenharmony_ci	struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req);
13862306a36Sopenharmony_ci	bool queue_ready = test_bit(NVME_LOOP_Q_LIVE, &queue->flags);
13962306a36Sopenharmony_ci	blk_status_t ret;
14062306a36Sopenharmony_ci
14162306a36Sopenharmony_ci	if (!nvme_check_ready(&queue->ctrl->ctrl, req, queue_ready))
14262306a36Sopenharmony_ci		return nvme_fail_nonready_command(&queue->ctrl->ctrl, req);
14362306a36Sopenharmony_ci
14462306a36Sopenharmony_ci	ret = nvme_setup_cmd(ns, req);
14562306a36Sopenharmony_ci	if (ret)
14662306a36Sopenharmony_ci		return ret;
14762306a36Sopenharmony_ci
14862306a36Sopenharmony_ci	nvme_start_request(req);
14962306a36Sopenharmony_ci	iod->cmd.common.flags |= NVME_CMD_SGL_METABUF;
15062306a36Sopenharmony_ci	iod->req.port = queue->ctrl->port;
15162306a36Sopenharmony_ci	if (!nvmet_req_init(&iod->req, &queue->nvme_cq,
15262306a36Sopenharmony_ci			&queue->nvme_sq, &nvme_loop_ops))
15362306a36Sopenharmony_ci		return BLK_STS_OK;
15462306a36Sopenharmony_ci
15562306a36Sopenharmony_ci	if (blk_rq_nr_phys_segments(req)) {
15662306a36Sopenharmony_ci		iod->sg_table.sgl = iod->first_sgl;
15762306a36Sopenharmony_ci		if (sg_alloc_table_chained(&iod->sg_table,
15862306a36Sopenharmony_ci				blk_rq_nr_phys_segments(req),
15962306a36Sopenharmony_ci				iod->sg_table.sgl, NVME_INLINE_SG_CNT)) {
16062306a36Sopenharmony_ci			nvme_cleanup_cmd(req);
16162306a36Sopenharmony_ci			return BLK_STS_RESOURCE;
16262306a36Sopenharmony_ci		}
16362306a36Sopenharmony_ci
16462306a36Sopenharmony_ci		iod->req.sg = iod->sg_table.sgl;
16562306a36Sopenharmony_ci		iod->req.sg_cnt = blk_rq_map_sg(req->q, req, iod->sg_table.sgl);
16662306a36Sopenharmony_ci		iod->req.transfer_len = blk_rq_payload_bytes(req);
16762306a36Sopenharmony_ci	}
16862306a36Sopenharmony_ci
16962306a36Sopenharmony_ci	queue_work(nvmet_wq, &iod->work);
17062306a36Sopenharmony_ci	return BLK_STS_OK;
17162306a36Sopenharmony_ci}
17262306a36Sopenharmony_ci
17362306a36Sopenharmony_cistatic void nvme_loop_submit_async_event(struct nvme_ctrl *arg)
17462306a36Sopenharmony_ci{
17562306a36Sopenharmony_ci	struct nvme_loop_ctrl *ctrl = to_loop_ctrl(arg);
17662306a36Sopenharmony_ci	struct nvme_loop_queue *queue = &ctrl->queues[0];
17762306a36Sopenharmony_ci	struct nvme_loop_iod *iod = &ctrl->async_event_iod;
17862306a36Sopenharmony_ci
17962306a36Sopenharmony_ci	memset(&iod->cmd, 0, sizeof(iod->cmd));
18062306a36Sopenharmony_ci	iod->cmd.common.opcode = nvme_admin_async_event;
18162306a36Sopenharmony_ci	iod->cmd.common.command_id = NVME_AQ_BLK_MQ_DEPTH;
18262306a36Sopenharmony_ci	iod->cmd.common.flags |= NVME_CMD_SGL_METABUF;
18362306a36Sopenharmony_ci
18462306a36Sopenharmony_ci	if (!nvmet_req_init(&iod->req, &queue->nvme_cq, &queue->nvme_sq,
18562306a36Sopenharmony_ci			&nvme_loop_ops)) {
18662306a36Sopenharmony_ci		dev_err(ctrl->ctrl.device, "failed async event work\n");
18762306a36Sopenharmony_ci		return;
18862306a36Sopenharmony_ci	}
18962306a36Sopenharmony_ci
19062306a36Sopenharmony_ci	queue_work(nvmet_wq, &iod->work);
19162306a36Sopenharmony_ci}
19262306a36Sopenharmony_ci
19362306a36Sopenharmony_cistatic int nvme_loop_init_iod(struct nvme_loop_ctrl *ctrl,
19462306a36Sopenharmony_ci		struct nvme_loop_iod *iod, unsigned int queue_idx)
19562306a36Sopenharmony_ci{
19662306a36Sopenharmony_ci	iod->req.cmd = &iod->cmd;
19762306a36Sopenharmony_ci	iod->req.cqe = &iod->cqe;
19862306a36Sopenharmony_ci	iod->queue = &ctrl->queues[queue_idx];
19962306a36Sopenharmony_ci	INIT_WORK(&iod->work, nvme_loop_execute_work);
20062306a36Sopenharmony_ci	return 0;
20162306a36Sopenharmony_ci}
20262306a36Sopenharmony_ci
20362306a36Sopenharmony_cistatic int nvme_loop_init_request(struct blk_mq_tag_set *set,
20462306a36Sopenharmony_ci		struct request *req, unsigned int hctx_idx,
20562306a36Sopenharmony_ci		unsigned int numa_node)
20662306a36Sopenharmony_ci{
20762306a36Sopenharmony_ci	struct nvme_loop_ctrl *ctrl = to_loop_ctrl(set->driver_data);
20862306a36Sopenharmony_ci	struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req);
20962306a36Sopenharmony_ci
21062306a36Sopenharmony_ci	nvme_req(req)->ctrl = &ctrl->ctrl;
21162306a36Sopenharmony_ci	nvme_req(req)->cmd = &iod->cmd;
21262306a36Sopenharmony_ci	return nvme_loop_init_iod(ctrl, blk_mq_rq_to_pdu(req),
21362306a36Sopenharmony_ci			(set == &ctrl->tag_set) ? hctx_idx + 1 : 0);
21462306a36Sopenharmony_ci}
21562306a36Sopenharmony_ci
21662306a36Sopenharmony_cistatic struct lock_class_key loop_hctx_fq_lock_key;
21762306a36Sopenharmony_ci
21862306a36Sopenharmony_cistatic int nvme_loop_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
21962306a36Sopenharmony_ci		unsigned int hctx_idx)
22062306a36Sopenharmony_ci{
22162306a36Sopenharmony_ci	struct nvme_loop_ctrl *ctrl = to_loop_ctrl(data);
22262306a36Sopenharmony_ci	struct nvme_loop_queue *queue = &ctrl->queues[hctx_idx + 1];
22362306a36Sopenharmony_ci
22462306a36Sopenharmony_ci	BUG_ON(hctx_idx >= ctrl->ctrl.queue_count);
22562306a36Sopenharmony_ci
22662306a36Sopenharmony_ci	/*
22762306a36Sopenharmony_ci	 * flush_end_io() can be called recursively for us, so use our own
22862306a36Sopenharmony_ci	 * lock class key for avoiding lockdep possible recursive locking,
22962306a36Sopenharmony_ci	 * then we can remove the dynamically allocated lock class for each
23062306a36Sopenharmony_ci	 * flush queue, that way may cause horrible boot delay.
23162306a36Sopenharmony_ci	 */
23262306a36Sopenharmony_ci	blk_mq_hctx_set_fq_lock_class(hctx, &loop_hctx_fq_lock_key);
23362306a36Sopenharmony_ci
23462306a36Sopenharmony_ci	hctx->driver_data = queue;
23562306a36Sopenharmony_ci	return 0;
23662306a36Sopenharmony_ci}
23762306a36Sopenharmony_ci
23862306a36Sopenharmony_cistatic int nvme_loop_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
23962306a36Sopenharmony_ci		unsigned int hctx_idx)
24062306a36Sopenharmony_ci{
24162306a36Sopenharmony_ci	struct nvme_loop_ctrl *ctrl = to_loop_ctrl(data);
24262306a36Sopenharmony_ci	struct nvme_loop_queue *queue = &ctrl->queues[0];
24362306a36Sopenharmony_ci
24462306a36Sopenharmony_ci	BUG_ON(hctx_idx != 0);
24562306a36Sopenharmony_ci
24662306a36Sopenharmony_ci	hctx->driver_data = queue;
24762306a36Sopenharmony_ci	return 0;
24862306a36Sopenharmony_ci}
24962306a36Sopenharmony_ci
25062306a36Sopenharmony_cistatic const struct blk_mq_ops nvme_loop_mq_ops = {
25162306a36Sopenharmony_ci	.queue_rq	= nvme_loop_queue_rq,
25262306a36Sopenharmony_ci	.complete	= nvme_loop_complete_rq,
25362306a36Sopenharmony_ci	.init_request	= nvme_loop_init_request,
25462306a36Sopenharmony_ci	.init_hctx	= nvme_loop_init_hctx,
25562306a36Sopenharmony_ci};
25662306a36Sopenharmony_ci
25762306a36Sopenharmony_cistatic const struct blk_mq_ops nvme_loop_admin_mq_ops = {
25862306a36Sopenharmony_ci	.queue_rq	= nvme_loop_queue_rq,
25962306a36Sopenharmony_ci	.complete	= nvme_loop_complete_rq,
26062306a36Sopenharmony_ci	.init_request	= nvme_loop_init_request,
26162306a36Sopenharmony_ci	.init_hctx	= nvme_loop_init_admin_hctx,
26262306a36Sopenharmony_ci};
26362306a36Sopenharmony_ci
26462306a36Sopenharmony_cistatic void nvme_loop_destroy_admin_queue(struct nvme_loop_ctrl *ctrl)
26562306a36Sopenharmony_ci{
26662306a36Sopenharmony_ci	if (!test_and_clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags))
26762306a36Sopenharmony_ci		return;
26862306a36Sopenharmony_ci	nvmet_sq_destroy(&ctrl->queues[0].nvme_sq);
26962306a36Sopenharmony_ci	nvme_remove_admin_tag_set(&ctrl->ctrl);
27062306a36Sopenharmony_ci}
27162306a36Sopenharmony_ci
27262306a36Sopenharmony_cistatic void nvme_loop_free_ctrl(struct nvme_ctrl *nctrl)
27362306a36Sopenharmony_ci{
27462306a36Sopenharmony_ci	struct nvme_loop_ctrl *ctrl = to_loop_ctrl(nctrl);
27562306a36Sopenharmony_ci
27662306a36Sopenharmony_ci	if (list_empty(&ctrl->list))
27762306a36Sopenharmony_ci		goto free_ctrl;
27862306a36Sopenharmony_ci
27962306a36Sopenharmony_ci	mutex_lock(&nvme_loop_ctrl_mutex);
28062306a36Sopenharmony_ci	list_del(&ctrl->list);
28162306a36Sopenharmony_ci	mutex_unlock(&nvme_loop_ctrl_mutex);
28262306a36Sopenharmony_ci
28362306a36Sopenharmony_ci	if (nctrl->tagset)
28462306a36Sopenharmony_ci		nvme_remove_io_tag_set(nctrl);
28562306a36Sopenharmony_ci	kfree(ctrl->queues);
28662306a36Sopenharmony_ci	nvmf_free_options(nctrl->opts);
28762306a36Sopenharmony_cifree_ctrl:
28862306a36Sopenharmony_ci	kfree(ctrl);
28962306a36Sopenharmony_ci}
29062306a36Sopenharmony_ci
29162306a36Sopenharmony_cistatic void nvme_loop_destroy_io_queues(struct nvme_loop_ctrl *ctrl)
29262306a36Sopenharmony_ci{
29362306a36Sopenharmony_ci	int i;
29462306a36Sopenharmony_ci
29562306a36Sopenharmony_ci	for (i = 1; i < ctrl->ctrl.queue_count; i++) {
29662306a36Sopenharmony_ci		clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[i].flags);
29762306a36Sopenharmony_ci		nvmet_sq_destroy(&ctrl->queues[i].nvme_sq);
29862306a36Sopenharmony_ci	}
29962306a36Sopenharmony_ci	ctrl->ctrl.queue_count = 1;
30062306a36Sopenharmony_ci}
30162306a36Sopenharmony_ci
30262306a36Sopenharmony_cistatic int nvme_loop_init_io_queues(struct nvme_loop_ctrl *ctrl)
30362306a36Sopenharmony_ci{
30462306a36Sopenharmony_ci	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
30562306a36Sopenharmony_ci	unsigned int nr_io_queues;
30662306a36Sopenharmony_ci	int ret, i;
30762306a36Sopenharmony_ci
30862306a36Sopenharmony_ci	nr_io_queues = min(opts->nr_io_queues, num_online_cpus());
30962306a36Sopenharmony_ci	ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
31062306a36Sopenharmony_ci	if (ret || !nr_io_queues)
31162306a36Sopenharmony_ci		return ret;
31262306a36Sopenharmony_ci
31362306a36Sopenharmony_ci	dev_info(ctrl->ctrl.device, "creating %d I/O queues.\n", nr_io_queues);
31462306a36Sopenharmony_ci
31562306a36Sopenharmony_ci	for (i = 1; i <= nr_io_queues; i++) {
31662306a36Sopenharmony_ci		ctrl->queues[i].ctrl = ctrl;
31762306a36Sopenharmony_ci		ret = nvmet_sq_init(&ctrl->queues[i].nvme_sq);
31862306a36Sopenharmony_ci		if (ret)
31962306a36Sopenharmony_ci			goto out_destroy_queues;
32062306a36Sopenharmony_ci
32162306a36Sopenharmony_ci		ctrl->ctrl.queue_count++;
32262306a36Sopenharmony_ci	}
32362306a36Sopenharmony_ci
32462306a36Sopenharmony_ci	return 0;
32562306a36Sopenharmony_ci
32662306a36Sopenharmony_ciout_destroy_queues:
32762306a36Sopenharmony_ci	nvme_loop_destroy_io_queues(ctrl);
32862306a36Sopenharmony_ci	return ret;
32962306a36Sopenharmony_ci}
33062306a36Sopenharmony_ci
33162306a36Sopenharmony_cistatic int nvme_loop_connect_io_queues(struct nvme_loop_ctrl *ctrl)
33262306a36Sopenharmony_ci{
33362306a36Sopenharmony_ci	int i, ret;
33462306a36Sopenharmony_ci
33562306a36Sopenharmony_ci	for (i = 1; i < ctrl->ctrl.queue_count; i++) {
33662306a36Sopenharmony_ci		ret = nvmf_connect_io_queue(&ctrl->ctrl, i);
33762306a36Sopenharmony_ci		if (ret)
33862306a36Sopenharmony_ci			return ret;
33962306a36Sopenharmony_ci		set_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[i].flags);
34062306a36Sopenharmony_ci	}
34162306a36Sopenharmony_ci
34262306a36Sopenharmony_ci	return 0;
34362306a36Sopenharmony_ci}
34462306a36Sopenharmony_ci
34562306a36Sopenharmony_cistatic int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl)
34662306a36Sopenharmony_ci{
34762306a36Sopenharmony_ci	int error;
34862306a36Sopenharmony_ci
34962306a36Sopenharmony_ci	ctrl->queues[0].ctrl = ctrl;
35062306a36Sopenharmony_ci	error = nvmet_sq_init(&ctrl->queues[0].nvme_sq);
35162306a36Sopenharmony_ci	if (error)
35262306a36Sopenharmony_ci		return error;
35362306a36Sopenharmony_ci	ctrl->ctrl.queue_count = 1;
35462306a36Sopenharmony_ci
35562306a36Sopenharmony_ci	error = nvme_alloc_admin_tag_set(&ctrl->ctrl, &ctrl->admin_tag_set,
35662306a36Sopenharmony_ci			&nvme_loop_admin_mq_ops,
35762306a36Sopenharmony_ci			sizeof(struct nvme_loop_iod) +
35862306a36Sopenharmony_ci			NVME_INLINE_SG_CNT * sizeof(struct scatterlist));
35962306a36Sopenharmony_ci	if (error)
36062306a36Sopenharmony_ci		goto out_free_sq;
36162306a36Sopenharmony_ci
36262306a36Sopenharmony_ci	/* reset stopped state for the fresh admin queue */
36362306a36Sopenharmony_ci	clear_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->ctrl.flags);
36462306a36Sopenharmony_ci
36562306a36Sopenharmony_ci	error = nvmf_connect_admin_queue(&ctrl->ctrl);
36662306a36Sopenharmony_ci	if (error)
36762306a36Sopenharmony_ci		goto out_cleanup_tagset;
36862306a36Sopenharmony_ci
36962306a36Sopenharmony_ci	set_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags);
37062306a36Sopenharmony_ci
37162306a36Sopenharmony_ci	error = nvme_enable_ctrl(&ctrl->ctrl);
37262306a36Sopenharmony_ci	if (error)
37362306a36Sopenharmony_ci		goto out_cleanup_tagset;
37462306a36Sopenharmony_ci
37562306a36Sopenharmony_ci	ctrl->ctrl.max_hw_sectors =
37662306a36Sopenharmony_ci		(NVME_LOOP_MAX_SEGMENTS - 1) << PAGE_SECTORS_SHIFT;
37762306a36Sopenharmony_ci
37862306a36Sopenharmony_ci	nvme_unquiesce_admin_queue(&ctrl->ctrl);
37962306a36Sopenharmony_ci
38062306a36Sopenharmony_ci	error = nvme_init_ctrl_finish(&ctrl->ctrl, false);
38162306a36Sopenharmony_ci	if (error)
38262306a36Sopenharmony_ci		goto out_cleanup_tagset;
38362306a36Sopenharmony_ci
38462306a36Sopenharmony_ci	return 0;
38562306a36Sopenharmony_ci
38662306a36Sopenharmony_ciout_cleanup_tagset:
38762306a36Sopenharmony_ci	clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags);
38862306a36Sopenharmony_ci	nvme_remove_admin_tag_set(&ctrl->ctrl);
38962306a36Sopenharmony_ciout_free_sq:
39062306a36Sopenharmony_ci	nvmet_sq_destroy(&ctrl->queues[0].nvme_sq);
39162306a36Sopenharmony_ci	return error;
39262306a36Sopenharmony_ci}
39362306a36Sopenharmony_ci
39462306a36Sopenharmony_cistatic void nvme_loop_shutdown_ctrl(struct nvme_loop_ctrl *ctrl)
39562306a36Sopenharmony_ci{
39662306a36Sopenharmony_ci	if (ctrl->ctrl.queue_count > 1) {
39762306a36Sopenharmony_ci		nvme_quiesce_io_queues(&ctrl->ctrl);
39862306a36Sopenharmony_ci		nvme_cancel_tagset(&ctrl->ctrl);
39962306a36Sopenharmony_ci		nvme_loop_destroy_io_queues(ctrl);
40062306a36Sopenharmony_ci	}
40162306a36Sopenharmony_ci
40262306a36Sopenharmony_ci	nvme_quiesce_admin_queue(&ctrl->ctrl);
40362306a36Sopenharmony_ci	if (ctrl->ctrl.state == NVME_CTRL_LIVE)
40462306a36Sopenharmony_ci		nvme_disable_ctrl(&ctrl->ctrl, true);
40562306a36Sopenharmony_ci
40662306a36Sopenharmony_ci	nvme_cancel_admin_tagset(&ctrl->ctrl);
40762306a36Sopenharmony_ci	nvme_loop_destroy_admin_queue(ctrl);
40862306a36Sopenharmony_ci}
40962306a36Sopenharmony_ci
41062306a36Sopenharmony_cistatic void nvme_loop_delete_ctrl_host(struct nvme_ctrl *ctrl)
41162306a36Sopenharmony_ci{
41262306a36Sopenharmony_ci	nvme_loop_shutdown_ctrl(to_loop_ctrl(ctrl));
41362306a36Sopenharmony_ci}
41462306a36Sopenharmony_ci
41562306a36Sopenharmony_cistatic void nvme_loop_delete_ctrl(struct nvmet_ctrl *nctrl)
41662306a36Sopenharmony_ci{
41762306a36Sopenharmony_ci	struct nvme_loop_ctrl *ctrl;
41862306a36Sopenharmony_ci
41962306a36Sopenharmony_ci	mutex_lock(&nvme_loop_ctrl_mutex);
42062306a36Sopenharmony_ci	list_for_each_entry(ctrl, &nvme_loop_ctrl_list, list) {
42162306a36Sopenharmony_ci		if (ctrl->ctrl.cntlid == nctrl->cntlid)
42262306a36Sopenharmony_ci			nvme_delete_ctrl(&ctrl->ctrl);
42362306a36Sopenharmony_ci	}
42462306a36Sopenharmony_ci	mutex_unlock(&nvme_loop_ctrl_mutex);
42562306a36Sopenharmony_ci}
42662306a36Sopenharmony_ci
42762306a36Sopenharmony_cistatic void nvme_loop_reset_ctrl_work(struct work_struct *work)
42862306a36Sopenharmony_ci{
42962306a36Sopenharmony_ci	struct nvme_loop_ctrl *ctrl =
43062306a36Sopenharmony_ci		container_of(work, struct nvme_loop_ctrl, ctrl.reset_work);
43162306a36Sopenharmony_ci	int ret;
43262306a36Sopenharmony_ci
43362306a36Sopenharmony_ci	nvme_stop_ctrl(&ctrl->ctrl);
43462306a36Sopenharmony_ci	nvme_loop_shutdown_ctrl(ctrl);
43562306a36Sopenharmony_ci
43662306a36Sopenharmony_ci	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
43762306a36Sopenharmony_ci		if (ctrl->ctrl.state != NVME_CTRL_DELETING &&
43862306a36Sopenharmony_ci		    ctrl->ctrl.state != NVME_CTRL_DELETING_NOIO)
43962306a36Sopenharmony_ci			/* state change failure for non-deleted ctrl? */
44062306a36Sopenharmony_ci			WARN_ON_ONCE(1);
44162306a36Sopenharmony_ci		return;
44262306a36Sopenharmony_ci	}
44362306a36Sopenharmony_ci
44462306a36Sopenharmony_ci	ret = nvme_loop_configure_admin_queue(ctrl);
44562306a36Sopenharmony_ci	if (ret)
44662306a36Sopenharmony_ci		goto out_disable;
44762306a36Sopenharmony_ci
44862306a36Sopenharmony_ci	ret = nvme_loop_init_io_queues(ctrl);
44962306a36Sopenharmony_ci	if (ret)
45062306a36Sopenharmony_ci		goto out_destroy_admin;
45162306a36Sopenharmony_ci
45262306a36Sopenharmony_ci	ret = nvme_loop_connect_io_queues(ctrl);
45362306a36Sopenharmony_ci	if (ret)
45462306a36Sopenharmony_ci		goto out_destroy_io;
45562306a36Sopenharmony_ci
45662306a36Sopenharmony_ci	blk_mq_update_nr_hw_queues(&ctrl->tag_set,
45762306a36Sopenharmony_ci			ctrl->ctrl.queue_count - 1);
45862306a36Sopenharmony_ci
45962306a36Sopenharmony_ci	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE))
46062306a36Sopenharmony_ci		WARN_ON_ONCE(1);
46162306a36Sopenharmony_ci
46262306a36Sopenharmony_ci	nvme_start_ctrl(&ctrl->ctrl);
46362306a36Sopenharmony_ci
46462306a36Sopenharmony_ci	return;
46562306a36Sopenharmony_ci
46662306a36Sopenharmony_ciout_destroy_io:
46762306a36Sopenharmony_ci	nvme_loop_destroy_io_queues(ctrl);
46862306a36Sopenharmony_ciout_destroy_admin:
46962306a36Sopenharmony_ci	nvme_loop_destroy_admin_queue(ctrl);
47062306a36Sopenharmony_ciout_disable:
47162306a36Sopenharmony_ci	dev_warn(ctrl->ctrl.device, "Removing after reset failure\n");
47262306a36Sopenharmony_ci	nvme_uninit_ctrl(&ctrl->ctrl);
47362306a36Sopenharmony_ci}
47462306a36Sopenharmony_ci
47562306a36Sopenharmony_cistatic const struct nvme_ctrl_ops nvme_loop_ctrl_ops = {
47662306a36Sopenharmony_ci	.name			= "loop",
47762306a36Sopenharmony_ci	.module			= THIS_MODULE,
47862306a36Sopenharmony_ci	.flags			= NVME_F_FABRICS,
47962306a36Sopenharmony_ci	.reg_read32		= nvmf_reg_read32,
48062306a36Sopenharmony_ci	.reg_read64		= nvmf_reg_read64,
48162306a36Sopenharmony_ci	.reg_write32		= nvmf_reg_write32,
48262306a36Sopenharmony_ci	.free_ctrl		= nvme_loop_free_ctrl,
48362306a36Sopenharmony_ci	.submit_async_event	= nvme_loop_submit_async_event,
48462306a36Sopenharmony_ci	.delete_ctrl		= nvme_loop_delete_ctrl_host,
48562306a36Sopenharmony_ci	.get_address		= nvmf_get_address,
48662306a36Sopenharmony_ci};
48762306a36Sopenharmony_ci
48862306a36Sopenharmony_cistatic int nvme_loop_create_io_queues(struct nvme_loop_ctrl *ctrl)
48962306a36Sopenharmony_ci{
49062306a36Sopenharmony_ci	int ret;
49162306a36Sopenharmony_ci
49262306a36Sopenharmony_ci	ret = nvme_loop_init_io_queues(ctrl);
49362306a36Sopenharmony_ci	if (ret)
49462306a36Sopenharmony_ci		return ret;
49562306a36Sopenharmony_ci
49662306a36Sopenharmony_ci	ret = nvme_alloc_io_tag_set(&ctrl->ctrl, &ctrl->tag_set,
49762306a36Sopenharmony_ci			&nvme_loop_mq_ops, 1,
49862306a36Sopenharmony_ci			sizeof(struct nvme_loop_iod) +
49962306a36Sopenharmony_ci			NVME_INLINE_SG_CNT * sizeof(struct scatterlist));
50062306a36Sopenharmony_ci	if (ret)
50162306a36Sopenharmony_ci		goto out_destroy_queues;
50262306a36Sopenharmony_ci
50362306a36Sopenharmony_ci	ret = nvme_loop_connect_io_queues(ctrl);
50462306a36Sopenharmony_ci	if (ret)
50562306a36Sopenharmony_ci		goto out_cleanup_tagset;
50662306a36Sopenharmony_ci
50762306a36Sopenharmony_ci	return 0;
50862306a36Sopenharmony_ci
50962306a36Sopenharmony_ciout_cleanup_tagset:
51062306a36Sopenharmony_ci	nvme_remove_io_tag_set(&ctrl->ctrl);
51162306a36Sopenharmony_ciout_destroy_queues:
51262306a36Sopenharmony_ci	nvme_loop_destroy_io_queues(ctrl);
51362306a36Sopenharmony_ci	return ret;
51462306a36Sopenharmony_ci}
51562306a36Sopenharmony_ci
51662306a36Sopenharmony_cistatic struct nvmet_port *nvme_loop_find_port(struct nvme_ctrl *ctrl)
51762306a36Sopenharmony_ci{
51862306a36Sopenharmony_ci	struct nvmet_port *p, *found = NULL;
51962306a36Sopenharmony_ci
52062306a36Sopenharmony_ci	mutex_lock(&nvme_loop_ports_mutex);
52162306a36Sopenharmony_ci	list_for_each_entry(p, &nvme_loop_ports, entry) {
52262306a36Sopenharmony_ci		/* if no transport address is specified use the first port */
52362306a36Sopenharmony_ci		if ((ctrl->opts->mask & NVMF_OPT_TRADDR) &&
52462306a36Sopenharmony_ci		    strcmp(ctrl->opts->traddr, p->disc_addr.traddr))
52562306a36Sopenharmony_ci			continue;
52662306a36Sopenharmony_ci		found = p;
52762306a36Sopenharmony_ci		break;
52862306a36Sopenharmony_ci	}
52962306a36Sopenharmony_ci	mutex_unlock(&nvme_loop_ports_mutex);
53062306a36Sopenharmony_ci	return found;
53162306a36Sopenharmony_ci}
53262306a36Sopenharmony_ci
53362306a36Sopenharmony_cistatic struct nvme_ctrl *nvme_loop_create_ctrl(struct device *dev,
53462306a36Sopenharmony_ci		struct nvmf_ctrl_options *opts)
53562306a36Sopenharmony_ci{
53662306a36Sopenharmony_ci	struct nvme_loop_ctrl *ctrl;
53762306a36Sopenharmony_ci	int ret;
53862306a36Sopenharmony_ci
53962306a36Sopenharmony_ci	ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
54062306a36Sopenharmony_ci	if (!ctrl)
54162306a36Sopenharmony_ci		return ERR_PTR(-ENOMEM);
54262306a36Sopenharmony_ci	ctrl->ctrl.opts = opts;
54362306a36Sopenharmony_ci	INIT_LIST_HEAD(&ctrl->list);
54462306a36Sopenharmony_ci
54562306a36Sopenharmony_ci	INIT_WORK(&ctrl->ctrl.reset_work, nvme_loop_reset_ctrl_work);
54662306a36Sopenharmony_ci
54762306a36Sopenharmony_ci	ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_loop_ctrl_ops,
54862306a36Sopenharmony_ci				0 /* no quirks, we're perfect! */);
54962306a36Sopenharmony_ci	if (ret) {
55062306a36Sopenharmony_ci		kfree(ctrl);
55162306a36Sopenharmony_ci		goto out;
55262306a36Sopenharmony_ci	}
55362306a36Sopenharmony_ci
55462306a36Sopenharmony_ci	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING))
55562306a36Sopenharmony_ci		WARN_ON_ONCE(1);
55662306a36Sopenharmony_ci
55762306a36Sopenharmony_ci	ret = -ENOMEM;
55862306a36Sopenharmony_ci
55962306a36Sopenharmony_ci	ctrl->ctrl.kato = opts->kato;
56062306a36Sopenharmony_ci	ctrl->port = nvme_loop_find_port(&ctrl->ctrl);
56162306a36Sopenharmony_ci
56262306a36Sopenharmony_ci	ctrl->queues = kcalloc(opts->nr_io_queues + 1, sizeof(*ctrl->queues),
56362306a36Sopenharmony_ci			GFP_KERNEL);
56462306a36Sopenharmony_ci	if (!ctrl->queues)
56562306a36Sopenharmony_ci		goto out_uninit_ctrl;
56662306a36Sopenharmony_ci
56762306a36Sopenharmony_ci	ret = nvme_loop_configure_admin_queue(ctrl);
56862306a36Sopenharmony_ci	if (ret)
56962306a36Sopenharmony_ci		goto out_free_queues;
57062306a36Sopenharmony_ci
57162306a36Sopenharmony_ci	if (opts->queue_size > ctrl->ctrl.maxcmd) {
57262306a36Sopenharmony_ci		/* warn if maxcmd is lower than queue_size */
57362306a36Sopenharmony_ci		dev_warn(ctrl->ctrl.device,
57462306a36Sopenharmony_ci			"queue_size %zu > ctrl maxcmd %u, clamping down\n",
57562306a36Sopenharmony_ci			opts->queue_size, ctrl->ctrl.maxcmd);
57662306a36Sopenharmony_ci		opts->queue_size = ctrl->ctrl.maxcmd;
57762306a36Sopenharmony_ci	}
57862306a36Sopenharmony_ci	ctrl->ctrl.sqsize = opts->queue_size - 1;
57962306a36Sopenharmony_ci
58062306a36Sopenharmony_ci	if (opts->nr_io_queues) {
58162306a36Sopenharmony_ci		ret = nvme_loop_create_io_queues(ctrl);
58262306a36Sopenharmony_ci		if (ret)
58362306a36Sopenharmony_ci			goto out_remove_admin_queue;
58462306a36Sopenharmony_ci	}
58562306a36Sopenharmony_ci
58662306a36Sopenharmony_ci	nvme_loop_init_iod(ctrl, &ctrl->async_event_iod, 0);
58762306a36Sopenharmony_ci
58862306a36Sopenharmony_ci	dev_info(ctrl->ctrl.device,
58962306a36Sopenharmony_ci		 "new ctrl: \"%s\"\n", ctrl->ctrl.opts->subsysnqn);
59062306a36Sopenharmony_ci
59162306a36Sopenharmony_ci	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE))
59262306a36Sopenharmony_ci		WARN_ON_ONCE(1);
59362306a36Sopenharmony_ci
59462306a36Sopenharmony_ci	mutex_lock(&nvme_loop_ctrl_mutex);
59562306a36Sopenharmony_ci	list_add_tail(&ctrl->list, &nvme_loop_ctrl_list);
59662306a36Sopenharmony_ci	mutex_unlock(&nvme_loop_ctrl_mutex);
59762306a36Sopenharmony_ci
59862306a36Sopenharmony_ci	nvme_start_ctrl(&ctrl->ctrl);
59962306a36Sopenharmony_ci
60062306a36Sopenharmony_ci	return &ctrl->ctrl;
60162306a36Sopenharmony_ci
60262306a36Sopenharmony_ciout_remove_admin_queue:
60362306a36Sopenharmony_ci	nvme_loop_destroy_admin_queue(ctrl);
60462306a36Sopenharmony_ciout_free_queues:
60562306a36Sopenharmony_ci	kfree(ctrl->queues);
60662306a36Sopenharmony_ciout_uninit_ctrl:
60762306a36Sopenharmony_ci	nvme_uninit_ctrl(&ctrl->ctrl);
60862306a36Sopenharmony_ci	nvme_put_ctrl(&ctrl->ctrl);
60962306a36Sopenharmony_ciout:
61062306a36Sopenharmony_ci	if (ret > 0)
61162306a36Sopenharmony_ci		ret = -EIO;
61262306a36Sopenharmony_ci	return ERR_PTR(ret);
61362306a36Sopenharmony_ci}
61462306a36Sopenharmony_ci
61562306a36Sopenharmony_cistatic int nvme_loop_add_port(struct nvmet_port *port)
61662306a36Sopenharmony_ci{
61762306a36Sopenharmony_ci	mutex_lock(&nvme_loop_ports_mutex);
61862306a36Sopenharmony_ci	list_add_tail(&port->entry, &nvme_loop_ports);
61962306a36Sopenharmony_ci	mutex_unlock(&nvme_loop_ports_mutex);
62062306a36Sopenharmony_ci	return 0;
62162306a36Sopenharmony_ci}
62262306a36Sopenharmony_ci
62362306a36Sopenharmony_cistatic void nvme_loop_remove_port(struct nvmet_port *port)
62462306a36Sopenharmony_ci{
62562306a36Sopenharmony_ci	mutex_lock(&nvme_loop_ports_mutex);
62662306a36Sopenharmony_ci	list_del_init(&port->entry);
62762306a36Sopenharmony_ci	mutex_unlock(&nvme_loop_ports_mutex);
62862306a36Sopenharmony_ci
62962306a36Sopenharmony_ci	/*
63062306a36Sopenharmony_ci	 * Ensure any ctrls that are in the process of being
63162306a36Sopenharmony_ci	 * deleted are in fact deleted before we return
63262306a36Sopenharmony_ci	 * and free the port. This is to prevent active
63362306a36Sopenharmony_ci	 * ctrls from using a port after it's freed.
63462306a36Sopenharmony_ci	 */
63562306a36Sopenharmony_ci	flush_workqueue(nvme_delete_wq);
63662306a36Sopenharmony_ci}
63762306a36Sopenharmony_ci
63862306a36Sopenharmony_cistatic const struct nvmet_fabrics_ops nvme_loop_ops = {
63962306a36Sopenharmony_ci	.owner		= THIS_MODULE,
64062306a36Sopenharmony_ci	.type		= NVMF_TRTYPE_LOOP,
64162306a36Sopenharmony_ci	.add_port	= nvme_loop_add_port,
64262306a36Sopenharmony_ci	.remove_port	= nvme_loop_remove_port,
64362306a36Sopenharmony_ci	.queue_response = nvme_loop_queue_response,
64462306a36Sopenharmony_ci	.delete_ctrl	= nvme_loop_delete_ctrl,
64562306a36Sopenharmony_ci};
64662306a36Sopenharmony_ci
64762306a36Sopenharmony_cistatic struct nvmf_transport_ops nvme_loop_transport = {
64862306a36Sopenharmony_ci	.name		= "loop",
64962306a36Sopenharmony_ci	.module		= THIS_MODULE,
65062306a36Sopenharmony_ci	.create_ctrl	= nvme_loop_create_ctrl,
65162306a36Sopenharmony_ci	.allowed_opts	= NVMF_OPT_TRADDR,
65262306a36Sopenharmony_ci};
65362306a36Sopenharmony_ci
65462306a36Sopenharmony_cistatic int __init nvme_loop_init_module(void)
65562306a36Sopenharmony_ci{
65662306a36Sopenharmony_ci	int ret;
65762306a36Sopenharmony_ci
65862306a36Sopenharmony_ci	ret = nvmet_register_transport(&nvme_loop_ops);
65962306a36Sopenharmony_ci	if (ret)
66062306a36Sopenharmony_ci		return ret;
66162306a36Sopenharmony_ci
66262306a36Sopenharmony_ci	ret = nvmf_register_transport(&nvme_loop_transport);
66362306a36Sopenharmony_ci	if (ret)
66462306a36Sopenharmony_ci		nvmet_unregister_transport(&nvme_loop_ops);
66562306a36Sopenharmony_ci
66662306a36Sopenharmony_ci	return ret;
66762306a36Sopenharmony_ci}
66862306a36Sopenharmony_ci
66962306a36Sopenharmony_cistatic void __exit nvme_loop_cleanup_module(void)
67062306a36Sopenharmony_ci{
67162306a36Sopenharmony_ci	struct nvme_loop_ctrl *ctrl, *next;
67262306a36Sopenharmony_ci
67362306a36Sopenharmony_ci	nvmf_unregister_transport(&nvme_loop_transport);
67462306a36Sopenharmony_ci	nvmet_unregister_transport(&nvme_loop_ops);
67562306a36Sopenharmony_ci
67662306a36Sopenharmony_ci	mutex_lock(&nvme_loop_ctrl_mutex);
67762306a36Sopenharmony_ci	list_for_each_entry_safe(ctrl, next, &nvme_loop_ctrl_list, list)
67862306a36Sopenharmony_ci		nvme_delete_ctrl(&ctrl->ctrl);
67962306a36Sopenharmony_ci	mutex_unlock(&nvme_loop_ctrl_mutex);
68062306a36Sopenharmony_ci
68162306a36Sopenharmony_ci	flush_workqueue(nvme_delete_wq);
68262306a36Sopenharmony_ci}
68362306a36Sopenharmony_ci
68462306a36Sopenharmony_cimodule_init(nvme_loop_init_module);
68562306a36Sopenharmony_cimodule_exit(nvme_loop_cleanup_module);
68662306a36Sopenharmony_ci
68762306a36Sopenharmony_ciMODULE_LICENSE("GPL v2");
68862306a36Sopenharmony_ciMODULE_ALIAS("nvmet-transport-254"); /* 254 == NVMF_TRTYPE_LOOP */
689