Lines Matching defs:dev

111 static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown);
112 static void nvme_delete_io_queues(struct nvme_dev *dev);
113 static void nvme_update_attrs(struct nvme_dev *dev);
123 struct device *dev;
190 struct nvme_dev *dev;
243 static inline unsigned int nvme_dbbuf_size(struct nvme_dev *dev)
245 return dev->nr_allocated_queues * 8 * dev->db_stride;
248 static void nvme_dbbuf_dma_alloc(struct nvme_dev *dev)
250 unsigned int mem_size = nvme_dbbuf_size(dev);
252 if (!(dev->ctrl.oacs & NVME_CTRL_OACS_DBBUF_SUPP))
255 if (dev->dbbuf_dbs) {
260 memset(dev->dbbuf_dbs, 0, mem_size);
261 memset(dev->dbbuf_eis, 0, mem_size);
265 dev->dbbuf_dbs = dma_alloc_coherent(dev->dev, mem_size,
266 &dev->dbbuf_dbs_dma_addr,
268 if (!dev->dbbuf_dbs)
270 dev->dbbuf_eis = dma_alloc_coherent(dev->dev, mem_size,
271 &dev->dbbuf_eis_dma_addr,
273 if (!dev->dbbuf_eis)
278 dma_free_coherent(dev->dev, mem_size, dev->dbbuf_dbs,
279 dev->dbbuf_dbs_dma_addr);
280 dev->dbbuf_dbs = NULL;
282 dev_warn(dev->dev, "unable to allocate dma for dbbuf\n");
285 static void nvme_dbbuf_dma_free(struct nvme_dev *dev)
287 unsigned int mem_size = nvme_dbbuf_size(dev);
289 if (dev->dbbuf_dbs) {
290 dma_free_coherent(dev->dev, mem_size,
291 dev->dbbuf_dbs, dev->dbbuf_dbs_dma_addr);
292 dev->dbbuf_dbs = NULL;
294 if (dev->dbbuf_eis) {
295 dma_free_coherent(dev->dev, mem_size,
296 dev->dbbuf_eis, dev->dbbuf_eis_dma_addr);
297 dev->dbbuf_eis = NULL;
301 static void nvme_dbbuf_init(struct nvme_dev *dev,
304 if (!dev->dbbuf_dbs || !qid)
307 nvmeq->dbbuf_sq_db = &dev->dbbuf_dbs[sq_idx(qid, dev->db_stride)];
308 nvmeq->dbbuf_cq_db = &dev->dbbuf_dbs[cq_idx(qid, dev->db_stride)];
309 nvmeq->dbbuf_sq_ei = &dev->dbbuf_eis[sq_idx(qid, dev->db_stride)];
310 nvmeq->dbbuf_cq_ei = &dev->dbbuf_eis[cq_idx(qid, dev->db_stride)];
324 static void nvme_dbbuf_set(struct nvme_dev *dev)
329 if (!dev->dbbuf_dbs)
333 c.dbbuf.prp1 = cpu_to_le64(dev->dbbuf_dbs_dma_addr);
334 c.dbbuf.prp2 = cpu_to_le64(dev->dbbuf_eis_dma_addr);
336 if (nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0)) {
337 dev_warn(dev->ctrl.device, "unable to set dbbuf\n");
339 nvme_dbbuf_dma_free(dev);
341 for (i = 1; i <= dev->online_queues; i++)
342 nvme_dbbuf_free(&dev->queues[i]);
398 struct nvme_dev *dev = to_nvme_dev(data);
399 struct nvme_queue *nvmeq = &dev->queues[0];
402 WARN_ON(dev->admin_tagset.tags[0] != hctx->tags);
411 struct nvme_dev *dev = to_nvme_dev(data);
412 struct nvme_queue *nvmeq = &dev->queues[hctx_idx + 1];
414 WARN_ON(dev->tagset.tags[hctx_idx] != hctx->tags);
430 static int queue_irq_offset(struct nvme_dev *dev)
433 if (dev->num_vecs > 1)
441 struct nvme_dev *dev = to_nvme_dev(set->driver_data);
444 offset = queue_irq_offset(dev);
448 map->nr_queues = dev->io_queues[i];
460 blk_mq_pci_map_queues(map, to_pci_dev(dev->dev), offset);
507 static inline bool nvme_pci_use_sgls(struct nvme_dev *dev, struct request *req,
515 if (!nvme_ctrl_sgl_supported(&dev->ctrl))
524 static void nvme_free_prps(struct nvme_dev *dev, struct request *req)
535 dma_pool_free(dev->prp_page_pool, prp_list, dma_addr);
540 static void nvme_unmap_data(struct nvme_dev *dev, struct request *req)
545 dma_unmap_page(dev->dev, iod->first_dma, iod->dma_len,
552 dma_unmap_sgtable(dev->dev, &iod->sgt, rq_dma_dir(req), 0);
555 dma_pool_free(dev->prp_small_pool, iod->list[0].sg_list,
558 dma_pool_free(dev->prp_page_pool, iod->list[0].sg_list,
561 nvme_free_prps(dev, req);
562 mempool_free(iod->sgt.sgl, dev->iod_mempool);
579 static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev,
615 pool = dev->prp_small_pool;
618 pool = dev->prp_page_pool;
660 nvme_free_prps(dev, req);
685 static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev,
705 pool = dev->prp_small_pool;
708 pool = dev->prp_page_pool;
730 static blk_status_t nvme_setup_prp_simple(struct nvme_dev *dev,
738 iod->first_dma = dma_map_bvec(dev->dev, bv, rq_dma_dir(req), 0);
739 if (dma_mapping_error(dev->dev, iod->first_dma))
751 static blk_status_t nvme_setup_sgl_simple(struct nvme_dev *dev,
757 iod->first_dma = dma_map_bvec(dev->dev, bv, rq_dma_dir(req), 0);
758 if (dma_mapping_error(dev->dev, iod->first_dma))
769 static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req,
782 return nvme_setup_prp_simple(dev, req,
786 nvme_ctrl_sgl_supported(&dev->ctrl))
787 return nvme_setup_sgl_simple(dev, req,
793 iod->sgt.sgl = mempool_alloc(dev->iod_mempool, GFP_ATOMIC);
801 rc = dma_map_sgtable(dev->dev, &iod->sgt, rq_dma_dir(req),
809 if (nvme_pci_use_sgls(dev, req, iod->sgt.nents))
810 ret = nvme_pci_setup_sgls(dev, req, &cmnd->rw);
812 ret = nvme_pci_setup_prps(dev, req, &cmnd->rw);
818 dma_unmap_sgtable(dev->dev, &iod->sgt, rq_dma_dir(req), 0);
820 mempool_free(iod->sgt.sgl, dev->iod_mempool);
824 static blk_status_t nvme_map_metadata(struct nvme_dev *dev, struct request *req,
829 iod->meta_dma = dma_map_bvec(dev->dev, rq_integrity_vec(req),
831 if (dma_mapping_error(dev->dev, iod->meta_dma))
837 static blk_status_t nvme_prep_rq(struct nvme_dev *dev, struct request *req)
851 ret = nvme_map_data(dev, req, &iod->cmd);
857 ret = nvme_map_metadata(dev, req, &iod->cmd);
865 nvme_unmap_data(dev, req);
878 struct nvme_dev *dev = nvmeq->dev;
890 if (unlikely(!nvme_check_ready(&dev->ctrl, req, true)))
891 return nvme_fail_nonready_command(&dev->ctrl, req);
893 ret = nvme_prep_rq(dev, req);
924 if (unlikely(!nvme_check_ready(&nvmeq->dev->ctrl, req, true)))
928 return nvme_prep_rq(nvmeq->dev, req) == BLK_STS_OK;
964 struct nvme_dev *dev = nvmeq->dev;
969 dma_unmap_page(dev->dev, iod->meta_dma,
974 nvme_unmap_data(dev, req);
1002 writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
1008 return nvmeq->dev->admin_tagset.tags[0];
1009 return nvmeq->dev->tagset.tags[nvmeq->qid - 1];
1026 nvme_complete_async_event(&nvmeq->dev->ctrl,
1033 dev_warn(nvmeq->dev->ctrl.device,
1107 struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
1133 struct nvme_dev *dev = to_nvme_dev(ctrl);
1134 struct nvme_queue *nvmeq = &dev->queues[0];
1146 static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
1153 return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
1156 static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
1176 return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
1179 static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
1182 struct nvme_ctrl *ctrl = &dev->ctrl;
1205 return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
1208 static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
1210 return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
1213 static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
1215 return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
1222 dev_warn(nvmeq->dev->ctrl.device,
1224 atomic_inc(&nvmeq->dev->ctrl.abort_limit);
1229 static bool nvme_should_reset(struct nvme_dev *dev, u32 csts)
1234 bool nssro = dev->subsystem && (csts & NVME_CSTS_NSSRO);
1237 switch (nvme_ctrl_state(&dev->ctrl)) {
1254 static void nvme_warn_reset(struct nvme_dev *dev, u32 csts)
1260 result = pci_read_config_word(to_pci_dev(dev->dev), PCI_STATUS,
1263 dev_warn(dev->ctrl.device,
1267 dev_warn(dev->ctrl.device,
1274 dev_warn(dev->ctrl.device,
1276 dev_warn(dev->ctrl.device,
1284 struct nvme_dev *dev = nvmeq->dev;
1287 u32 csts = readl(dev->bar + NVME_REG_CSTS);
1293 if (pci_channel_offline(to_pci_dev(dev->dev)))
1299 if (nvme_should_reset(dev, csts)) {
1300 nvme_warn_reset(dev, csts);
1313 dev_warn(dev->ctrl.device,
1325 switch (nvme_ctrl_state(&dev->ctrl)) {
1327 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING);
1330 dev_warn_ratelimited(dev->ctrl.device,
1334 nvme_dev_disable(dev, true);
1348 dev_warn(dev->ctrl.device,
1355 if (atomic_dec_return(&dev->ctrl.abort_limit) < 0) {
1356 atomic_inc(&dev->ctrl.abort_limit);
1365 dev_warn(nvmeq->dev->ctrl.device,
1371 abort_req = blk_mq_alloc_request(dev->ctrl.admin_q, nvme_req_op(&cmd),
1374 atomic_inc(&dev->ctrl.abort_limit);
1391 if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_RESETTING))
1394 nvme_dev_disable(dev, false);
1395 if (nvme_try_sched_reset(&dev->ctrl))
1396 nvme_unquiesce_io_queues(&dev->ctrl);
1402 dma_free_coherent(nvmeq->dev->dev, CQ_SIZE(nvmeq),
1408 pci_free_p2pmem(to_pci_dev(nvmeq->dev->dev),
1411 dma_free_coherent(nvmeq->dev->dev, SQ_SIZE(nvmeq),
1416 static void nvme_free_queues(struct nvme_dev *dev, int lowest)
1420 for (i = dev->ctrl.queue_count - 1; i >= lowest; i--) {
1421 dev->ctrl.queue_count--;
1422 nvme_free_queue(&dev->queues[i]);
1426 static void nvme_suspend_queue(struct nvme_dev *dev, unsigned int qid)
1428 struct nvme_queue *nvmeq = &dev->queues[qid];
1436 nvmeq->dev->online_queues--;
1437 if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)
1438 nvme_quiesce_admin_queue(&nvmeq->dev->ctrl);
1440 pci_free_irq(to_pci_dev(dev->dev), nvmeq->cq_vector, nvmeq);
1443 static void nvme_suspend_io_queues(struct nvme_dev *dev)
1447 for (i = dev->ctrl.queue_count - 1; i > 0; i--)
1448 nvme_suspend_queue(dev, i);
1457 static void nvme_reap_pending_cqes(struct nvme_dev *dev)
1461 for (i = dev->ctrl.queue_count - 1; i > 0; i--) {
1462 spin_lock(&dev->queues[i].cq_poll_lock);
1463 nvme_poll_cq(&dev->queues[i], NULL);
1464 spin_unlock(&dev->queues[i].cq_poll_lock);
1468 static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues,
1471 int q_depth = dev->q_depth;
1475 if (q_size_aligned * nr_io_queues > dev->cmb_size) {
1476 u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues);
1493 static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq,
1496 struct pci_dev *pdev = to_pci_dev(dev->dev);
1498 if (qid && dev->cmb_use_sqes && (dev->cmbsz & NVME_CMBSZ_SQS)) {
1512 nvmeq->sq_cmds = dma_alloc_coherent(dev->dev, SQ_SIZE(nvmeq),
1519 static int nvme_alloc_queue(struct nvme_dev *dev, int qid, int depth)
1521 struct nvme_queue *nvmeq = &dev->queues[qid];
1523 if (dev->ctrl.queue_count > qid)
1526 nvmeq->sqes = qid ? dev->io_sqes : NVME_ADM_SQES;
1528 nvmeq->cqes = dma_alloc_coherent(dev->dev, CQ_SIZE(nvmeq),
1533 if (nvme_alloc_sq_cmds(dev, nvmeq, qid))
1536 nvmeq->dev = dev;
1541 nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1543 dev->ctrl.queue_count++;
1548 dma_free_coherent(dev->dev, CQ_SIZE(nvmeq), (void *)nvmeq->cqes,
1556 struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
1557 int nr = nvmeq->dev->ctrl.instance;
1570 struct nvme_dev *dev = nvmeq->dev;
1576 nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1578 nvme_dbbuf_init(dev, nvmeq, qid);
1579 dev->online_queues++;
1586 static int nvme_setup_io_queues_trylock(struct nvme_dev *dev)
1591 if (!mutex_trylock(&dev->shutdown_lock))
1597 if (nvme_ctrl_state(&dev->ctrl) != NVME_CTRL_CONNECTING) {
1598 mutex_unlock(&dev->shutdown_lock);
1607 struct nvme_dev *dev = nvmeq->dev;
1618 vector = dev->num_vecs == 1 ? 0 : qid;
1622 result = adapter_alloc_cq(dev, qid, nvmeq, vector);
1626 result = adapter_alloc_sq(dev, qid, nvmeq);
1634 result = nvme_setup_io_queues_trylock(dev);
1645 mutex_unlock(&dev->shutdown_lock);
1649 dev->online_queues--;
1650 mutex_unlock(&dev->shutdown_lock);
1651 adapter_delete_sq(dev, qid);
1653 adapter_delete_cq(dev, qid);
1677 static void nvme_dev_remove_admin(struct nvme_dev *dev)
1679 if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) {
1685 nvme_unquiesce_admin_queue(&dev->ctrl);
1686 nvme_remove_admin_tag_set(&dev->ctrl);
1690 static unsigned long db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues)
1692 return NVME_REG_DBS + ((nr_io_queues + 1) * 8 * dev->db_stride);
1695 static int nvme_remap_bar(struct nvme_dev *dev, unsigned long size)
1697 struct pci_dev *pdev = to_pci_dev(dev->dev);
1699 if (size <= dev->bar_mapped_size)
1703 if (dev->bar)
1704 iounmap(dev->bar);
1705 dev->bar = ioremap(pci_resource_start(pdev, 0), size);
1706 if (!dev->bar) {
1707 dev->bar_mapped_size = 0;
1710 dev->bar_mapped_size = size;
1711 dev->dbs = dev->bar + NVME_REG_DBS;
1716 static int nvme_pci_configure_admin_queue(struct nvme_dev *dev)
1722 result = nvme_remap_bar(dev, db_bar_size(dev, 0));
1726 dev->subsystem = readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 1, 0) ?
1727 NVME_CAP_NSSRC(dev->ctrl.cap) : 0;
1729 if (dev->subsystem &&
1730 (readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_NSSRO))
1731 writel(NVME_CSTS_NSSRO, dev->bar + NVME_REG_CSTS);
1740 result = nvme_disable_ctrl(&dev->ctrl, false);
1744 result = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH);
1748 dev->ctrl.numa_node = dev_to_node(dev->dev);
1750 nvmeq = &dev->queues[0];
1754 writel(aqa, dev->bar + NVME_REG_AQA);
1755 lo_hi_writeq(nvmeq->sq_dma_addr, dev->bar + NVME_REG_ASQ);
1756 lo_hi_writeq(nvmeq->cq_dma_addr, dev->bar + NVME_REG_ACQ);
1758 result = nvme_enable_ctrl(&dev->ctrl);
1766 dev->online_queues--;
1774 static int nvme_create_io_queues(struct nvme_dev *dev)
1779 for (i = dev->ctrl.queue_count; i <= dev->max_qid; i++) {
1780 if (nvme_alloc_queue(dev, i, dev->q_depth)) {
1786 max = min(dev->max_qid, dev->ctrl.queue_count - 1);
1787 if (max != 1 && dev->io_queues[HCTX_TYPE_POLL]) {
1788 rw_queues = dev->io_queues[HCTX_TYPE_DEFAULT] +
1789 dev->io_queues[HCTX_TYPE_READ];
1794 for (i = dev->online_queues; i <= max; i++) {
1797 ret = nvme_create_queue(&dev->queues[i], i, polled);
1811 static u64 nvme_cmb_size_unit(struct nvme_dev *dev)
1813 u8 szu = (dev->cmbsz >> NVME_CMBSZ_SZU_SHIFT) & NVME_CMBSZ_SZU_MASK;
1818 static u32 nvme_cmb_size(struct nvme_dev *dev)
1820 return (dev->cmbsz >> NVME_CMBSZ_SZ_SHIFT) & NVME_CMBSZ_SZ_MASK;
1823 static void nvme_map_cmb(struct nvme_dev *dev)
1827 struct pci_dev *pdev = to_pci_dev(dev->dev);
1830 if (dev->cmb_size)
1833 if (NVME_CAP_CMBS(dev->ctrl.cap))
1834 writel(NVME_CMBMSC_CRE, dev->bar + NVME_REG_CMBMSC);
1836 dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ);
1837 if (!dev->cmbsz)
1839 dev->cmbloc = readl(dev->bar + NVME_REG_CMBLOC);
1841 size = nvme_cmb_size_unit(dev) * nvme_cmb_size(dev);
1842 offset = nvme_cmb_size_unit(dev) * NVME_CMB_OFST(dev->cmbloc);
1843 bar = NVME_CMB_BIR(dev->cmbloc);
1853 if (NVME_CAP_CMBS(dev->ctrl.cap)) {
1856 dev->bar + NVME_REG_CMBMSC);
1868 dev_warn(dev->ctrl.device,
1873 dev->cmb_size = size;
1874 dev->cmb_use_sqes = use_cmb_sqes && (dev->cmbsz & NVME_CMBSZ_SQS);
1876 if ((dev->cmbsz & (NVME_CMBSZ_WDS | NVME_CMBSZ_RDS)) ==
1880 nvme_update_attrs(dev);
1883 static int nvme_set_host_mem(struct nvme_dev *dev, u32 bits)
1885 u32 host_mem_size = dev->host_mem_size >> NVME_CTRL_PAGE_SHIFT;
1886 u64 dma_addr = dev->host_mem_descs_dma;
1896 c.features.dword15 = cpu_to_le32(dev->nr_host_mem_descs);
1898 ret = nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
1900 dev_warn(dev->ctrl.device,
1904 dev->hmb = bits & NVME_HOST_MEM_ENABLE;
1909 static void nvme_free_host_mem(struct nvme_dev *dev)
1913 for (i = 0; i < dev->nr_host_mem_descs; i++) {
1914 struct nvme_host_mem_buf_desc *desc = &dev->host_mem_descs[i];
1917 dma_free_attrs(dev->dev, size, dev->host_mem_desc_bufs[i],
1922 kfree(dev->host_mem_desc_bufs);
1923 dev->host_mem_desc_bufs = NULL;
1924 dma_free_coherent(dev->dev,
1925 dev->nr_host_mem_descs * sizeof(*dev->host_mem_descs),
1926 dev->host_mem_descs, dev->host_mem_descs_dma);
1927 dev->host_mem_descs = NULL;
1928 dev->nr_host_mem_descs = 0;
1931 static int __nvme_alloc_host_mem(struct nvme_dev *dev, u64 preferred,
1945 if (dev->ctrl.hmmaxd && dev->ctrl.hmmaxd < max_entries)
1946 max_entries = dev->ctrl.hmmaxd;
1948 descs = dma_alloc_coherent(dev->dev, max_entries * sizeof(*descs),
1961 bufs[i] = dma_alloc_attrs(dev->dev, len, &dma_addr, GFP_KERNEL,
1974 dev->nr_host_mem_descs = i;
1975 dev->host_mem_size = size;
1976 dev->host_mem_descs = descs;
1977 dev->host_mem_descs_dma = descs_dma;
1978 dev->host_mem_desc_bufs = bufs;
1985 dma_free_attrs(dev->dev, size, bufs[i],
1992 dma_free_coherent(dev->dev, max_entries * sizeof(*descs), descs,
1995 dev->host_mem_descs = NULL;
1999 static int nvme_alloc_host_mem(struct nvme_dev *dev, u64 min, u64 preferred)
2002 u64 hmminds = max_t(u32, dev->ctrl.hmminds * 4096, PAGE_SIZE * 2);
2007 if (!__nvme_alloc_host_mem(dev, preferred, chunk_size)) {
2008 if (!min || dev->host_mem_size >= min)
2010 nvme_free_host_mem(dev);
2017 static int nvme_setup_host_mem(struct nvme_dev *dev)
2020 u64 preferred = (u64)dev->ctrl.hmpre * 4096;
2021 u64 min = (u64)dev->ctrl.hmmin * 4096;
2025 if (!dev->ctrl.hmpre)
2030 dev_warn(dev->ctrl.device,
2033 nvme_free_host_mem(dev);
2040 if (dev->host_mem_descs) {
2041 if (dev->host_mem_size >= min)
2044 nvme_free_host_mem(dev);
2047 if (!dev->host_mem_descs) {
2048 if (nvme_alloc_host_mem(dev, min, preferred)) {
2049 dev_warn(dev->ctrl.device,
2054 dev_info(dev->ctrl.device,
2056 dev->host_mem_size >> ilog2(SZ_1M));
2059 ret = nvme_set_host_mem(dev, enable_bits);
2061 nvme_free_host_mem(dev);
2065 static ssize_t cmb_show(struct device *dev, struct device_attribute *attr,
2068 struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
2075 static ssize_t cmbloc_show(struct device *dev, struct device_attribute *attr,
2078 struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
2084 static ssize_t cmbsz_show(struct device *dev, struct device_attribute *attr,
2087 struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
2093 static ssize_t hmb_show(struct device *dev, struct device_attribute *attr,
2096 struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
2101 static ssize_t hmb_store(struct device *dev, struct device_attribute *attr,
2104 struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
2134 struct nvme_dev *dev = to_nvme_dev(ctrl);
2139 if (!dev->cmbsz)
2167 static void nvme_update_attrs(struct nvme_dev *dev)
2169 sysfs_update_group(&dev->ctrl.device->kobj, &nvme_pci_dev_attrs_group);
2178 struct nvme_dev *dev = affd->priv;
2179 unsigned int nr_read_queues, nr_write_queues = dev->nr_write_queues;
2203 dev->io_queues[HCTX_TYPE_DEFAULT] = nrirqs - nr_read_queues;
2205 dev->io_queues[HCTX_TYPE_READ] = nr_read_queues;
2210 static int nvme_setup_irqs(struct nvme_dev *dev, unsigned int nr_io_queues)
2212 struct pci_dev *pdev = to_pci_dev(dev->dev);
2216 .priv = dev,
2224 poll_queues = min(dev->nr_poll_queues, nr_io_queues - 1);
2225 dev->io_queues[HCTX_TYPE_POLL] = poll_queues;
2231 dev->io_queues[HCTX_TYPE_DEFAULT] = 1;
2232 dev->io_queues[HCTX_TYPE_READ] = 0;
2240 if (!(dev->ctrl.quirks & NVME_QUIRK_SINGLE_VECTOR))
2246 static unsigned int nvme_max_io_queues(struct nvme_dev *dev)
2252 if (dev->ctrl.quirks & NVME_QUIRK_SHARED_TAGS)
2254 return num_possible_cpus() + dev->nr_write_queues + dev->nr_poll_queues;
2257 static int nvme_setup_io_queues(struct nvme_dev *dev)
2259 struct nvme_queue *adminq = &dev->queues[0];
2260 struct pci_dev *pdev = to_pci_dev(dev->dev);
2269 dev->nr_write_queues = write_queues;
2270 dev->nr_poll_queues = poll_queues;
2272 nr_io_queues = dev->nr_allocated_queues - 1;
2273 result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues);
2287 result = nvme_setup_io_queues_trylock(dev);
2293 if (dev->cmb_use_sqes) {
2294 result = nvme_cmb_qdepth(dev, nr_io_queues,
2297 dev->q_depth = result;
2298 dev->ctrl.sqsize = result - 1;
2300 dev->cmb_use_sqes = false;
2305 size = db_bar_size(dev, nr_io_queues);
2306 result = nvme_remap_bar(dev, size);
2314 adminq->q_db = dev->dbs;
2327 result = nvme_setup_irqs(dev, nr_io_queues);
2333 dev->num_vecs = result;
2335 dev->max_qid = result + dev->io_queues[HCTX_TYPE_POLL];
2347 mutex_unlock(&dev->shutdown_lock);
2349 result = nvme_create_io_queues(dev);
2350 if (result || dev->online_queues < 2)
2353 if (dev->online_queues - 1 < dev->max_qid) {
2354 nr_io_queues = dev->online_queues - 1;
2355 nvme_delete_io_queues(dev);
2356 result = nvme_setup_io_queues_trylock(dev);
2359 nvme_suspend_io_queues(dev);
2362 dev_info(dev->ctrl.device, "%d/%d/%d default/read/poll queues\n",
2363 dev->io_queues[HCTX_TYPE_DEFAULT],
2364 dev->io_queues[HCTX_TYPE_READ],
2365 dev->io_queues[HCTX_TYPE_POLL]);
2368 mutex_unlock(&dev->shutdown_lock);
2395 struct request_queue *q = nvmeq->dev->ctrl.admin_q;
2418 static bool __nvme_delete_io_queues(struct nvme_dev *dev, u8 opcode)
2420 int nr_queues = dev->online_queues - 1, sent = 0;
2426 if (nvme_delete_queue(&dev->queues[nr_queues], opcode))
2432 struct nvme_queue *nvmeq = &dev->queues[nr_queues + sent];
2446 static void nvme_delete_io_queues(struct nvme_dev *dev)
2448 if (__nvme_delete_io_queues(dev, nvme_admin_delete_sq))
2449 __nvme_delete_io_queues(dev, nvme_admin_delete_cq);
2452 static unsigned int nvme_pci_nr_maps(struct nvme_dev *dev)
2454 if (dev->io_queues[HCTX_TYPE_POLL])
2456 if (dev->io_queues[HCTX_TYPE_READ])
2461 static void nvme_pci_update_nr_queues(struct nvme_dev *dev)
2463 blk_mq_update_nr_hw_queues(&dev->tagset, dev->online_queues - 1);
2465 nvme_free_queues(dev, dev->online_queues);
2468 static int nvme_pci_enable(struct nvme_dev *dev)
2471 struct pci_dev *pdev = to_pci_dev(dev->dev);
2478 if (readl(dev->bar + NVME_REG_CSTS) == -1) {
2492 dev->ctrl.cap = lo_hi_readq(dev->bar + NVME_REG_CAP);
2494 dev->q_depth = min_t(u32, NVME_CAP_MQES(dev->ctrl.cap) + 1,
2496 dev->db_stride = 1 << NVME_CAP_STRIDE(dev->ctrl.cap);
2497 dev->dbs = dev->bar + 4096;
2504 if (dev->ctrl.quirks & NVME_QUIRK_128_BYTES_SQES)
2505 dev->io_sqes = 7;
2507 dev->io_sqes = NVME_NVM_IOSQES;
2514 dev->q_depth = 2;
2515 dev_warn(dev->ctrl.device, "detected Apple NVMe controller, "
2517 dev->q_depth);
2520 NVME_CAP_MQES(dev->ctrl.cap) == 0) {
2521 dev->q_depth = 64;
2522 dev_err(dev->ctrl.device, "detected PM1725 NVMe controller, "
2523 "set queue depth=%u\n", dev->q_depth);
2530 if ((dev->ctrl.quirks & NVME_QUIRK_SHARED_TAGS) &&
2531 (dev->q_depth < (NVME_AQ_DEPTH + 2))) {
2532 dev->q_depth = NVME_AQ_DEPTH + 2;
2533 dev_warn(dev->ctrl.device, "IO queue depth clamped to %d\n",
2534 dev->q_depth);
2536 dev->ctrl.sqsize = dev->q_depth - 1; /* 0's based queue depth */
2538 nvme_map_cmb(dev);
2542 result = nvme_pci_configure_admin_queue(dev);
2554 static void nvme_dev_unmap(struct nvme_dev *dev)
2556 if (dev->bar)
2557 iounmap(dev->bar);
2558 pci_release_mem_regions(to_pci_dev(dev->dev));
2561 static bool nvme_pci_ctrl_is_dead(struct nvme_dev *dev)
2563 struct pci_dev *pdev = to_pci_dev(dev->dev);
2571 csts = readl(dev->bar + NVME_REG_CSTS);
2575 static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown)
2577 enum nvme_ctrl_state state = nvme_ctrl_state(&dev->ctrl);
2578 struct pci_dev *pdev = to_pci_dev(dev->dev);
2581 mutex_lock(&dev->shutdown_lock);
2582 dead = nvme_pci_ctrl_is_dead(dev);
2585 nvme_start_freeze(&dev->ctrl);
2591 nvme_wait_freeze_timeout(&dev->ctrl, NVME_IO_TIMEOUT);
2594 nvme_quiesce_io_queues(&dev->ctrl);
2596 if (!dead && dev->ctrl.queue_count > 0) {
2597 nvme_delete_io_queues(dev);
2598 nvme_disable_ctrl(&dev->ctrl, shutdown);
2599 nvme_poll_irqdisable(&dev->queues[0]);
2601 nvme_suspend_io_queues(dev);
2602 nvme_suspend_queue(dev, 0);
2606 nvme_reap_pending_cqes(dev);
2608 nvme_cancel_tagset(&dev->ctrl);
2609 nvme_cancel_admin_tagset(&dev->ctrl);
2617 nvme_unquiesce_io_queues(&dev->ctrl);
2618 if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q))
2619 nvme_unquiesce_admin_queue(&dev->ctrl);
2621 mutex_unlock(&dev->shutdown_lock);
2624 static int nvme_disable_prepare_reset(struct nvme_dev *dev, bool shutdown)
2626 if (!nvme_wait_reset(&dev->ctrl))
2628 nvme_dev_disable(dev, shutdown);
2632 static int nvme_setup_prp_pools(struct nvme_dev *dev)
2634 dev->prp_page_pool = dma_pool_create("prp list page", dev->dev,
2637 if (!dev->prp_page_pool)
2641 dev->prp_small_pool = dma_pool_create("prp list 256", dev->dev,
2643 if (!dev->prp_small_pool) {
2644 dma_pool_destroy(dev->prp_page_pool);
2650 static void nvme_release_prp_pools(struct nvme_dev *dev)
2652 dma_pool_destroy(dev->prp_page_pool);
2653 dma_pool_destroy(dev->prp_small_pool);
2656 static int nvme_pci_alloc_iod_mempool(struct nvme_dev *dev)
2660 dev->iod_mempool = mempool_create_node(1,
2663 dev_to_node(dev->dev));
2664 if (!dev->iod_mempool)
2669 static void nvme_free_tagset(struct nvme_dev *dev)
2671 if (dev->tagset.tags)
2672 nvme_remove_io_tag_set(&dev->ctrl);
2673 dev->ctrl.tagset = NULL;
2679 struct nvme_dev *dev = to_nvme_dev(ctrl);
2681 nvme_free_tagset(dev);
2682 put_device(dev->dev);
2683 kfree(dev->queues);
2684 kfree(dev);
2689 struct nvme_dev *dev =
2691 bool was_suspend = !!(dev->ctrl.ctrl_config & NVME_CC_SHN_NORMAL);
2694 if (nvme_ctrl_state(&dev->ctrl) != NVME_CTRL_RESETTING) {
2695 dev_warn(dev->ctrl.device, "ctrl state %d is not RESETTING\n",
2696 dev->ctrl.state);
2705 if (dev->ctrl.ctrl_config & NVME_CC_ENABLE)
2706 nvme_dev_disable(dev, false);
2707 nvme_sync_queues(&dev->ctrl);
2709 mutex_lock(&dev->shutdown_lock);
2710 result = nvme_pci_enable(dev);
2713 nvme_unquiesce_admin_queue(&dev->ctrl);
2714 mutex_unlock(&dev->shutdown_lock);
2720 if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_CONNECTING)) {
2721 dev_warn(dev->ctrl.device,
2727 result = nvme_init_ctrl_finish(&dev->ctrl, was_suspend);
2731 nvme_dbbuf_dma_alloc(dev);
2733 result = nvme_setup_host_mem(dev);
2737 result = nvme_setup_io_queues(dev);
2746 if (dev->online_queues > 1) {
2747 nvme_unquiesce_io_queues(&dev->ctrl);
2748 nvme_wait_freeze(&dev->ctrl);
2749 nvme_pci_update_nr_queues(dev);
2750 nvme_dbbuf_set(dev);
2751 nvme_unfreeze(&dev->ctrl);
2753 dev_warn(dev->ctrl.device, "IO queues lost\n");
2754 nvme_mark_namespaces_dead(&dev->ctrl);
2755 nvme_unquiesce_io_queues(&dev->ctrl);
2756 nvme_remove_namespaces(&dev->ctrl);
2757 nvme_free_tagset(dev);
2764 if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_LIVE)) {
2765 dev_warn(dev->ctrl.device,
2771 nvme_start_ctrl(&dev->ctrl);
2775 mutex_unlock(&dev->shutdown_lock);
2781 dev_warn(dev->ctrl.device, "Disabling device after reset failure: %d\n",
2783 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING);
2784 nvme_dev_disable(dev, true);
2785 nvme_sync_queues(&dev->ctrl);
2786 nvme_mark_namespaces_dead(&dev->ctrl);
2787 nvme_unquiesce_io_queues(&dev->ctrl);
2788 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DEAD);
2811 struct pci_dev *pdev = to_pci_dev(to_nvme_dev(ctrl)->dev);
2813 return snprintf(buf, size, "%s\n", dev_name(&pdev->dev));
2818 struct pci_dev *pdev = to_pci_dev(to_nvme_dev(ctrl)->dev);
2832 struct nvme_dev *dev = to_nvme_dev(ctrl);
2834 return dma_pci_p2pdma_supported(dev->dev);
2852 static int nvme_dev_map(struct nvme_dev *dev)
2854 struct pci_dev *pdev = to_pci_dev(dev->dev);
2859 if (nvme_remap_bar(dev, NVME_REG_DBS + 4096))
2927 int node = dev_to_node(&pdev->dev);
2928 struct nvme_dev *dev;
2931 dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, node);
2932 if (!dev)
2934 INIT_WORK(&dev->ctrl.reset_work, nvme_reset_work);
2935 mutex_init(&dev->shutdown_lock);
2937 dev->nr_write_queues = write_queues;
2938 dev->nr_poll_queues = poll_queues;
2939 dev->nr_allocated_queues = nvme_max_io_queues(dev) + 1;
2940 dev->queues = kcalloc_node(dev->nr_allocated_queues,
2942 if (!dev->queues)
2945 dev->dev = get_device(&pdev->dev);
2950 acpi_storage_d3(&pdev->dev)) {
2955 dev_info(&pdev->dev,
2959 ret = nvme_init_ctrl(&dev->ctrl, &pdev->dev, &nvme_pci_ctrl_ops,
2964 if (dev->ctrl.quirks & NVME_QUIRK_DMA_ADDRESS_BITS_48)
2965 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48));
2967 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
2968 dma_set_min_align_mask(&pdev->dev, NVME_CTRL_PAGE_SIZE - 1);
2969 dma_set_max_seg_size(&pdev->dev, 0xffffffff);
2975 dev->ctrl.max_hw_sectors = min_t(u32,
2976 NVME_MAX_KB_SZ << 1, dma_opt_mapping_size(&pdev->dev) >> 9);
2977 dev->ctrl.max_segments = NVME_MAX_SEGS;
2983 dev->ctrl.max_integrity_segments = 1;
2984 return dev;
2987 put_device(dev->dev);
2988 kfree(dev->queues);
2990 kfree(dev);
2996 struct nvme_dev *dev;
2999 dev = nvme_pci_alloc_dev(pdev, id);
3000 if (IS_ERR(dev))
3001 return PTR_ERR(dev);
3003 result = nvme_dev_map(dev);
3007 result = nvme_setup_prp_pools(dev);
3011 result = nvme_pci_alloc_iod_mempool(dev);
3015 dev_info(dev->ctrl.device, "pci function %s\n", dev_name(&pdev->dev));
3017 result = nvme_pci_enable(dev);
3021 result = nvme_alloc_admin_tag_set(&dev->ctrl, &dev->admin_tagset,
3030 if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_CONNECTING)) {
3031 dev_warn(dev->ctrl.device,
3037 result = nvme_init_ctrl_finish(&dev->ctrl, false);
3041 nvme_dbbuf_dma_alloc(dev);
3043 result = nvme_setup_host_mem(dev);
3047 result = nvme_setup_io_queues(dev);
3051 if (dev->online_queues > 1) {
3052 nvme_alloc_io_tag_set(&dev->ctrl, &dev->tagset, &nvme_mq_ops,
3053 nvme_pci_nr_maps(dev), sizeof(struct nvme_iod));
3054 nvme_dbbuf_set(dev);
3057 if (!dev->ctrl.tagset)
3058 dev_warn(dev->ctrl.device, "IO queues not created\n");
3060 if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_LIVE)) {
3061 dev_warn(dev->ctrl.device,
3067 pci_set_drvdata(pdev, dev);
3069 nvme_start_ctrl(&dev->ctrl);
3070 nvme_put_ctrl(&dev->ctrl);
3071 flush_work(&dev->ctrl.scan_work);
3075 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING);
3076 nvme_dev_disable(dev, true);
3077 nvme_free_host_mem(dev);
3078 nvme_dev_remove_admin(dev);
3079 nvme_dbbuf_dma_free(dev);
3080 nvme_free_queues(dev, 0);
3082 mempool_destroy(dev->iod_mempool);
3084 nvme_release_prp_pools(dev);
3086 nvme_dev_unmap(dev);
3088 nvme_uninit_ctrl(&dev->ctrl);
3089 nvme_put_ctrl(&dev->ctrl);
3095 struct nvme_dev *dev = pci_get_drvdata(pdev);
3102 nvme_disable_prepare_reset(dev, false);
3103 nvme_sync_queues(&dev->ctrl);
3108 struct nvme_dev *dev = pci_get_drvdata(pdev);
3110 if (!nvme_try_sched_reset(&dev->ctrl))
3111 flush_work(&dev->ctrl.reset_work);
3116 struct nvme_dev *dev = pci_get_drvdata(pdev);
3118 nvme_disable_prepare_reset(dev, true);
3128 struct nvme_dev *dev = pci_get_drvdata(pdev);
3130 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING);
3134 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DEAD);
3135 nvme_dev_disable(dev, true);
3138 flush_work(&dev->ctrl.reset_work);
3139 nvme_stop_ctrl(&dev->ctrl);
3140 nvme_remove_namespaces(&dev->ctrl);
3141 nvme_dev_disable(dev, true);
3142 nvme_free_host_mem(dev);
3143 nvme_dev_remove_admin(dev);
3144 nvme_dbbuf_dma_free(dev);
3145 nvme_free_queues(dev, 0);
3146 mempool_destroy(dev->iod_mempool);
3147 nvme_release_prp_pools(dev);
3148 nvme_dev_unmap(dev);
3149 nvme_uninit_ctrl(&dev->ctrl);
3163 static int nvme_resume(struct device *dev)
3165 struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev));
3179 static int nvme_suspend(struct device *dev)
3181 struct pci_dev *pdev = to_pci_dev(dev);
3255 static int nvme_simple_suspend(struct device *dev)
3257 struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev));
3262 static int nvme_simple_resume(struct device *dev)
3264 struct pci_dev *pdev = to_pci_dev(dev);
3283 struct nvme_dev *dev = pci_get_drvdata(pdev);
3294 dev_warn(dev->ctrl.device,
3296 if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_RESETTING)) {
3297 nvme_dev_disable(dev, true);
3300 nvme_dev_disable(dev, false);
3303 dev_warn(dev->ctrl.device,
3312 struct nvme_dev *dev = pci_get_drvdata(pdev);
3314 dev_info(dev->ctrl.device, "restart after slot reset\n");
3316 if (!nvme_try_sched_reset(&dev->ctrl))
3317 nvme_unquiesce_io_queues(&dev->ctrl);
3323 struct nvme_dev *dev = pci_get_drvdata(pdev);
3325 flush_work(&dev->ctrl.reset_work);