Lines Matching defs:anv
226 struct apple_nvme *anv = cookie;
228 dev_warn(anv->dev, "RTKit crashed; unable to recover without a reboot");
229 nvme_reset_ctrl(&anv->ctrl);
235 struct apple_nvme *anv = cookie;
244 dma_alloc_coherent(anv->dev, bfr->size, &bfr->iova, GFP_KERNEL);
248 ret = apple_sart_add_allowed_region(anv->sart, bfr->iova, bfr->size);
250 dma_free_coherent(anv->dev, bfr->size, bfr->buffer, bfr->iova);
261 struct apple_nvme *anv = cookie;
263 apple_sart_remove_allowed_region(anv->sart, bfr->iova, bfr->size);
264 dma_free_coherent(anv->dev, bfr->size, bfr->buffer, bfr->iova);
275 struct apple_nvme *anv = queue_to_apple_nvme(q);
277 writel(tag, anv->mmio_nvme + APPLE_NVMMU_TCB_INVAL);
278 if (readl(anv->mmio_nvme + APPLE_NVMMU_TCB_STAT))
279 dev_warn_ratelimited(anv->dev,
286 struct apple_nvme *anv = queue_to_apple_nvme(q);
313 spin_lock_irq(&anv->lock);
315 spin_unlock_irq(&anv->lock);
342 static void apple_nvme_free_prps(struct apple_nvme *anv, struct request *req)
353 dma_pool_free(anv->prp_page_pool, prp_list, dma_addr);
358 static void apple_nvme_unmap_data(struct apple_nvme *anv, struct request *req)
363 dma_unmap_page(anv->dev, iod->first_dma, iod->dma_len,
370 dma_unmap_sg(anv->dev, iod->sg, iod->nents, rq_dma_dir(req));
372 dma_pool_free(anv->prp_small_pool, apple_nvme_iod_list(req)[0],
375 apple_nvme_free_prps(anv, req);
376 mempool_free(iod->sg, anv->iod_mempool);
393 static blk_status_t apple_nvme_setup_prps(struct apple_nvme *anv,
431 pool = anv->prp_small_pool;
434 pool = anv->prp_page_pool;
478 apple_nvme_free_prps(anv, req);
487 static blk_status_t apple_nvme_setup_prp_simple(struct apple_nvme *anv,
496 iod->first_dma = dma_map_bvec(anv->dev, bv, rq_dma_dir(req), 0);
497 if (dma_mapping_error(anv->dev, iod->first_dma))
507 static blk_status_t apple_nvme_map_data(struct apple_nvme *anv,
519 return apple_nvme_setup_prp_simple(anv, req, &cmnd->rw,
524 iod->sg = mempool_alloc(anv->iod_mempool, GFP_ATOMIC);
532 nr_mapped = dma_map_sg_attrs(anv->dev, iod->sg, iod->nents,
537 ret = apple_nvme_setup_prps(anv, req, &cmnd->rw);
543 dma_unmap_sg(anv->dev, iod->sg, iod->nents, rq_dma_dir(req));
545 mempool_free(iod->sg, anv->iod_mempool);
552 struct apple_nvme *anv = queue_to_apple_nvme(iod->q);
555 apple_nvme_unmap_data(anv, req);
577 apple_nvme_queue_tagset(struct apple_nvme *anv, struct apple_nvme_queue *q)
580 return anv->admin_tagset.tags[0];
582 return anv->tagset.tags[0];
588 struct apple_nvme *anv = queue_to_apple_nvme(q);
595 req = nvme_find_rq(apple_nvme_queue_tagset(anv, q), command_id);
597 dev_warn(anv->dev, "invalid id %d completed", command_id);
660 struct apple_nvme *anv = data;
664 spin_lock_irqsave(&anv->lock, flags);
665 if (apple_nvme_handle_cq(&anv->ioq, false))
667 if (apple_nvme_handle_cq(&anv->adminq, false))
669 spin_unlock_irqrestore(&anv->lock, flags);
676 static int apple_nvme_create_cq(struct apple_nvme *anv)
685 c.create_cq.prp1 = cpu_to_le64(anv->ioq.cq_dma_addr);
691 return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0);
694 static int apple_nvme_remove_cq(struct apple_nvme *anv)
701 return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0);
704 static int apple_nvme_create_sq(struct apple_nvme *anv)
713 c.create_sq.prp1 = cpu_to_le64(anv->ioq.sq_dma_addr);
719 return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0);
722 static int apple_nvme_remove_sq(struct apple_nvme *anv)
729 return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0);
737 struct apple_nvme *anv = queue_to_apple_nvme(q);
753 if (!nvme_check_ready(&anv->ctrl, req, true))
754 return nvme_fail_nonready_command(&anv->ctrl, req);
761 ret = apple_nvme_map_data(anv, req, cmnd);
787 struct apple_nvme *anv = queue_to_apple_nvme(q);
792 nreq->ctrl = &anv->ctrl;
798 static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown)
800 u32 csts = readl(anv->mmio_nvme + NVME_REG_CSTS);
804 if (apple_rtkit_is_crashed(anv->rtk))
811 if (anv->ctrl.state == NVME_CTRL_LIVE ||
812 anv->ctrl.state == NVME_CTRL_RESETTING) {
814 nvme_start_freeze(&anv->ctrl);
822 nvme_wait_freeze_timeout(&anv->ctrl, NVME_IO_TIMEOUT);
824 nvme_quiesce_io_queues(&anv->ctrl);
827 if (READ_ONCE(anv->ioq.enabled)) {
828 apple_nvme_remove_sq(anv);
829 apple_nvme_remove_cq(anv);
847 nvme_disable_ctrl(&anv->ctrl, shutdown);
848 nvme_disable_ctrl(&anv->ctrl, false);
851 WRITE_ONCE(anv->ioq.enabled, false);
852 WRITE_ONCE(anv->adminq.enabled, false);
854 nvme_quiesce_admin_queue(&anv->ctrl);
857 spin_lock_irqsave(&anv->lock, flags);
858 apple_nvme_handle_cq(&anv->ioq, true);
859 apple_nvme_handle_cq(&anv->adminq, true);
860 spin_unlock_irqrestore(&anv->lock, flags);
862 nvme_cancel_tagset(&anv->ctrl);
863 nvme_cancel_admin_tagset(&anv->ctrl);
871 nvme_unquiesce_io_queues(&anv->ctrl);
872 nvme_unquiesce_admin_queue(&anv->ctrl);
880 struct apple_nvme *anv = queue_to_apple_nvme(q);
882 u32 csts = readl(anv->mmio_nvme + NVME_REG_CSTS);
884 if (anv->ctrl.state != NVME_CTRL_LIVE) {
899 dev_warn(anv->dev,
912 if (!apple_rtkit_is_crashed(anv->rtk) && !(csts & NVME_CSTS_CFS)) {
913 spin_lock_irqsave(&anv->lock, flags);
915 spin_unlock_irqrestore(&anv->lock, flags);
917 dev_warn(anv->dev,
928 dev_warn(anv->dev, "I/O %d(aq:%d) timeout: resetting controller\n",
931 apple_nvme_disable(anv, false);
932 nvme_reset_ctrl(&anv->ctrl);
940 struct apple_nvme *anv = queue_to_apple_nvme(q);
944 spin_lock_irqsave(&anv->lock, flags);
946 spin_unlock_irqrestore(&anv->lock, flags);
986 struct apple_nvme *anv =
989 if (anv->ctrl.state != NVME_CTRL_RESETTING) {
990 dev_warn(anv->dev, "ctrl state %d is not RESETTING\n",
991 anv->ctrl.state);
997 if (apple_rtkit_is_crashed(anv->rtk)) {
998 dev_err(anv->dev,
1005 if (apple_rtkit_is_running(anv->rtk)) {
1007 if (anv->ctrl.ctrl_config & NVME_CC_ENABLE)
1008 apple_nvme_disable(anv, false);
1009 dev_dbg(anv->dev, "Trying to shut down RTKit before reset.");
1010 ret = apple_rtkit_shutdown(anv->rtk);
1015 writel(0, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL);
1017 ret = reset_control_assert(anv->reset);
1021 ret = apple_rtkit_reinit(anv->rtk);
1025 ret = reset_control_deassert(anv->reset);
1030 anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL);
1031 ret = apple_rtkit_boot(anv->rtk);
1033 dev_err(anv->dev, "ANS did not boot");
1037 ret = readl_poll_timeout(anv->mmio_nvme + APPLE_ANS_BOOT_STATUS,
1042 dev_err(anv->dev, "ANS did not initialize");
1046 dev_dbg(anv->dev, "ANS booted successfully.");
1052 anv->ctrl.max_hw_sectors = min_t(u32, NVME_MAX_KB_SZ << 1,
1053 dma_max_mapping_size(anv->dev) >> 9);
1054 anv->ctrl.max_segments = NVME_MAX_SEGS;
1056 dma_set_max_seg_size(anv->dev, 0xffffffff);
1066 anv->mmio_nvme + APPLE_ANS_LINEAR_SQ_CTRL);
1070 anv->mmio_nvme + APPLE_ANS_MAX_PEND_CMDS_CTRL);
1074 anv->mmio_nvme + APPLE_NVMMU_NUM_TCBS);
1082 writel(readl(anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL) &
1084 anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL);
1089 writel(aqa, anv->mmio_nvme + NVME_REG_AQA);
1090 writeq(anv->adminq.sq_dma_addr, anv->mmio_nvme + NVME_REG_ASQ);
1091 writeq(anv->adminq.cq_dma_addr, anv->mmio_nvme + NVME_REG_ACQ);
1094 writeq(anv->adminq.tcb_dma_addr,
1095 anv->mmio_nvme + APPLE_NVMMU_ASQ_TCB_BASE);
1096 writeq(anv->ioq.tcb_dma_addr,
1097 anv->mmio_nvme + APPLE_NVMMU_IOSQ_TCB_BASE);
1099 anv->ctrl.sqsize =
1101 anv->ctrl.cap = readq(anv->mmio_nvme + NVME_REG_CAP);
1103 dev_dbg(anv->dev, "Enabling controller now");
1104 ret = nvme_enable_ctrl(&anv->ctrl);
1108 dev_dbg(anv->dev, "Starting admin queue");
1109 apple_nvme_init_queue(&anv->adminq);
1110 nvme_unquiesce_admin_queue(&anv->ctrl);
1112 if (!nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_CONNECTING)) {
1113 dev_warn(anv->ctrl.device,
1119 ret = nvme_init_ctrl_finish(&anv->ctrl, false);
1123 dev_dbg(anv->dev, "Creating IOCQ");
1124 ret = apple_nvme_create_cq(anv);
1127 dev_dbg(anv->dev, "Creating IOSQ");
1128 ret = apple_nvme_create_sq(anv);
1132 apple_nvme_init_queue(&anv->ioq);
1134 ret = nvme_set_queue_count(&anv->ctrl, &nr_io_queues);
1142 anv->ctrl.queue_count = nr_io_queues + 1;
1144 nvme_unquiesce_io_queues(&anv->ctrl);
1145 nvme_wait_freeze(&anv->ctrl);
1146 blk_mq_update_nr_hw_queues(&anv->tagset, 1);
1147 nvme_unfreeze(&anv->ctrl);
1149 if (!nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_LIVE)) {
1150 dev_warn(anv->ctrl.device,
1156 nvme_start_ctrl(&anv->ctrl);
1158 dev_dbg(anv->dev, "ANS boot and NVMe init completed.");
1162 apple_nvme_remove_sq(anv);
1164 apple_nvme_remove_cq(anv);
1166 dev_warn(anv->ctrl.device, "Reset failure status: %d\n", ret);
1167 nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_DELETING);
1168 nvme_get_ctrl(&anv->ctrl);
1169 apple_nvme_disable(anv, false);
1170 nvme_mark_namespaces_dead(&anv->ctrl);
1171 if (!queue_work(nvme_wq, &anv->remove_work))
1172 nvme_put_ctrl(&anv->ctrl);
1177 struct apple_nvme *anv =
1180 nvme_put_ctrl(&anv->ctrl);
1181 device_release_driver(anv->dev);
1211 struct apple_nvme *anv = ctrl_to_apple_nvme(ctrl);
1213 if (anv->ctrl.admin_q)
1214 blk_put_queue(anv->ctrl.admin_q);
1215 put_device(anv->dev);
1231 struct apple_nvme *anv = data;
1233 flush_work(&anv->ctrl.reset_work);
1234 flush_work(&anv->ctrl.scan_work);
1235 nvme_put_ctrl(&anv->ctrl);
1243 static int apple_nvme_alloc_tagsets(struct apple_nvme *anv)
1247 anv->admin_tagset.ops = &apple_nvme_mq_admin_ops;
1248 anv->admin_tagset.nr_hw_queues = 1;
1249 anv->admin_tagset.queue_depth = APPLE_NVME_AQ_MQ_TAG_DEPTH;
1250 anv->admin_tagset.timeout = NVME_ADMIN_TIMEOUT;
1251 anv->admin_tagset.numa_node = NUMA_NO_NODE;
1252 anv->admin_tagset.cmd_size = sizeof(struct apple_nvme_iod);
1253 anv->admin_tagset.flags = BLK_MQ_F_NO_SCHED;
1254 anv->admin_tagset.driver_data = &anv->adminq;
1256 ret = blk_mq_alloc_tag_set(&anv->admin_tagset);
1259 ret = devm_add_action_or_reset(anv->dev, devm_apple_nvme_put_tag_set,
1260 &anv->admin_tagset);
1264 anv->tagset.ops = &apple_nvme_mq_ops;
1265 anv->tagset.nr_hw_queues = 1;
1266 anv->tagset.nr_maps = 1;
1272 anv->tagset.reserved_tags = APPLE_NVME_AQ_DEPTH;
1273 anv->tagset.queue_depth = APPLE_ANS_MAX_QUEUE_DEPTH - 1;
1274 anv->tagset.timeout = NVME_IO_TIMEOUT;
1275 anv->tagset.numa_node = NUMA_NO_NODE;
1276 anv->tagset.cmd_size = sizeof(struct apple_nvme_iod);
1277 anv->tagset.flags = BLK_MQ_F_SHOULD_MERGE;
1278 anv->tagset.driver_data = &anv->ioq;
1280 ret = blk_mq_alloc_tag_set(&anv->tagset);
1283 ret = devm_add_action_or_reset(anv->dev, devm_apple_nvme_put_tag_set,
1284 &anv->tagset);
1288 anv->ctrl.admin_tagset = &anv->admin_tagset;
1289 anv->ctrl.tagset = &anv->tagset;
1294 static int apple_nvme_queue_alloc(struct apple_nvme *anv,
1299 q->cqes = dmam_alloc_coherent(anv->dev,
1305 q->sqes = dmam_alloc_coherent(anv->dev,
1315 q->tcbs = dmam_alloc_coherent(anv->dev,
1330 static void apple_nvme_detach_genpd(struct apple_nvme *anv)
1334 if (anv->pd_count <= 1)
1337 for (i = anv->pd_count - 1; i >= 0; i--) {
1338 if (anv->pd_link[i])
1339 device_link_del(anv->pd_link[i]);
1340 if (!IS_ERR_OR_NULL(anv->pd_dev[i]))
1341 dev_pm_domain_detach(anv->pd_dev[i], true);
1345 static int apple_nvme_attach_genpd(struct apple_nvme *anv)
1347 struct device *dev = anv->dev;
1350 anv->pd_count = of_count_phandle_with_args(
1352 if (anv->pd_count <= 1)
1355 anv->pd_dev = devm_kcalloc(dev, anv->pd_count, sizeof(*anv->pd_dev),
1357 if (!anv->pd_dev)
1360 anv->pd_link = devm_kcalloc(dev, anv->pd_count, sizeof(*anv->pd_link),
1362 if (!anv->pd_link)
1365 for (i = 0; i < anv->pd_count; i++) {
1366 anv->pd_dev[i] = dev_pm_domain_attach_by_id(dev, i);
1367 if (IS_ERR(anv->pd_dev[i])) {
1368 apple_nvme_detach_genpd(anv);
1369 return PTR_ERR(anv->pd_dev[i]);
1372 anv->pd_link[i] = device_link_add(dev, anv->pd_dev[i],
1376 if (!anv->pd_link[i]) {
1377 apple_nvme_detach_genpd(anv);
1393 struct apple_nvme *anv;
1396 anv = devm_kzalloc(dev, sizeof(*anv), GFP_KERNEL);
1397 if (!anv)
1400 anv->dev = get_device(dev);
1401 anv->adminq.is_adminq = true;
1402 platform_set_drvdata(pdev, anv);
1404 ret = apple_nvme_attach_genpd(anv);
1414 anv->irq = platform_get_irq(pdev, 0);
1415 if (anv->irq < 0) {
1416 ret = anv->irq;
1419 if (!anv->irq) {
1424 anv->mmio_coproc = devm_platform_ioremap_resource_byname(pdev, "ans");
1425 if (IS_ERR(anv->mmio_coproc)) {
1426 ret = PTR_ERR(anv->mmio_coproc);
1429 anv->mmio_nvme = devm_platform_ioremap_resource_byname(pdev, "nvme");
1430 if (IS_ERR(anv->mmio_nvme)) {
1431 ret = PTR_ERR(anv->mmio_nvme);
1435 anv->adminq.sq_db = anv->mmio_nvme + APPLE_ANS_LINEAR_ASQ_DB;
1436 anv->adminq.cq_db = anv->mmio_nvme + APPLE_ANS_ACQ_DB;
1437 anv->ioq.sq_db = anv->mmio_nvme + APPLE_ANS_LINEAR_IOSQ_DB;
1438 anv->ioq.cq_db = anv->mmio_nvme + APPLE_ANS_IOCQ_DB;
1440 anv->sart = devm_apple_sart_get(dev);
1441 if (IS_ERR(anv->sart)) {
1442 ret = dev_err_probe(dev, PTR_ERR(anv->sart),
1447 anv->reset = devm_reset_control_array_get_exclusive(anv->dev);
1448 if (IS_ERR(anv->reset)) {
1449 ret = dev_err_probe(dev, PTR_ERR(anv->reset),
1454 INIT_WORK(&anv->ctrl.reset_work, apple_nvme_reset_work);
1455 INIT_WORK(&anv->remove_work, apple_nvme_remove_dead_ctrl_work);
1456 spin_lock_init(&anv->lock);
1458 ret = apple_nvme_queue_alloc(anv, &anv->adminq);
1461 ret = apple_nvme_queue_alloc(anv, &anv->ioq);
1465 anv->prp_page_pool = dmam_pool_create("prp list page", anv->dev,
1468 if (!anv->prp_page_pool) {
1473 anv->prp_small_pool =
1474 dmam_pool_create("prp list 256", anv->dev, 256, 256, 0);
1475 if (!anv->prp_small_pool) {
1481 anv->iod_mempool =
1483 if (!anv->iod_mempool) {
1487 ret = devm_add_action_or_reset(anv->dev,
1488 devm_apple_nvme_mempool_destroy, anv->iod_mempool);
1492 ret = apple_nvme_alloc_tagsets(anv);
1496 ret = devm_request_irq(anv->dev, anv->irq, apple_nvme_irq, 0,
1497 "nvme-apple", anv);
1503 anv->rtk =
1504 devm_apple_rtkit_init(dev, anv, NULL, 0, &apple_nvme_rtkit_ops);
1505 if (IS_ERR(anv->rtk)) {
1506 ret = dev_err_probe(dev, PTR_ERR(anv->rtk),
1511 ret = nvme_init_ctrl(&anv->ctrl, anv->dev, &nvme_ctrl_ops,
1518 anv->ctrl.admin_q = blk_mq_init_queue(&anv->admin_tagset);
1519 if (IS_ERR(anv->ctrl.admin_q)) {
1524 nvme_reset_ctrl(&anv->ctrl);
1525 async_schedule(apple_nvme_async_probe, anv);
1530 put_device(anv->dev);
1536 struct apple_nvme *anv = platform_get_drvdata(pdev);
1538 nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_DELETING);
1539 flush_work(&anv->ctrl.reset_work);
1540 nvme_stop_ctrl(&anv->ctrl);
1541 nvme_remove_namespaces(&anv->ctrl);
1542 apple_nvme_disable(anv, true);
1543 nvme_uninit_ctrl(&anv->ctrl);
1545 if (apple_rtkit_is_running(anv->rtk))
1546 apple_rtkit_shutdown(anv->rtk);
1548 apple_nvme_detach_genpd(anv);
1555 struct apple_nvme *anv = platform_get_drvdata(pdev);
1557 apple_nvme_disable(anv, true);
1558 if (apple_rtkit_is_running(anv->rtk))
1559 apple_rtkit_shutdown(anv->rtk);
1564 struct apple_nvme *anv = dev_get_drvdata(dev);
1566 return nvme_reset_ctrl(&anv->ctrl);
1571 struct apple_nvme *anv = dev_get_drvdata(dev);
1574 apple_nvme_disable(anv, true);
1576 if (apple_rtkit_is_running(anv->rtk))
1577 ret = apple_rtkit_shutdown(anv->rtk);
1579 writel(0, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL);