Lines Matching defs:hdev
33 * @hdev: pointer to habanalabs device structure.
46 static u64 hl_set_dram_bar(struct hl_device *hdev, u64 addr, struct pci_mem_region *region,
49 struct asic_fixed_properties *prop = &hdev->asic_prop;
58 old_base = hdev->asic_funcs->set_dram_bar_base(hdev, bar_base_addr);
67 int hl_access_sram_dram_region(struct hl_device *hdev, u64 addr, u64 *val,
70 struct pci_mem_region *region = &hdev->pci_mem_region[region_type];
75 old_base = hl_set_dram_bar(hdev, addr, region, &bar_region_base);
80 acc_addr = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
105 rc = hl_set_dram_bar(hdev, old_base, region, NULL);
113 static void *hl_dma_alloc_common(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle,
121 ptr = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, size, dma_handle, flag);
124 ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, size, flag, dma_handle);
129 trace_habanalabs_dma_alloc(hdev->dev, (u64) (uintptr_t) ptr, *dma_handle, size,
135 static void hl_asic_dma_free_common(struct hl_device *hdev, size_t size, void *cpu_addr,
144 hdev->asic_funcs->asic_dma_free_coherent(hdev, size, cpu_addr, dma_handle);
147 hdev->asic_funcs->asic_dma_pool_free(hdev, cpu_addr, dma_handle);
151 trace_habanalabs_dma_free(hdev->dev, store_cpu_addr, dma_handle, size, caller);
154 void *hl_asic_dma_alloc_coherent_caller(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle,
157 return hl_dma_alloc_common(hdev, size, dma_handle, flag, DMA_ALLOC_COHERENT, caller);
160 void hl_asic_dma_free_coherent_caller(struct hl_device *hdev, size_t size, void *cpu_addr,
163 hl_asic_dma_free_common(hdev, size, cpu_addr, dma_handle, DMA_ALLOC_COHERENT, caller);
166 void *hl_asic_dma_pool_zalloc_caller(struct hl_device *hdev, size_t size, gfp_t mem_flags,
169 return hl_dma_alloc_common(hdev, size, dma_handle, mem_flags, DMA_ALLOC_POOL, caller);
172 void hl_asic_dma_pool_free_caller(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr,
175 hl_asic_dma_free_common(hdev, 0, vaddr, dma_addr, DMA_ALLOC_POOL, caller);
178 void *hl_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle)
180 return hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, size, dma_handle);
183 void hl_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr)
185 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, size, vaddr);
188 int hl_dma_map_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir)
190 struct asic_fixed_properties *prop = &hdev->asic_prop;
194 rc = dma_map_sgtable(&hdev->pdev->dev, sgt, dir, 0);
206 void hl_dma_unmap_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir)
208 struct asic_fixed_properties *prop = &hdev->asic_prop;
217 dma_unmap_sgtable(&hdev->pdev->dev, sgt, dir, 0);
223 * @hdev: pointer to habanalabs device structure
228 int hl_access_cfg_region(struct hl_device *hdev, u64 addr, u64 *val,
231 struct pci_mem_region *cfg_region = &hdev->pci_mem_region[PCI_REGION_CFG];
235 dev_err(hdev->dev, "address %#llx not a multiple of %zu\n", addr, sizeof(u32));
257 dev_err(hdev->dev, "access type %d is not supported\n", acc_type);
267 * @hdev: pointer to habanalabs device structure
273 int hl_access_dev_mem(struct hl_device *hdev, enum pci_region region_type,
278 return hl_access_cfg_region(hdev, addr, val, acc_type);
281 return hl_access_sram_dram_region(hdev, addr, val, acc_type,
314 enum hl_device_status hl_device_status(struct hl_device *hdev)
318 if (hdev->reset_info.in_reset) {
319 if (hdev->reset_info.in_compute_reset)
323 } else if (hdev->reset_info.needs_reset) {
325 } else if (hdev->disabled) {
327 } else if (!hdev->init_done) {
336 bool hl_device_operational(struct hl_device *hdev,
341 current_status = hl_device_status(hdev);
358 bool hl_ctrl_device_operational(struct hl_device *hdev,
363 current_status = hl_device_status(hdev);
380 static void print_idle_status_mask(struct hl_device *hdev, const char *message,
384 dev_err(hdev->dev, "%s (mask %#llx_%016llx_%016llx_%016llx)\n",
387 dev_err(hdev->dev, "%s (mask %#llx_%016llx_%016llx)\n",
390 dev_err(hdev->dev, "%s (mask %#llx_%016llx)\n",
393 dev_err(hdev->dev, "%s (mask %#llx)\n", message, idle_mask[0]);
401 struct hl_device *hdev;
405 hdev = hpriv->hdev;
407 hdev->asic_funcs->send_device_activity(hdev, false);
422 reset_device = hdev->reset_upon_device_release || hdev->reset_info.watchdog_active;
427 if (!hdev->reset_info.in_reset && !reset_device && hdev->pdev && !hdev->pldm)
428 device_is_idle = hdev->asic_funcs->is_device_idle(hdev, idle_mask,
431 print_idle_status_mask(hdev, "device is not idle after user context is closed",
445 mutex_lock(&hdev->fpriv_list_lock);
447 mutex_unlock(&hdev->fpriv_list_lock);
450 hl_device_reset(hdev, HL_DRV_RESET_DEV_RELEASE);
453 int rc = hdev->asic_funcs->scrub_device_mem(hdev);
456 dev_err(hdev->dev, "failed to scrub memory from hpriv release (%d)\n", rc);
463 mutex_lock(&hdev->fpriv_list_lock);
464 hdev->is_compute_ctx_active = false;
465 mutex_unlock(&hdev->fpriv_list_lock);
467 hdev->compute_ctx_in_release = 0;
488 static void print_device_in_use_info(struct hl_device *hdev, const char *message)
499 active_cs_num = hl_get_active_cs_num(hdev);
505 dmabuf_export_cnt = atomic_read(&hdev->dmabuf_export_cnt);
515 dev_notice(hdev->dev, "%s%s\n", message, buf);
529 struct hl_device *hdev = hpriv->hdev;
533 if (!hdev) {
539 hl_ctx_mgr_fini(hdev, &hpriv->ctx_mgr);
546 hdev->compute_ctx_in_release = 1;
549 print_device_in_use_info(hdev, "User process closed FD but device still in use");
550 hl_device_reset(hdev, HL_DRV_RESET_HARD);
553 hdev->last_open_session_duration_jif = jiffies - hdev->last_successful_open_jif;
561 struct hl_device *hdev = hpriv->hdev;
565 if (!hdev) {
570 mutex_lock(&hdev->fpriv_ctrl_list_lock);
572 mutex_unlock(&hdev->fpriv_ctrl_list_lock);
598 struct hl_device *hdev = hpriv->hdev;
601 if (!hdev) {
645 * @hdev: pointer to habanalabs device structure
655 static int device_init_cdev(struct hl_device *hdev, struct class *class,
668 (*dev)->devt = MKDEV(hdev->major, minor);
671 dev_set_drvdata(*dev, hdev);
677 static int cdev_sysfs_debugfs_add(struct hl_device *hdev)
681 rc = cdev_device_add(&hdev->cdev, hdev->dev);
683 dev_err(hdev->dev,
688 rc = cdev_device_add(&hdev->cdev_ctrl, hdev->dev_ctrl);
690 dev_err(hdev->dev,
696 rc = hl_sysfs_init(hdev);
698 dev_err(hdev->dev, "failed to initialize sysfs\n");
702 hl_debugfs_add_device(hdev);
704 hdev->cdev_sysfs_debugfs_created = true;
709 cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);
711 cdev_device_del(&hdev->cdev, hdev->dev);
715 static void cdev_sysfs_debugfs_remove(struct hl_device *hdev)
717 if (!hdev->cdev_sysfs_debugfs_created)
720 hl_debugfs_remove_device(hdev);
721 hl_sysfs_fini(hdev);
722 cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);
723 cdev_device_del(&hdev->cdev, hdev->dev);
726 put_device(hdev->dev);
727 put_device(hdev->dev_ctrl);
734 struct hl_device *hdev = device_reset_work->hdev;
740 rc = hl_device_reset(hdev, flags);
742 if ((rc == -EBUSY) && !hdev->device_fini_pending) {
743 struct hl_ctx *ctx = hl_get_compute_ctx(hdev);
749 dev_info(hdev->dev,
754 dev_info(hdev->dev, "Could not reset device. will try again in %u seconds",
758 queue_delayed_work(hdev->reset_wq, &device_reset_work->reset_work,
767 struct hl_device *hdev = watchdog_work->hdev;
770 dev_dbg(hdev->dev, "Device wasn't released in time. Initiate hard-reset.\n");
774 hl_device_reset(hdev, flags);
780 * @hdev: pointer to habanalabs device structure
785 static int device_early_init(struct hl_device *hdev)
790 switch (hdev->asic_type) {
792 goya_set_asic_funcs(hdev);
793 strscpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name));
796 gaudi_set_asic_funcs(hdev);
797 strscpy(hdev->asic_name, "GAUDI", sizeof(hdev->asic_name));
800 gaudi_set_asic_funcs(hdev);
801 strscpy(hdev->asic_name, "GAUDI SEC", sizeof(hdev->asic_name));
804 gaudi2_set_asic_funcs(hdev);
805 strscpy(hdev->asic_name, "GAUDI2", sizeof(hdev->asic_name));
808 gaudi2_set_asic_funcs(hdev);
809 strscpy(hdev->asic_name, "GAUDI2B", sizeof(hdev->asic_name));
812 gaudi2_set_asic_funcs(hdev);
813 strscpy(hdev->asic_name, "GAUDI2C", sizeof(hdev->asic_name));
816 dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
817 hdev->asic_type);
821 rc = hdev->asic_funcs->early_init(hdev);
825 rc = hl_asid_init(hdev);
829 if (hdev->asic_prop.completion_queues_count) {
830 hdev->cq_wq = kcalloc(hdev->asic_prop.completion_queues_count,
833 if (!hdev->cq_wq) {
839 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) {
840 snprintf(workq_name, 32, "hl%u-free-jobs-%u", hdev->cdev_idx, (u32) i);
841 hdev->cq_wq[i] = create_singlethread_workqueue(workq_name);
842 if (hdev->cq_wq[i] == NULL) {
843 dev_err(hdev->dev, "Failed to allocate CQ workqueue\n");
849 snprintf(workq_name, 32, "hl%u-events", hdev->cdev_idx);
850 hdev->eq_wq = create_singlethread_workqueue(workq_name);
851 if (hdev->eq_wq == NULL) {
852 dev_err(hdev->dev, "Failed to allocate EQ workqueue\n");
857 snprintf(workq_name, 32, "hl%u-cs-completions", hdev->cdev_idx);
858 hdev->cs_cmplt_wq = alloc_workqueue(workq_name, WQ_UNBOUND, 0);
859 if (!hdev->cs_cmplt_wq) {
860 dev_err(hdev->dev,
866 snprintf(workq_name, 32, "hl%u-ts-free-obj", hdev->cdev_idx);
867 hdev->ts_free_obj_wq = alloc_workqueue(workq_name, WQ_UNBOUND, 0);
868 if (!hdev->ts_free_obj_wq) {
869 dev_err(hdev->dev,
875 snprintf(workq_name, 32, "hl%u-prefetch", hdev->cdev_idx);
876 hdev->prefetch_wq = alloc_workqueue(workq_name, WQ_UNBOUND, 0);
877 if (!hdev->prefetch_wq) {
878 dev_err(hdev->dev, "Failed to allocate MMU prefetch workqueue\n");
883 hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info), GFP_KERNEL);
884 if (!hdev->hl_chip_info) {
889 rc = hl_mmu_if_set_funcs(hdev);
893 hl_mem_mgr_init(hdev->dev, &hdev->kernel_mem_mgr);
895 snprintf(workq_name, 32, "hl%u_device_reset", hdev->cdev_idx);
896 hdev->reset_wq = create_singlethread_workqueue(workq_name);
897 if (!hdev->reset_wq) {
899 dev_err(hdev->dev, "Failed to create device reset WQ\n");
903 INIT_DELAYED_WORK(&hdev->device_reset_work.reset_work, device_hard_reset_pending);
904 hdev->device_reset_work.hdev = hdev;
905 hdev->device_fini_pending = 0;
907 INIT_DELAYED_WORK(&hdev->device_release_watchdog_work.reset_work,
909 hdev->device_release_watchdog_work.hdev = hdev;
911 mutex_init(&hdev->send_cpu_message_lock);
912 mutex_init(&hdev->debug_lock);
913 INIT_LIST_HEAD(&hdev->cs_mirror_list);
914 spin_lock_init(&hdev->cs_mirror_lock);
915 spin_lock_init(&hdev->reset_info.lock);
916 INIT_LIST_HEAD(&hdev->fpriv_list);
917 INIT_LIST_HEAD(&hdev->fpriv_ctrl_list);
918 mutex_init(&hdev->fpriv_list_lock);
919 mutex_init(&hdev->fpriv_ctrl_list_lock);
920 mutex_init(&hdev->clk_throttling.lock);
925 hl_mem_mgr_fini(&hdev->kernel_mem_mgr);
926 hl_mem_mgr_idr_destroy(&hdev->kernel_mem_mgr);
928 kfree(hdev->hl_chip_info);
930 destroy_workqueue(hdev->prefetch_wq);
932 destroy_workqueue(hdev->ts_free_obj_wq);
934 destroy_workqueue(hdev->cs_cmplt_wq);
936 destroy_workqueue(hdev->eq_wq);
938 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
939 if (hdev->cq_wq[i])
940 destroy_workqueue(hdev->cq_wq[i]);
941 kfree(hdev->cq_wq);
943 hl_asid_fini(hdev);
945 if (hdev->asic_funcs->early_fini)
946 hdev->asic_funcs->early_fini(hdev);
954 * @hdev: pointer to habanalabs device structure
957 static void device_early_fini(struct hl_device *hdev)
961 mutex_destroy(&hdev->debug_lock);
962 mutex_destroy(&hdev->send_cpu_message_lock);
964 mutex_destroy(&hdev->fpriv_list_lock);
965 mutex_destroy(&hdev->fpriv_ctrl_list_lock);
967 mutex_destroy(&hdev->clk_throttling.lock);
969 hl_mem_mgr_fini(&hdev->kernel_mem_mgr);
970 hl_mem_mgr_idr_destroy(&hdev->kernel_mem_mgr);
972 kfree(hdev->hl_chip_info);
974 destroy_workqueue(hdev->prefetch_wq);
975 destroy_workqueue(hdev->ts_free_obj_wq);
976 destroy_workqueue(hdev->cs_cmplt_wq);
977 destroy_workqueue(hdev->eq_wq);
978 destroy_workqueue(hdev->reset_wq);
980 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
981 destroy_workqueue(hdev->cq_wq[i]);
982 kfree(hdev->cq_wq);
984 hl_asid_fini(hdev);
986 if (hdev->asic_funcs->early_fini)
987 hdev->asic_funcs->early_fini(hdev);
990 static bool is_pci_link_healthy(struct hl_device *hdev)
994 if (!hdev->pdev)
997 pci_read_config_word(hdev->pdev, PCI_VENDOR_ID, &vendor_id);
1004 struct hl_device *hdev = container_of(work, struct hl_device,
1009 if (!hl_device_operational(hdev, NULL))
1012 if (!hdev->asic_funcs->send_heartbeat(hdev))
1015 if (hl_device_operational(hdev, NULL))
1016 dev_err(hdev->dev, "Device heartbeat failed! PCI link is %s\n",
1017 is_pci_link_healthy(hdev) ? "healthy" : "broken");
1021 hl_handle_fw_err(hdev, &info);
1022 hl_device_cond_reset(hdev, HL_DRV_RESET_HARD | HL_DRV_RESET_HEARTBEAT, event_mask);
1037 if (!hdev->reset_info.in_reset)
1038 hdev->reset_info.prev_reset_trigger = HL_RESET_TRIGGER_DEFAULT;
1040 schedule_delayed_work(&hdev->work_heartbeat,
1047 * @hdev: pointer to habanalabs device structure
1052 static int device_late_init(struct hl_device *hdev)
1056 if (hdev->asic_funcs->late_init) {
1057 rc = hdev->asic_funcs->late_init(hdev);
1059 dev_err(hdev->dev,
1065 hdev->high_pll = hdev->asic_prop.high_pll;
1067 if (hdev->heartbeat) {
1068 INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat);
1069 schedule_delayed_work(&hdev->work_heartbeat,
1073 hdev->late_init_done = true;
1081 * @hdev: pointer to habanalabs device structure
1084 static void device_late_fini(struct hl_device *hdev)
1086 if (!hdev->late_init_done)
1089 if (hdev->heartbeat)
1090 cancel_delayed_work_sync(&hdev->work_heartbeat);
1092 if (hdev->asic_funcs->late_fini)
1093 hdev->asic_funcs->late_fini(hdev);
1095 hdev->late_init_done = false;
1098 int hl_device_utilization(struct hl_device *hdev, u32 *utilization)
1103 max_power = hdev->max_power;
1104 dc_power = hdev->asic_prop.dc_power_default;
1107 dev_warn(hdev->dev, "device utilization is not supported\n");
1110 rc = hl_fw_cpucp_power_get(hdev, &curr_power);
1123 int hl_device_set_debug_mode(struct hl_device *hdev, struct hl_ctx *ctx, bool enable)
1127 mutex_lock(&hdev->debug_lock);
1130 if (!hdev->in_debug) {
1131 dev_err(hdev->dev,
1137 if (!hdev->reset_info.hard_reset_pending)
1138 hdev->asic_funcs->halt_coresight(hdev, ctx);
1140 hdev->in_debug = 0;
1145 if (hdev->in_debug) {
1146 dev_err(hdev->dev,
1152 hdev->in_debug = 1;
1155 mutex_unlock(&hdev->debug_lock);
1160 static void take_release_locks(struct hl_device *hdev)
1165 hdev->asic_funcs->hw_queues_lock(hdev);
1166 hdev->asic_funcs->hw_queues_unlock(hdev);
1169 mutex_lock(&hdev->send_cpu_message_lock);
1170 mutex_unlock(&hdev->send_cpu_message_lock);
1173 mutex_lock(&hdev->fpriv_list_lock);
1174 mutex_unlock(&hdev->fpriv_list_lock);
1175 mutex_lock(&hdev->fpriv_ctrl_list_lock);
1176 mutex_unlock(&hdev->fpriv_ctrl_list_lock);
1179 static void hl_abort_waiting_for_completions(struct hl_device *hdev)
1181 hl_abort_waiting_for_cs_completions(hdev);
1186 hl_release_pending_user_interrupts(hdev);
1189 static void cleanup_resources(struct hl_device *hdev, bool hard_reset, bool fw_reset,
1193 device_late_fini(hdev);
1200 hdev->asic_funcs->halt_engines(hdev, hard_reset, fw_reset);
1203 hl_cs_rollback_all(hdev, skip_wq_flush);
1206 flush_workqueue(hdev->prefetch_wq);
1208 hl_abort_waiting_for_completions(hdev);
1214 * @hdev: pointer to habanalabs device structure
1220 int hl_device_suspend(struct hl_device *hdev)
1224 pci_save_state(hdev->pdev);
1227 spin_lock(&hdev->reset_info.lock);
1228 if (hdev->reset_info.in_reset) {
1229 spin_unlock(&hdev->reset_info.lock);
1230 dev_err(hdev->dev, "Can't suspend while in reset\n");
1233 hdev->reset_info.in_reset = 1;
1234 spin_unlock(&hdev->reset_info.lock);
1237 hdev->disabled = true;
1239 take_release_locks(hdev);
1241 rc = hdev->asic_funcs->suspend(hdev);
1243 dev_err(hdev->dev,
1247 pci_disable_device(hdev->pdev);
1248 pci_set_power_state(hdev->pdev, PCI_D3hot);
1256 * @hdev: pointer to habanalabs device structure
1262 int hl_device_resume(struct hl_device *hdev)
1266 pci_set_power_state(hdev->pdev, PCI_D0);
1267 pci_restore_state(hdev->pdev);
1268 rc = pci_enable_device_mem(hdev->pdev);
1270 dev_err(hdev->dev,
1275 pci_set_master(hdev->pdev);
1277 rc = hdev->asic_funcs->resume(hdev);
1279 dev_err(hdev->dev, "Failed to resume device after suspend\n");
1287 spin_lock(&hdev->reset_info.lock);
1288 hdev->reset_info.in_reset = 0;
1289 spin_unlock(&hdev->reset_info.lock);
1291 rc = hl_device_reset(hdev, HL_DRV_RESET_HARD);
1293 dev_err(hdev->dev, "Failed to reset device during resume\n");
1300 pci_disable_device(hdev->pdev);
1305 static int device_kill_open_processes(struct hl_device *hdev, u32 timeout, bool control_dev)
1313 fd_lock = control_dev ? &hdev->fpriv_ctrl_list_lock : &hdev->fpriv_list_lock;
1314 fd_list = control_dev ? &hdev->fpriv_ctrl_list : &hdev->fpriv_list;
1325 if (hdev->process_kill_trial_cnt) {
1343 dev_info(hdev->dev, "Killing user process pid=%d\n",
1355 dev_dbg(hdev->dev,
1373 dev_dbg(hdev->dev,
1386 if (hdev->process_kill_trial_cnt == HL_PENDING_RESET_MAX_TRIALS)
1389 hdev->process_kill_trial_cnt++;
1394 static void device_disable_open_processes(struct hl_device *hdev, bool control_dev)
1400 fd_lock = control_dev ? &hdev->fpriv_ctrl_list_lock : &hdev->fpriv_list_lock;
1401 fd_list = control_dev ? &hdev->fpriv_ctrl_list : &hdev->fpriv_list;
1405 hpriv->hdev = NULL;
1409 static void send_disable_pci_access(struct hl_device *hdev, u32 flags)
1425 if (hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0)) {
1426 dev_warn(hdev->dev, "Failed to disable FW's PCI access\n");
1431 if (hdev->cpu_queues_enable)
1432 synchronize_irq(pci_irq_vector(hdev->pdev,
1433 hdev->asic_prop.eq_interrupt_id));
1437 static void handle_reset_trigger(struct hl_device *hdev, u32 flags)
1442 if (hdev->is_compute_ctx_active)
1452 hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_HEARTBEAT;
1455 hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_TDR;
1458 hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
1461 hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
1469 if (hdev->reset_info.prev_reset_trigger != cur_reset_trigger) {
1470 hdev->reset_info.prev_reset_trigger = cur_reset_trigger;
1471 hdev->reset_info.reset_trigger_repeated = 0;
1473 hdev->reset_info.reset_trigger_repeated = 1;
1480 * @hdev: pointer to habanalabs device structure
1493 int hl_device_reset(struct hl_device *hdev, u32 flags)
1501 if (!hdev->init_done) {
1502 dev_err(hdev->dev, "Can't reset before initialization is done\n");
1512 reset_upon_device_release = hdev->reset_upon_device_release && from_dev_release;
1514 if (!hard_reset && (hl_device_status(hdev) == HL_DEVICE_STATUS_MALFUNCTION)) {
1515 dev_dbg(hdev->dev, "soft-reset isn't supported on a malfunctioning device\n");
1519 if (!hard_reset && !hdev->asic_prop.supports_compute_reset) {
1520 dev_dbg(hdev->dev, "asic doesn't support compute reset - do hard-reset instead\n");
1526 dev_crit(hdev->dev,
1534 if (!hard_reset && !hdev->asic_prop.allow_inference_soft_reset) {
1535 dev_dbg(hdev->dev,
1542 if (from_hard_reset_thread && hdev->process_kill_trial_cnt)
1552 spin_lock(&hdev->reset_info.lock);
1553 if (hdev->reset_info.in_reset) {
1555 if (hard_reset && hdev->reset_info.in_compute_reset)
1556 hdev->reset_info.hard_reset_schedule_flags = flags;
1557 spin_unlock(&hdev->reset_info.lock);
1564 hdev->reset_info.in_compute_reset = !hard_reset;
1566 hdev->reset_info.in_reset = 1;
1568 spin_unlock(&hdev->reset_info.lock);
1574 if ((hard_reset || from_dev_release) && hdev->reset_info.watchdog_active) {
1576 &hdev->device_release_watchdog_work;
1578 hdev->reset_info.watchdog_active = 0;
1583 hdev->reset_info.in_compute_reset = 0;
1594 handle_reset_trigger(hdev, flags);
1595 send_disable_pci_access(hdev, flags);
1598 hdev->disabled = true;
1600 take_release_locks(hdev);
1603 dev_info(hdev->dev, "Going to reset device\n");
1605 dev_dbg(hdev->dev, "Going to reset device after release by user\n");
1607 dev_dbg(hdev->dev, "Going to reset engines of inference device\n");
1611 hdev->reset_info.hard_reset_pending = true;
1613 hdev->process_kill_trial_cnt = 0;
1615 hdev->device_reset_work.flags = flags;
1621 queue_delayed_work(hdev->reset_wq, &hdev->device_reset_work.reset_work, 0);
1626 cleanup_resources(hdev, hard_reset, fw_reset, from_dev_release);
1634 rc = device_kill_open_processes(hdev, 0, false);
1637 if (hdev->device_fini_pending) {
1638 dev_crit(hdev->dev,
1640 dev_name(&(hdev)->pdev->dev));
1649 dev_crit(hdev->dev,
1651 dev_name(&(hdev)->pdev->dev));
1658 flush_workqueue(hdev->eq_wq);
1662 hw_fini_rc = hdev->asic_funcs->hw_fini(hdev, hard_reset, fw_reset);
1665 hdev->fw_loader.fw_comp_loaded = FW_TYPE_NONE;
1668 if (hdev->kernel_ctx && hl_ctx_put(hdev->kernel_ctx) == 1)
1669 hdev->kernel_ctx = NULL;
1671 hl_vm_fini(hdev);
1672 hl_mmu_fini(hdev);
1673 hl_eq_reset(hdev, &hdev->event_queue);
1677 hl_hw_queue_reset(hdev, hard_reset);
1678 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
1679 hl_cq_reset(hdev, &hdev->completion_queue[i]);
1682 ctx = hl_get_compute_ctx(hdev);
1696 hdev->device_cpu_disabled = false;
1697 hdev->reset_info.hard_reset_pending = false;
1699 if (hdev->reset_info.reset_trigger_repeated &&
1700 (hdev->reset_info.prev_reset_trigger ==
1705 dev_crit(hdev->dev,
1707 dev_name(&(hdev)->pdev->dev));
1712 if (hdev->kernel_ctx) {
1713 dev_crit(hdev->dev,
1715 dev_name(&(hdev)->pdev->dev));
1720 rc = hl_mmu_init(hdev);
1722 dev_err(hdev->dev,
1728 hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx),
1730 if (!hdev->kernel_ctx) {
1732 hl_mmu_fini(hdev);
1736 hdev->is_compute_ctx_active = false;
1738 rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
1740 dev_err(hdev->dev,
1742 kfree(hdev->kernel_ctx);
1743 hdev->kernel_ctx = NULL;
1744 hl_mmu_fini(hdev);
1753 hdev->disabled = false;
1757 rc = hl_fw_read_preboot_status(hdev);
1762 rc = hdev->asic_funcs->hw_init(hdev);
1764 dev_err(hdev->dev, "failed to initialize the H/W after reset\n");
1769 if (!hdev->asic_funcs->is_device_idle(hdev, idle_mask,
1771 print_idle_status_mask(hdev, "device is not idle after reset", idle_mask);
1777 rc = hdev->asic_funcs->test_queues(hdev);
1779 dev_err(hdev->dev, "Failed to detect if device is alive after reset\n");
1784 rc = device_late_init(hdev);
1786 dev_err(hdev->dev, "Failed late init after hard reset\n");
1790 rc = hl_vm_init(hdev);
1792 dev_err(hdev->dev, "Failed to init memory module after hard reset\n");
1796 if (!hdev->asic_prop.fw_security_enabled)
1797 hl_fw_set_max_power(hdev);
1799 rc = hdev->asic_funcs->compute_reset_late_init(hdev);
1802 dev_err(hdev->dev,
1805 dev_err(hdev->dev, "Failed late init after compute reset\n");
1810 rc = hdev->asic_funcs->scrub_device_mem(hdev);
1812 dev_err(hdev->dev, "scrub mem failed from device reset (%d)\n", rc);
1816 spin_lock(&hdev->reset_info.lock);
1817 hdev->reset_info.in_compute_reset = 0;
1823 if (!hard_reset && hdev->reset_info.hard_reset_schedule_flags)
1826 hdev->reset_info.in_reset = 0;
1828 spin_unlock(&hdev->reset_info.lock);
1830 hdev->reset_info.needs_reset = false;
1833 dev_info(hdev->dev,
1835 dev_name(&(hdev)->pdev->dev));
1837 dev_dbg(hdev->dev,
1839 dev_name(&(hdev)->pdev->dev));
1842 hdev->reset_info.hard_reset_cnt++;
1849 hdev->asic_funcs->enable_events_from_fw(hdev);
1852 hdev->reset_info.compute_reset_cnt++;
1855 dev_info(hdev->dev, "Performing hard reset scheduled during compute reset\n");
1856 flags = hdev->reset_info.hard_reset_schedule_flags;
1857 hdev->reset_info.hard_reset_schedule_flags = 0;
1866 hdev->disabled = true;
1868 spin_lock(&hdev->reset_info.lock);
1869 hdev->reset_info.in_compute_reset = 0;
1872 dev_err(hdev->dev,
1874 dev_name(&(hdev)->pdev->dev));
1875 hdev->reset_info.hard_reset_cnt++;
1878 dev_err(hdev->dev, "Failed to reset device after user release\n");
1881 dev_err(hdev->dev, "Failed to do compute reset\n");
1882 hdev->reset_info.compute_reset_cnt++;
1885 spin_unlock(&hdev->reset_info.lock);
1891 hdev->reset_info.in_reset = 0;
1893 spin_unlock(&hdev->reset_info.lock);
1900 * @hdev: pointer to habanalabs device structure.
1907 int hl_device_cond_reset(struct hl_device *hdev, u32 flags, u64 event_mask)
1917 dev_err(hdev->dev, "Resetting device without a reset indication to user\n");
1921 ctx = hl_get_compute_ctx(hdev);
1928 spin_lock(&hdev->reset_info.lock);
1929 if (hdev->reset_info.in_reset) {
1930 spin_unlock(&hdev->reset_info.lock);
1934 if (hdev->reset_info.watchdog_active)
1937 hdev->device_release_watchdog_work.flags = flags;
1938 dev_dbg(hdev->dev, "Device is going to be hard-reset in %u sec unless being released\n",
1939 hdev->device_release_watchdog_timeout_sec);
1940 schedule_delayed_work(&hdev->device_release_watchdog_work.reset_work,
1941 msecs_to_jiffies(hdev->device_release_watchdog_timeout_sec * 1000));
1942 hdev->reset_info.watchdog_active = 1;
1944 spin_unlock(&hdev->reset_info.lock);
1946 hl_notifier_event_send_all(hdev, event_mask);
1950 hl_abort_waiting_for_completions(hdev);
1956 hl_notifier_event_send_all(hdev, event_mask);
1960 return hl_device_reset(hdev, flags);
1977 * @hdev: pointer to habanalabs device structure
1981 void hl_notifier_event_send_all(struct hl_device *hdev, u64 event_mask)
1986 dev_warn(hdev->dev, "Skip sending zero event");
1990 mutex_lock(&hdev->fpriv_list_lock);
1992 list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node)
1995 mutex_unlock(&hdev->fpriv_list_lock);
1998 mutex_lock(&hdev->fpriv_ctrl_list_lock);
2000 list_for_each_entry(hpriv, &hdev->fpriv_ctrl_list, dev_node)
2003 mutex_unlock(&hdev->fpriv_ctrl_list_lock);
2006 static int create_cdev(struct hl_device *hdev)
2011 hdev->cdev_idx = hdev->id / 2;
2013 name = kasprintf(GFP_KERNEL, "hl%d", hdev->cdev_idx);
2020 rc = device_init_cdev(hdev, hdev->hclass, hdev->id, &hl_ops, name,
2021 &hdev->cdev, &hdev->dev);
2028 name = kasprintf(GFP_KERNEL, "hl_controlD%d", hdev->cdev_idx);
2035 rc = device_init_cdev(hdev, hdev->hclass, hdev->id_control, &hl_ctrl_ops,
2036 name, &hdev->cdev_ctrl, &hdev->dev_ctrl);
2046 put_device(hdev->dev);
2054 * @hdev: pointer to habanalabs device structure
2060 int hl_device_init(struct hl_device *hdev)
2065 rc = create_cdev(hdev);
2070 rc = device_early_init(hdev);
2074 user_interrupt_cnt = hdev->asic_prop.user_dec_intr_count +
2075 hdev->asic_prop.user_interrupt_count;
2078 hdev->user_interrupt = kcalloc(user_interrupt_cnt, sizeof(*hdev->user_interrupt),
2080 if (!hdev->user_interrupt) {
2090 rc = hdev->asic_funcs->sw_init(hdev);
2096 hl_multi_cs_completion_init(hdev);
2103 rc = hl_hw_queues_create(hdev);
2105 dev_err(hdev->dev, "failed to initialize kernel queues\n");
2109 cq_cnt = hdev->asic_prop.completion_queues_count;
2117 hdev->completion_queue = kcalloc(cq_cnt,
2118 sizeof(*hdev->completion_queue),
2121 if (!hdev->completion_queue) {
2122 dev_err(hdev->dev,
2130 rc = hl_cq_init(hdev, &hdev->completion_queue[i],
2131 hdev->asic_funcs->get_queue_id_for_cq(hdev, i));
2133 dev_err(hdev->dev,
2137 hdev->completion_queue[i].cq_idx = i;
2140 hdev->shadow_cs_queue = kcalloc(hdev->asic_prop.max_pending_cs,
2142 if (!hdev->shadow_cs_queue) {
2152 rc = hl_eq_init(hdev, &hdev->event_queue);
2154 dev_err(hdev->dev, "failed to initialize event queue\n");
2159 rc = hl_mmu_init(hdev);
2161 dev_err(hdev->dev, "Failed to initialize MMU S/W structures\n");
2166 hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL);
2167 if (!hdev->kernel_ctx) {
2172 hdev->is_compute_ctx_active = false;
2174 hdev->asic_funcs->state_dump_init(hdev);
2176 hdev->device_release_watchdog_timeout_sec = HL_DEVICE_RELEASE_WATCHDOG_TIMEOUT_SEC;
2178 hdev->memory_scrub_val = MEM_SCRUB_DEFAULT_VAL;
2180 rc = hl_debugfs_device_init(hdev);
2182 dev_err(hdev->dev, "failed to initialize debugfs entry structure\n");
2183 kfree(hdev->kernel_ctx);
2190 rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
2192 dev_err(hdev->dev, "failed to initialize kernel context\n");
2193 kfree(hdev->kernel_ctx);
2197 rc = hl_cb_pool_init(hdev);
2199 dev_err(hdev->dev, "failed to initialize CB pool\n");
2203 rc = hl_dec_init(hdev);
2205 dev_err(hdev->dev, "Failed to initialize the decoder module\n");
2219 hdev->disabled = false;
2221 rc = hdev->asic_funcs->hw_init(hdev);
2223 dev_err(hdev->dev, "failed to initialize the H/W\n");
2229 rc = hdev->asic_funcs->test_queues(hdev);
2231 dev_err(hdev->dev, "Failed to detect if device is alive\n");
2236 rc = device_late_init(hdev);
2238 dev_err(hdev->dev, "Failed late initialization\n");
2243 dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n",
2244 hdev->asic_name,
2245 hdev->asic_prop.dram_size / SZ_1G);
2247 rc = hl_vm_init(hdev);
2249 dev_err(hdev->dev, "Failed to initialize memory module\n");
2259 rc = cdev_sysfs_debugfs_add(hdev);
2261 dev_err(hdev->dev, "Failed to add char devices and sysfs/debugfs files\n");
2269 if (hdev->asic_prop.set_max_power_on_device_init &&
2270 !hdev->asic_prop.fw_security_enabled)
2271 hl_fw_set_max_power(hdev);
2279 rc = hl_hwmon_init(hdev);
2281 dev_err(hdev->dev, "Failed to initialize hwmon\n");
2286 dev_notice(hdev->dev,
2288 dev_name(&(hdev)->pdev->dev));
2290 hdev->init_done = true;
2297 hdev->asic_funcs->enable_events_from_fw(hdev);
2302 hl_cb_pool_fini(hdev);
2304 if (hl_ctx_put(hdev->kernel_ctx) != 1)
2305 dev_err(hdev->dev,
2308 hl_debugfs_device_fini(hdev);
2310 hl_mmu_fini(hdev);
2312 hl_eq_fini(hdev, &hdev->event_queue);
2314 kfree(hdev->shadow_cs_queue);
2317 hl_cq_fini(hdev, &hdev->completion_queue[i]);
2318 kfree(hdev->completion_queue);
2320 hl_hw_queues_destroy(hdev);
2322 hdev->asic_funcs->sw_fini(hdev);
2324 kfree(hdev->user_interrupt);
2326 device_early_fini(hdev);
2328 put_device(hdev->dev_ctrl);
2329 put_device(hdev->dev);
2331 hdev->disabled = true;
2333 cdev_sysfs_debugfs_add(hdev);
2334 dev_err(&hdev->pdev->dev,
2336 hdev->cdev_idx, dev_name(&hdev->pdev->dev));
2344 * @hdev: pointer to habanalabs device structure
2348 void hl_device_fini(struct hl_device *hdev)
2355 dev_info(hdev->dev, "Removing device\n");
2357 hdev->device_fini_pending = 1;
2358 flush_delayed_work(&hdev->device_reset_work.reset_work);
2360 if (hdev->pldm)
2375 spin_lock(&hdev->reset_info.lock);
2376 device_in_reset = !!hdev->reset_info.in_reset;
2378 hdev->reset_info.in_reset = 1;
2379 spin_unlock(&hdev->reset_info.lock);
2384 spin_lock(&hdev->reset_info.lock);
2385 device_in_reset = !!hdev->reset_info.in_reset;
2387 hdev->reset_info.in_reset = 1;
2388 spin_unlock(&hdev->reset_info.lock);
2391 dev_crit(hdev->dev,
2393 dev_name(&(hdev)->pdev->dev));
2398 cancel_delayed_work_sync(&hdev->device_release_watchdog_work.reset_work);
2407 hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0);
2410 hdev->disabled = true;
2412 take_release_locks(hdev);
2414 hdev->reset_info.hard_reset_pending = true;
2416 hl_hwmon_fini(hdev);
2418 cleanup_resources(hdev, true, false, false);
2424 dev_info(hdev->dev,
2428 hdev->process_kill_trial_cnt = 0;
2429 rc = device_kill_open_processes(hdev, HL_WAIT_PROCESS_KILL_ON_DEVICE_FINI, false);
2431 dev_crit(hdev->dev, "Failed to kill all open processes\n");
2432 device_disable_open_processes(hdev, false);
2435 hdev->process_kill_trial_cnt = 0;
2436 rc = device_kill_open_processes(hdev, 0, true);
2438 dev_crit(hdev->dev, "Failed to kill all control device open processes\n");
2439 device_disable_open_processes(hdev, true);
2442 hl_cb_pool_fini(hdev);
2445 rc = hdev->asic_funcs->hw_fini(hdev, true, false);
2447 dev_err(hdev->dev, "hw_fini failed in device fini while removing device %d\n", rc);
2449 hdev->fw_loader.fw_comp_loaded = FW_TYPE_NONE;
2452 if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1))
2453 dev_err(hdev->dev, "kernel ctx is still alive\n");
2455 hl_dec_fini(hdev);
2457 hl_vm_fini(hdev);
2459 hl_mmu_fini(hdev);
2461 vfree(hdev->captured_err_info.page_fault_info.user_mappings);
2463 hl_eq_fini(hdev, &hdev->event_queue);
2465 kfree(hdev->shadow_cs_queue);
2467 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
2468 hl_cq_fini(hdev, &hdev->completion_queue[i]);
2469 kfree(hdev->completion_queue);
2470 kfree(hdev->user_interrupt);
2472 hl_hw_queues_destroy(hdev);
2475 hdev->asic_funcs->sw_fini(hdev);
2477 device_early_fini(hdev);
2480 cdev_sysfs_debugfs_remove(hdev);
2482 hl_debugfs_device_fini(hdev);
2494 * @hdev: pointer to habanalabs device structure
2500 inline u32 hl_rreg(struct hl_device *hdev, u32 reg)
2502 u32 val = readl(hdev->rmmio + reg);
2505 trace_habanalabs_rreg32(hdev->dev, reg, val);
2513 * @hdev: pointer to habanalabs device structure
2520 inline void hl_wreg(struct hl_device *hdev, u32 reg, u32 val)
2523 trace_habanalabs_wreg32(hdev->dev, reg, val);
2525 writel(val, hdev->rmmio + reg);
2528 void hl_capture_razwi(struct hl_device *hdev, u64 addr, u16 *engine_id, u16 num_of_engines,
2531 struct razwi_info *razwi_info = &hdev->captured_err_info.razwi_info;
2534 dev_err(hdev->dev,
2541 if (atomic_cmpxchg(&hdev->captured_err_info.razwi_info.razwi_detected, 0, 1))
2554 void hl_handle_razwi(struct hl_device *hdev, u64 addr, u16 *engine_id, u16 num_of_engines,
2557 hl_capture_razwi(hdev, addr, engine_id, num_of_engines, flags);
2563 static void hl_capture_user_mappings(struct hl_device *hdev, bool is_pmmu)
2565 struct page_fault_info *pgf_info = &hdev->captured_err_info.page_fault_info;
2577 ctx = hl_get_compute_ctx(hdev);
2579 dev_err(hdev->dev, "Can't get user context for user mappings\n");
2625 void hl_capture_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_pmmu)
2627 struct page_fault_info *pgf_info = &hdev->captured_err_info.page_fault_info;
2636 hl_capture_user_mappings(hdev, is_pmmu);
2641 void hl_handle_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_pmmu,
2644 hl_capture_page_fault(hdev, addr, eng_id, is_pmmu);
2650 static void hl_capture_hw_err(struct hl_device *hdev, u16 event_id)
2652 struct hw_err_info *info = &hdev->captured_err_info.hw_err;
2664 void hl_handle_critical_hw_err(struct hl_device *hdev, u16 event_id, u64 *event_mask)
2666 hl_capture_hw_err(hdev, event_id);
2672 static void hl_capture_fw_err(struct hl_device *hdev, struct hl_info_fw_err_info *fw_info)
2674 struct fw_err_info *info = &hdev->captured_err_info.fw_err;
2688 void hl_handle_fw_err(struct hl_device *hdev, struct hl_info_fw_err_info *info)
2690 hl_capture_fw_err(hdev, info);