Lines Matching defs:hdev
353 static int gaudi_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 asid,
355 static int gaudi_send_job_on_qman0(struct hl_device *hdev,
357 static int gaudi_memset_device_memory(struct hl_device *hdev, u64 addr,
359 static int gaudi_run_tpc_kernel(struct hl_device *hdev, u64 tpc_kernel,
361 static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev);
362 static int gaudi_cpucp_info_get(struct hl_device *hdev);
363 static void gaudi_disable_clock_gating(struct hl_device *hdev);
364 static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid);
366 static int gaudi_get_fixed_properties(struct hl_device *hdev)
368 struct asic_fixed_properties *prop = &hdev->asic_prop;
421 if (hdev->pldm)
482 static int gaudi_pci_bars_map(struct hl_device *hdev)
488 rc = hl_pci_bars_map(hdev, name, is_wc);
492 hdev->rmmio = hdev->pcie_bar[CFG_BAR_ID] +
498 static u64 gaudi_set_hbm_bar_base(struct hl_device *hdev, u64 addr)
500 struct gaudi_device *gaudi = hdev->asic_specific;
512 rc = hl_pci_set_inbound_region(hdev, 2, &pci_region);
524 static int gaudi_init_iatu(struct hl_device *hdev)
534 rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region);
542 rc = hl_pci_set_inbound_region(hdev, 1, &inbound_region);
550 rc = hl_pci_set_inbound_region(hdev, 2, &inbound_region);
554 hdev->asic_funcs->set_dma_mask_from_fw(hdev);
559 rc = hl_pci_set_outbound_region(hdev, &outbound_region);
565 static int gaudi_early_init(struct hl_device *hdev)
567 struct asic_fixed_properties *prop = &hdev->asic_prop;
568 struct pci_dev *pdev = hdev->pdev;
571 rc = gaudi_get_fixed_properties(hdev);
573 dev_err(hdev->dev, "Failed to get fixed properties\n");
579 dev_err(hdev->dev,
590 dev_err(hdev->dev,
602 rc = hl_pci_init(hdev, mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS,
609 dev_info(hdev->dev, "firmware-level security is disabled\n");
614 kfree(hdev->asic_prop.hw_queues_props);
618 static int gaudi_early_fini(struct hl_device *hdev)
620 kfree(hdev->asic_prop.hw_queues_props);
621 hl_pci_fini(hdev);
629 * @hdev: pointer to hl_device structure
632 static void gaudi_fetch_psoc_frequency(struct hl_device *hdev)
634 struct asic_fixed_properties *prop = &hdev->asic_prop;
656 dev_warn(hdev->dev,
667 static int _gaudi_init_tpc_mem(struct hl_device *hdev,
670 struct asic_fixed_properties *prop = &hdev->asic_prop;
679 cb = hl_cb_kernel_create(hdev, PAGE_SIZE, false);
702 job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true);
704 dev_err(hdev->dev, "Failed to allocate a new job\n");
717 hl_debugfs_add_job(hdev, job);
719 rc = gaudi_send_job_on_qman0(hdev, job);
725 rc = gaudi_run_tpc_kernel(hdev, dst_addr, tpc_id);
731 hl_userptr_delete_list(hdev, &job->userptr_list);
732 hl_debugfs_remove_job(hdev, job);
738 hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT);
745 * @hdev: Pointer to hl_device structure.
751 static int gaudi_init_tpc_mem(struct hl_device *hdev)
760 rc = request_firmware(&fw, GAUDI_TPC_FW_FILE, hdev->dev);
767 dev_err(hdev->dev, "Failed to load firmware file %s\n",
773 cpu_addr = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, fw_size,
776 dev_err(hdev->dev,
785 rc = _gaudi_init_tpc_mem(hdev, dma_handle, fw_size);
787 hdev->asic_funcs->asic_dma_free_coherent(hdev, fw->size, cpu_addr,
795 static int gaudi_late_init(struct hl_device *hdev)
797 struct gaudi_device *gaudi = hdev->asic_specific;
800 rc = gaudi->cpucp_info_get(hdev);
802 dev_err(hdev->dev, "Failed to get cpucp info\n");
806 rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_ENABLE_PCI_ACCESS);
808 dev_err(hdev->dev, "Failed to enable PCI access from CPU\n");
814 gaudi_fetch_psoc_frequency(hdev);
816 rc = gaudi_mmu_clear_pgt_range(hdev);
818 dev_err(hdev->dev, "Failed to clear MMU page tables range\n");
822 rc = gaudi_init_tpc_mem(hdev);
824 dev_err(hdev->dev, "Failed to initialize TPC memories\n");
831 hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS);
836 static void gaudi_late_fini(struct hl_device *hdev)
841 if (!hdev->hl_chip_info->info)
844 channel_info_arr = hdev->hl_chip_info->info;
854 hdev->hl_chip_info->info = NULL;
857 static int gaudi_alloc_cpu_accessible_dma_mem(struct hl_device *hdev)
874 hdev->asic_funcs->asic_dma_alloc_coherent(hdev,
890 dev_err(hdev->dev,
896 hdev->cpu_accessible_dma_mem = virt_addr_arr[i];
897 hdev->cpu_accessible_dma_address = dma_addr_arr[i];
898 hdev->cpu_pci_msb_addr =
899 GAUDI_CPU_PCI_MSB_ADDR(hdev->cpu_accessible_dma_address);
901 GAUDI_PCI_TO_CPU_ADDR(hdev->cpu_accessible_dma_address);
905 hdev->asic_funcs->asic_dma_free_coherent(hdev,
913 static void gaudi_free_internal_qmans_pq_mem(struct hl_device *hdev)
915 struct gaudi_device *gaudi = hdev->asic_specific;
923 hdev->asic_funcs->asic_dma_free_coherent(hdev, q->pq_size,
929 static int gaudi_alloc_internal_qmans_pq_mem(struct hl_device *hdev)
931 struct gaudi_device *gaudi = hdev->asic_specific;
953 dev_err(hdev->dev, "Bad internal queue index %d", i);
958 q->pq_kernel_addr = hdev->asic_funcs->asic_dma_alloc_coherent(
959 hdev, q->pq_size,
971 gaudi_free_internal_qmans_pq_mem(hdev);
975 static int gaudi_sw_init(struct hl_device *hdev)
989 dev_err(hdev->dev,
1005 hdev->asic_specific = gaudi;
1008 hdev->dma_pool = dma_pool_create(dev_name(hdev->dev),
1009 &hdev->pdev->dev, GAUDI_DMA_POOL_BLK_SIZE, 8, 0);
1010 if (!hdev->dma_pool) {
1011 dev_err(hdev->dev, "failed to create DMA pool\n");
1016 rc = gaudi_alloc_cpu_accessible_dma_mem(hdev);
1020 hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1);
1021 if (!hdev->cpu_accessible_dma_pool) {
1022 dev_err(hdev->dev,
1028 rc = gen_pool_add(hdev->cpu_accessible_dma_pool,
1029 (uintptr_t) hdev->cpu_accessible_dma_mem,
1032 dev_err(hdev->dev,
1038 rc = gaudi_alloc_internal_qmans_pq_mem(hdev);
1045 hdev->supports_sync_stream = true;
1046 hdev->supports_coresight = true;
1051 gen_pool_destroy(hdev->cpu_accessible_dma_pool);
1053 GAUDI_CPU_TO_PCI_ADDR(hdev->cpu_accessible_dma_address,
1054 hdev->cpu_pci_msb_addr);
1055 hdev->asic_funcs->asic_dma_free_coherent(hdev,
1057 hdev->cpu_accessible_dma_mem,
1058 hdev->cpu_accessible_dma_address);
1060 dma_pool_destroy(hdev->dma_pool);
1066 static int gaudi_sw_fini(struct hl_device *hdev)
1068 struct gaudi_device *gaudi = hdev->asic_specific;
1070 gaudi_free_internal_qmans_pq_mem(hdev);
1072 gen_pool_destroy(hdev->cpu_accessible_dma_pool);
1074 GAUDI_CPU_TO_PCI_ADDR(hdev->cpu_accessible_dma_address,
1075 hdev->cpu_pci_msb_addr);
1076 hdev->asic_funcs->asic_dma_free_coherent(hdev,
1078 hdev->cpu_accessible_dma_mem,
1079 hdev->cpu_accessible_dma_address);
1081 dma_pool_destroy(hdev->dma_pool);
1092 struct hl_device *hdev = arg;
1095 if (hdev->disabled)
1098 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
1099 hl_irq_handler_cq(irq, &hdev->completion_queue[i]);
1101 hl_irq_handler_eq(irq, &hdev->event_queue);
1110 static int gaudi_pci_irq_vector(struct hl_device *hdev, unsigned int nr,
1116 dev_crit(hdev->dev, "CPU EQ must use IRQ %d\n",
1122 return pci_irq_vector(hdev->pdev, msi_vec);
1125 static int gaudi_enable_msi_single(struct hl_device *hdev)
1129 dev_info(hdev->dev, "Working in single MSI IRQ mode\n");
1131 irq = gaudi_pci_irq_vector(hdev, 0, false);
1133 "gaudi single msi", hdev);
1135 dev_err(hdev->dev,
1141 static int gaudi_enable_msi_multi(struct hl_device *hdev)
1143 int cq_cnt = hdev->asic_prop.completion_queues_count;
1147 irq = gaudi_pci_irq_vector(hdev, i, false);
1149 &hdev->completion_queue[i]);
1151 dev_err(hdev->dev, "Failed to request IRQ %d", irq);
1156 irq = gaudi_pci_irq_vector(hdev, GAUDI_EVENT_QUEUE_MSI_IDX, true);
1158 &hdev->event_queue);
1160 dev_err(hdev->dev, "Failed to request IRQ %d", irq);
1168 free_irq(gaudi_pci_irq_vector(hdev, i, false),
1169 &hdev->completion_queue[i]);
1173 static int gaudi_enable_msi(struct hl_device *hdev)
1175 struct gaudi_device *gaudi = hdev->asic_specific;
1181 rc = pci_alloc_irq_vectors(hdev->pdev, 1, GAUDI_MSI_ENTRIES,
1184 dev_err(hdev->dev, "MSI: Failed to enable support %d\n", rc);
1190 rc = gaudi_enable_msi_single(hdev);
1193 rc = gaudi_enable_msi_multi(hdev);
1204 pci_free_irq_vectors(hdev->pdev);
1208 static void gaudi_sync_irqs(struct hl_device *hdev)
1210 struct gaudi_device *gaudi = hdev->asic_specific;
1211 int i, cq_cnt = hdev->asic_prop.completion_queues_count;
1219 synchronize_irq(gaudi_pci_irq_vector(hdev, i, false));
1221 synchronize_irq(gaudi_pci_irq_vector(hdev,
1225 synchronize_irq(gaudi_pci_irq_vector(hdev, 0, false));
1229 static void gaudi_disable_msi(struct hl_device *hdev)
1231 struct gaudi_device *gaudi = hdev->asic_specific;
1232 int i, irq, cq_cnt = hdev->asic_prop.completion_queues_count;
1237 gaudi_sync_irqs(hdev);
1240 irq = gaudi_pci_irq_vector(hdev, GAUDI_EVENT_QUEUE_MSI_IDX,
1242 free_irq(irq, &hdev->event_queue);
1245 irq = gaudi_pci_irq_vector(hdev, i, false);
1246 free_irq(irq, &hdev->completion_queue[i]);
1249 free_irq(gaudi_pci_irq_vector(hdev, 0, false), hdev);
1252 pci_free_irq_vectors(hdev->pdev);
1257 static void gaudi_init_scrambler_sram(struct hl_device *hdev)
1259 struct gaudi_device *gaudi = hdev->asic_specific;
1264 if (!hdev->sram_scrambler_enable)
1321 static void gaudi_init_scrambler_hbm(struct hl_device *hdev)
1323 struct gaudi_device *gaudi = hdev->asic_specific;
1328 if (!hdev->dram_scrambler_enable)
1385 static void gaudi_init_e2e(struct hl_device *hdev)
1507 if (!hdev->dram_scrambler_enable) {
1750 static void gaudi_init_hbm_cred(struct hl_device *hdev)
1806 static void gaudi_init_golden_registers(struct hl_device *hdev)
1811 gaudi_init_e2e(hdev);
1813 gaudi_init_hbm_cred(hdev);
1815 hdev->asic_funcs->disable_clock_gating(hdev);
1829 writeq(0, hdev->pcie_bar[SRAM_BAR_ID] + i);
1837 static void gaudi_init_pci_dma_qman(struct hl_device *hdev, int dma_id,
1894 if (hdev->stop_on_err) {
1924 static void gaudi_init_dma_core(struct hl_device *hdev, int dma_id)
1937 if (hdev->stop_on_err)
1955 static void gaudi_enable_qman(struct hl_device *hdev, int dma_id,
1963 static void gaudi_init_pci_dma_qmans(struct hl_device *hdev)
1965 struct gaudi_device *gaudi = hdev->asic_specific;
1989 q = &hdev->kernel_queues[q_idx];
1992 gaudi_init_pci_dma_qman(hdev, dma_id, j,
1996 gaudi_init_dma_core(hdev, dma_id);
1998 gaudi_enable_qman(hdev, dma_id, PCI_DMA_QMAN_ENABLE);
2004 static void gaudi_init_hbm_dma_qman(struct hl_device *hdev, int dma_id,
2051 if (hdev->stop_on_err) {
2085 static void gaudi_init_hbm_dma_qmans(struct hl_device *hdev)
2087 struct gaudi_device *gaudi = hdev->asic_specific;
2107 gaudi_init_hbm_dma_qman(hdev, dma_id, j,
2112 gaudi_init_hbm_dma_qman(hdev, dma_id, 4, 0);
2114 gaudi_init_dma_core(hdev, dma_id);
2116 gaudi_enable_qman(hdev, dma_id, HBM_DMA_QMAN_ENABLE);
2122 static void gaudi_init_mme_qman(struct hl_device *hdev, u32 mme_offset,
2170 if (hdev->stop_on_err) {
2203 static void gaudi_init_mme_qmans(struct hl_device *hdev)
2205 struct gaudi_device *gaudi = hdev->asic_specific;
2225 gaudi_init_mme_qman(hdev, mme_offset, (i & 0x3),
2233 gaudi_init_mme_qman(hdev, mme_offset, 4, 0);
2234 gaudi_init_mme_qman(hdev, 0, 4, 0);
2242 static void gaudi_init_tpc_qman(struct hl_device *hdev, u32 tpc_offset,
2290 if (hdev->stop_on_err) {
2324 static void gaudi_init_tpc_qmans(struct hl_device *hdev)
2326 struct gaudi_device *gaudi = hdev->asic_specific;
2346 gaudi_init_tpc_qman(hdev, tpc_offset, i,
2351 gaudi_init_tpc_qman(hdev, tpc_offset, 4, 0);
2369 static void gaudi_disable_pci_dma_qmans(struct hl_device *hdev)
2371 struct gaudi_device *gaudi = hdev->asic_specific;
2381 static void gaudi_disable_hbm_dma_qmans(struct hl_device *hdev)
2383 struct gaudi_device *gaudi = hdev->asic_specific;
2395 static void gaudi_disable_mme_qmans(struct hl_device *hdev)
2397 struct gaudi_device *gaudi = hdev->asic_specific;
2406 static void gaudi_disable_tpc_qmans(struct hl_device *hdev)
2408 struct gaudi_device *gaudi = hdev->asic_specific;
2421 static void gaudi_stop_pci_dma_qmans(struct hl_device *hdev)
2423 struct gaudi_device *gaudi = hdev->asic_specific;
2434 static void gaudi_stop_hbm_dma_qmans(struct hl_device *hdev)
2436 struct gaudi_device *gaudi = hdev->asic_specific;
2450 static void gaudi_stop_mme_qmans(struct hl_device *hdev)
2452 struct gaudi_device *gaudi = hdev->asic_specific;
2462 static void gaudi_stop_tpc_qmans(struct hl_device *hdev)
2464 struct gaudi_device *gaudi = hdev->asic_specific;
2479 static void gaudi_pci_dma_stall(struct hl_device *hdev)
2481 struct gaudi_device *gaudi = hdev->asic_specific;
2491 static void gaudi_hbm_dma_stall(struct hl_device *hdev)
2493 struct gaudi_device *gaudi = hdev->asic_specific;
2505 static void gaudi_mme_stall(struct hl_device *hdev)
2507 struct gaudi_device *gaudi = hdev->asic_specific;
2531 static void gaudi_tpc_stall(struct hl_device *hdev)
2533 struct gaudi_device *gaudi = hdev->asic_specific;
2548 static void gaudi_set_clock_gating(struct hl_device *hdev)
2550 struct gaudi_device *gaudi = hdev->asic_specific;
2558 if (hdev->in_debug)
2562 enable = !!(hdev->clock_gating_mask &
2573 enable = !!(hdev->clock_gating_mask &
2583 enable = !!(hdev->clock_gating_mask & (BIT_ULL(GAUDI_ENGINE_ID_MME_0)));
2587 enable = !!(hdev->clock_gating_mask & (BIT_ULL(GAUDI_ENGINE_ID_MME_2)));
2592 enable = !!(hdev->clock_gating_mask &
2606 static void gaudi_disable_clock_gating(struct hl_device *hdev)
2608 struct gaudi_device *gaudi = hdev->asic_specific;
2637 static void gaudi_enable_timestamp(struct hl_device *hdev)
2650 static void gaudi_disable_timestamp(struct hl_device *hdev)
2656 static void gaudi_halt_engines(struct hl_device *hdev, bool hard_reset)
2660 dev_info(hdev->dev,
2663 if (hdev->pldm)
2669 gaudi_stop_mme_qmans(hdev);
2670 gaudi_stop_tpc_qmans(hdev);
2671 gaudi_stop_hbm_dma_qmans(hdev);
2672 gaudi_stop_pci_dma_qmans(hdev);
2674 hdev->asic_funcs->disable_clock_gating(hdev);
2678 gaudi_pci_dma_stall(hdev);
2679 gaudi_hbm_dma_stall(hdev);
2680 gaudi_tpc_stall(hdev);
2681 gaudi_mme_stall(hdev);
2685 gaudi_disable_mme_qmans(hdev);
2686 gaudi_disable_tpc_qmans(hdev);
2687 gaudi_disable_hbm_dma_qmans(hdev);
2688 gaudi_disable_pci_dma_qmans(hdev);
2690 gaudi_disable_timestamp(hdev);
2692 gaudi_disable_msi(hdev);
2695 static int gaudi_mmu_init(struct hl_device *hdev)
2697 struct asic_fixed_properties *prop = &hdev->asic_prop;
2698 struct gaudi_device *gaudi = hdev->asic_specific;
2702 if (!hdev->mmu_enable)
2708 hdev->dram_supports_virtual_memory = false;
2714 rc = gaudi_mmu_update_asid_hop0_addr(hdev, i, hop0_addr);
2716 dev_err(hdev->dev,
2726 hdev->asic_funcs->mmu_invalidate_cache(hdev, true, 0);
2732 hdev->mmu_huge_page_opt ? 0x30440 : 0x40440);
2748 static int gaudi_load_firmware_to_device(struct hl_device *hdev)
2753 gaudi_init_scrambler_hbm(hdev);
2755 dst = hdev->pcie_bar[HBM_BAR_ID] + LINUX_FW_OFFSET;
2757 return hl_fw_load_fw_to_device(hdev, GAUDI_LINUX_FW_FILE, dst);
2760 static int gaudi_load_boot_fit_to_device(struct hl_device *hdev)
2764 dst = hdev->pcie_bar[SRAM_BAR_ID] + BOOT_FIT_SRAM_OFFSET;
2766 return hl_fw_load_fw_to_device(hdev, GAUDI_BOOT_FIT_FILE, dst);
2769 static void gaudi_read_device_fw_version(struct hl_device *hdev,
2779 dest = hdev->asic_prop.uboot_ver;
2784 dest = hdev->asic_prop.preboot_ver;
2788 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
2795 memcpy_fromio(dest, hdev->pcie_bar[SRAM_BAR_ID] + ver_off,
2798 dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n",
2804 static int gaudi_init_cpu(struct hl_device *hdev)
2806 struct gaudi_device *gaudi = hdev->asic_specific;
2809 if (!hdev->cpu_enable)
2819 WREG32(mmCPU_IF_CPU_MSB_ADDR, hdev->cpu_pci_msb_addr);
2821 rc = hl_fw_init_cpu(hdev, mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS,
2825 !hdev->bmc_enable, GAUDI_CPU_TIMEOUT_USEC,
2836 static int gaudi_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout)
2838 struct gaudi_device *gaudi = hdev->asic_specific;
2842 &hdev->kernel_queues[GAUDI_QUEUE_ID_CPU_PQ];
2845 if (!hdev->cpu_queues_enable)
2851 eq = &hdev->event_queue;
2860 lower_32_bits(hdev->cpu_accessible_dma_address));
2862 upper_32_bits(hdev->cpu_accessible_dma_address));
2882 hdev,
2890 dev_err(hdev->dev,
2899 static void gaudi_pre_hw_init(struct hl_device *hdev)
2951 static int gaudi_hw_init(struct hl_device *hdev)
2955 dev_info(hdev->dev, "Starting initialization of H/W\n");
2957 gaudi_pre_hw_init(hdev);
2959 gaudi_init_pci_dma_qmans(hdev);
2961 gaudi_init_hbm_dma_qmans(hdev);
2963 rc = gaudi_init_cpu(hdev);
2965 dev_err(hdev->dev, "failed to initialize CPU\n");
2970 gaudi_init_scrambler_sram(hdev);
2973 gaudi_init_scrambler_hbm(hdev);
2975 gaudi_init_golden_registers(hdev);
2977 rc = gaudi_mmu_init(hdev);
2981 gaudi_init_security(hdev);
2983 gaudi_init_mme_qmans(hdev);
2985 gaudi_init_tpc_qmans(hdev);
2987 hdev->asic_funcs->set_clock_gating(hdev);
2989 gaudi_enable_timestamp(hdev);
2992 rc = gaudi_enable_msi(hdev);
2997 rc = gaudi_init_cpu_queues(hdev, GAUDI_CPU_TIMEOUT_USEC);
2999 dev_err(hdev->dev, "failed to initialize CPU H/W queues %d\n",
3010 gaudi_disable_msi(hdev);
3012 gaudi_disable_mme_qmans(hdev);
3013 gaudi_disable_pci_dma_qmans(hdev);
3018 static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset)
3020 struct gaudi_device *gaudi = hdev->asic_specific;
3024 dev_err(hdev->dev, "GAUDI doesn't support soft-reset\n");
3028 if (hdev->pldm) {
3068 dev_info(hdev->dev,
3080 dev_err(hdev->dev,
3098 static int gaudi_suspend(struct hl_device *hdev)
3102 rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS);
3104 dev_err(hdev->dev, "Failed to disable PCI access from CPU\n");
3109 static int gaudi_resume(struct hl_device *hdev)
3111 return gaudi_init_iatu(hdev);
3114 static int gaudi_cb_mmap(struct hl_device *hdev, struct vm_area_struct *vma,
3122 rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr,
3125 dev_err(hdev->dev, "dma_mmap_coherent error %d", rc);
3130 static void gaudi_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi)
3132 struct gaudi_device *gaudi = hdev->asic_specific;
3367 dev_err(hdev->dev, "h/w queue %d is invalid. Can't set pi\n",
3382 static void gaudi_pqe_write(struct hl_device *hdev, __le64 *pqe,
3392 static void *gaudi_dma_alloc_coherent(struct hl_device *hdev, size_t size,
3395 void *kernel_addr = dma_alloc_coherent(&hdev->pdev->dev, size,
3405 static void gaudi_dma_free_coherent(struct hl_device *hdev, size_t size,
3411 dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, fixed_dma_handle);
3414 static void *gaudi_get_int_queue_base(struct hl_device *hdev,
3418 struct gaudi_device *gaudi = hdev->asic_specific;
3423 dev_err(hdev->dev, "Got invalid queue id %d\n", queue_id);
3434 static int gaudi_send_cpu_message(struct hl_device *hdev, u32 *msg,
3437 struct gaudi_device *gaudi = hdev->asic_specific;
3448 return hl_fw_send_cpu_message(hdev, GAUDI_QUEUE_ID_CPU_PQ, msg, len,
3452 static int gaudi_test_queue(struct hl_device *hdev, u32 hw_queue_id)
3461 if (hdev->pldm)
3468 fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL,
3471 dev_err(hdev->dev,
3479 fence_pkt = hdev->asic_funcs->asic_dma_pool_zalloc(hdev,
3483 dev_err(hdev->dev,
3498 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id,
3502 dev_err(hdev->dev,
3508 rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp, (tmp == fence_val),
3511 hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
3514 dev_err(hdev->dev,
3521 hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_pkt,
3524 hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr,
3529 static int gaudi_test_cpu_queue(struct hl_device *hdev)
3531 struct gaudi_device *gaudi = hdev->asic_specific;
3540 return hl_fw_test_cpu_queue(hdev);
3543 static int gaudi_test_queues(struct hl_device *hdev)
3547 for (i = 0 ; i < hdev->asic_prop.max_queues ; i++) {
3548 if (hdev->asic_prop.hw_queues_props[i].type == QUEUE_TYPE_EXT) {
3549 rc = gaudi_test_queue(hdev, i);
3555 rc = gaudi_test_cpu_queue(hdev);
3562 static void *gaudi_dma_pool_zalloc(struct hl_device *hdev, size_t size,
3570 kernel_addr = dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle);
3579 static void gaudi_dma_pool_free(struct hl_device *hdev, void *vaddr,
3585 dma_pool_free(hdev->dma_pool, vaddr, fixed_dma_addr);
3588 static void *gaudi_cpu_accessible_dma_pool_alloc(struct hl_device *hdev,
3591 return hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle);
3594 static void gaudi_cpu_accessible_dma_pool_free(struct hl_device *hdev,
3597 hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr);
3600 static int gaudi_dma_map_sg(struct hl_device *hdev, struct scatterlist *sgl,
3606 if (!dma_map_sg(&hdev->pdev->dev, sgl, nents, dir))
3616 static void gaudi_dma_unmap_sg(struct hl_device *hdev, struct scatterlist *sgl,
3626 dma_unmap_sg(&hdev->pdev->dev, sgl, nents, dir);
3629 static u32 gaudi_get_dma_desc_list_size(struct hl_device *hdev,
3671 static int gaudi_pin_memory_before_cs(struct hl_device *hdev,
3679 if (hl_userptr_is_pinned(hdev, addr, le32_to_cpu(user_dma_pkt->tsize),
3687 rc = hl_pin_host_memory(hdev, addr, le32_to_cpu(user_dma_pkt->tsize),
3694 rc = hdev->asic_funcs->asic_dma_map_sg(hdev, userptr->sgt->sgl,
3697 dev_err(hdev->dev, "failed to map sgt with DMA region\n");
3706 gaudi_get_dma_desc_list_size(hdev, userptr->sgt);
3712 hl_unpin_host_memory(hdev, userptr);
3718 static int gaudi_validate_dma_pkt_host(struct hl_device *hdev,
3736 dev_dbg(hdev->dev, "DMA direction is HOST --> DEVICE\n");
3740 dev_dbg(hdev->dev, "DMA direction is DEVICE --> HOST\n");
3750 rc = gaudi_pin_memory_before_cs(hdev, parser, user_dma_pkt,
3756 static int gaudi_validate_dma_pkt_no_mmu(struct hl_device *hdev,
3765 dev_dbg(hdev->dev, "DMA packet details:\n");
3766 dev_dbg(hdev->dev, "source == 0x%llx\n",
3768 dev_dbg(hdev->dev, "destination == 0x%llx\n", dst_addr);
3769 dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize));
3784 return gaudi_validate_dma_pkt_host(hdev, parser, user_dma_pkt,
3788 static int gaudi_validate_load_and_exe_pkt(struct hl_device *hdev,
3797 dev_err(hdev->dev,
3807 static int gaudi_validate_cb(struct hl_device *hdev,
3829 dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id);
3837 dev_err(hdev->dev,
3845 dev_err(hdev->dev,
3851 dev_err(hdev->dev, "User not allowed to use CP_DMA\n");
3856 dev_err(hdev->dev, "User not allowed to use STOP\n");
3861 dev_err(hdev->dev,
3867 rc = gaudi_validate_load_and_exe_pkt(hdev, parser,
3876 rc = gaudi_validate_dma_pkt_no_mmu(hdev, parser,
3891 dev_err(hdev->dev, "Invalid packet header 0x%x\n",
3911 static int gaudi_patch_dma_packet(struct hl_device *hdev,
3950 (!hl_userptr_is_pinned(hdev, addr,
3953 dev_err(hdev->dev, "Userptr 0x%llx + 0x%x NOT mapped\n",
4016 dev_err(hdev->dev,
4030 static int gaudi_patch_cb(struct hl_device *hdev,
4054 dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id);
4062 dev_err(hdev->dev,
4070 rc = gaudi_patch_dma_packet(hdev, parser,
4078 dev_err(hdev->dev,
4084 dev_err(hdev->dev, "User not allowed to use CP_DMA\n");
4089 dev_err(hdev->dev, "User not allowed to use STOP\n");
4107 dev_err(hdev->dev, "Invalid packet header 0x%x\n",
4120 static int gaudi_parse_cb_mmu(struct hl_device *hdev,
4136 rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx,
4141 dev_err(hdev->dev,
4148 parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr,
4171 rc = gaudi_validate_cb(hdev, parser, true);
4180 dev_err(hdev->dev, "user CB size mismatch\n");
4193 hl_cb_destroy(hdev, &hdev->kernel_cb_mgr,
4199 static int gaudi_parse_cb_no_mmu(struct hl_device *hdev,
4205 rc = gaudi_validate_cb(hdev, parser, false);
4210 rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx,
4214 dev_err(hdev->dev,
4220 parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr,
4230 rc = gaudi_patch_cb(hdev, parser);
4242 hl_cb_destroy(hdev, &hdev->kernel_cb_mgr,
4247 hl_userptr_delete_list(hdev, parser->job_userptr_list);
4251 static int gaudi_parse_cb_no_ext_queue(struct hl_device *hdev,
4254 struct asic_fixed_properties *asic_prop = &hdev->asic_prop;
4276 dev_err(hdev->dev,
4283 static int gaudi_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser)
4285 struct gaudi_device *gaudi = hdev->asic_specific;
4288 return gaudi_parse_cb_no_ext_queue(hdev, parser);
4291 return gaudi_parse_cb_mmu(hdev, parser);
4293 return gaudi_parse_cb_no_mmu(hdev, parser);
4296 static void gaudi_add_end_of_cb_packets(struct hl_device *hdev,
4301 struct gaudi_device *gaudi = hdev->asic_specific;
4330 static void gaudi_update_eq_ci(struct hl_device *hdev, u32 val)
4335 static int gaudi_memset_device_memory(struct hl_device *hdev, u64 addr,
4344 cb = hl_cb_kernel_create(hdev, PAGE_SIZE, false);
4363 job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true);
4365 dev_err(hdev->dev, "Failed to allocate a new job\n");
4372 if (err_cause && !hdev->init_done) {
4373 dev_dbg(hdev->dev,
4387 hl_debugfs_add_job(hdev, job);
4389 rc = gaudi_send_job_on_qman0(hdev, job);
4390 hl_debugfs_remove_job(hdev, job);
4397 dev_err(hdev->dev, "DMA Failed, cause 0x%x\n", err_cause);
4399 if (!hdev->init_done) {
4400 dev_dbg(hdev->dev,
4409 hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT);
4414 static void gaudi_restore_sm_registers(struct hl_device *hdev)
4441 static void gaudi_restore_dma_registers(struct hl_device *hdev)
4468 static void gaudi_restore_qm_registers(struct hl_device *hdev)
4489 static void gaudi_restore_user_registers(struct hl_device *hdev)
4491 gaudi_restore_sm_registers(hdev);
4492 gaudi_restore_dma_registers(hdev);
4493 gaudi_restore_qm_registers(hdev);
4496 static int gaudi_context_switch(struct hl_device *hdev, u32 asid)
4498 struct asic_fixed_properties *prop = &hdev->asic_prop;
4500 u32 size = hdev->pldm ? 0x10000 :
4505 rc = gaudi_memset_device_memory(hdev, addr, size, val);
4507 dev_err(hdev->dev, "Failed to clear SRAM in context switch\n");
4511 gaudi_mmu_prepare(hdev, asid);
4513 gaudi_restore_user_registers(hdev);
4518 static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev)
4520 struct asic_fixed_properties *prop = &hdev->asic_prop;
4521 struct gaudi_device *gaudi = hdev->asic_specific;
4528 return gaudi_memset_device_memory(hdev, addr, size, 0);
4531 static void gaudi_restore_phase_topology(struct hl_device *hdev)
4536 static int gaudi_debugfs_read32(struct hl_device *hdev, u64 addr, u32 *val)
4538 struct asic_fixed_properties *prop = &hdev->asic_prop;
4539 struct gaudi_device *gaudi = hdev->asic_specific;
4546 (hdev->clock_gating_mask &
4549 dev_err_ratelimited(hdev->dev,
4558 *val = readl(hdev->pcie_bar[SRAM_BAR_ID] +
4560 } else if (addr < DRAM_PHYS_BASE + hdev->asic_prop.dram_size) {
4564 hbm_bar_addr = gaudi_set_hbm_bar_base(hdev, bar_base_addr);
4566 *val = readl(hdev->pcie_bar[HBM_BAR_ID] +
4569 hbm_bar_addr = gaudi_set_hbm_bar_base(hdev,
4583 static int gaudi_debugfs_write32(struct hl_device *hdev, u64 addr, u32 val)
4585 struct asic_fixed_properties *prop = &hdev->asic_prop;
4586 struct gaudi_device *gaudi = hdev->asic_specific;
4593 (hdev->clock_gating_mask &
4596 dev_err_ratelimited(hdev->dev,
4605 writel(val, hdev->pcie_bar[SRAM_BAR_ID] +
4607 } else if (addr < DRAM_PHYS_BASE + hdev->asic_prop.dram_size) {
4611 hbm_bar_addr = gaudi_set_hbm_bar_base(hdev, bar_base_addr);
4613 writel(val, hdev->pcie_bar[HBM_BAR_ID] +
4616 hbm_bar_addr = gaudi_set_hbm_bar_base(hdev,
4630 static int gaudi_debugfs_read64(struct hl_device *hdev, u64 addr, u64 *val)
4632 struct asic_fixed_properties *prop = &hdev->asic_prop;
4633 struct gaudi_device *gaudi = hdev->asic_specific;
4640 (hdev->clock_gating_mask &
4643 dev_err_ratelimited(hdev->dev,
4655 *val = readq(hdev->pcie_bar[SRAM_BAR_ID] +
4658 DRAM_PHYS_BASE + hdev->asic_prop.dram_size - sizeof(u64)) {
4662 hbm_bar_addr = gaudi_set_hbm_bar_base(hdev, bar_base_addr);
4664 *val = readq(hdev->pcie_bar[HBM_BAR_ID] +
4667 hbm_bar_addr = gaudi_set_hbm_bar_base(hdev,
4681 static int gaudi_debugfs_write64(struct hl_device *hdev, u64 addr, u64 val)
4683 struct asic_fixed_properties *prop = &hdev->asic_prop;
4684 struct gaudi_device *gaudi = hdev->asic_specific;
4691 (hdev->clock_gating_mask &
4694 dev_err_ratelimited(hdev->dev,
4705 writeq(val, hdev->pcie_bar[SRAM_BAR_ID] +
4708 DRAM_PHYS_BASE + hdev->asic_prop.dram_size - sizeof(u64)) {
4712 hbm_bar_addr = gaudi_set_hbm_bar_base(hdev, bar_base_addr);
4714 writeq(val, hdev->pcie_bar[HBM_BAR_ID] +
4717 hbm_bar_addr = gaudi_set_hbm_bar_base(hdev,
4731 static u64 gaudi_read_pte(struct hl_device *hdev, u64 addr)
4733 struct gaudi_device *gaudi = hdev->asic_specific;
4735 if (hdev->hard_reset_pending)
4738 return readq(hdev->pcie_bar[HBM_BAR_ID] +
4742 static void gaudi_write_pte(struct hl_device *hdev, u64 addr, u64 val)
4744 struct gaudi_device *gaudi = hdev->asic_specific;
4746 if (hdev->hard_reset_pending)
4749 writeq(val, hdev->pcie_bar[HBM_BAR_ID] +
4753 void gaudi_mmu_prepare_reg(struct hl_device *hdev, u64 reg, u32 asid)
4760 static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid)
4762 struct gaudi_device *gaudi = hdev->asic_specific;
4774 hdev->asic_funcs->disable_clock_gating(hdev);
4776 gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_0, asid);
4777 gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_1, asid);
4778 gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_2, asid);
4779 gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_3, asid);
4780 gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_4, asid);
4782 gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_0, asid);
4783 gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_1, asid);
4784 gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_2, asid);
4785 gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_3, asid);
4786 gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_4, asid);
4788 gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_0, asid);
4789 gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_1, asid);
4790 gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_2, asid);
4791 gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_3, asid);
4792 gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_4, asid);
4794 gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_0, asid);
4795 gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_1, asid);
4796 gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_2, asid);
4797 gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_3, asid);
4798 gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_4, asid);
4800 gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_0, asid);
4801 gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_1, asid);
4802 gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_2, asid);
4803 gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_3, asid);
4804 gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_4, asid);
4806 gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_0, asid);
4807 gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_1, asid);
4808 gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_2, asid);
4809 gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_3, asid);
4810 gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_4, asid);
4812 gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_0, asid);
4813 gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_1, asid);
4814 gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_2, asid);
4815 gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_3, asid);
4816 gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_4, asid);
4818 gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_0, asid);
4819 gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_1, asid);
4820 gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_2, asid);
4821 gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_3, asid);
4822 gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_4, asid);
4824 gaudi_mmu_prepare_reg(hdev, mmDMA0_CORE_NON_SECURE_PROPS, asid);
4825 gaudi_mmu_prepare_reg(hdev, mmDMA1_CORE_NON_SECURE_PROPS, asid);
4826 gaudi_mmu_prepare_reg(hdev, mmDMA2_CORE_NON_SECURE_PROPS, asid);
4827 gaudi_mmu_prepare_reg(hdev, mmDMA3_CORE_NON_SECURE_PROPS, asid);
4828 gaudi_mmu_prepare_reg(hdev, mmDMA4_CORE_NON_SECURE_PROPS, asid);
4829 gaudi_mmu_prepare_reg(hdev, mmDMA5_CORE_NON_SECURE_PROPS, asid);
4830 gaudi_mmu_prepare_reg(hdev, mmDMA6_CORE_NON_SECURE_PROPS, asid);
4831 gaudi_mmu_prepare_reg(hdev, mmDMA7_CORE_NON_SECURE_PROPS, asid);
4833 gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_0, asid);
4834 gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_1, asid);
4835 gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_2, asid);
4836 gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_3, asid);
4837 gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_4, asid);
4838 gaudi_mmu_prepare_reg(hdev, mmTPC0_CFG_ARUSER_LO, asid);
4839 gaudi_mmu_prepare_reg(hdev, mmTPC0_CFG_AWUSER_LO, asid);
4841 gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_0, asid);
4842 gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_1, asid);
4843 gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_2, asid);
4844 gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_3, asid);
4845 gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_4, asid);
4846 gaudi_mmu_prepare_reg(hdev, mmTPC1_CFG_ARUSER_LO, asid);
4847 gaudi_mmu_prepare_reg(hdev, mmTPC1_CFG_AWUSER_LO, asid);
4849 gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_0, asid);
4850 gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_1, asid);
4851 gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_2, asid);
4852 gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_3, asid);
4853 gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_4, asid);
4854 gaudi_mmu_prepare_reg(hdev, mmTPC2_CFG_ARUSER_LO, asid);
4855 gaudi_mmu_prepare_reg(hdev, mmTPC2_CFG_AWUSER_LO, asid);
4857 gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_0, asid);
4858 gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_1, asid);
4859 gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_2, asid);
4860 gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_3, asid);
4861 gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_4, asid);
4862 gaudi_mmu_prepare_reg(hdev, mmTPC3_CFG_ARUSER_LO, asid);
4863 gaudi_mmu_prepare_reg(hdev, mmTPC3_CFG_AWUSER_LO, asid);
4865 gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_0, asid);
4866 gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_1, asid);
4867 gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_2, asid);
4868 gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_3, asid);
4869 gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_4, asid);
4870 gaudi_mmu_prepare_reg(hdev, mmTPC4_CFG_ARUSER_LO, asid);
4871 gaudi_mmu_prepare_reg(hdev, mmTPC4_CFG_AWUSER_LO, asid);
4873 gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_0, asid);
4874 gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_1, asid);
4875 gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_2, asid);
4876 gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_3, asid);
4877 gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_4, asid);
4878 gaudi_mmu_prepare_reg(hdev, mmTPC5_CFG_ARUSER_LO, asid);
4879 gaudi_mmu_prepare_reg(hdev, mmTPC5_CFG_AWUSER_LO, asid);
4881 gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_0, asid);
4882 gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_1, asid);
4883 gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_2, asid);
4884 gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_3, asid);
4885 gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_4, asid);
4886 gaudi_mmu_prepare_reg(hdev, mmTPC6_CFG_ARUSER_LO, asid);
4887 gaudi_mmu_prepare_reg(hdev, mmTPC6_CFG_AWUSER_LO, asid);
4889 gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_0, asid);
4890 gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_1, asid);
4891 gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_2, asid);
4892 gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_3, asid);
4893 gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_4, asid);
4894 gaudi_mmu_prepare_reg(hdev, mmTPC7_CFG_ARUSER_LO, asid);
4895 gaudi_mmu_prepare_reg(hdev, mmTPC7_CFG_AWUSER_LO, asid);
4897 gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_0, asid);
4898 gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_1, asid);
4899 gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_2, asid);
4900 gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_3, asid);
4901 gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_4, asid);
4902 gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_0, asid);
4903 gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_1, asid);
4904 gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_2, asid);
4905 gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_3, asid);
4906 gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_4, asid);
4908 gaudi_mmu_prepare_reg(hdev, mmMME0_SBAB_ARUSER0, asid);
4909 gaudi_mmu_prepare_reg(hdev, mmMME0_SBAB_ARUSER1, asid);
4910 gaudi_mmu_prepare_reg(hdev, mmMME1_SBAB_ARUSER0, asid);
4911 gaudi_mmu_prepare_reg(hdev, mmMME1_SBAB_ARUSER1, asid);
4912 gaudi_mmu_prepare_reg(hdev, mmMME2_SBAB_ARUSER0, asid);
4913 gaudi_mmu_prepare_reg(hdev, mmMME2_SBAB_ARUSER1, asid);
4914 gaudi_mmu_prepare_reg(hdev, mmMME3_SBAB_ARUSER0, asid);
4915 gaudi_mmu_prepare_reg(hdev, mmMME3_SBAB_ARUSER1, asid);
4916 gaudi_mmu_prepare_reg(hdev, mmMME0_ACC_WBC, asid);
4917 gaudi_mmu_prepare_reg(hdev, mmMME1_ACC_WBC, asid);
4918 gaudi_mmu_prepare_reg(hdev, mmMME2_ACC_WBC, asid);
4919 gaudi_mmu_prepare_reg(hdev, mmMME3_ACC_WBC, asid);
4921 hdev->asic_funcs->set_clock_gating(hdev);
4926 static int gaudi_send_job_on_qman0(struct hl_device *hdev,
4936 if (hdev->pldm)
4941 if (!hdev->asic_funcs->is_device_idle(hdev, NULL, NULL)) {
4942 dev_err_ratelimited(hdev->dev,
4947 fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL,
4950 dev_err(hdev->dev,
4972 rc = hl_hw_queue_send_cb_no_cmpl(hdev, GAUDI_QUEUE_ID_DMA_0_0,
4975 dev_err(hdev->dev, "Failed to send CB on QMAN0, %d\n", rc);
4979 rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp,
4983 hl_hw_queue_inc_ci_kernel(hdev, GAUDI_QUEUE_ID_DMA_0_0);
4986 dev_err(hdev->dev, "QMAN0 Job timeout (0x%x)\n", tmp);
4994 hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr,
5015 static const char *gaudi_get_razwi_initiator_dma_name(struct hl_device *hdev,
5092 static const char *gaudi_get_razwi_initiator_name(struct hl_device *hdev,
5139 return gaudi_get_razwi_initiator_dma_name(hdev, x_y, is_write);
5170 dev_err(hdev->dev,
5181 static void gaudi_print_razwi_info(struct hl_device *hdev)
5184 dev_err_ratelimited(hdev->dev,
5186 gaudi_get_razwi_initiator_name(hdev, true));
5191 dev_err_ratelimited(hdev->dev,
5193 gaudi_get_razwi_initiator_name(hdev, false));
5198 static void gaudi_print_mmu_error_info(struct hl_device *hdev)
5200 struct gaudi_device *gaudi = hdev->asic_specific;
5213 dev_err_ratelimited(hdev->dev, "MMU page fault on va 0x%llx\n",
5225 dev_err_ratelimited(hdev->dev,
5250 static int gaudi_extract_ecc_info(struct hl_device *hdev,
5254 struct gaudi_device *gaudi = hdev->asic_specific;
5272 hdev->asic_funcs->disable_clock_gating(hdev);
5290 dev_err(hdev->dev, "ECC error information cannot be found\n");
5314 hdev->asic_funcs->set_clock_gating(hdev);
5322 static void gaudi_handle_qman_err_generic(struct hl_device *hdev,
5345 dev_err_ratelimited(hdev->dev,
5364 dev_err_ratelimited(hdev->dev,
5372 static void gaudi_handle_ecc_event(struct hl_device *hdev, u16 event_type,
5458 rc = gaudi_extract_ecc_info(hdev, ¶ms, &ecc_address,
5464 dev_err(hdev->dev,
5469 static void gaudi_handle_qman_err(struct hl_device *hdev, u16 event_type)
5504 gaudi_handle_qman_err_generic(hdev, desc, glbl_sts_addr, arb_err_addr);
5507 static void gaudi_print_irq_info(struct hl_device *hdev, u16 event_type,
5513 dev_err_ratelimited(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n",
5517 gaudi_print_razwi_info(hdev);
5518 gaudi_print_mmu_error_info(hdev);
5522 static int gaudi_soft_reset_late_init(struct hl_device *hdev)
5524 struct gaudi_device *gaudi = hdev->asic_specific;
5529 return hl_fw_unmask_irq_arr(hdev, gaudi->events, sizeof(gaudi->events));
5532 static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device)
5543 dev_err(hdev->dev,
5550 dev_err(hdev->dev,
5563 dev_err(hdev->dev,
5570 dev_err(hdev->dev,
5592 dev_err(hdev->dev,
5600 dev_err(hdev->dev,
5631 static bool gaudi_tpc_read_interrupts(struct hl_device *hdev, u8 tpc_id,
5634 struct gaudi_device *gaudi = hdev->asic_specific;
5645 hdev->asic_funcs->disable_clock_gating(hdev);
5652 dev_err_ratelimited(hdev->dev,
5664 hdev->asic_funcs->set_clock_gating(hdev);
5681 static void gaudi_print_clk_change_info(struct hl_device *hdev,
5686 hdev->clk_throttling_reason |= HL_CLK_THROTTLE_POWER;
5687 dev_info_ratelimited(hdev->dev,
5692 hdev->clk_throttling_reason &= ~HL_CLK_THROTTLE_POWER;
5693 dev_info_ratelimited(hdev->dev,
5698 hdev->clk_throttling_reason |= HL_CLK_THROTTLE_THERMAL;
5699 dev_info_ratelimited(hdev->dev,
5704 hdev->clk_throttling_reason &= ~HL_CLK_THROTTLE_THERMAL;
5705 dev_info_ratelimited(hdev->dev,
5710 dev_err(hdev->dev, "Received invalid clock change event %d\n",
5716 static void gaudi_handle_eqe(struct hl_device *hdev,
5719 struct gaudi_device *gaudi = hdev->asic_specific;
5727 dev_err(hdev->dev, "Event type %u exceeds maximum of %u",
5757 gaudi_print_irq_info(hdev, event_type, true);
5758 gaudi_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data);
5759 if (hdev->hard_reset_on_fw_events)
5760 hl_device_reset(hdev, true, false);
5767 gaudi_print_irq_info(hdev, event_type, false);
5768 if (hdev->hard_reset_on_fw_events)
5769 hl_device_reset(hdev, true, false);
5776 gaudi_print_irq_info(hdev, event_type, false);
5777 gaudi_hbm_read_interrupts(hdev,
5779 if (hdev->hard_reset_on_fw_events)
5780 hl_device_reset(hdev, true, false);
5787 gaudi_print_irq_info(hdev, event_type, false);
5788 gaudi_hbm_read_interrupts(hdev,
5800 gaudi_print_irq_info(hdev, event_type, true);
5801 reset_required = gaudi_tpc_read_interrupts(hdev,
5805 dev_err(hdev->dev, "hard reset required due to %s\n",
5808 if (hdev->hard_reset_on_fw_events)
5809 hl_device_reset(hdev, true, false);
5811 hl_fw_unmask_irq(hdev, event_type);
5823 gaudi_print_irq_info(hdev, event_type, true);
5824 reset_required = gaudi_tpc_read_interrupts(hdev,
5828 dev_err(hdev->dev, "hard reset required due to %s\n",
5831 if (hdev->hard_reset_on_fw_events)
5832 hl_device_reset(hdev, true, false);
5834 hl_fw_unmask_irq(hdev, event_type);
5859 gaudi_print_irq_info(hdev, event_type, true);
5860 gaudi_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data);
5861 hl_fw_unmask_irq(hdev, event_type);
5884 gaudi_print_irq_info(hdev, event_type, true);
5885 gaudi_handle_qman_err(hdev, event_type);
5886 hl_fw_unmask_irq(hdev, event_type);
5890 gaudi_print_irq_info(hdev, event_type, true);
5891 if (hdev->hard_reset_on_fw_events)
5892 hl_device_reset(hdev, true, false);
5904 gaudi_print_irq_info(hdev, event_type, false);
5905 hl_fw_unmask_irq(hdev, event_type);
5909 gaudi_print_clk_change_info(hdev, event_type);
5910 hl_fw_unmask_irq(hdev, event_type);
5915 dev_err(hdev->dev,
5921 dev_err(hdev->dev, "Received invalid H/W interrupt %d\n",
5927 static void *gaudi_get_events_stat(struct hl_device *hdev, bool aggregate,
5930 struct gaudi_device *gaudi = hdev->asic_specific;
5941 static int gaudi_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard,
5944 struct gaudi_device *gaudi = hdev->asic_specific;
5949 hdev->hard_reset_pending)
5952 if (hdev->pldm)
5957 mutex_lock(&hdev->mmu_cache_lock);
5965 hdev,
5974 mutex_unlock(&hdev->mmu_cache_lock);
5977 dev_err_ratelimited(hdev->dev,
5979 hl_device_reset(hdev, true, false);
5985 static int gaudi_mmu_invalidate_cache_range(struct hl_device *hdev,
5988 struct gaudi_device *gaudi = hdev->asic_specific;
5995 hdev->hard_reset_pending)
5998 mutex_lock(&hdev->mmu_cache_lock);
6000 if (hdev->pldm)
6020 hdev,
6027 mutex_unlock(&hdev->mmu_cache_lock);
6030 dev_err_ratelimited(hdev->dev,
6032 hl_device_reset(hdev, true, false);
6038 static int gaudi_mmu_update_asid_hop0_addr(struct hl_device *hdev,
6044 if (hdev->pldm)
6055 hdev,
6063 dev_err(hdev->dev,
6071 static int gaudi_send_heartbeat(struct hl_device *hdev)
6073 struct gaudi_device *gaudi = hdev->asic_specific;
6078 return hl_fw_send_heartbeat(hdev);
6081 static int gaudi_cpucp_info_get(struct hl_device *hdev)
6083 struct gaudi_device *gaudi = hdev->asic_specific;
6084 struct asic_fixed_properties *prop = &hdev->asic_prop;
6090 rc = hl_fw_cpucp_info_get(hdev);
6098 hdev->card_type = le32_to_cpu(hdev->asic_prop.cpucp_info.card_type);
6100 if (hdev->card_type == cpucp_card_type_pci)
6102 else if (hdev->card_type == cpucp_card_type_pmc)
6105 hdev->max_power = prop->max_power_default;
6110 static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask,
6113 struct gaudi_device *gaudi = hdev->asic_specific;
6123 hdev->asic_funcs->disable_clock_gating(hdev);
6211 hdev->asic_funcs->set_clock_gating(hdev);
6218 static void gaudi_hw_queues_lock(struct hl_device *hdev)
6221 struct gaudi_device *gaudi = hdev->asic_specific;
6226 static void gaudi_hw_queues_unlock(struct hl_device *hdev)
6229 struct gaudi_device *gaudi = hdev->asic_specific;
6234 static u32 gaudi_get_pci_id(struct hl_device *hdev)
6236 return hdev->pdev->device;
6239 static int gaudi_get_eeprom_data(struct hl_device *hdev, void *data,
6242 struct gaudi_device *gaudi = hdev->asic_specific;
6247 return hl_fw_get_eeprom_data(hdev, data, max_size);
6254 static int gaudi_run_tpc_kernel(struct hl_device *hdev, u64 tpc_kernel,
6257 struct gaudi_device *gaudi = hdev->asic_specific;
6264 if (hdev->pldm)
6271 hdev->asic_funcs->disable_clock_gating(hdev);
6300 hdev,
6309 dev_err(hdev->dev,
6312 hdev->asic_funcs->set_clock_gating(hdev);
6325 hdev,
6334 dev_err(hdev->dev,
6337 hdev->asic_funcs->set_clock_gating(hdev);
6343 hdev,
6350 hdev->asic_funcs->set_clock_gating(hdev);
6354 dev_err(hdev->dev,
6363 static enum hl_device_hw_state gaudi_get_hw_state(struct hl_device *hdev)
6373 static u32 gaudi_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx)
6378 static u32 gaudi_get_signal_cb_size(struct hl_device *hdev)
6384 static u32 gaudi_get_wait_cb_size(struct hl_device *hdev)
6391 static void gaudi_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id)
6485 static void gaudi_gen_wait_cb(struct hl_device *hdev, void *data, u16 sob_id,
6533 dev_crit(hdev->dev, "wrong queue id %d for wait packet\n",
6583 static void gaudi_reset_sob(struct hl_device *hdev, void *data)
6587 dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx,
6596 static void gaudi_set_dma_mask_from_fw(struct hl_device *hdev)
6600 hdev->power9_64bit_dma_enable = 1;
6601 hdev->dma_mask = 64;
6603 hdev->power9_64bit_dma_enable = 0;
6604 hdev->dma_mask = 48;
6608 static u64 gaudi_get_device_time(struct hl_device *hdev)
6693 * @hdev: pointer to hl_device structure
6696 void gaudi_set_asic_funcs(struct hl_device *hdev)
6698 hdev->asic_funcs = &gaudi_funcs;