Lines Matching defs:hdev

475 static int gaudi_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 asid,
477 static int gaudi_send_job_on_qman0(struct hl_device *hdev,
479 static int gaudi_memset_device_memory(struct hl_device *hdev, u64 addr,
481 static int gaudi_memset_registers(struct hl_device *hdev, u64 reg_base,
483 static int gaudi_run_tpc_kernel(struct hl_device *hdev, u64 tpc_kernel,
485 static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev);
486 static int gaudi_cpucp_info_get(struct hl_device *hdev);
487 static void gaudi_disable_clock_gating(struct hl_device *hdev);
488 static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid);
489 static u32 gaudi_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id,
491 static u32 gaudi_gen_wait_cb(struct hl_device *hdev,
494 get_collective_mode(struct hl_device *hdev, u32 queue_id)
514 static inline void set_default_power_values(struct hl_device *hdev)
516 struct asic_fixed_properties *prop = &hdev->asic_prop;
518 if (hdev->card_type == cpucp_card_type_pmc) {
531 static int gaudi_set_fixed_properties(struct hl_device *hdev)
533 struct asic_fixed_properties *prop = &hdev->asic_prop;
568 get_collective_mode(hdev, i);
608 if (hdev->pldm)
655 set_default_power_values(hdev);
707 static int gaudi_pci_bars_map(struct hl_device *hdev)
713 rc = hl_pci_bars_map(hdev, name, is_wc);
717 hdev->rmmio = hdev->pcie_bar[CFG_BAR_ID] +
723 static u64 gaudi_set_hbm_bar_base(struct hl_device *hdev, u64 addr)
725 struct gaudi_device *gaudi = hdev->asic_specific;
733 if (hdev->asic_prop.iatu_done_by_fw)
740 rc = hl_pci_set_inbound_region(hdev, 2, &pci_region);
752 static int gaudi_init_iatu(struct hl_device *hdev)
758 if (hdev->asic_prop.iatu_done_by_fw)
765 rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region);
773 rc = hl_pci_set_inbound_region(hdev, 1, &inbound_region);
781 rc = hl_pci_set_inbound_region(hdev, 2, &inbound_region);
788 rc = hl_pci_set_outbound_region(hdev, &outbound_region);
794 static enum hl_device_hw_state gaudi_get_hw_state(struct hl_device *hdev)
799 static int gaudi_early_init(struct hl_device *hdev)
801 struct asic_fixed_properties *prop = &hdev->asic_prop;
802 struct pci_dev *pdev = hdev->pdev;
807 rc = gaudi_set_fixed_properties(hdev);
809 dev_err(hdev->dev, "Failed setting fixed properties\n");
817 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n",
826 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n",
833 hdev->dram_pci_bar_start = pci_resource_start(pdev, HBM_BAR_ID);
836 if (hdev->asic_prop.fw_security_enabled) {
837 hdev->asic_prop.iatu_done_by_fw = true;
843 hdev->asic_prop.gic_interrupts_enable = false;
847 rc = hl_pci_elbi_read(hdev, CFG_BASE + mmCPU_BOOT_DEV_STS0,
855 hdev->asic_prop.iatu_done_by_fw = true;
858 rc = hl_pci_init(hdev);
865 rc = hl_fw_read_preboot_status(hdev);
867 if (hdev->reset_on_preboot_fail)
869 hdev->asic_funcs->hw_fini(hdev, true, false);
873 if (gaudi_get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {
874 dev_dbg(hdev->dev, "H/W state is dirty, must reset before initializing\n");
875 rc = hdev->asic_funcs->hw_fini(hdev, true, false);
877 dev_err(hdev->dev, "failed to reset HW in dirty state (%d)\n", rc);
885 hl_pci_fini(hdev);
887 kfree(hdev->asic_prop.hw_queues_props);
891 static int gaudi_early_fini(struct hl_device *hdev)
893 kfree(hdev->asic_prop.hw_queues_props);
894 hl_pci_fini(hdev);
902 * @hdev: pointer to hl_device structure
905 static int gaudi_fetch_psoc_frequency(struct hl_device *hdev)
908 struct asic_fixed_properties *prop = &hdev->asic_prop;
912 if ((hdev->fw_components & FW_TYPE_LINUX) &&
914 struct gaudi_device *gaudi = hdev->asic_specific;
919 rc = hl_fw_cpucp_pll_info_get(hdev, HL_GAUDI_CPU_PLL, pll_freq_arr);
948 dev_warn(hdev->dev, "Received invalid div select value: %#x", div_sel);
962 static int _gaudi_init_tpc_mem(struct hl_device *hdev,
965 struct asic_fixed_properties *prop = &hdev->asic_prop;
974 cb = hl_cb_kernel_create(hdev, PAGE_SIZE, false);
998 job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true);
1000 dev_err(hdev->dev, "Failed to allocate a new job\n");
1013 hl_debugfs_add_job(hdev, job);
1015 rc = gaudi_send_job_on_qman0(hdev, job);
1021 rc = gaudi_run_tpc_kernel(hdev, dst_addr, tpc_id);
1027 hl_userptr_delete_list(hdev, &job->userptr_list);
1028 hl_debugfs_remove_job(hdev, job);
1034 hl_cb_destroy(&hdev->kernel_mem_mgr, cb->buf->handle);
1041 * @hdev: Pointer to hl_device structure.
1047 static int gaudi_init_tpc_mem(struct hl_device *hdev)
1056 rc = request_firmware(&fw, GAUDI_TPC_FW_FILE, hdev->dev);
1063 dev_err(hdev->dev, "Failed to load firmware file %s\n",
1069 cpu_addr = hl_asic_dma_alloc_coherent(hdev, fw_size, &dma_handle, GFP_KERNEL | __GFP_ZERO);
1071 dev_err(hdev->dev,
1080 rc = _gaudi_init_tpc_mem(hdev, dma_handle, fw_size);
1082 hl_asic_dma_free_coherent(hdev, fw->size, cpu_addr, dma_handle);
1089 static void gaudi_collective_map_sobs(struct hl_device *hdev, u32 stream)
1091 struct gaudi_device *gaudi = hdev->asic_specific;
1103 q = &hdev->kernel_queues[queue_id + (4 * i)];
1111 q = &hdev->kernel_queues[queue_id];
1116 q = &hdev->kernel_queues[queue_id];
1125 struct hl_device *hdev = hw_sob_group->hdev;
1139 struct hl_device *hdev = hw_sob_group->hdev;
1141 dev_crit(hdev->dev,
1164 static int gaudi_collective_init(struct hl_device *hdev)
1170 gaudi = hdev->asic_specific;
1172 sob_id = hdev->asic_prop.collective_first_sob;
1180 prop->hw_sob_group[i].hdev = hdev;
1189 gaudi_collective_map_sobs(hdev, i);
1197 static void gaudi_reset_sob_group(struct hl_device *hdev, u16 sob_group)
1199 struct gaudi_device *gaudi = hdev->asic_specific;
1206 static void gaudi_collective_master_init_job(struct hl_device *hdev,
1215 gaudi = hdev->asic_specific;
1218 prop = &hdev->kernel_queues[queue_id].sync_stream_prop;
1226 dev_dbg(hdev->dev,
1239 cb_size += gaudi_gen_wait_cb(hdev, &wait_prop);
1244 dev_dbg(hdev->dev,
1254 cb_size += gaudi_gen_wait_cb(hdev, &wait_prop);
1257 static void gaudi_collective_slave_init_job(struct hl_device *hdev,
1265 prop = &hdev->kernel_queues[queue_id].sync_stream_prop;
1272 hl_hw_queue_encaps_sig_set_sob_info(hdev, job->cs, job,
1275 dev_dbg(hdev->dev, "collective wait: Sequence %llu found, sob_id: %u, wait for sob_val: %u\n",
1290 dev_dbg(hdev->dev,
1295 cb_size += gaudi_gen_wait_cb(hdev, &wait_prop);
1297 dev_dbg(hdev->dev,
1301 cb_size += gaudi_gen_signal_cb(hdev, job->user_cb,
1315 struct hl_device *hdev;
1320 hdev = ctx->hdev;
1321 gaudi = hdev->asic_specific;
1371 if (hdev->kernel_queues[queue_id].collective_mode ==
1373 gaudi_collective_master_init_job(hdev, job, stream,
1376 gaudi_collective_slave_init_job(hdev, job, cs_cmpl);
1399 gaudi_collective_map_sobs(hdev, stream);
1401 dev_dbg(hdev->dev, "switched to SOB group %d, stream: %d\n",
1425 static int gaudi_collective_wait_create_job(struct hl_device *hdev,
1437 cntr = &hdev->aggregated_cs_counters;
1462 hw_queue_prop = &hdev->asic_prop.hw_queues_props[queue_id];
1463 job = hl_cs_allocate_job(hdev, hw_queue_prop->type, true);
1467 dev_err(hdev->dev, "Failed to allocate a new job\n");
1472 cb = hl_cb_kernel_create(hdev, cb_size, !patched_cb);
1506 hl_cb_destroy(&hdev->kernel_mem_mgr, cb->buf->handle);
1516 hl_debugfs_add_job(hdev, job);
1521 static int gaudi_collective_wait_create_jobs(struct hl_device *hdev,
1526 struct gaudi_device *gaudi = hdev->asic_specific;
1534 hw_queue_prop = &hdev->asic_prop.hw_queues_props[wait_queue_id];
1536 dev_err(hdev->dev,
1545 dev_err(hdev->dev,
1573 rc = gaudi_collective_wait_create_job(hdev, ctx, cs,
1594 rc = gaudi_collective_wait_create_job(hdev, ctx, cs,
1606 static int gaudi_late_init(struct hl_device *hdev)
1608 struct gaudi_device *gaudi = hdev->asic_specific;
1611 rc = gaudi->cpucp_info_get(hdev);
1613 dev_err(hdev->dev, "Failed to get cpucp info\n");
1617 if ((hdev->card_type == cpucp_card_type_pci) &&
1618 (hdev->nic_ports_mask & 0x3)) {
1619 dev_info(hdev->dev,
1621 hdev->nic_ports_mask &= ~0x3;
1638 rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_ENABLE_PCI_ACCESS, 0x0);
1640 dev_err(hdev->dev, "Failed to enable PCI access from CPU\n");
1645 rc = hdev->asic_funcs->scrub_device_mem(hdev);
1649 rc = gaudi_fetch_psoc_frequency(hdev);
1651 dev_err(hdev->dev, "Failed to fetch psoc frequency\n");
1655 rc = gaudi_mmu_clear_pgt_range(hdev);
1657 dev_err(hdev->dev, "Failed to clear MMU page tables range\n");
1661 rc = gaudi_init_tpc_mem(hdev);
1663 dev_err(hdev->dev, "Failed to initialize TPC memories\n");
1667 rc = gaudi_collective_init(hdev);
1669 dev_err(hdev->dev, "Failed to init collective\n");
1676 gaudi_mmu_prepare(hdev, 1);
1678 hl_fw_set_pll_profile(hdev);
1683 hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0);
1688 static void gaudi_late_fini(struct hl_device *hdev)
1690 hl_hwmon_release_resources(hdev);
1693 static int gaudi_alloc_cpu_accessible_dma_mem(struct hl_device *hdev)
1709 virt_addr_arr[i] = hl_asic_dma_alloc_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE,
1724 dev_err(hdev->dev,
1730 hdev->cpu_accessible_dma_mem = virt_addr_arr[i];
1731 hdev->cpu_accessible_dma_address = dma_addr_arr[i];
1732 hdev->cpu_pci_msb_addr =
1733 GAUDI_CPU_PCI_MSB_ADDR(hdev->cpu_accessible_dma_address);
1735 if (!hdev->asic_prop.fw_security_enabled)
1736 GAUDI_PCI_TO_CPU_ADDR(hdev->cpu_accessible_dma_address);
1740 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, virt_addr_arr[j],
1746 static void gaudi_free_internal_qmans_pq_mem(struct hl_device *hdev)
1748 struct gaudi_device *gaudi = hdev->asic_specific;
1756 hl_asic_dma_free_coherent(hdev, q->pq_size, q->pq_kernel_addr, q->pq_dma_addr);
1760 static int gaudi_alloc_internal_qmans_pq_mem(struct hl_device *hdev)
1762 struct gaudi_device *gaudi = hdev->asic_specific;
1786 dev_err(hdev->dev, "Bad internal queue index %d", i);
1791 q->pq_kernel_addr = hl_asic_dma_alloc_coherent(hdev, q->pq_size, &q->pq_dma_addr,
1802 gaudi_free_internal_qmans_pq_mem(hdev);
1806 static void gaudi_set_pci_memory_regions(struct hl_device *hdev)
1808 struct asic_fixed_properties *prop = &hdev->asic_prop;
1812 region = &hdev->pci_mem_region[PCI_REGION_CFG];
1821 region = &hdev->pci_mem_region[PCI_REGION_SRAM];
1830 region = &hdev->pci_mem_region[PCI_REGION_DRAM];
1832 region->region_size = hdev->asic_prop.dram_size;
1839 region = &hdev->pci_mem_region[PCI_REGION_SP_SRAM];
1848 static int gaudi_sw_init(struct hl_device *hdev)
1862 dev_err(hdev->dev,
1876 hdev->asic_specific = gaudi;
1879 hdev->dma_pool = dma_pool_create(dev_name(hdev->dev),
1880 &hdev->pdev->dev, GAUDI_DMA_POOL_BLK_SIZE, 8, 0);
1881 if (!hdev->dma_pool) {
1882 dev_err(hdev->dev, "failed to create DMA pool\n");
1887 rc = gaudi_alloc_cpu_accessible_dma_mem(hdev);
1891 hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1);
1892 if (!hdev->cpu_accessible_dma_pool) {
1893 dev_err(hdev->dev,
1899 rc = gen_pool_add(hdev->cpu_accessible_dma_pool,
1900 (uintptr_t) hdev->cpu_accessible_dma_mem,
1903 dev_err(hdev->dev,
1909 rc = gaudi_alloc_internal_qmans_pq_mem(hdev);
1915 hdev->supports_sync_stream = true;
1916 hdev->supports_coresight = true;
1917 hdev->supports_staged_submission = true;
1918 hdev->supports_wait_for_multi_cs = true;
1920 hdev->asic_funcs->set_pci_memory_regions(hdev);
1921 hdev->stream_master_qid_arr =
1922 hdev->asic_funcs->get_stream_master_qid_arr();
1923 hdev->stream_master_qid_arr_size = GAUDI_STREAM_MASTER_ARR_SIZE;
1928 gen_pool_destroy(hdev->cpu_accessible_dma_pool);
1930 if (!hdev->asic_prop.fw_security_enabled)
1931 GAUDI_CPU_TO_PCI_ADDR(hdev->cpu_accessible_dma_address,
1932 hdev->cpu_pci_msb_addr);
1933 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem,
1934 hdev->cpu_accessible_dma_address);
1936 dma_pool_destroy(hdev->dma_pool);
1942 static int gaudi_sw_fini(struct hl_device *hdev)
1944 struct gaudi_device *gaudi = hdev->asic_specific;
1946 gaudi_free_internal_qmans_pq_mem(hdev);
1948 gen_pool_destroy(hdev->cpu_accessible_dma_pool);
1950 if (!hdev->asic_prop.fw_security_enabled)
1951 GAUDI_CPU_TO_PCI_ADDR(hdev->cpu_accessible_dma_address,
1952 hdev->cpu_pci_msb_addr);
1954 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem,
1955 hdev->cpu_accessible_dma_address);
1957 dma_pool_destroy(hdev->dma_pool);
1966 struct hl_device *hdev = arg;
1969 if (hdev->disabled)
1972 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
1973 hl_irq_handler_cq(irq, &hdev->completion_queue[i]);
1975 hl_irq_handler_eq(irq, &hdev->event_queue);
1984 static int gaudi_pci_irq_vector(struct hl_device *hdev, unsigned int nr,
1990 dev_crit(hdev->dev, "CPU EQ must use IRQ %d\n",
1996 return pci_irq_vector(hdev->pdev, msi_vec);
1999 static int gaudi_enable_msi_single(struct hl_device *hdev)
2003 dev_dbg(hdev->dev, "Working in single MSI IRQ mode\n");
2005 irq = gaudi_pci_irq_vector(hdev, 0, false);
2007 "gaudi single msi", hdev);
2009 dev_err(hdev->dev,
2015 static int gaudi_enable_msi(struct hl_device *hdev)
2017 struct gaudi_device *gaudi = hdev->asic_specific;
2023 rc = pci_alloc_irq_vectors(hdev->pdev, 1, 1, PCI_IRQ_MSI);
2025 dev_err(hdev->dev, "MSI: Failed to enable support %d\n", rc);
2029 rc = gaudi_enable_msi_single(hdev);
2038 pci_free_irq_vectors(hdev->pdev);
2042 static void gaudi_sync_irqs(struct hl_device *hdev)
2044 struct gaudi_device *gaudi = hdev->asic_specific;
2050 synchronize_irq(gaudi_pci_irq_vector(hdev, 0, false));
2053 static void gaudi_disable_msi(struct hl_device *hdev)
2055 struct gaudi_device *gaudi = hdev->asic_specific;
2060 gaudi_sync_irqs(hdev);
2061 free_irq(gaudi_pci_irq_vector(hdev, 0, false), hdev);
2062 pci_free_irq_vectors(hdev->pdev);
2067 static void gaudi_init_scrambler_sram(struct hl_device *hdev)
2069 struct gaudi_device *gaudi = hdev->asic_specific;
2071 if (hdev->asic_prop.fw_security_enabled)
2074 if (hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &
2135 static void gaudi_init_scrambler_hbm(struct hl_device *hdev)
2137 struct gaudi_device *gaudi = hdev->asic_specific;
2139 if (hdev->asic_prop.fw_security_enabled)
2142 if (hdev->asic_prop.fw_bootfit_cpu_boot_dev_sts0 &
2203 static void gaudi_init_e2e(struct hl_device *hdev)
2205 if (hdev->asic_prop.fw_security_enabled)
2208 if (hdev->asic_prop.fw_bootfit_cpu_boot_dev_sts0 &
2453 static void gaudi_init_hbm_cred(struct hl_device *hdev)
2457 if (hdev->asic_prop.fw_security_enabled)
2460 if (hdev->asic_prop.fw_bootfit_cpu_boot_dev_sts0 &
2516 static void gaudi_init_golden_registers(struct hl_device *hdev)
2521 gaudi_init_e2e(hdev);
2522 gaudi_init_hbm_cred(hdev);
2536 writeq(0, hdev->pcie_bar[SRAM_BAR_ID] + i);
2544 static void gaudi_init_pci_dma_qman(struct hl_device *hdev, int dma_id,
2548 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
2601 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?
2607 if (hdev->stop_on_err)
2635 static void gaudi_init_dma_core(struct hl_device *hdev, int dma_id)
2638 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
2651 if (hdev->stop_on_err)
2656 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?
2675 static void gaudi_enable_qman(struct hl_device *hdev, int dma_id,
2683 static void gaudi_init_pci_dma_qmans(struct hl_device *hdev)
2685 struct gaudi_device *gaudi = hdev->asic_specific;
2709 q = &hdev->kernel_queues[q_idx];
2712 gaudi_init_pci_dma_qman(hdev, dma_id, j,
2716 gaudi_init_dma_core(hdev, dma_id);
2718 gaudi_enable_qman(hdev, dma_id, PCI_DMA_QMAN_ENABLE);
2724 static void gaudi_init_hbm_dma_qman(struct hl_device *hdev, int dma_id,
2728 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
2772 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?
2785 if (hdev->stop_on_err)
2829 static void gaudi_init_hbm_dma_qmans(struct hl_device *hdev)
2831 struct gaudi_device *gaudi = hdev->asic_specific;
2851 gaudi_init_hbm_dma_qman(hdev, dma_id, j,
2856 gaudi_init_hbm_dma_qman(hdev, dma_id, 4, 0);
2858 gaudi_init_dma_core(hdev, dma_id);
2860 gaudi_enable_qman(hdev, dma_id, HBM_DMA_QMAN_ENABLE);
2866 static void gaudi_init_mme_qman(struct hl_device *hdev, u32 mme_offset,
2870 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
2905 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?
2921 if (hdev->stop_on_err)
2953 static void gaudi_init_mme_qmans(struct hl_device *hdev)
2955 struct gaudi_device *gaudi = hdev->asic_specific;
2975 gaudi_init_mme_qman(hdev, mme_offset, (i & 0x3),
2983 gaudi_init_mme_qman(hdev, mme_offset, 4, 0);
2984 gaudi_init_mme_qman(hdev, 0, 4, 0);
2992 static void gaudi_init_tpc_qman(struct hl_device *hdev, u32 tpc_offset,
2996 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
3041 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?
3054 if (hdev->stop_on_err)
3098 static void gaudi_init_tpc_qmans(struct hl_device *hdev)
3100 struct gaudi_device *gaudi = hdev->asic_specific;
3120 gaudi_init_tpc_qman(hdev, tpc_offset, i,
3125 gaudi_init_tpc_qman(hdev, tpc_offset, 4, 0);
3143 static void gaudi_init_nic_qman(struct hl_device *hdev, u32 nic_offset,
3147 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
3198 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?
3204 if (hdev->stop_on_err)
3231 static void gaudi_init_nic_qmans(struct hl_device *hdev)
3233 struct gaudi_device *gaudi = hdev->asic_specific;
3243 if (!hdev->nic_ports_mask)
3249 dev_dbg(hdev->dev, "Initializing NIC QMANs\n");
3252 if (!(hdev->nic_ports_mask & (1 << nic_id))) {
3266 gaudi_init_nic_qman(hdev, nic_offset, (i & 0x3),
3283 static void gaudi_disable_pci_dma_qmans(struct hl_device *hdev)
3285 struct gaudi_device *gaudi = hdev->asic_specific;
3295 static void gaudi_disable_hbm_dma_qmans(struct hl_device *hdev)
3297 struct gaudi_device *gaudi = hdev->asic_specific;
3309 static void gaudi_disable_mme_qmans(struct hl_device *hdev)
3311 struct gaudi_device *gaudi = hdev->asic_specific;
3320 static void gaudi_disable_tpc_qmans(struct hl_device *hdev)
3322 struct gaudi_device *gaudi = hdev->asic_specific;
3335 static void gaudi_disable_nic_qmans(struct hl_device *hdev)
3337 struct gaudi_device *gaudi = hdev->asic_specific;
3359 static void gaudi_stop_pci_dma_qmans(struct hl_device *hdev)
3361 struct gaudi_device *gaudi = hdev->asic_specific;
3372 static void gaudi_stop_hbm_dma_qmans(struct hl_device *hdev)
3374 struct gaudi_device *gaudi = hdev->asic_specific;
3388 static void gaudi_stop_mme_qmans(struct hl_device *hdev)
3390 struct gaudi_device *gaudi = hdev->asic_specific;
3400 static void gaudi_stop_tpc_qmans(struct hl_device *hdev)
3402 struct gaudi_device *gaudi = hdev->asic_specific;
3417 static void gaudi_stop_nic_qmans(struct hl_device *hdev)
3419 struct gaudi_device *gaudi = hdev->asic_specific;
3484 static void gaudi_pci_dma_stall(struct hl_device *hdev)
3486 struct gaudi_device *gaudi = hdev->asic_specific;
3496 static void gaudi_hbm_dma_stall(struct hl_device *hdev)
3498 struct gaudi_device *gaudi = hdev->asic_specific;
3510 static void gaudi_mme_stall(struct hl_device *hdev)
3512 struct gaudi_device *gaudi = hdev->asic_specific;
3536 static void gaudi_tpc_stall(struct hl_device *hdev)
3538 struct gaudi_device *gaudi = hdev->asic_specific;
3553 static void gaudi_disable_clock_gating(struct hl_device *hdev)
3558 if (hdev->asic_prop.fw_security_enabled)
3581 static void gaudi_enable_timestamp(struct hl_device *hdev)
3594 static void gaudi_disable_timestamp(struct hl_device *hdev)
3600 static void gaudi_halt_engines(struct hl_device *hdev, bool hard_reset, bool fw_reset)
3604 if (hdev->pldm)
3612 gaudi_stop_nic_qmans(hdev);
3613 gaudi_stop_mme_qmans(hdev);
3614 gaudi_stop_tpc_qmans(hdev);
3615 gaudi_stop_hbm_dma_qmans(hdev);
3616 gaudi_stop_pci_dma_qmans(hdev);
3620 gaudi_pci_dma_stall(hdev);
3621 gaudi_hbm_dma_stall(hdev);
3622 gaudi_tpc_stall(hdev);
3623 gaudi_mme_stall(hdev);
3627 gaudi_disable_nic_qmans(hdev);
3628 gaudi_disable_mme_qmans(hdev);
3629 gaudi_disable_tpc_qmans(hdev);
3630 gaudi_disable_hbm_dma_qmans(hdev);
3631 gaudi_disable_pci_dma_qmans(hdev);
3633 gaudi_disable_timestamp(hdev);
3636 gaudi_disable_msi(hdev);
3639 static int gaudi_mmu_init(struct hl_device *hdev)
3641 struct asic_fixed_properties *prop = &hdev->asic_prop;
3642 struct gaudi_device *gaudi = hdev->asic_specific;
3653 rc = gaudi_mmu_update_asid_hop0_addr(hdev, i, hop0_addr);
3655 dev_err(hdev->dev,
3668 rc = hl_mmu_invalidate_cache(hdev, true, 0);
3688 static int gaudi_load_firmware_to_device(struct hl_device *hdev)
3692 dst = hdev->pcie_bar[HBM_BAR_ID] + LINUX_FW_OFFSET;
3694 return hl_fw_load_fw_to_device(hdev, GAUDI_LINUX_FW_FILE, dst, 0, 0);
3697 static int gaudi_load_boot_fit_to_device(struct hl_device *hdev)
3701 dst = hdev->pcie_bar[SRAM_BAR_ID] + BOOT_FIT_SRAM_OFFSET;
3703 return hl_fw_load_fw_to_device(hdev, GAUDI_BOOT_FIT_FILE, dst, 0, 0);
3706 static void gaudi_init_dynamic_firmware_loader(struct hl_device *hdev)
3711 dynamic_loader = &hdev->fw_loader.dynamic_loader;
3729 static void gaudi_init_static_firmware_loader(struct hl_device *hdev)
3733 static_loader = &hdev->fw_loader.static_loader;
3747 static_loader->cpu_reset_wait_msec = hdev->pldm ?
3752 static void gaudi_init_firmware_preload_params(struct hl_device *hdev)
3754 struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load;
3764 static void gaudi_init_firmware_loader(struct hl_device *hdev)
3766 struct asic_fixed_properties *prop = &hdev->asic_prop;
3767 struct fw_load_mgr *fw_loader = &hdev->fw_loader;
3775 fw_loader->skip_bmc = !hdev->bmc_enable;
3780 gaudi_init_dynamic_firmware_loader(hdev);
3782 gaudi_init_static_firmware_loader(hdev);
3785 static int gaudi_init_cpu(struct hl_device *hdev)
3787 struct gaudi_device *gaudi = hdev->asic_specific;
3790 if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU))
3800 if (!hdev->asic_prop.fw_security_enabled)
3801 WREG32(mmCPU_IF_CPU_MSB_ADDR, hdev->cpu_pci_msb_addr);
3803 rc = hl_fw_init_cpu(hdev);
3813 static int gaudi_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout)
3816 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
3817 struct asic_fixed_properties *prop = &hdev->asic_prop;
3818 struct gaudi_device *gaudi = hdev->asic_specific;
3822 &hdev->kernel_queues[GAUDI_QUEUE_ID_CPU_PQ];
3825 if (!hdev->cpu_queues_enable)
3831 eq = &hdev->event_queue;
3840 lower_32_bits(hdev->cpu_accessible_dma_address));
3842 upper_32_bits(hdev->cpu_accessible_dma_address));
3863 hdev,
3871 dev_err(hdev->dev,
3886 static void gaudi_pre_hw_init(struct hl_device *hdev)
3891 if (!hdev->asic_prop.fw_security_enabled) {
3914 static int gaudi_hw_init(struct hl_device *hdev)
3916 struct gaudi_device *gaudi = hdev->asic_specific;
3919 gaudi_pre_hw_init(hdev);
3925 if (hdev->asic_prop.iatu_done_by_fw)
3932 if (gaudi_set_hbm_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) {
3933 dev_err(hdev->dev,
3938 rc = gaudi_init_cpu(hdev);
3940 dev_err(hdev->dev, "failed to initialize CPU\n");
3947 gaudi_disable_clock_gating(hdev);
3950 gaudi_init_scrambler_sram(hdev);
3953 gaudi_init_scrambler_hbm(hdev);
3955 gaudi_init_golden_registers(hdev);
3957 rc = gaudi_mmu_init(hdev);
3961 gaudi_init_security(hdev);
3963 gaudi_init_pci_dma_qmans(hdev);
3965 gaudi_init_hbm_dma_qmans(hdev);
3967 gaudi_init_mme_qmans(hdev);
3969 gaudi_init_tpc_qmans(hdev);
3971 gaudi_init_nic_qmans(hdev);
3973 gaudi_enable_timestamp(hdev);
3976 rc = gaudi_enable_msi(hdev);
3981 rc = gaudi_init_cpu_queues(hdev, GAUDI_CPU_TIMEOUT_USEC);
3983 dev_err(hdev->dev, "failed to initialize CPU H/W queues %d\n",
3994 gaudi_disable_msi(hdev);
3996 gaudi_disable_mme_qmans(hdev);
3997 gaudi_disable_pci_dma_qmans(hdev);
4002 static int gaudi_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset)
4005 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
4007 struct gaudi_device *gaudi = hdev->asic_specific;
4011 dev_err(hdev->dev, "GAUDI doesn't support soft-reset\n");
4015 if (hdev->pldm) {
4024 dev_dbg(hdev->dev,
4031 driver_performs_reset = !!(!hdev->asic_prop.fw_security_enabled &&
4032 !hdev->asic_prop.hard_reset_done_by_fw);
4045 if (hdev->fw_loader.fw_comp_loaded & FW_TYPE_LINUX) {
4046 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?
4064 if (hdev->reset_info.curr_reset_cause == HL_RESET_CAUSE_HEARTBEAT) {
4065 if (hdev->asic_prop.hard_reset_done_by_fw)
4066 hl_fw_ask_hard_reset_without_linux(hdev);
4068 hl_fw_ask_halt_machine_without_linux(hdev);
4071 if (hdev->asic_prop.hard_reset_done_by_fw)
4072 hl_fw_ask_hard_reset_without_linux(hdev);
4074 hl_fw_ask_halt_machine_without_linux(hdev);
4116 dev_dbg(hdev->dev,
4120 dev_dbg(hdev->dev,
4134 dev_err(hdev->dev, "Timeout while waiting for device to reset 0x%x\n", status);
4147 hdev->device_cpu_is_halted = false;
4152 static int gaudi_suspend(struct hl_device *hdev)
4156 rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0);
4158 dev_err(hdev->dev, "Failed to disable PCI access from CPU\n");
4163 static int gaudi_resume(struct hl_device *hdev)
4165 return gaudi_init_iatu(hdev);
4168 static int gaudi_mmap(struct hl_device *hdev, struct vm_area_struct *vma,
4176 rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr,
4179 dev_err(hdev->dev, "dma_mmap_coherent error %d", rc);
4184 static void gaudi_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi)
4187 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
4189 struct gaudi_device *gaudi = hdev->asic_specific;
4503 dev_err(hdev->dev, "h/w queue %d is invalid. Can't set pi\n",
4517 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?
4526 static void gaudi_pqe_write(struct hl_device *hdev, __le64 *pqe,
4536 static void *gaudi_dma_alloc_coherent(struct hl_device *hdev, size_t size,
4539 void *kernel_addr = dma_alloc_coherent(&hdev->pdev->dev, size,
4549 static void gaudi_dma_free_coherent(struct hl_device *hdev, size_t size,
4555 dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, fixed_dma_handle);
4558 static int gaudi_scrub_device_dram(struct hl_device *hdev, u64 val)
4560 struct asic_fixed_properties *prop = &hdev->asic_prop;
4572 dev_dbg(hdev->dev,
4600 hdev,
4608 dev_err(hdev->dev,
4619 static int gaudi_scrub_device_mem(struct hl_device *hdev)
4621 struct asic_fixed_properties *prop = &hdev->asic_prop;
4622 u64 wait_to_idle_time = hdev->pdev ? HBM_SCRUBBING_TIMEOUT_US :
4624 u64 addr, size, val = hdev->memory_scrub_val;
4628 if (!hdev->memory_scrub)
4632 while (!hdev->asic_funcs->is_device_idle(hdev, NULL, 0, NULL)) {
4634 dev_err(hdev->dev, "waiting for idle timeout\n");
4642 size = hdev->pldm ? 0x10000 : prop->sram_size - SRAM_USER_BASE_OFFSET;
4644 dev_dbg(hdev->dev, "Scrubbing SRAM: 0x%09llx - 0x%09llx val: 0x%llx\n",
4646 rc = gaudi_memset_device_memory(hdev, addr, size, val);
4648 dev_err(hdev->dev, "Failed to clear SRAM (%d)\n", rc);
4653 rc = gaudi_scrub_device_dram(hdev, val);
4655 dev_err(hdev->dev, "Failed to clear HBM (%d)\n", rc);
4662 static void *gaudi_get_int_queue_base(struct hl_device *hdev,
4666 struct gaudi_device *gaudi = hdev->asic_specific;
4671 dev_err(hdev->dev, "Got invalid queue id %d\n", queue_id);
4682 static int gaudi_send_cpu_message(struct hl_device *hdev, u32 *msg,
4685 struct gaudi_device *gaudi = hdev->asic_specific;
4696 return hl_fw_send_cpu_message(hdev, GAUDI_QUEUE_ID_CPU_PQ, msg, len,
4700 static int gaudi_test_queue(struct hl_device *hdev, u32 hw_queue_id)
4709 if (hdev->pldm)
4716 fence_ptr = hl_asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL, &fence_dma_addr);
4718 dev_err(hdev->dev,
4726 fence_pkt = hl_asic_dma_pool_zalloc(hdev, sizeof(struct packet_msg_prot), GFP_KERNEL,
4729 dev_err(hdev->dev,
4744 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id,
4748 dev_err(hdev->dev,
4754 rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp, (tmp == fence_val),
4757 hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
4760 dev_err(hdev->dev,
4767 hl_asic_dma_pool_free(hdev, (void *) fence_pkt, pkt_dma_addr);
4769 hl_asic_dma_pool_free(hdev, (void *) fence_ptr, fence_dma_addr);
4773 static int gaudi_test_cpu_queue(struct hl_device *hdev)
4775 struct gaudi_device *gaudi = hdev->asic_specific;
4784 return hl_fw_test_cpu_queue(hdev);
4787 static int gaudi_test_queues(struct hl_device *hdev)
4791 for (i = 0 ; i < hdev->asic_prop.max_queues ; i++) {
4792 if (hdev->asic_prop.hw_queues_props[i].type == QUEUE_TYPE_EXT) {
4793 rc = gaudi_test_queue(hdev, i);
4799 rc = gaudi_test_cpu_queue(hdev);
4806 static void *gaudi_dma_pool_zalloc(struct hl_device *hdev, size_t size,
4814 kernel_addr = dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle);
4823 static void gaudi_dma_pool_free(struct hl_device *hdev, void *vaddr,
4829 dma_pool_free(hdev->dma_pool, vaddr, fixed_dma_addr);
4832 static void *gaudi_cpu_accessible_dma_pool_alloc(struct hl_device *hdev,
4835 return hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle);
4838 static void gaudi_cpu_accessible_dma_pool_free(struct hl_device *hdev,
4841 hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr);
4844 static u32 gaudi_get_dma_desc_list_size(struct hl_device *hdev, struct sg_table *sgt)
4884 static int gaudi_pin_memory_before_cs(struct hl_device *hdev,
4892 if (hl_userptr_is_pinned(hdev, addr, le32_to_cpu(user_dma_pkt->tsize),
4900 rc = hl_pin_host_memory(hdev, addr, le32_to_cpu(user_dma_pkt->tsize),
4907 rc = hdev->asic_funcs->asic_dma_map_sgtable(hdev, userptr->sgt, dir);
4909 dev_err(hdev->dev, "failed to map sgt with DMA region\n");
4918 gaudi_get_dma_desc_list_size(hdev, userptr->sgt);
4924 hl_unpin_host_memory(hdev, userptr);
4930 static int gaudi_validate_dma_pkt_host(struct hl_device *hdev,
4948 dev_dbg(hdev->dev, "DMA direction is HOST --> DEVICE\n");
4952 dev_dbg(hdev->dev, "DMA direction is DEVICE --> HOST\n");
4962 rc = gaudi_pin_memory_before_cs(hdev, parser, user_dma_pkt,
4968 static int gaudi_validate_dma_pkt_no_mmu(struct hl_device *hdev,
4977 dev_dbg(hdev->dev, "DMA packet details:\n");
4978 dev_dbg(hdev->dev, "source == 0x%llx\n",
4980 dev_dbg(hdev->dev, "destination == 0x%llx\n", dst_addr);
4981 dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize));
4996 return gaudi_validate_dma_pkt_host(hdev, parser, user_dma_pkt,
5000 static int gaudi_validate_load_and_exe_pkt(struct hl_device *hdev,
5009 dev_err(hdev->dev,
5019 static int gaudi_validate_cb(struct hl_device *hdev,
5041 dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id);
5049 dev_err(hdev->dev,
5057 dev_err(hdev->dev,
5063 dev_err(hdev->dev, "User not allowed to use CP_DMA\n");
5068 dev_err(hdev->dev, "User not allowed to use STOP\n");
5073 dev_err(hdev->dev,
5079 rc = gaudi_validate_load_and_exe_pkt(hdev, parser,
5088 rc = gaudi_validate_dma_pkt_no_mmu(hdev, parser,
5103 dev_err(hdev->dev, "Invalid packet header 0x%x\n",
5126 static int gaudi_patch_dma_packet(struct hl_device *hdev,
5165 (!hl_userptr_is_pinned(hdev, addr,
5168 dev_err(hdev->dev, "Userptr 0x%llx + 0x%x NOT mapped\n",
5231 dev_err(hdev->dev,
5245 static int gaudi_patch_cb(struct hl_device *hdev,
5269 dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id);
5277 dev_err(hdev->dev,
5285 rc = gaudi_patch_dma_packet(hdev, parser,
5293 dev_err(hdev->dev,
5299 dev_err(hdev->dev, "User not allowed to use CP_DMA\n");
5304 dev_err(hdev->dev, "User not allowed to use STOP\n");
5322 dev_err(hdev->dev, "Invalid packet header 0x%x\n",
5335 static int gaudi_parse_cb_mmu(struct hl_device *hdev,
5355 rc = hl_cb_create(hdev, &hdev->kernel_mem_mgr, hdev->kernel_ctx,
5360 dev_err(hdev->dev,
5366 parser->patched_cb = hl_cb_get(&hdev->kernel_mem_mgr, handle);
5369 dev_crit(hdev->dev, "DMA CB handle invalid 0x%llx\n", handle);
5393 rc = gaudi_validate_cb(hdev, parser, true);
5402 dev_err(hdev->dev, "user CB size mismatch\n");
5415 hl_cb_destroy(&hdev->kernel_mem_mgr, handle);
5420 static int gaudi_parse_cb_no_mmu(struct hl_device *hdev,
5426 rc = gaudi_validate_cb(hdev, parser, false);
5431 rc = hl_cb_create(hdev, &hdev->kernel_mem_mgr, hdev->kernel_ctx,
5435 dev_err(hdev->dev,
5440 parser->patched_cb = hl_cb_get(&hdev->kernel_mem_mgr, handle);
5443 dev_crit(hdev->dev, "DMA CB handle invalid 0x%llx\n", handle);
5448 rc = gaudi_patch_cb(hdev, parser);
5460 hl_cb_destroy(&hdev->kernel_mem_mgr, handle);
5464 hl_userptr_delete_list(hdev, parser->job_userptr_list);
5468 static int gaudi_parse_cb_no_ext_queue(struct hl_device *hdev,
5471 struct asic_fixed_properties *asic_prop = &hdev->asic_prop;
5472 struct gaudi_device *gaudi = hdev->asic_specific;
5481 dev_err(hdev->dev, "h/w queue %d is disabled\n", parser->hw_queue_id);
5506 dev_err(hdev->dev,
5513 static int gaudi_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser)
5515 struct gaudi_device *gaudi = hdev->asic_specific;
5518 return gaudi_parse_cb_no_ext_queue(hdev, parser);
5521 return gaudi_parse_cb_mmu(hdev, parser);
5523 return gaudi_parse_cb_no_mmu(hdev, parser);
5526 static void gaudi_add_end_of_cb_packets(struct hl_device *hdev, void *kernel_address,
5559 msi_addr = hdev->pdev ? mmPCIE_CORE_MSI_REQ : mmPCIE_MSI_INTR_0 + msi_vec * 4;
5563 static void gaudi_update_eq_ci(struct hl_device *hdev, u32 val)
5568 static int gaudi_memset_device_memory(struct hl_device *hdev, u64 addr,
5577 cb = hl_cb_kernel_create(hdev, PAGE_SIZE, false);
5596 job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true);
5598 dev_err(hdev->dev, "Failed to allocate a new job\n");
5605 if (err_cause && !hdev->init_done) {
5606 dev_dbg(hdev->dev,
5620 hl_debugfs_add_job(hdev, job);
5622 rc = gaudi_send_job_on_qman0(hdev, job);
5623 hl_debugfs_remove_job(hdev, job);
5630 dev_err(hdev->dev, "DMA Failed, cause 0x%x\n", err_cause);
5632 if (!hdev->init_done) {
5633 dev_dbg(hdev->dev,
5642 hl_cb_destroy(&hdev->kernel_mem_mgr, cb->buf->handle);
5647 static int gaudi_memset_registers(struct hl_device *hdev, u64 reg_base,
5659 dev_err(hdev->dev, "CB size must be smaller than %uMB", SZ_2M);
5663 cb = hl_cb_kernel_create(hdev, cb_size, false);
5681 job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true);
5683 dev_err(hdev->dev, "Failed to allocate a new job\n");
5696 hl_debugfs_add_job(hdev, job);
5698 rc = gaudi_send_job_on_qman0(hdev, job);
5699 hl_debugfs_remove_job(hdev, job);
5705 hl_cb_destroy(&hdev->kernel_mem_mgr, cb->buf->handle);
5710 static int gaudi_restore_sm_registers(struct hl_device *hdev)
5718 rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0);
5720 dev_err(hdev->dev, "failed resetting SM registers");
5726 rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0);
5728 dev_err(hdev->dev, "failed resetting SM registers");
5734 rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0);
5736 dev_err(hdev->dev, "failed resetting SM registers");
5742 rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0);
5744 dev_err(hdev->dev, "failed resetting SM registers");
5750 rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0);
5752 dev_err(hdev->dev, "failed resetting SM registers");
5758 rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0);
5760 dev_err(hdev->dev, "failed resetting SM registers");
5767 rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0);
5769 dev_err(hdev->dev, "failed resetting SM registers");
5776 rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0);
5778 dev_err(hdev->dev, "failed resetting SM registers");
5785 static void gaudi_restore_dma_registers(struct hl_device *hdev)
5812 static void gaudi_restore_qm_registers(struct hl_device *hdev)
5839 static int gaudi_restore_user_registers(struct hl_device *hdev)
5843 rc = gaudi_restore_sm_registers(hdev);
5847 gaudi_restore_dma_registers(hdev);
5848 gaudi_restore_qm_registers(hdev);
5853 static int gaudi_context_switch(struct hl_device *hdev, u32 asid)
5858 static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev)
5860 u32 size = hdev->asic_prop.mmu_pgt_size +
5861 hdev->asic_prop.mmu_cache_mng_size;
5862 struct gaudi_device *gaudi = hdev->asic_specific;
5863 u64 addr = hdev->asic_prop.mmu_pgt_addr;
5868 return gaudi_memset_device_memory(hdev, addr, size, 0);
5871 static void gaudi_restore_phase_topology(struct hl_device *hdev)
5876 static int gaudi_dma_core_transfer(struct hl_device *hdev, int dma_id, u64 addr,
5894 hdev,
5902 dev_err(hdev->dev,
5911 dev_err(hdev->dev, "DMA Failed, cause 0x%x\n", err_cause);
5912 dev_dbg(hdev->dev,
5923 static int gaudi_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size,
5934 kernel_addr = hl_asic_dma_alloc_coherent(hdev, SZ_2M, &dma_addr, GFP_KERNEL | __GFP_ZERO);
5939 hdev->asic_funcs->hw_queues_lock(hdev);
5961 dev_err_ratelimited(hdev->dev,
5981 dev_dbg(hdev->dev,
5996 rc = gaudi_dma_core_transfer(hdev, dma_id, addr, size_to_dma,
6021 hdev->asic_funcs->hw_queues_unlock(hdev);
6023 hl_asic_dma_free_coherent(hdev, SZ_2M, kernel_addr, dma_addr);
6028 static u64 gaudi_read_pte(struct hl_device *hdev, u64 addr)
6030 struct gaudi_device *gaudi = hdev->asic_specific;
6032 if (hdev->reset_info.hard_reset_pending)
6035 return readq(hdev->pcie_bar[HBM_BAR_ID] +
6039 static void gaudi_write_pte(struct hl_device *hdev, u64 addr, u64 val)
6041 struct gaudi_device *gaudi = hdev->asic_specific;
6043 if (hdev->reset_info.hard_reset_pending)
6046 writeq(val, hdev->pcie_bar[HBM_BAR_ID] +
6050 void gaudi_mmu_prepare_reg(struct hl_device *hdev, u64 reg, u32 asid)
6057 static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid)
6059 struct gaudi_device *gaudi = hdev->asic_specific;
6065 dev_crit(hdev->dev, "asid %u is too big\n", asid);
6069 gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_0, asid);
6070 gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_1, asid);
6071 gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_2, asid);
6072 gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_3, asid);
6073 gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_4, asid);
6075 gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_0, asid);
6076 gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_1, asid);
6077 gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_2, asid);
6078 gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_3, asid);
6079 gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_4, asid);
6081 gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_0, asid);
6082 gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_1, asid);
6083 gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_2, asid);
6084 gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_3, asid);
6085 gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_4, asid);
6087 gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_0, asid);
6088 gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_1, asid);
6089 gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_2, asid);
6090 gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_3, asid);
6091 gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_4, asid);
6093 gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_0, asid);
6094 gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_1, asid);
6095 gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_2, asid);
6096 gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_3, asid);
6097 gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_4, asid);
6099 gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_0, asid);
6100 gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_1, asid);
6101 gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_2, asid);
6102 gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_3, asid);
6103 gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_4, asid);
6105 gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_0, asid);
6106 gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_1, asid);
6107 gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_2, asid);
6108 gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_3, asid);
6109 gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_4, asid);
6111 gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_0, asid);
6112 gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_1, asid);
6113 gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_2, asid);
6114 gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_3, asid);
6115 gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_4, asid);
6117 gaudi_mmu_prepare_reg(hdev, mmDMA0_CORE_NON_SECURE_PROPS, asid);
6118 gaudi_mmu_prepare_reg(hdev, mmDMA1_CORE_NON_SECURE_PROPS, asid);
6119 gaudi_mmu_prepare_reg(hdev, mmDMA2_CORE_NON_SECURE_PROPS, asid);
6120 gaudi_mmu_prepare_reg(hdev, mmDMA3_CORE_NON_SECURE_PROPS, asid);
6121 gaudi_mmu_prepare_reg(hdev, mmDMA4_CORE_NON_SECURE_PROPS, asid);
6122 gaudi_mmu_prepare_reg(hdev, mmDMA5_CORE_NON_SECURE_PROPS, asid);
6123 gaudi_mmu_prepare_reg(hdev, mmDMA6_CORE_NON_SECURE_PROPS, asid);
6124 gaudi_mmu_prepare_reg(hdev, mmDMA7_CORE_NON_SECURE_PROPS, asid);
6126 gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_0, asid);
6127 gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_1, asid);
6128 gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_2, asid);
6129 gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_3, asid);
6130 gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_4, asid);
6131 gaudi_mmu_prepare_reg(hdev, mmTPC0_CFG_ARUSER_LO, asid);
6132 gaudi_mmu_prepare_reg(hdev, mmTPC0_CFG_AWUSER_LO, asid);
6134 gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_0, asid);
6135 gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_1, asid);
6136 gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_2, asid);
6137 gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_3, asid);
6138 gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_4, asid);
6139 gaudi_mmu_prepare_reg(hdev, mmTPC1_CFG_ARUSER_LO, asid);
6140 gaudi_mmu_prepare_reg(hdev, mmTPC1_CFG_AWUSER_LO, asid);
6142 gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_0, asid);
6143 gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_1, asid);
6144 gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_2, asid);
6145 gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_3, asid);
6146 gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_4, asid);
6147 gaudi_mmu_prepare_reg(hdev, mmTPC2_CFG_ARUSER_LO, asid);
6148 gaudi_mmu_prepare_reg(hdev, mmTPC2_CFG_AWUSER_LO, asid);
6150 gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_0, asid);
6151 gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_1, asid);
6152 gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_2, asid);
6153 gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_3, asid);
6154 gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_4, asid);
6155 gaudi_mmu_prepare_reg(hdev, mmTPC3_CFG_ARUSER_LO, asid);
6156 gaudi_mmu_prepare_reg(hdev, mmTPC3_CFG_AWUSER_LO, asid);
6158 gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_0, asid);
6159 gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_1, asid);
6160 gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_2, asid);
6161 gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_3, asid);
6162 gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_4, asid);
6163 gaudi_mmu_prepare_reg(hdev, mmTPC4_CFG_ARUSER_LO, asid);
6164 gaudi_mmu_prepare_reg(hdev, mmTPC4_CFG_AWUSER_LO, asid);
6166 gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_0, asid);
6167 gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_1, asid);
6168 gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_2, asid);
6169 gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_3, asid);
6170 gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_4, asid);
6171 gaudi_mmu_prepare_reg(hdev, mmTPC5_CFG_ARUSER_LO, asid);
6172 gaudi_mmu_prepare_reg(hdev, mmTPC5_CFG_AWUSER_LO, asid);
6174 gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_0, asid);
6175 gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_1, asid);
6176 gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_2, asid);
6177 gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_3, asid);
6178 gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_4, asid);
6179 gaudi_mmu_prepare_reg(hdev, mmTPC6_CFG_ARUSER_LO, asid);
6180 gaudi_mmu_prepare_reg(hdev, mmTPC6_CFG_AWUSER_LO, asid);
6182 gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_0, asid);
6183 gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_1, asid);
6184 gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_2, asid);
6185 gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_3, asid);
6186 gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_4, asid);
6187 gaudi_mmu_prepare_reg(hdev, mmTPC7_CFG_ARUSER_LO, asid);
6188 gaudi_mmu_prepare_reg(hdev, mmTPC7_CFG_AWUSER_LO, asid);
6190 gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_0, asid);
6191 gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_1, asid);
6192 gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_2, asid);
6193 gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_3, asid);
6194 gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_4, asid);
6195 gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_0, asid);
6196 gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_1, asid);
6197 gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_2, asid);
6198 gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_3, asid);
6199 gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_4, asid);
6201 gaudi_mmu_prepare_reg(hdev, mmMME0_SBAB_ARUSER0, asid);
6202 gaudi_mmu_prepare_reg(hdev, mmMME0_SBAB_ARUSER1, asid);
6203 gaudi_mmu_prepare_reg(hdev, mmMME1_SBAB_ARUSER0, asid);
6204 gaudi_mmu_prepare_reg(hdev, mmMME1_SBAB_ARUSER1, asid);
6205 gaudi_mmu_prepare_reg(hdev, mmMME2_SBAB_ARUSER0, asid);
6206 gaudi_mmu_prepare_reg(hdev, mmMME2_SBAB_ARUSER1, asid);
6207 gaudi_mmu_prepare_reg(hdev, mmMME3_SBAB_ARUSER0, asid);
6208 gaudi_mmu_prepare_reg(hdev, mmMME3_SBAB_ARUSER1, asid);
6209 gaudi_mmu_prepare_reg(hdev, mmMME0_ACC_WBC, asid);
6210 gaudi_mmu_prepare_reg(hdev, mmMME1_ACC_WBC, asid);
6211 gaudi_mmu_prepare_reg(hdev, mmMME2_ACC_WBC, asid);
6212 gaudi_mmu_prepare_reg(hdev, mmMME3_ACC_WBC, asid);
6215 gaudi_mmu_prepare_reg(hdev, mmNIC0_QM0_GLBL_NON_SECURE_PROPS_0,
6217 gaudi_mmu_prepare_reg(hdev, mmNIC0_QM0_GLBL_NON_SECURE_PROPS_1,
6219 gaudi_mmu_prepare_reg(hdev, mmNIC0_QM0_GLBL_NON_SECURE_PROPS_2,
6221 gaudi_mmu_prepare_reg(hdev, mmNIC0_QM0_GLBL_NON_SECURE_PROPS_3,
6223 gaudi_mmu_prepare_reg(hdev, mmNIC0_QM0_GLBL_NON_SECURE_PROPS_4,
6228 gaudi_mmu_prepare_reg(hdev, mmNIC0_QM1_GLBL_NON_SECURE_PROPS_0,
6230 gaudi_mmu_prepare_reg(hdev, mmNIC0_QM1_GLBL_NON_SECURE_PROPS_1,
6232 gaudi_mmu_prepare_reg(hdev, mmNIC0_QM1_GLBL_NON_SECURE_PROPS_2,
6234 gaudi_mmu_prepare_reg(hdev, mmNIC0_QM1_GLBL_NON_SECURE_PROPS_3,
6236 gaudi_mmu_prepare_reg(hdev, mmNIC0_QM1_GLBL_NON_SECURE_PROPS_4,
6241 gaudi_mmu_prepare_reg(hdev, mmNIC1_QM0_GLBL_NON_SECURE_PROPS_0,
6243 gaudi_mmu_prepare_reg(hdev, mmNIC1_QM0_GLBL_NON_SECURE_PROPS_1,
6245 gaudi_mmu_prepare_reg(hdev, mmNIC1_QM0_GLBL_NON_SECURE_PROPS_2,
6247 gaudi_mmu_prepare_reg(hdev, mmNIC1_QM0_GLBL_NON_SECURE_PROPS_3,
6249 gaudi_mmu_prepare_reg(hdev, mmNIC1_QM0_GLBL_NON_SECURE_PROPS_4,
6254 gaudi_mmu_prepare_reg(hdev, mmNIC1_QM1_GLBL_NON_SECURE_PROPS_0,
6256 gaudi_mmu_prepare_reg(hdev, mmNIC1_QM1_GLBL_NON_SECURE_PROPS_1,
6258 gaudi_mmu_prepare_reg(hdev, mmNIC1_QM1_GLBL_NON_SECURE_PROPS_2,
6260 gaudi_mmu_prepare_reg(hdev, mmNIC1_QM1_GLBL_NON_SECURE_PROPS_3,
6262 gaudi_mmu_prepare_reg(hdev, mmNIC1_QM1_GLBL_NON_SECURE_PROPS_4,
6267 gaudi_mmu_prepare_reg(hdev, mmNIC2_QM0_GLBL_NON_SECURE_PROPS_0,
6269 gaudi_mmu_prepare_reg(hdev, mmNIC2_QM0_GLBL_NON_SECURE_PROPS_1,
6271 gaudi_mmu_prepare_reg(hdev, mmNIC2_QM0_GLBL_NON_SECURE_PROPS_2,
6273 gaudi_mmu_prepare_reg(hdev, mmNIC2_QM0_GLBL_NON_SECURE_PROPS_3,
6275 gaudi_mmu_prepare_reg(hdev, mmNIC2_QM0_GLBL_NON_SECURE_PROPS_4,
6280 gaudi_mmu_prepare_reg(hdev, mmNIC2_QM1_GLBL_NON_SECURE_PROPS_0,
6282 gaudi_mmu_prepare_reg(hdev, mmNIC2_QM1_GLBL_NON_SECURE_PROPS_1,
6284 gaudi_mmu_prepare_reg(hdev, mmNIC2_QM1_GLBL_NON_SECURE_PROPS_2,
6286 gaudi_mmu_prepare_reg(hdev, mmNIC2_QM1_GLBL_NON_SECURE_PROPS_3,
6288 gaudi_mmu_prepare_reg(hdev, mmNIC2_QM1_GLBL_NON_SECURE_PROPS_4,
6293 gaudi_mmu_prepare_reg(hdev, mmNIC3_QM0_GLBL_NON_SECURE_PROPS_0,
6295 gaudi_mmu_prepare_reg(hdev, mmNIC3_QM0_GLBL_NON_SECURE_PROPS_1,
6297 gaudi_mmu_prepare_reg(hdev, mmNIC3_QM0_GLBL_NON_SECURE_PROPS_2,
6299 gaudi_mmu_prepare_reg(hdev, mmNIC3_QM0_GLBL_NON_SECURE_PROPS_3,
6301 gaudi_mmu_prepare_reg(hdev, mmNIC3_QM0_GLBL_NON_SECURE_PROPS_4,
6306 gaudi_mmu_prepare_reg(hdev, mmNIC3_QM1_GLBL_NON_SECURE_PROPS_0,
6308 gaudi_mmu_prepare_reg(hdev, mmNIC3_QM1_GLBL_NON_SECURE_PROPS_1,
6310 gaudi_mmu_prepare_reg(hdev, mmNIC3_QM1_GLBL_NON_SECURE_PROPS_2,
6312 gaudi_mmu_prepare_reg(hdev, mmNIC3_QM1_GLBL_NON_SECURE_PROPS_3,
6314 gaudi_mmu_prepare_reg(hdev, mmNIC3_QM1_GLBL_NON_SECURE_PROPS_4,
6319 gaudi_mmu_prepare_reg(hdev, mmNIC4_QM0_GLBL_NON_SECURE_PROPS_0,
6321 gaudi_mmu_prepare_reg(hdev, mmNIC4_QM0_GLBL_NON_SECURE_PROPS_1,
6323 gaudi_mmu_prepare_reg(hdev, mmNIC4_QM0_GLBL_NON_SECURE_PROPS_2,
6325 gaudi_mmu_prepare_reg(hdev, mmNIC4_QM0_GLBL_NON_SECURE_PROPS_3,
6327 gaudi_mmu_prepare_reg(hdev, mmNIC4_QM0_GLBL_NON_SECURE_PROPS_4,
6332 gaudi_mmu_prepare_reg(hdev, mmNIC4_QM1_GLBL_NON_SECURE_PROPS_0,
6334 gaudi_mmu_prepare_reg(hdev, mmNIC4_QM1_GLBL_NON_SECURE_PROPS_1,
6336 gaudi_mmu_prepare_reg(hdev, mmNIC4_QM1_GLBL_NON_SECURE_PROPS_2,
6338 gaudi_mmu_prepare_reg(hdev, mmNIC4_QM1_GLBL_NON_SECURE_PROPS_3,
6340 gaudi_mmu_prepare_reg(hdev, mmNIC4_QM1_GLBL_NON_SECURE_PROPS_4,
6344 gaudi_mmu_prepare_reg(hdev, mmPSOC_GLOBAL_CONF_TRACE_ARUSER, asid);
6345 gaudi_mmu_prepare_reg(hdev, mmPSOC_GLOBAL_CONF_TRACE_AWUSER, asid);
6348 static int gaudi_send_job_on_qman0(struct hl_device *hdev,
6358 if (hdev->pldm)
6363 fence_ptr = hl_asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL, &fence_dma_addr);
6365 dev_err(hdev->dev,
6388 rc = hl_hw_queue_send_cb_no_cmpl(hdev, GAUDI_QUEUE_ID_DMA_0_0,
6391 dev_err(hdev->dev, "Failed to send CB on QMAN0, %d\n", rc);
6395 rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp,
6399 hl_hw_queue_inc_ci_kernel(hdev, GAUDI_QUEUE_ID_DMA_0_0);
6402 dev_err(hdev->dev, "QMAN0 Job timeout (0x%x)\n", tmp);
6409 hl_asic_dma_pool_free(hdev, (void *) fence_ptr, fence_dma_addr);
6429 static const char *gaudi_get_razwi_initiator_dma_name(struct hl_device *hdev, u32 x_y,
6527 static const char *gaudi_get_razwi_initiator_name(struct hl_device *hdev, bool is_write,
6585 return gaudi_get_razwi_initiator_dma_name(hdev, x_y, is_write,
6633 dev_err(hdev->dev,
6644 static void gaudi_print_and_get_razwi_info(struct hl_device *hdev, u16 *engine_id_1,
6649 dev_err_ratelimited(hdev->dev,
6651 gaudi_get_razwi_initiator_name(hdev, true, engine_id_1, engine_id_2));
6657 dev_err_ratelimited(hdev->dev,
6659 gaudi_get_razwi_initiator_name(hdev, false, engine_id_1, engine_id_2));
6665 static void gaudi_print_and_get_mmu_error_info(struct hl_device *hdev, u64 *addr, u64 *event_mask)
6667 struct gaudi_device *gaudi = hdev->asic_specific;
6679 dev_err_ratelimited(hdev->dev, "MMU page fault on va 0x%llx\n", *addr);
6680 hl_handle_page_fault(hdev, *addr, 0, true, event_mask);
6691 dev_err_ratelimited(hdev->dev, "MMU access error on va 0x%llx\n", *addr);
6715 static int gaudi_extract_ecc_info(struct hl_device *hdev,
6748 dev_err(hdev->dev, "ECC error information cannot be found\n");
6795 * @hdev: pointer to the habanalabs device structure
6800 static void gaudi_handle_sw_config_stream_data(struct hl_device *hdev, u32 stream,
6817 dev_info(hdev->dev, "stop on err: stream: %u, addr: %#llx, size: %u\n",
6821 hdev->captured_err_info.undef_opcode.cq_addr = cq_ptr;
6822 hdev->captured_err_info.undef_opcode.cq_size = size;
6823 hdev->captured_err_info.undef_opcode.stream_id = stream;
6830 * @hdev: pointer to the habanalabs device structure
6837 static void gaudi_handle_last_pqes_on_err(struct hl_device *hdev, u32 qid_base,
6847 q = &hdev->kernel_queues[qid_base + stream];
6856 hdev->asic_funcs->hw_queues_lock(hdev);
6859 gaudi_handle_sw_config_stream_data(hdev, stream, qman_base, event_mask);
6881 dev_info(hdev->dev, "stop on err PQE(stream %u): ci: %u, addr: %#llx, size: %u\n",
6889 struct undefined_opcode_info *undef_opcode = &hdev->captured_err_info.undef_opcode;
6901 hdev->asic_funcs->hw_queues_unlock(hdev);
6907 * @hdev: pointer to the habanalabs device structure
6917 static void handle_qman_data_on_err(struct hl_device *hdev, u32 qid_base,
6923 gaudi_handle_last_pqes_on_err(hdev, qid_base, stream,
6929 gaudi_handle_sw_config_stream_data(hdev, stream, qman_base, event_mask);
6932 gaudi_handle_last_pqes_on_err(hdev, qid_base, i,
6936 static void gaudi_handle_qman_err_generic(struct hl_device *hdev,
6964 dev_err_ratelimited(hdev->dev,
6973 hdev->captured_err_info.undef_opcode.write_enable) {
6974 memset(&hdev->captured_err_info.undef_opcode, 0,
6975 sizeof(hdev->captured_err_info.undef_opcode));
6977 hdev->captured_err_info.undef_opcode.write_enable = false;
6982 if (!hdev->stop_on_err)
6985 handle_qman_data_on_err(hdev, qid_base, i, qman_base, *event_mask);
6995 dev_err_ratelimited(hdev->dev,
7003 static void gaudi_print_sm_sei_info(struct hl_device *hdev, u16 event_type,
7013 dev_err_ratelimited(hdev->dev,
7019 dev_err_ratelimited(hdev->dev,
7025 dev_err_ratelimited(hdev->dev,
7031 dev_err_ratelimited(hdev->dev, "Unknown SM SEI cause %u",
7037 static void gaudi_handle_ecc_event(struct hl_device *hdev, u16 event_type,
7046 if (hdev->asic_prop.fw_security_enabled) {
7123 rc = gaudi_extract_ecc_info(hdev, &params, &ecc_address,
7129 dev_err(hdev->dev,
7134 static void gaudi_handle_qman_err(struct hl_device *hdev, u16 event_type, u64 *event_mask)
7222 gaudi_handle_qman_err_generic(hdev, desc, qman_base, qid_base, event_mask);
7225 static void gaudi_print_irq_info(struct hl_device *hdev, u16 event_type,
7242 dev_err_ratelimited(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n",
7246 gaudi_print_and_get_razwi_info(hdev, &engine_id[0], &engine_id[1], &is_read,
7248 gaudi_print_and_get_mmu_error_info(hdev, &razwi_addr, event_mask);
7263 hl_handle_razwi(hdev, razwi_addr, engine_id, num_of_razwi_eng,
7268 static void gaudi_print_out_of_sync_info(struct hl_device *hdev,
7271 struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI_QUEUE_ID_CPU_PQ];
7273 dev_err(hdev->dev, "Out of sync with FW, FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d\n",
7277 static void gaudi_print_fw_alive_info(struct hl_device *hdev,
7280 dev_err(hdev->dev,
7288 static void gaudi_print_nic_axi_irq_info(struct hl_device *hdev, u16 event_type,
7318 dev_err(hdev->dev, "unknown NIC AXI cause %d\n",
7326 dev_err_ratelimited(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n",
7330 static int gaudi_compute_reset_late_init(struct hl_device *hdev)
7336 static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device,
7342 if (hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &
7345 dev_err(hdev->dev, "No FW ECC data");
7364 dev_err(hdev->dev,
7367 dev_err(hdev->dev,
7375 if (hdev->asic_prop.fw_security_enabled) {
7376 dev_info(hdev->dev, "Cannot access MC regs for ECC data while security is enabled\n");
7386 dev_err(hdev->dev,
7393 dev_err(hdev->dev,
7406 dev_err(hdev->dev,
7413 dev_err(hdev->dev,
7435 dev_err(hdev->dev,
7443 dev_err(hdev->dev,
7474 static bool gaudi_tpc_read_interrupts(struct hl_device *hdev, u8 tpc_id,
7485 dev_err_ratelimited(hdev->dev,
7510 static void gaudi_print_clk_change_info(struct hl_device *hdev, u16 event_type, u64 *event_mask)
7514 mutex_lock(&hdev->clk_throttling.lock);
7518 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_POWER;
7519 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_POWER;
7520 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].start = ktime_get();
7521 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = zero_time;
7522 dev_info_ratelimited(hdev->dev,
7527 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_POWER;
7528 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = ktime_get();
7529 dev_info_ratelimited(hdev->dev,
7534 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_THERMAL;
7535 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_THERMAL;
7536 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].start = ktime_get();
7537 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = zero_time;
7539 dev_info_ratelimited(hdev->dev,
7544 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_THERMAL;
7545 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = ktime_get();
7547 dev_info_ratelimited(hdev->dev,
7552 dev_err(hdev->dev, "Received invalid clock change event %d\n",
7557 mutex_unlock(&hdev->clk_throttling.lock);
7560 static void gaudi_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry)
7562 struct gaudi_device *gaudi = hdev->asic_specific;
7574 dev_err(hdev->dev, "Event type %u exceeds maximum of %u",
7606 gaudi_print_irq_info(hdev, event_type, true, &event_mask);
7607 gaudi_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data);
7616 gaudi_print_irq_info(hdev, event_type, false, &event_mask);
7625 gaudi_print_irq_info(hdev, event_type, false, &event_mask);
7626 gaudi_hbm_read_interrupts(hdev,
7637 gaudi_print_irq_info(hdev, event_type, false, &event_mask);
7638 gaudi_hbm_read_interrupts(hdev,
7641 hl_fw_unmask_irq(hdev, event_type);
7659 gaudi_print_irq_info(hdev, event_type, true, &event_mask);
7660 reset_required = gaudi_tpc_read_interrupts(hdev,
7665 dev_err(hdev->dev, "reset required due to %s\n",
7671 hl_fw_unmask_irq(hdev, event_type);
7684 gaudi_print_irq_info(hdev, event_type, true, &event_mask);
7685 reset_required = gaudi_tpc_read_interrupts(hdev,
7690 dev_err(hdev->dev, "reset required due to %s\n",
7696 hl_fw_unmask_irq(hdev, event_type);
7723 gaudi_print_irq_info(hdev, event_type, true, &event_mask);
7724 gaudi_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data);
7725 hl_fw_unmask_irq(hdev, event_type);
7733 gaudi_print_irq_info(hdev, event_type, true, &event_mask);
7734 hl_fw_unmask_irq(hdev, event_type);
7740 gaudi_print_irq_info(hdev, event_type, true, &event_mask);
7741 hl_fw_unmask_irq(hdev, event_type);
7769 gaudi_print_irq_info(hdev, event_type, true, &event_mask);
7770 gaudi_handle_qman_err(hdev, event_type, &event_mask);
7771 hl_fw_unmask_irq(hdev, event_type);
7776 gaudi_print_irq_info(hdev, event_type, true, &event_mask);
7789 gaudi_print_irq_info(hdev, event_type, false, &event_mask);
7790 hl_fw_unmask_irq(hdev, event_type);
7795 gaudi_print_nic_axi_irq_info(hdev, event_type, &data);
7796 hl_fw_unmask_irq(hdev, event_type);
7801 gaudi_print_irq_info(hdev, event_type, false, &event_mask);
7802 gaudi_print_sm_sei_info(hdev, event_type,
7804 rc = hl_state_dump(hdev);
7807 dev_err(hdev->dev,
7809 hl_fw_unmask_irq(hdev, event_type);
7816 gaudi_print_clk_change_info(hdev, event_type, &event_mask);
7817 hl_fw_unmask_irq(hdev, event_type);
7822 dev_err(hdev->dev,
7829 gaudi_print_irq_info(hdev, event_type, false, &event_mask);
7834 gaudi_print_irq_info(hdev, event_type, false, &event_mask);
7835 gaudi_print_out_of_sync_info(hdev, &eq_entry->pkt_sync_err);
7840 gaudi_print_irq_info(hdev, event_type, false, &event_mask);
7841 gaudi_print_fw_alive_info(hdev, &eq_entry->fw_alive);
7845 hl_handle_fw_err(hdev, &fw_err_info);
7849 dev_err(hdev->dev, "Received invalid H/W interrupt %d\n",
7855 hl_notifier_event_send_all(hdev, event_mask);
7862 if (hdev->asic_prop.fw_security_enabled && !reset_direct) {
7868 } else if (hdev->hard_reset_on_fw_events) {
7878 hl_handle_critical_hw_err(hdev, event_type, &event_mask);
7880 hl_device_cond_reset(hdev, flags, event_mask);
7882 hl_fw_unmask_irq(hdev, event_type);
7885 hl_notifier_event_send_all(hdev, event_mask);
7889 static void *gaudi_get_events_stat(struct hl_device *hdev, bool aggregate, u32 *size)
7891 struct gaudi_device *gaudi = hdev->asic_specific;
7902 static int gaudi_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, u32 flags)
7904 struct gaudi_device *gaudi = hdev->asic_specific;
7909 hdev->reset_info.hard_reset_pending)
7912 if (hdev->pldm)
7923 hdev,
7935 static int gaudi_mmu_invalidate_cache_range(struct hl_device *hdev,
7942 return hdev->asic_funcs->mmu_invalidate_cache(hdev, is_hard, flags);
7945 static int gaudi_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 asid, u64 phys_addr)
7950 if (hdev->pldm)
7961 hdev,
7969 dev_err(hdev->dev,
7977 static int gaudi_send_heartbeat(struct hl_device *hdev)
7979 struct gaudi_device *gaudi = hdev->asic_specific;
7984 return hl_fw_send_heartbeat(hdev);
7987 static int gaudi_cpucp_info_get(struct hl_device *hdev)
7989 struct gaudi_device *gaudi = hdev->asic_specific;
7990 struct asic_fixed_properties *prop = &hdev->asic_prop;
7996 rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0,
8006 hdev->card_type = le32_to_cpu(hdev->asic_prop.cpucp_info.card_type);
8008 set_default_power_values(hdev);
8013 static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
8016 struct gaudi_device *gaudi = hdev->asic_specific;
8150 static void gaudi_hw_queues_lock(struct hl_device *hdev)
8153 struct gaudi_device *gaudi = hdev->asic_specific;
8158 static void gaudi_hw_queues_unlock(struct hl_device *hdev)
8161 struct gaudi_device *gaudi = hdev->asic_specific;
8166 static u32 gaudi_get_pci_id(struct hl_device *hdev)
8168 return hdev->pdev->device;
8171 static int gaudi_get_eeprom_data(struct hl_device *hdev, void *data,
8174 struct gaudi_device *gaudi = hdev->asic_specific;
8179 return hl_fw_get_eeprom_data(hdev, data, max_size);
8182 static int gaudi_get_monitor_dump(struct hl_device *hdev, void *data)
8184 struct gaudi_device *gaudi = hdev->asic_specific;
8189 return hl_fw_get_monitor_dump(hdev, data);
8196 static int gaudi_run_tpc_kernel(struct hl_device *hdev, u64 tpc_kernel, u32 tpc_id)
8204 if (hdev->pldm)
8236 hdev,
8245 dev_err(hdev->dev,
8259 hdev,
8268 dev_err(hdev->dev,
8275 hdev,
8283 dev_err(hdev->dev,
8292 static int gaudi_internal_cb_pool_init(struct hl_device *hdev,
8295 struct gaudi_device *gaudi = hdev->asic_specific;
8301 hdev->internal_cb_pool_virt_addr = hl_asic_dma_alloc_coherent(hdev,
8303 &hdev->internal_cb_pool_dma_addr,
8306 if (!hdev->internal_cb_pool_virt_addr)
8313 hdev->internal_cb_pool = gen_pool_create(min_alloc_order, -1);
8314 if (!hdev->internal_cb_pool) {
8315 dev_err(hdev->dev,
8321 rc = gen_pool_add(hdev->internal_cb_pool,
8322 (uintptr_t) hdev->internal_cb_pool_virt_addr,
8325 dev_err(hdev->dev,
8331 hdev->internal_cb_va_base = hl_reserve_va_block(hdev, ctx,
8335 if (!hdev->internal_cb_va_base) {
8340 mutex_lock(&hdev->mmu_lock);
8342 rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base,
8343 hdev->internal_cb_pool_dma_addr,
8348 rc = hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR);
8352 mutex_unlock(&hdev->mmu_lock);
8357 hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base,
8360 mutex_unlock(&hdev->mmu_lock);
8361 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base,
8364 gen_pool_destroy(hdev->internal_cb_pool);
8366 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr,
8367 hdev->internal_cb_pool_dma_addr);
8372 static void gaudi_internal_cb_pool_fini(struct hl_device *hdev,
8375 struct gaudi_device *gaudi = hdev->asic_specific;
8380 mutex_lock(&hdev->mmu_lock);
8381 hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base,
8383 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base,
8385 hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR);
8386 mutex_unlock(&hdev->mmu_lock);
8388 gen_pool_destroy(hdev->internal_cb_pool);
8390 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr,
8391 hdev->internal_cb_pool_dma_addr);
8401 rc = gaudi_internal_cb_pool_init(ctx->hdev, ctx);
8405 rc = gaudi_restore_user_registers(ctx->hdev);
8407 gaudi_internal_cb_pool_fini(ctx->hdev, ctx);
8417 gaudi_internal_cb_pool_fini(ctx->hdev, ctx);
8425 static u32 gaudi_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx)
8430 static u32 gaudi_get_signal_cb_size(struct hl_device *hdev)
8436 static u32 gaudi_get_wait_cb_size(struct hl_device *hdev)
8443 static u32 gaudi_get_sob_addr(struct hl_device *hdev, u32 sob_id)
8448 static u32 gaudi_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id,
8496 static u32 gaudi_add_arm_monitor_pkt(struct hl_device *hdev,
8506 dev_err(hdev->dev,
8566 static int gaudi_get_fence_addr(struct hl_device *hdev, u32 queue_id, u64 *addr)
8729 static u32 gaudi_gen_wait_cb(struct hl_device *hdev,
8737 if (gaudi_get_fence_addr(hdev, prop->q_idx, &fence_addr)) {
8738 dev_crit(hdev->dev, "wrong queue id %d for wait packet\n",
8744 size += gaudi_add_arm_monitor_pkt(hdev, buf + size, prop->sob_base,
8751 static void gaudi_reset_sob(struct hl_device *hdev, void *data)
8755 dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx,
8764 static u64 gaudi_get_device_time(struct hl_device *hdev)
8771 static int gaudi_get_hw_block_id(struct hl_device *hdev, u64 block_addr,
8777 static int gaudi_block_mmap(struct hl_device *hdev,
8784 static void gaudi_enable_events_from_fw(struct hl_device *hdev)
8787 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
8788 u32 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?
8796 static int gaudi_ack_mmu_page_fault_or_access_error(struct hl_device *hdev, u64 mmu_cap_mask)
8844 static int gaudi_gen_sync_to_engine_map(struct hl_device *hdev,
8847 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
8933 struct hl_device *hdev,
8941 name = hl_state_dump_get_monitor_name(hdev, mon);
8972 struct hl_device *hdev, u64 base_offset, u64 status_base_offset,
8976 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
9048 static void gaudi_state_dump_init(struct hl_device *hdev)
9050 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
9075 static int gaudi_set_dram_properties(struct hl_device *hdev)
9080 static int gaudi_set_binning_masks(struct hl_device *hdev)
9085 static void gaudi_check_if_razwi_happened(struct hl_device *hdev)
9091 struct hl_device *hdev = dev_get_drvdata(dev);
9094 cpucp_info = &hdev->asic_prop.cpucp_info;
9106 static void gaudi_add_device_attr(struct hl_device *hdev, struct attribute_group *dev_clk_attr_grp,
9109 hl_sysfs_add_dev_clk_attr(hdev, dev_clk_attr_grp);
9113 static int gaudi_send_device_activity(struct hl_device *hdev, bool open)
9218 * @hdev: pointer to hl_device structure
9221 void gaudi_set_asic_funcs(struct hl_device *hdev)
9223 hdev->asic_funcs = &gaudi_funcs;