Lines Matching defs:hdev
2104 static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val);
2105 static bool gaudi2_is_queue_enabled(struct hl_device *hdev, u32 hw_queue_id);
2106 static bool gaudi2_is_arc_enabled(struct hl_device *hdev, u64 arc_id);
2107 static void gaudi2_clr_arc_id_cap(struct hl_device *hdev, u64 arc_id);
2108 static void gaudi2_set_arc_id_cap(struct hl_device *hdev, u64 arc_id);
2109 static void gaudi2_memset_device_lbw(struct hl_device *hdev, u32 addr, u32 size, u32 val);
2110 static int gaudi2_send_job_to_kdma(struct hl_device *hdev, u64 src_addr, u64 dst_addr, u32 size,
2112 static bool gaudi2_get_tpc_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
2114 static bool gaudi2_get_mme_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
2116 static bool gaudi2_get_edma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
2118 static u64 gaudi2_mmu_scramble_addr(struct hl_device *hdev, u64 raw_addr);
2119 static u64 gaudi2_mmu_descramble_addr(struct hl_device *hdev, u64 scrambled_addr);
2121 static void gaudi2_init_scrambler_hbm(struct hl_device *hdev)
2126 static u32 gaudi2_get_signal_cb_size(struct hl_device *hdev)
2131 static u32 gaudi2_get_wait_cb_size(struct hl_device *hdev)
2136 void gaudi2_iterate_tpcs(struct hl_device *hdev, struct iterate_module_ctx *ctx)
2138 struct asic_fixed_properties *prop = &hdev->asic_prop;
2154 ctx->fn(hdev, dcore, inst, offset, ctx);
2156 dev_err(hdev->dev, "TPC iterator failed for DCORE%d TPC%d\n",
2168 ctx->fn(hdev, 0, NUM_DCORE0_TPC - 1, offset, ctx);
2170 dev_err(hdev->dev, "TPC iterator failed for DCORE0 TPC6\n");
2181 static int set_number_of_functional_hbms(struct hl_device *hdev)
2183 struct asic_fixed_properties *prop = &hdev->asic_prop;
2184 u8 faulty_hbms = hweight64(hdev->dram_binning);
2188 dev_dbg(hdev->dev, "All HBM are in use (no binning)\n");
2200 dev_err(hdev->dev,
2202 MAX_FAULTY_HBMS, hdev->dram_binning);
2214 static int gaudi2_set_dram_properties(struct hl_device *hdev)
2216 struct asic_fixed_properties *prop = &hdev->asic_prop;
2220 rc = set_number_of_functional_hbms(hdev);
2269 static int gaudi2_set_fixed_properties(struct hl_device *hdev)
2271 struct asic_fixed_properties *prop = &hdev->asic_prop;
2325 if (hdev->pldm)
2359 hdev->pmmu_huge_range = true;
2468 static int gaudi2_pci_bars_map(struct hl_device *hdev)
2474 rc = hl_pci_bars_map(hdev, name, is_wc);
2478 hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] + (CFG_BASE - STM_FLASH_BASE_ADDR);
2483 static u64 gaudi2_set_hbm_bar_base(struct hl_device *hdev, u64 addr)
2485 struct gaudi2_device *gaudi2 = hdev->asic_specific;
2493 if (hdev->asic_prop.iatu_done_by_fw)
2500 rc = hl_pci_set_inbound_region(hdev, 2, &pci_region);
2512 static int gaudi2_init_iatu(struct hl_device *hdev)
2519 if (hdev->asic_prop.iatu_done_by_fw)
2530 rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region);
2538 hdev->pcie_bar_phys[SRAM_CFG_BAR_ID] = (u64)bar_addr_high << 32 | bar_addr_low;
2546 rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region);
2556 rc = hl_pci_set_inbound_region(hdev, 1, &inbound_region);
2564 rc = hl_pci_set_inbound_region(hdev, 2, &inbound_region);
2571 rc = hl_pci_set_outbound_region(hdev, &outbound_region);
2576 static enum hl_device_hw_state gaudi2_get_hw_state(struct hl_device *hdev)
2581 static int gaudi2_tpc_binning_init_prop(struct hl_device *hdev)
2583 struct asic_fixed_properties *prop = &hdev->asic_prop;
2589 if (hweight64(hdev->tpc_binning) > MAX_CLUSTER_BINNING_FAULTY_TPCS) {
2590 dev_err(hdev->dev, "TPC binning is supported for max of %d faulty TPCs, provided mask 0x%llx\n",
2592 hdev->tpc_binning);
2596 prop->tpc_binning_mask = hdev->tpc_binning;
2602 static int gaudi2_set_tpc_binning_masks(struct hl_device *hdev)
2604 struct asic_fixed_properties *prop = &hdev->asic_prop;
2610 rc = gaudi2_tpc_binning_init_prop(hdev);
2638 dev_err(hdev->dev,
2660 static int gaudi2_set_dec_binning_masks(struct hl_device *hdev)
2662 struct asic_fixed_properties *prop = &hdev->asic_prop;
2665 num_faulty = hweight32(hdev->decoder_binning);
2672 dev_err(hdev->dev, "decoder binning is supported for max of single faulty decoder, provided mask 0x%x\n",
2673 hdev->decoder_binning);
2677 prop->decoder_binning_mask = (hdev->decoder_binning & GAUDI2_DECODER_FULL_MASK);
2687 static void gaudi2_set_dram_binning_masks(struct hl_device *hdev)
2689 struct asic_fixed_properties *prop = &hdev->asic_prop;
2692 if (!hdev->dram_binning) {
2699 prop->faulty_dram_cluster_map |= hdev->dram_binning;
2700 prop->dram_binning_mask = hdev->dram_binning;
2704 static int gaudi2_set_edma_binning_masks(struct hl_device *hdev)
2706 struct asic_fixed_properties *prop = &hdev->asic_prop;
2710 num_faulty = hweight32(hdev->edma_binning);
2717 dev_err(hdev->dev,
2719 hdev->edma_binning);
2723 if (!hdev->edma_binning) {
2729 seq = __ffs((unsigned long)hdev->edma_binning);
2733 prop->edma_binning_mask = hdev->edma_binning;
2746 static int gaudi2_set_xbar_edge_enable_mask(struct hl_device *hdev, u32 xbar_edge_iso_mask)
2748 struct asic_fixed_properties *prop = &hdev->asic_prop;
2768 dev_err(hdev->dev, "we cannot have more than %d faulty XBAR EDGE\n",
2782 static int gaudi2_set_cluster_binning_masks_common(struct hl_device *hdev, u8 xbar_edge_iso_mask)
2791 hdev->asic_prop.faulty_dram_cluster_map = 0;
2793 gaudi2_set_dram_binning_masks(hdev);
2795 rc = gaudi2_set_edma_binning_masks(hdev);
2799 rc = gaudi2_set_xbar_edge_enable_mask(hdev, xbar_edge_iso_mask);
2805 hdev->asic_prop.hmmu_hif_enabled_mask = GAUDI2_HIF_HMMU_FULL_MASK;
2810 static int gaudi2_set_cluster_binning_masks(struct hl_device *hdev)
2812 struct asic_fixed_properties *prop = &hdev->asic_prop;
2815 rc = gaudi2_set_cluster_binning_masks_common(hdev, prop->cpucp_info.xbar_binning_mask);
2829 static int gaudi2_set_binning_masks(struct hl_device *hdev)
2833 rc = gaudi2_set_cluster_binning_masks(hdev);
2837 rc = gaudi2_set_tpc_binning_masks(hdev);
2841 rc = gaudi2_set_dec_binning_masks(hdev);
2848 static int gaudi2_cpucp_info_get(struct hl_device *hdev)
2850 struct gaudi2_device *gaudi2 = hdev->asic_specific;
2851 struct asic_fixed_properties *prop = &hdev->asic_prop;
2862 if (hdev->reset_info.in_compute_reset)
2865 rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0,
2876 dev_err(hdev->dev,
2890 hdev->dram_binning = prop->cpucp_info.dram_binning_mask;
2891 hdev->edma_binning = prop->cpucp_info.edma_binning_mask;
2892 hdev->tpc_binning = le64_to_cpu(prop->cpucp_info.tpc_binning_mask);
2893 hdev->decoder_binning = lower_32_bits(le64_to_cpu(prop->cpucp_info.decoder_binning_mask));
2895 dev_dbg(hdev->dev, "Read binning masks: tpc: 0x%llx, dram: 0x%llx, edma: 0x%x, dec: 0x%x\n",
2896 hdev->tpc_binning, hdev->dram_binning, hdev->edma_binning,
2897 hdev->decoder_binning);
2903 rc = hdev->asic_funcs->set_dram_properties(hdev);
2907 rc = hdev->asic_funcs->set_binning_masks(hdev);
2911 max_power = hl_fw_get_max_power(hdev);
2920 static int gaudi2_fetch_psoc_frequency(struct hl_device *hdev)
2922 struct gaudi2_device *gaudi2 = hdev->asic_specific;
2929 rc = hl_fw_cpucp_pll_info_get(hdev, HL_GAUDI2_CPU_PLL, pll_freq_arr);
2933 hdev->asic_prop.psoc_timestamp_frequency = pll_freq_arr[3];
2938 static int gaudi2_early_init(struct hl_device *hdev)
2940 struct asic_fixed_properties *prop = &hdev->asic_prop;
2941 struct pci_dev *pdev = hdev->pdev;
2945 rc = gaudi2_set_fixed_properties(hdev);
2953 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n",
2961 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n",
2968 hdev->dram_pci_bar_start = pci_resource_start(pdev, DRAM_BAR_ID);
2973 if (hdev->pldm)
2974 hdev->asic_prop.iatu_done_by_fw = false;
2976 hdev->asic_prop.iatu_done_by_fw = true;
2978 rc = hl_pci_init(hdev);
2985 rc = hl_fw_read_preboot_status(hdev);
2987 if (hdev->reset_on_preboot_fail)
2989 hdev->asic_funcs->hw_fini(hdev, true, false);
2993 if (gaudi2_get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {
2994 dev_dbg(hdev->dev, "H/W state is dirty, must reset before initializing\n");
2995 rc = hdev->asic_funcs->hw_fini(hdev, true, false);
2997 dev_err(hdev->dev, "failed to reset HW in dirty state (%d)\n", rc);
3005 hl_pci_fini(hdev);
3007 kfree(hdev->asic_prop.hw_queues_props);
3011 static int gaudi2_early_fini(struct hl_device *hdev)
3013 kfree(hdev->asic_prop.hw_queues_props);
3014 hl_pci_fini(hdev);
3039 static void gaudi2_init_arcs(struct hl_device *hdev)
3041 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
3042 struct gaudi2_device *gaudi2 = hdev->asic_specific;
3047 if (gaudi2_is_arc_enabled(hdev, i))
3050 gaudi2_set_arc_id_cap(hdev, i);
3054 if (!gaudi2_is_queue_enabled(hdev, i))
3058 if (gaudi2_is_arc_enabled(hdev, arc_id))
3062 !(hdev->nic_ports_mask & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0)))
3069 gaudi2_set_arc_id_cap(hdev, arc_id);
3073 hdev->asic_prop.engine_core_interrupt_reg_addr =
3077 static int gaudi2_scrub_arc_dccm(struct hl_device *hdev, u32 cpu_id)
3085 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id],
3097 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id],
3107 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id],
3117 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id],
3126 static int gaudi2_scrub_arcs_dccm(struct hl_device *hdev)
3132 if (!gaudi2_is_arc_enabled(hdev, arc_id))
3135 rc = gaudi2_scrub_arc_dccm(hdev, arc_id);
3143 static int gaudi2_late_init(struct hl_device *hdev)
3145 struct gaudi2_device *gaudi2 = hdev->asic_specific;
3148 hdev->asic_prop.supports_advanced_cpucp_rc = true;
3150 rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_ENABLE_PCI_ACCESS,
3153 dev_err(hdev->dev, "Failed to enable PCI access from CPU\n");
3157 rc = gaudi2_fetch_psoc_frequency(hdev);
3159 dev_err(hdev->dev, "Failed to fetch psoc frequency\n");
3163 gaudi2_init_arcs(hdev);
3165 rc = gaudi2_scrub_arcs_dccm(hdev);
3167 dev_err(hdev->dev, "Failed to scrub arcs DCCM\n");
3171 gaudi2_init_security(hdev);
3176 hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0);
3181 static void gaudi2_late_fini(struct hl_device *hdev)
3183 hl_hwmon_release_resources(hdev);
3202 static void gaudi2_user_mapped_blocks_init(struct hl_device *hdev)
3204 struct gaudi2_device *gaudi2 = hdev->asic_specific;
3274 static int gaudi2_alloc_cpu_accessible_dma_mem(struct hl_device *hdev)
3286 virt_addr_arr[i] = hl_asic_dma_alloc_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE,
3299 dev_err(hdev->dev,
3305 hdev->cpu_accessible_dma_mem = virt_addr_arr[i];
3306 hdev->cpu_accessible_dma_address = dma_addr_arr[i];
3310 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, virt_addr_arr[j],
3316 static void gaudi2_set_pci_memory_regions(struct hl_device *hdev)
3318 struct asic_fixed_properties *prop = &hdev->asic_prop;
3322 region = &hdev->pci_mem_region[PCI_REGION_CFG];
3331 region = &hdev->pci_mem_region[PCI_REGION_SRAM];
3340 region = &hdev->pci_mem_region[PCI_REGION_DRAM];
3342 region->region_size = hdev->asic_prop.dram_size;
3349 static void gaudi2_user_interrupt_setup(struct hl_device *hdev)
3351 struct asic_fixed_properties *prop = &hdev->asic_prop;
3355 HL_USR_INTR_STRUCT_INIT(hdev->tpc_interrupt, hdev, 0, HL_USR_INTERRUPT_TPC);
3358 HL_USR_INTR_STRUCT_INIT(hdev->unexpected_error_interrupt, hdev, 0,
3362 HL_USR_INTR_STRUCT_INIT(hdev->common_user_cq_interrupt, hdev,
3366 HL_USR_INTR_STRUCT_INIT(hdev->common_decoder_interrupt, hdev,
3380 HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i,
3384 HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i, HL_USR_INTERRUPT_CQ);
3394 static void gaudi2_special_blocks_free(struct hl_device *hdev)
3396 struct asic_fixed_properties *prop = &hdev->asic_prop;
3405 static void gaudi2_special_blocks_iterator_free(struct hl_device *hdev)
3407 gaudi2_special_blocks_free(hdev);
3410 static bool gaudi2_special_block_skip(struct hl_device *hdev,
3417 static int gaudi2_special_blocks_config(struct hl_device *hdev)
3419 struct asic_fixed_properties *prop = &hdev->asic_prop;
3482 static int gaudi2_special_blocks_iterator_config(struct hl_device *hdev)
3484 return gaudi2_special_blocks_config(hdev);
3487 static void gaudi2_test_queues_msgs_free(struct hl_device *hdev)
3489 struct gaudi2_device *gaudi2 = hdev->asic_specific;
3498 hl_asic_dma_pool_free(hdev, msg_info[i].kern_addr, msg_info[i].dma_addr);
3503 static int gaudi2_test_queues_msgs_alloc(struct hl_device *hdev)
3505 struct gaudi2_device *gaudi2 = hdev->asic_specific;
3512 (void *)hl_asic_dma_pool_zalloc(hdev, sizeof(struct packet_msg_short),
3515 dev_err(hdev->dev,
3525 gaudi2_test_queues_msgs_free(hdev);
3529 static int gaudi2_sw_init(struct hl_device *hdev)
3531 struct asic_fixed_properties *prop = &hdev->asic_prop;
3545 dev_err(hdev->dev, "H/W events array exceeds the limit of %u events\n",
3559 hdev->asic_specific = gaudi2;
3565 hdev->dma_pool = dma_pool_create(dev_name(hdev->dev), &hdev->pdev->dev,
3567 if (!hdev->dma_pool) {
3568 dev_err(hdev->dev, "failed to create DMA pool\n");
3573 rc = gaudi2_alloc_cpu_accessible_dma_mem(hdev);
3577 hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1);
3578 if (!hdev->cpu_accessible_dma_pool) {
3579 dev_err(hdev->dev, "Failed to create CPU accessible DMA pool\n");
3584 rc = gen_pool_add(hdev->cpu_accessible_dma_pool, (uintptr_t) hdev->cpu_accessible_dma_mem,
3587 dev_err(hdev->dev, "Failed to add memory to CPU accessible DMA pool\n");
3592 gaudi2->virt_msix_db_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, prop->pmmu.page_size,
3595 dev_err(hdev->dev, "Failed to allocate DMA memory for virtual MSI-X doorbell\n");
3602 gaudi2->scratchpad_kernel_address = hl_asic_dma_alloc_coherent(hdev, PAGE_SIZE,
3610 gaudi2_user_mapped_blocks_init(hdev);
3613 gaudi2_user_interrupt_setup(hdev);
3615 hdev->supports_coresight = true;
3616 hdev->supports_sync_stream = true;
3617 hdev->supports_cb_mapping = true;
3618 hdev->supports_wait_for_multi_cs = false;
3623 if (hl_is_fw_sw_ver_below(hdev, 1, 11))
3624 hdev->event_queue.check_eqe_index = false;
3626 hdev->event_queue.check_eqe_index = true;
3628 hdev->asic_funcs->set_pci_memory_regions(hdev);
3630 rc = gaudi2_special_blocks_iterator_config(hdev);
3634 rc = gaudi2_test_queues_msgs_alloc(hdev);
3641 gaudi2_special_blocks_iterator_free(hdev);
3643 hl_asic_dma_free_coherent(hdev, PAGE_SIZE, gaudi2->scratchpad_kernel_address,
3646 hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr);
3648 gen_pool_destroy(hdev->cpu_accessible_dma_pool);
3650 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem,
3651 hdev->cpu_accessible_dma_address);
3653 dma_pool_destroy(hdev->dma_pool);
3659 static int gaudi2_sw_fini(struct hl_device *hdev)
3661 struct asic_fixed_properties *prop = &hdev->asic_prop;
3662 struct gaudi2_device *gaudi2 = hdev->asic_specific;
3664 gaudi2_test_queues_msgs_free(hdev);
3666 gaudi2_special_blocks_iterator_free(hdev);
3668 hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr);
3670 gen_pool_destroy(hdev->cpu_accessible_dma_pool);
3672 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem,
3673 hdev->cpu_accessible_dma_address);
3675 hl_asic_dma_free_coherent(hdev, PAGE_SIZE, gaudi2->scratchpad_kernel_address,
3678 dma_pool_destroy(hdev->dma_pool);
3685 static void gaudi2_stop_qman_common(struct hl_device *hdev, u32 reg_base)
3695 static void gaudi2_flush_qman_common(struct hl_device *hdev, u32 reg_base)
3702 static void gaudi2_flush_qman_arc_common(struct hl_device *hdev, u32 reg_base)
3710 * @hdev: pointer to the habanalabs device structure
3716 static void gaudi2_clear_qm_fence_counters_common(struct hl_device *hdev, u32 queue_id,
3734 gaudi2_memset_device_lbw(hdev, addr, size, val);
3737 static void gaudi2_qman_manual_flush_common(struct hl_device *hdev, u32 queue_id)
3741 gaudi2_clear_qm_fence_counters_common(hdev, queue_id, true);
3742 gaudi2_flush_qman_common(hdev, reg_base);
3743 gaudi2_flush_qman_arc_common(hdev, reg_base);
3746 static void gaudi2_stop_dma_qmans(struct hl_device *hdev)
3748 struct gaudi2_device *gaudi2 = hdev->asic_specific;
3755 gaudi2_stop_qman_common(hdev, mmPDMA0_QM_BASE);
3756 gaudi2_stop_qman_common(hdev, mmPDMA1_QM_BASE);
3774 gaudi2_stop_qman_common(hdev, qm_base);
3779 static void gaudi2_stop_mme_qmans(struct hl_device *hdev)
3781 struct gaudi2_device *gaudi2 = hdev->asic_specific;
3790 gaudi2_stop_qman_common(hdev, mmDCORE0_MME_QM_BASE + (i * offset));
3794 static void gaudi2_stop_tpc_qmans(struct hl_device *hdev)
3796 struct gaudi2_device *gaudi2 = hdev->asic_specific;
3808 gaudi2_stop_qman_common(hdev, reg_base);
3812 static void gaudi2_stop_rot_qmans(struct hl_device *hdev)
3814 struct gaudi2_device *gaudi2 = hdev->asic_specific;
3826 gaudi2_stop_qman_common(hdev, reg_base);
3830 static void gaudi2_stop_nic_qmans(struct hl_device *hdev)
3832 struct gaudi2_device *gaudi2 = hdev->asic_specific;
3842 if (!(hdev->nic_ports_mask & BIT(i)))
3846 gaudi2_stop_qman_common(hdev, reg_base);
3850 static void gaudi2_stall_dma_common(struct hl_device *hdev, u32 reg_base)
3858 static void gaudi2_dma_stall(struct hl_device *hdev)
3860 struct gaudi2_device *gaudi2 = hdev->asic_specific;
3866 gaudi2_stall_dma_common(hdev, mmPDMA0_CORE_BASE);
3867 gaudi2_stall_dma_common(hdev, mmPDMA1_CORE_BASE);
3885 gaudi2_stall_dma_common(hdev, core_base);
3890 static void gaudi2_mme_stall(struct hl_device *hdev)
3892 struct gaudi2_device *gaudi2 = hdev->asic_specific;
3902 static void gaudi2_tpc_stall(struct hl_device *hdev)
3904 struct gaudi2_device *gaudi2 = hdev->asic_specific;
3920 static void gaudi2_rotator_stall(struct hl_device *hdev)
3922 struct gaudi2_device *gaudi2 = hdev->asic_specific;
3941 static void gaudi2_disable_qman_common(struct hl_device *hdev, u32 reg_base)
3946 static void gaudi2_disable_dma_qmans(struct hl_device *hdev)
3948 struct gaudi2_device *gaudi2 = hdev->asic_specific;
3954 gaudi2_disable_qman_common(hdev, mmPDMA0_QM_BASE);
3955 gaudi2_disable_qman_common(hdev, mmPDMA1_QM_BASE);
3973 gaudi2_disable_qman_common(hdev, qm_base);
3978 static void gaudi2_disable_mme_qmans(struct hl_device *hdev)
3980 struct gaudi2_device *gaudi2 = hdev->asic_specific;
3987 gaudi2_disable_qman_common(hdev, mmDCORE0_MME_QM_BASE + (i * offset));
3990 static void gaudi2_disable_tpc_qmans(struct hl_device *hdev)
3992 struct gaudi2_device *gaudi2 = hdev->asic_specific;
4004 gaudi2_disable_qman_common(hdev, reg_base);
4008 static void gaudi2_disable_rot_qmans(struct hl_device *hdev)
4010 struct gaudi2_device *gaudi2 = hdev->asic_specific;
4022 gaudi2_disable_qman_common(hdev, reg_base);
4026 static void gaudi2_disable_nic_qmans(struct hl_device *hdev)
4028 struct gaudi2_device *gaudi2 = hdev->asic_specific;
4038 if (!(hdev->nic_ports_mask & BIT(i)))
4042 gaudi2_disable_qman_common(hdev, reg_base);
4046 static void gaudi2_enable_timestamp(struct hl_device *hdev)
4059 static void gaudi2_disable_timestamp(struct hl_device *hdev)
4085 static void gaudi2_dec_disable_msix(struct hl_device *hdev, u32 max_irq_num)
4091 irq = pci_irq_vector(hdev->pdev, i);
4094 dec = hdev->dec + relative_idx / 2;
4102 (void *) &hdev->user_interrupt[dec->core_id]));
4106 static int gaudi2_dec_enable_msix(struct hl_device *hdev)
4115 irq = pci_irq_vector(hdev->pdev, i);
4125 dec = hdev->dec + relative_idx / 2;
4133 (void *) &hdev->user_interrupt[dec->core_id]);
4137 dev_err(hdev->dev, "Failed to request IRQ %d", irq);
4145 gaudi2_dec_disable_msix(hdev, (GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + irq_init_cnt));
4149 static int gaudi2_enable_msix(struct hl_device *hdev)
4151 struct asic_fixed_properties *prop = &hdev->asic_prop;
4152 struct gaudi2_device *gaudi2 = hdev->asic_specific;
4159 rc = pci_alloc_irq_vectors(hdev->pdev, GAUDI2_MSIX_ENTRIES, GAUDI2_MSIX_ENTRIES,
4162 dev_err(hdev->dev, "MSI-X: Failed to enable support -- %d/%d\n",
4167 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION);
4168 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION];
4171 dev_err(hdev->dev, "Failed to request IRQ %d", irq);
4175 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE);
4177 &hdev->event_queue);
4179 dev_err(hdev->dev, "Failed to request IRQ %d", irq);
4183 rc = gaudi2_dec_enable_msix(hdev);
4185 dev_err(hdev->dev, "Failed to enable decoder IRQ");
4189 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT);
4192 gaudi2_irq_name(GAUDI2_IRQ_NUM_TPC_ASSERT), &hdev->tpc_interrupt);
4194 dev_err(hdev->dev, "Failed to request IRQ %d", irq);
4198 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR);
4201 &hdev->unexpected_error_interrupt);
4203 dev_err(hdev->dev, "Failed to request IRQ %d", irq);
4211 irq = pci_irq_vector(hdev->pdev, i);
4214 gaudi2_irq_name(i), &hdev->user_interrupt[j]);
4217 dev_err(hdev->dev, "Failed to request IRQ %d", irq);
4230 irq = pci_irq_vector(hdev->pdev, i);
4231 free_irq(irq, &hdev->user_interrupt[j]);
4233 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR);
4234 free_irq(irq, &hdev->unexpected_error_interrupt);
4236 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT);
4237 free_irq(irq, &hdev->tpc_interrupt);
4239 gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_DEC_LAST + 1);
4241 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE);
4245 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION);
4249 pci_free_irq_vectors(hdev->pdev);
4254 static void gaudi2_sync_irqs(struct hl_device *hdev)
4256 struct gaudi2_device *gaudi2 = hdev->asic_specific;
4264 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION));
4267 irq = pci_irq_vector(hdev->pdev, i);
4271 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT));
4272 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR));
4274 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = 0 ; j < hdev->asic_prop.user_interrupt_count;
4276 irq = pci_irq_vector(hdev->pdev, i);
4280 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE));
4283 static void gaudi2_disable_msix(struct hl_device *hdev)
4285 struct asic_fixed_properties *prop = &hdev->asic_prop;
4286 struct gaudi2_device *gaudi2 = hdev->asic_specific;
4293 gaudi2_sync_irqs(hdev);
4295 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE);
4296 free_irq(irq, &hdev->event_queue);
4298 gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM + 1);
4300 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT);
4301 free_irq(irq, &hdev->tpc_interrupt);
4303 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR);
4304 free_irq(irq, &hdev->unexpected_error_interrupt);
4307 k < hdev->asic_prop.user_interrupt_count ; i++, j++, k++) {
4309 irq = pci_irq_vector(hdev->pdev, i);
4310 free_irq(irq, &hdev->user_interrupt[j]);
4313 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION);
4314 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION];
4317 pci_free_irq_vectors(hdev->pdev);
4322 static void gaudi2_stop_dcore_dec(struct hl_device *hdev, int dcore_id)
4329 if (hdev->pldm)
4336 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit)))
4349 hdev,
4356 dev_err(hdev->dev,
4362 static void gaudi2_stop_pcie_dec(struct hl_device *hdev)
4369 if (hdev->pldm)
4376 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit)))
4389 hdev,
4396 dev_err(hdev->dev,
4402 static void gaudi2_stop_dec(struct hl_device *hdev)
4404 struct gaudi2_device *gaudi2 = hdev->asic_specific;
4411 gaudi2_stop_dcore_dec(hdev, dcore_id);
4413 gaudi2_stop_pcie_dec(hdev);
4416 static void gaudi2_set_arc_running_mode(struct hl_device *hdev, u32 cpu_id, u32 run_mode)
4429 static void gaudi2_halt_arcs(struct hl_device *hdev)
4434 if (gaudi2_is_arc_enabled(hdev, arc_id))
4435 gaudi2_set_arc_running_mode(hdev, arc_id, HL_ENGINE_CORE_HALT);
4439 static int gaudi2_verify_arc_running_mode(struct hl_device *hdev, u32 cpu_id, u32 run_mode)
4444 if (hdev->pldm)
4453 rc = hl_poll_timeout(hdev, reg_base + ARC_HALT_ACK_OFFSET,
4466 static void gaudi2_reset_arcs(struct hl_device *hdev)
4468 struct gaudi2_device *gaudi2 = hdev->asic_specific;
4475 if (gaudi2_is_arc_enabled(hdev, arc_id))
4476 gaudi2_clr_arc_id_cap(hdev, arc_id);
4479 static void gaudi2_nic_qmans_manual_flush(struct hl_device *hdev)
4481 struct gaudi2_device *gaudi2 = hdev->asic_specific;
4491 if (!(hdev->nic_ports_mask & BIT(i)))
4494 gaudi2_qman_manual_flush_common(hdev, queue_id);
4498 static int gaudi2_set_engine_cores(struct hl_device *hdev, u32 *core_ids,
4504 if (gaudi2_is_arc_enabled(hdev, core_ids[i]))
4505 gaudi2_set_arc_running_mode(hdev, core_ids[i], core_command);
4509 if (gaudi2_is_arc_enabled(hdev, core_ids[i])) {
4510 rc = gaudi2_verify_arc_running_mode(hdev, core_ids[i], core_command);
4513 dev_err(hdev->dev, "failed to %s arc: %d\n",
4524 static int gaudi2_set_tpc_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command)
4526 struct gaudi2_device *gaudi2 = hdev->asic_specific;
4551 static int gaudi2_set_mme_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command)
4553 struct gaudi2_device *gaudi2 = hdev->asic_specific;
4569 static int gaudi2_set_edma_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command)
4571 struct gaudi2_device *gaudi2 = hdev->asic_specific;
4596 static int gaudi2_set_engine_modes(struct hl_device *hdev,
4607 rc = gaudi2_set_tpc_engine_mode(hdev, engine_ids[i], engine_command);
4616 rc = gaudi2_set_mme_engine_mode(hdev, engine_ids[i], engine_command);
4625 rc = gaudi2_set_edma_engine_mode(hdev, engine_ids[i], engine_command);
4631 dev_err(hdev->dev, "Invalid engine ID %u\n", engine_ids[i]);
4639 static int gaudi2_set_engines(struct hl_device *hdev, u32 *engine_ids,
4645 return gaudi2_set_engine_cores(hdev, engine_ids, num_engines, engine_command);
4649 return gaudi2_set_engine_modes(hdev, engine_ids, num_engines, engine_command);
4652 dev_err(hdev->dev, "failed to execute command id %u\n", engine_command);
4657 static void gaudi2_halt_engines(struct hl_device *hdev, bool hard_reset, bool fw_reset)
4661 if (hdev->pldm)
4669 gaudi2_stop_dma_qmans(hdev);
4670 gaudi2_stop_mme_qmans(hdev);
4671 gaudi2_stop_tpc_qmans(hdev);
4672 gaudi2_stop_rot_qmans(hdev);
4673 gaudi2_stop_nic_qmans(hdev);
4676 gaudi2_halt_arcs(hdev);
4677 gaudi2_dma_stall(hdev);
4678 gaudi2_mme_stall(hdev);
4679 gaudi2_tpc_stall(hdev);
4680 gaudi2_rotator_stall(hdev);
4684 gaudi2_stop_dec(hdev);
4691 gaudi2_nic_qmans_manual_flush(hdev);
4693 gaudi2_disable_dma_qmans(hdev);
4694 gaudi2_disable_mme_qmans(hdev);
4695 gaudi2_disable_tpc_qmans(hdev);
4696 gaudi2_disable_rot_qmans(hdev);
4697 gaudi2_disable_nic_qmans(hdev);
4698 gaudi2_disable_timestamp(hdev);
4702 gaudi2_disable_msix(hdev);
4706 gaudi2_sync_irqs(hdev);
4709 static void gaudi2_init_firmware_preload_params(struct hl_device *hdev)
4711 struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load;
4721 static void gaudi2_init_firmware_loader(struct hl_device *hdev)
4723 struct fw_load_mgr *fw_loader = &hdev->fw_loader;
4743 dynamic_loader = &hdev->fw_loader.dynamic_loader;
4750 static int gaudi2_init_cpu(struct hl_device *hdev)
4752 struct gaudi2_device *gaudi2 = hdev->asic_specific;
4755 if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU))
4761 rc = hl_fw_init_cpu(hdev);
4770 static int gaudi2_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout)
4772 struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ];
4773 struct asic_fixed_properties *prop = &hdev->asic_prop;
4774 struct gaudi2_device *gaudi2 = hdev->asic_specific;
4780 if (!hdev->cpu_queues_enable)
4786 eq = &hdev->event_queue;
4788 dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
4796 WREG32(mmCPU_IF_CQ_BASE_ADDR_LOW, lower_32_bits(hdev->cpu_accessible_dma_address));
4797 WREG32(mmCPU_IF_CQ_BASE_ADDR_HIGH, upper_32_bits(hdev->cpu_accessible_dma_address));
4816 hdev,
4824 dev_err(hdev->dev, "Failed to communicate with device CPU (timeout)\n");
4839 static void gaudi2_init_qman_pq(struct hl_device *hdev, u32 reg_base,
4846 q = &hdev->kernel_queues[queue_id_base + pq_id];
4859 static void gaudi2_init_qman_cp(struct hl_device *hdev, u32 reg_base)
4881 static void gaudi2_init_qman_pqc(struct hl_device *hdev, u32 reg_base,
4884 struct gaudi2_device *gaudi2 = hdev->asic_specific;
4911 static u32 gaudi2_get_dyn_sp_reg(struct hl_device *hdev, u32 queue_id_base)
4913 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
4953 dev_err(hdev->dev, "Unexpected h/w queue %d\n", queue_id_base);
4960 static void gaudi2_init_qman_common(struct hl_device *hdev, u32 reg_base,
4968 irq_handler_offset = gaudi2_get_dyn_sp_reg(hdev, queue_id_base);
4996 static void gaudi2_init_qman(struct hl_device *hdev, u32 reg_base,
5002 hdev->kernel_queues[queue_id_base + pq_id].cq_id = GAUDI2_RESERVED_CQ_CS_COMPLETION;
5004 gaudi2_init_qman_pq(hdev, reg_base, queue_id_base);
5005 gaudi2_init_qman_cp(hdev, reg_base);
5006 gaudi2_init_qman_pqc(hdev, reg_base, queue_id_base);
5007 gaudi2_init_qman_common(hdev, reg_base, queue_id_base);
5010 static void gaudi2_init_dma_core(struct hl_device *hdev, u32 reg_base,
5023 dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
5040 static void gaudi2_init_kdma(struct hl_device *hdev)
5042 struct gaudi2_device *gaudi2 = hdev->asic_specific;
5050 gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_KDMA, true);
5055 static void gaudi2_init_pdma(struct hl_device *hdev)
5057 struct gaudi2_device *gaudi2 = hdev->asic_specific;
5064 gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_PDMA0, false);
5067 gaudi2_init_qman(hdev, reg_base, GAUDI2_QUEUE_ID_PDMA_0_0);
5070 gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_PDMA1, false);
5073 gaudi2_init_qman(hdev, reg_base, GAUDI2_QUEUE_ID_PDMA_1_0);
5078 static void gaudi2_init_edma_instance(struct hl_device *hdev, u8 seq)
5086 gaudi2_init_dma_core(hdev, reg_base, base_edma_core_id, false);
5089 gaudi2_init_qman(hdev, reg_base, base_edma_qman_id);
5092 static void gaudi2_init_edma(struct hl_device *hdev)
5094 struct asic_fixed_properties *prop = &hdev->asic_prop;
5095 struct gaudi2_device *gaudi2 = hdev->asic_specific;
5108 gaudi2_init_edma_instance(hdev, seq);
5117 * @hdev: pointer to habanalabs device structure.
5130 static void gaudi2_arm_monitors_for_virt_msix_db(struct hl_device *hdev, u32 sob_id,
5134 struct gaudi2_device *gaudi2 = hdev->asic_specific;
5196 static void gaudi2_prepare_sm_for_virt_msix_db(struct hl_device *hdev)
5199 struct asic_fixed_properties *prop = &hdev->asic_prop;
5209 gaudi2_arm_monitors_for_virt_msix_db(hdev, sob_id, first_mon_id, interrupt_id);
5214 gaudi2_arm_monitors_for_virt_msix_db(hdev, sob_id, first_mon_id, interrupt_id);
5218 static void gaudi2_init_sm(struct hl_device *hdev)
5220 struct gaudi2_device *gaudi2 = hdev->asic_specific;
5243 hdev->completion_queue[i].bus_address;
5258 gaudi2_prepare_sm_for_virt_msix_db(hdev);
5261 static void gaudi2_init_mme_acc(struct hl_device *hdev, u32 reg_base)
5263 struct gaudi2_device *gaudi2 = hdev->asic_specific;
5283 static void gaudi2_init_dcore_mme(struct hl_device *hdev, int dcore_id,
5302 dev_err(hdev->dev, "Invalid dcore id %u\n", dcore_id);
5308 gaudi2_init_mme_acc(hdev, reg_base);
5312 gaudi2_init_qman(hdev, reg_base, queue_id_base);
5315 static void gaudi2_init_mme(struct hl_device *hdev)
5317 struct gaudi2_device *gaudi2 = hdev->asic_specific;
5324 gaudi2_init_dcore_mme(hdev, i, false);
5330 static void gaudi2_init_tpc_cfg(struct hl_device *hdev, u32 reg_base)
5344 static void gaudi2_init_tpc_config(struct hl_device *hdev, int dcore, int inst,
5347 struct gaudi2_device *gaudi2 = hdev->asic_specific;
5360 gaudi2_init_tpc_cfg(hdev, mmDCORE0_TPC0_CFG_BASE + offset);
5361 gaudi2_init_qman(hdev, mmDCORE0_TPC0_QM_BASE + offset, queue_id_base);
5366 static void gaudi2_init_tpc(struct hl_device *hdev)
5368 struct gaudi2_device *gaudi2 = hdev->asic_specific;
5372 if (!hdev->asic_prop.tpc_enabled_mask)
5384 gaudi2_iterate_tpcs(hdev, &tpc_iter);
5387 static void gaudi2_init_rotator(struct hl_device *hdev)
5389 struct gaudi2_device *gaudi2 = hdev->asic_specific;
5396 gaudi2_init_qman(hdev, reg_base, queue_id);
5402 static void gaudi2_init_vdec_brdg_ctrl(struct hl_device *hdev, u64 base_addr, u32 decoder_id)
5419 static void gaudi2_init_dec(struct hl_device *hdev)
5421 struct gaudi2_device *gaudi2 = hdev->asic_specific;
5425 if (!hdev->asic_prop.decoder_enabled_mask)
5435 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit)))
5443 gaudi2_init_vdec_brdg_ctrl(hdev, base_addr, dec_bit);
5450 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit)))
5456 gaudi2_init_vdec_brdg_ctrl(hdev, base_addr, dec_bit);
5462 static int gaudi2_mmu_update_asid_hop0_addr(struct hl_device *hdev,
5468 if (hdev->pldm || !hdev->pdev)
5479 hdev,
5487 dev_err(hdev->dev, "Timeout during MMU hop0 config of asid %d\n", asid);
5494 static void gaudi2_mmu_send_invalidate_cache_cmd(struct hl_device *hdev, u32 stlb_base,
5508 static int gaudi2_mmu_invalidate_cache_status_poll(struct hl_device *hdev, u32 stlb_base,
5514 timeout_usec = (hdev->pldm) ? GAUDI2_PLDM_MMU_TIMEOUT_USEC :
5520 hdev,
5544 hdev,
5554 bool gaudi2_is_hmmu_enabled(struct hl_device *hdev, int dcore_id, int hmmu_id)
5556 struct gaudi2_device *gaudi2 = hdev->asic_specific;
5576 static void gaudi2_mmu_invalidate_cache_trigger(struct hl_device *hdev, u32 stlb_base,
5607 gaudi2_mmu_send_invalidate_cache_cmd(hdev, stlb_base, start_offset,
5611 static inline void gaudi2_hmmu_invalidate_cache_trigger(struct hl_device *hdev,
5617 gaudi2_mmu_invalidate_cache_trigger(hdev, stlb_base, inv_params);
5620 static inline int gaudi2_hmmu_invalidate_cache_status_poll(struct hl_device *hdev,
5626 return gaudi2_mmu_invalidate_cache_status_poll(hdev, stlb_base, inv_params);
5629 static int gaudi2_hmmus_invalidate_cache(struct hl_device *hdev,
5637 if (!gaudi2_is_hmmu_enabled(hdev, dcore_id, hmmu_id))
5640 gaudi2_hmmu_invalidate_cache_trigger(hdev, dcore_id, hmmu_id, inv_params);
5649 if (!gaudi2_is_hmmu_enabled(hdev, dcore_id, hmmu_id))
5652 rc = gaudi2_hmmu_invalidate_cache_status_poll(hdev, dcore_id, hmmu_id,
5662 static int gaudi2_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, u32 flags)
5664 struct gaudi2_device *gaudi2 = hdev->asic_specific;
5668 if (hdev->reset_info.hard_reset_pending)
5676 gaudi2_mmu_invalidate_cache_trigger(hdev, mmPMMU_HBW_STLB_BASE, &invld_params);
5677 rc = gaudi2_mmu_invalidate_cache_status_poll(hdev, mmPMMU_HBW_STLB_BASE,
5681 rc = gaudi2_hmmus_invalidate_cache(hdev, &invld_params);
5687 static int gaudi2_mmu_invalidate_cache_range(struct hl_device *hdev, bool is_hard,
5691 struct gaudi2_device *gaudi2 = hdev->asic_specific;
5696 if (hdev->reset_info.hard_reset_pending)
5722 gaudi2_mmu_invalidate_cache_trigger(hdev, mmPMMU_HBW_STLB_BASE, &invld_params);
5723 rc = gaudi2_mmu_invalidate_cache_status_poll(hdev, mmPMMU_HBW_STLB_BASE,
5729 invld_params.start_va = gaudi2_mmu_scramble_addr(hdev, start_va);
5730 invld_params.end_va = gaudi2_mmu_scramble_addr(hdev, end_va);
5733 rc = gaudi2_hmmus_invalidate_cache(hdev, &invld_params);
5739 static int gaudi2_mmu_update_hop0_addr(struct hl_device *hdev, u32 stlb_base)
5741 struct asic_fixed_properties *prop = &hdev->asic_prop;
5747 if (hdev->pldm)
5751 hop0_addr = hdev->mmu_priv.hr.mmu_asid_hop0[asid].phys_addr;
5752 rc = gaudi2_mmu_update_asid_hop0_addr(hdev, stlb_base, asid, hop0_addr);
5754 dev_err(hdev->dev, "failed to set hop0 addr for asid %d\n", asid);
5762 static int gaudi2_mmu_init_common(struct hl_device *hdev, u32 mmu_base, u32 stlb_base)
5767 if (hdev->pldm || !hdev->pdev)
5775 hdev,
5783 dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU SRAM init\n");
5785 rc = gaudi2_mmu_update_hop0_addr(hdev, stlb_base);
5792 hdev,
5800 dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU invalidate all\n");
5807 static int gaudi2_pci_mmu_init(struct hl_device *hdev)
5809 struct gaudi2_device *gaudi2 = hdev->asic_specific;
5848 rc = gaudi2_mmu_init_common(hdev, mmu_base, stlb_base);
5857 static int gaudi2_dcore_hmmu_init(struct hl_device *hdev, int dcore_id,
5860 struct asic_fixed_properties *prop = &hdev->asic_prop;
5861 struct gaudi2_device *gaudi2 = hdev->asic_specific;
5900 rc = gaudi2_mmu_init_common(hdev, mmu_base, stlb_base);
5909 static int gaudi2_hbm_mmu_init(struct hl_device *hdev)
5915 rc = gaudi2_dcore_hmmu_init(hdev, dcore_id, hmmu_id);
5923 static int gaudi2_mmu_init(struct hl_device *hdev)
5927 rc = gaudi2_pci_mmu_init(hdev);
5931 rc = gaudi2_hbm_mmu_init(hdev);
5938 static int gaudi2_hw_init(struct hl_device *hdev)
5940 struct gaudi2_device *gaudi2 = hdev->asic_specific;
5957 if (hdev->asic_prop.iatu_done_by_fw)
5964 if (gaudi2_set_hbm_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) {
5965 dev_err(hdev->dev, "failed to map HBM bar to DRAM base address\n");
5969 rc = gaudi2_init_cpu(hdev);
5971 dev_err(hdev->dev, "failed to initialize CPU\n");
5975 gaudi2_init_scrambler_hbm(hdev);
5976 gaudi2_init_kdma(hdev);
5978 rc = gaudi2_init_cpu_queues(hdev, GAUDI2_CPU_TIMEOUT_USEC);
5980 dev_err(hdev->dev, "failed to initialize CPU H/W queues %d\n", rc);
5984 rc = gaudi2->cpucp_info_get(hdev);
5986 dev_err(hdev->dev, "Failed to get cpucp info\n");
5990 rc = gaudi2_mmu_init(hdev);
5994 gaudi2_init_pdma(hdev);
5995 gaudi2_init_edma(hdev);
5996 gaudi2_init_sm(hdev);
5997 gaudi2_init_tpc(hdev);
5998 gaudi2_init_mme(hdev);
5999 gaudi2_init_rotator(hdev);
6000 gaudi2_init_dec(hdev);
6001 gaudi2_enable_timestamp(hdev);
6003 rc = gaudi2_coresight_init(hdev);
6007 rc = gaudi2_enable_msix(hdev);
6017 gaudi2_disable_dma_qmans(hdev);
6018 gaudi2_disable_mme_qmans(hdev);
6019 gaudi2_disable_tpc_qmans(hdev);
6020 gaudi2_disable_rot_qmans(hdev);
6021 gaudi2_disable_nic_qmans(hdev);
6023 gaudi2_disable_timestamp(hdev);
6031 * @hdev: pointer to the habanalabs device structure
6036 static void gaudi2_send_hard_reset_cmd(struct hl_device *hdev)
6038 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
6040 struct gaudi2_device *gaudi2 = hdev->asic_specific;
6043 preboot_only = (hdev->fw_loader.fw_comp_loaded == FW_TYPE_PREBOOT_CPU);
6044 heartbeat_reset = (hdev->reset_info.curr_reset_cause == HL_RESET_CAUSE_HEARTBEAT);
6096 if (hdev->asic_prop.hard_reset_done_by_fw)
6097 hl_fw_ask_hard_reset_without_linux(hdev);
6099 hl_fw_ask_halt_machine_without_linux(hdev);
6106 * @hdev: pointer to the habanalabs device structure
6110 static void gaudi2_execute_hard_reset(struct hl_device *hdev)
6112 if (hdev->asic_prop.hard_reset_done_by_fw) {
6113 gaudi2_send_hard_reset_cmd(hdev);
6123 gaudi2_send_hard_reset_cmd(hdev);
6128 static int gaudi2_get_soft_rst_done_indication(struct hl_device *hdev, u32 poll_timeout_us)
6135 hdev,
6143 dev_err(hdev->dev, "Timeout while waiting for FW to complete soft reset (0x%x)\n",
6151 * @hdev: pointer to the habanalabs device structure
6157 static int gaudi2_execute_soft_reset(struct hl_device *hdev, bool driver_performs_reset,
6160 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
6164 if (hl_is_fw_sw_ver_below(hdev, 1, 10)) {
6174 rc = gaudi2_get_soft_rst_done_indication(hdev, poll_timeout_us);
6176 rc = hl_fw_send_soft_reset(hdev);
6185 gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 1,
6188 gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 2,
6196 static void gaudi2_poll_btm_indication(struct hl_device *hdev, u32 poll_timeout_us)
6206 hdev,
6214 dev_err(hdev->dev, "Timeout while waiting for device to reset 0x%x\n", reg_val);
6217 static int gaudi2_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset)
6219 struct gaudi2_device *gaudi2 = hdev->asic_specific;
6224 if (hdev->pldm) {
6236 gaudi2_reset_arcs(hdev);
6239 driver_performs_reset = !hdev->asic_prop.hard_reset_done_by_fw;
6240 gaudi2_execute_hard_reset(hdev);
6248 driver_performs_reset = (hdev->fw_components == FW_TYPE_PREBOOT_CPU &&
6249 !hdev->asic_prop.fw_security_enabled);
6250 rc = gaudi2_execute_soft_reset(hdev, driver_performs_reset, poll_timeout_us);
6280 if (hdev->fw_components & FW_TYPE_PREBOOT_CPU)
6281 hl_fw_wait_preboot_ready(hdev);
6283 gaudi2_poll_btm_indication(hdev, poll_timeout_us);
6317 static int gaudi2_suspend(struct hl_device *hdev)
6321 rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0);
6323 dev_err(hdev->dev, "Failed to disable PCI access from CPU\n");
6328 static int gaudi2_resume(struct hl_device *hdev)
6330 return gaudi2_init_iatu(hdev);
6333 static int gaudi2_mmap(struct hl_device *hdev, struct vm_area_struct *vma,
6343 rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr, dma_addr, size);
6345 dev_err(hdev->dev, "dma_mmap_coherent error %d", rc);
6353 dev_err(hdev->dev, "remap_pfn_range error %d", rc);
6360 static bool gaudi2_is_queue_enabled(struct hl_device *hdev, u32 hw_queue_id)
6362 struct gaudi2_device *gaudi2 = hdev->asic_specific;
6466 static bool gaudi2_is_arc_enabled(struct hl_device *hdev, u64 arc_id)
6468 struct gaudi2_device *gaudi2 = hdev->asic_specific;
6486 static void gaudi2_clr_arc_id_cap(struct hl_device *hdev, u64 arc_id)
6488 struct gaudi2_device *gaudi2 = hdev->asic_specific;
6509 static void gaudi2_set_arc_id_cap(struct hl_device *hdev, u64 arc_id)
6511 struct gaudi2_device *gaudi2 = hdev->asic_specific;
6532 static void gaudi2_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi)
6534 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
6563 static void gaudi2_pqe_write(struct hl_device *hdev, __le64 *pqe, struct hl_bd *bd)
6572 static void *gaudi2_dma_alloc_coherent(struct hl_device *hdev, size_t size,
6575 return dma_alloc_coherent(&hdev->pdev->dev, size, dma_handle, flags);
6578 static void gaudi2_dma_free_coherent(struct hl_device *hdev, size_t size,
6581 dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, dma_handle);
6584 static int gaudi2_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len,
6587 struct gaudi2_device *gaudi2 = hdev->asic_specific;
6598 return hl_fw_send_cpu_message(hdev, GAUDI2_QUEUE_ID_CPU_PQ, msg, len, timeout, result);
6601 static void *gaudi2_dma_pool_zalloc(struct hl_device *hdev, size_t size,
6607 return dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle);
6610 static void gaudi2_dma_pool_free(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr)
6612 dma_pool_free(hdev->dma_pool, vaddr, dma_addr);
6615 static void *gaudi2_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
6618 return hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle);
6621 static void gaudi2_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr)
6623 hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr);
6626 static dma_addr_t gaudi2_dma_map_single(struct hl_device *hdev, void *addr, int len,
6631 dma_addr = dma_map_single(&hdev->pdev->dev, addr, len, dir);
6632 if (unlikely(dma_mapping_error(&hdev->pdev->dev, dma_addr)))
6638 static void gaudi2_dma_unmap_single(struct hl_device *hdev, dma_addr_t addr, int len,
6641 dma_unmap_single(&hdev->pdev->dev, addr, len, dir);
6644 static int gaudi2_validate_cb_address(struct hl_device *hdev, struct hl_cs_parser *parser)
6646 struct asic_fixed_properties *asic_prop = &hdev->asic_prop;
6647 struct gaudi2_device *gaudi2 = hdev->asic_specific;
6649 if (!gaudi2_is_queue_enabled(hdev, parser->hw_queue_id)) {
6650 dev_err(hdev->dev, "h/w queue %d is disabled\n", parser->hw_queue_id);
6688 if (!hdev->pdev)
6691 if (!device_iommu_mapped(&hdev->pdev->dev))
6695 dev_err(hdev->dev, "CB address %p + 0x%x for internal QMAN is not valid\n",
6701 static int gaudi2_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser)
6703 struct gaudi2_device *gaudi2 = hdev->asic_specific;
6706 return gaudi2_validate_cb_address(hdev, parser);
6709 dev_err(hdev->dev, "PMMU not initialized - Unsupported mode in Gaudi2\n");
6716 static int gaudi2_send_heartbeat(struct hl_device *hdev)
6718 struct gaudi2_device *gaudi2 = hdev->asic_specific;
6723 return hl_fw_send_heartbeat(hdev);
6729 static void gaudi2_kdma_set_mmbp_asid(struct hl_device *hdev,
6744 static void gaudi2_arm_cq_monitor(struct hl_device *hdev, u32 sob_id, u32 mon_id, u32 cq_id,
6774 static int gaudi2_send_job_to_kdma(struct hl_device *hdev,
6784 gaudi2_arm_cq_monitor(hdev, GAUDI2_RESERVED_SOB_KDMA_COMPLETION,
6812 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_KDMA_COMPLETION];
6816 if (hdev->pldm)
6824 hdev,
6835 dev_err(hdev->dev, "Timeout while waiting for KDMA to be idle\n");
6845 static void gaudi2_memset_device_lbw(struct hl_device *hdev, u32 addr, u32 size, u32 val)
6853 static void gaudi2_qman_set_test_mode(struct hl_device *hdev, u32 hw_queue_id, bool enable)
6866 static inline u32 gaudi2_test_queue_hw_queue_id_to_sob_id(struct hl_device *hdev, u32 hw_queue_id)
6868 return hdev->asic_prop.first_available_user_sob[0] +
6872 static void gaudi2_test_queue_clear(struct hl_device *hdev, u32 hw_queue_id)
6874 u32 sob_offset = gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4;
6881 static int gaudi2_test_queue_send_msg_short(struct hl_device *hdev, u32 hw_queue_id, u32 sob_val,
6884 u32 sob_offset = gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4;
6899 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, pkt_size, msg_info->dma_addr);
6901 dev_err(hdev->dev,
6907 static int gaudi2_test_queue_wait_completion(struct hl_device *hdev, u32 hw_queue_id, u32 sob_val)
6909 u32 sob_offset = gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4;
6914 if (hdev->pldm)
6920 hdev,
6928 dev_err(hdev->dev, "H/W queue %d test failed (SOB_OBJ_0 == 0x%x)\n",
6936 static int gaudi2_test_cpu_queue(struct hl_device *hdev)
6938 struct gaudi2_device *gaudi2 = hdev->asic_specific;
6947 return hl_fw_test_cpu_queue(hdev);
6950 static int gaudi2_test_queues(struct hl_device *hdev)
6952 struct gaudi2_device *gaudi2 = hdev->asic_specific;
6959 if (!gaudi2_is_queue_enabled(hdev, i))
6963 gaudi2_qman_set_test_mode(hdev, i, true);
6964 gaudi2_test_queue_clear(hdev, i);
6965 rc = gaudi2_test_queue_send_msg_short(hdev, i, sob_val, msg_info);
6970 rc = gaudi2_test_cpu_queue(hdev);
6976 if (!gaudi2_is_queue_enabled(hdev, i))
6979 rc = gaudi2_test_queue_wait_completion(hdev, i, sob_val);
6984 gaudi2_test_queue_clear(hdev, i);
6985 gaudi2_qman_set_test_mode(hdev, i, false);
6992 static int gaudi2_compute_reset_late_init(struct hl_device *hdev)
6994 struct gaudi2_device *gaudi2 = hdev->asic_specific;
6998 gaudi2_init_arcs(hdev);
7000 rc = gaudi2_scrub_arcs_dccm(hdev);
7002 dev_err(hdev->dev, "Failed to scrub arcs DCCM\n");
7006 gaudi2_init_security(hdev);
7010 return hl_fw_unmask_irq_arr(hdev, gaudi2->hw_events, irq_arr_size);
7013 static bool gaudi2_get_edma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
7017 struct asic_fixed_properties *prop = &hdev->asic_prop;
7063 static bool gaudi2_get_pdma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
7103 static bool gaudi2_get_nic_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
7114 if (e && hdev->nic_ports_mask)
7125 if (!(hdev->nic_ports_mask & BIT(i)))
7149 static bool gaudi2_get_mme_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
7192 static void gaudi2_is_tpc_engine_idle(struct hl_device *hdev, int dcore, int inst, u32 offset,
7225 static bool gaudi2_get_tpc_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
7228 struct asic_fixed_properties *prop = &hdev->asic_prop;
7248 gaudi2_iterate_tpcs(hdev, &tpc_iter);
7253 static bool gaudi2_get_decoder_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
7256 struct asic_fixed_properties *prop = &hdev->asic_prop;
7322 static bool gaudi2_get_rotator_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
7360 static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
7365 is_idle &= gaudi2_get_edma_idle_status(hdev, mask_arr, mask_len, e);
7366 is_idle &= gaudi2_get_pdma_idle_status(hdev, mask_arr, mask_len, e);
7367 is_idle &= gaudi2_get_nic_idle_status(hdev, mask_arr, mask_len, e);
7368 is_idle &= gaudi2_get_mme_idle_status(hdev, mask_arr, mask_len, e);
7369 is_idle &= gaudi2_get_tpc_idle_status(hdev, mask_arr, mask_len, e);
7370 is_idle &= gaudi2_get_decoder_idle_status(hdev, mask_arr, mask_len, e);
7371 is_idle &= gaudi2_get_rotator_idle_status(hdev, mask_arr, mask_len, e);
7376 static void gaudi2_hw_queues_lock(struct hl_device *hdev)
7379 struct gaudi2_device *gaudi2 = hdev->asic_specific;
7384 static void gaudi2_hw_queues_unlock(struct hl_device *hdev)
7387 struct gaudi2_device *gaudi2 = hdev->asic_specific;
7392 static u32 gaudi2_get_pci_id(struct hl_device *hdev)
7394 return hdev->pdev->device;
7397 static int gaudi2_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size)
7399 struct gaudi2_device *gaudi2 = hdev->asic_specific;
7404 return hl_fw_get_eeprom_data(hdev, data, max_size);
7407 static void gaudi2_update_eq_ci(struct hl_device *hdev, u32 val)
7412 static void *gaudi2_get_events_stat(struct hl_device *hdev, bool aggregate, u32 *size)
7414 struct gaudi2_device *gaudi2 = hdev->asic_specific;
7425 static void gaudi2_mmu_vdec_dcore_prepare(struct hl_device *hdev, int dcore_id,
7447 static void gaudi2_mmu_dcore_prepare(struct hl_device *hdev, int dcore_id, u32 asid)
7451 struct asic_fixed_properties *prop = &hdev->asic_prop;
7512 gaudi2_mmu_vdec_dcore_prepare(hdev, dcore_id, vdec_id, rw_asid, 0);
7516 static void gudi2_mmu_vdec_shared_prepare(struct hl_device *hdev,
7537 static void gudi2_mmu_arc_farm_arc_dup_eng_prepare(struct hl_device *hdev, int arc_farm_id,
7546 static void gaudi2_arc_mmu_prepare(struct hl_device *hdev, u32 cpu_id, u32 asid)
7590 static int gaudi2_arc_mmu_prepare_all(struct hl_device *hdev, u32 asid)
7594 if (hdev->fw_components & FW_TYPE_BOOT_CPU)
7595 return hl_fw_cpucp_engine_core_asid_set(hdev, asid);
7598 gaudi2_arc_mmu_prepare(hdev, i, asid);
7601 if (!gaudi2_is_queue_enabled(hdev, i))
7604 gaudi2_arc_mmu_prepare(hdev, gaudi2_queue_id_to_arc_id[i], asid);
7610 static int gaudi2_mmu_shared_prepare(struct hl_device *hdev, u32 asid)
7612 struct asic_fixed_properties *prop = &hdev->asic_prop;
7641 gudi2_mmu_vdec_shared_prepare(hdev, 0, rw_asid, 0);
7644 gudi2_mmu_vdec_shared_prepare(hdev, 1, rw_asid, 0);
7648 gudi2_mmu_arc_farm_arc_dup_eng_prepare(hdev, i, rw_asid, 0);
7650 rc = gaudi2_arc_mmu_prepare_all(hdev, asid);
7657 static void gaudi2_tpc_mmu_prepare(struct hl_device *hdev, int dcore, int inst, u32 offset,
7669 static int gaudi2_mmu_prepare(struct hl_device *hdev, u32 asid)
7671 struct gaudi2_device *gaudi2 = hdev->asic_specific;
7680 dev_crit(hdev->dev, "asid %u is too big\n", asid);
7687 rc = gaudi2_mmu_shared_prepare(hdev, asid);
7694 gaudi2_iterate_tpcs(hdev, &tpc_iter);
7696 gaudi2_mmu_dcore_prepare(hdev, i, asid);
7717 static void gaudi2_print_event(struct hl_device *hdev, u16 event_type,
7728 dev_err_ratelimited(hdev->dev, "%s: %pV\n",
7732 dev_err(hdev->dev, "%s: %pV\n",
7739 static bool gaudi2_handle_ecc_event(struct hl_device *hdev, u16 event_type,
7749 gaudi2_print_event(hdev, event_type, !ecc_data->is_critical,
7756 static void print_lower_qman_data_on_err(struct hl_device *hdev, u64 qman_base)
7775 dev_info(hdev->dev,
7780 static int gaudi2_handle_qman_err_generic(struct hl_device *hdev, u16 event_type,
7807 gaudi2_print_event(hdev, event_type, true,
7816 print_lower_qman_data_on_err(hdev, qman_base);
7826 gaudi2_print_event(hdev, event_type, true,
7837 static void gaudi2_razwi_rr_hbw_shared_printf_info(struct hl_device *hdev,
7857 hl_handle_razwi(hdev, (u64)razwi_hi << 32 | razwi_lo, &eng_id, 1,
7860 dev_err_ratelimited(hdev->dev,
7865 static void gaudi2_razwi_rr_lbw_shared_printf_info(struct hl_device *hdev,
7884 hl_handle_razwi(hdev, razwi_addr, &eng_id, 1, rd_wr_flag | HL_RAZWI_LBW, event_mask);
7885 dev_err_ratelimited(hdev->dev,
7891 static enum gaudi2_engine_id gaudi2_razwi_calc_engine_id(struct hl_device *hdev,
7939 static void gaudi2_ack_module_razwi_event_handler(struct hl_device *hdev,
7953 if (hdev->tpc_binning) {
7954 binned_idx = __ffs(hdev->tpc_binning);
7961 if (hl_is_fw_sw_ver_below(hdev, 1, 9) &&
7962 !hdev->asic_prop.fw_security_enabled &&
8026 if (hdev->decoder_binning) {
8027 binned_idx = __ffs(hdev->decoder_binning);
8061 eng_id = gaudi2_razwi_calc_engine_id(hdev, module, module_idx);
8063 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, hbw_rtr_mstr_if_base_addr, true,
8071 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, hbw_rtr_mstr_if_base_addr, false,
8079 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, lbw_rtr_mstr_if_base_addr, true,
8087 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, lbw_rtr_mstr_if_base_addr, false,
8095 static void gaudi2_check_if_razwi_happened(struct hl_device *hdev)
8097 struct asic_fixed_properties *prop = &hdev->asic_prop;
8103 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_TPC, mod_idx, 0, NULL);
8109 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mod_idx,
8115 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_EDMA, mod_idx, 0, NULL);
8119 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_PDMA, mod_idx, 0, NULL);
8123 if (hdev->nic_ports_mask & BIT(mod_idx))
8124 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_NIC, mod_idx >> 1, 0,
8130 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_DEC, mod_idx, 0, NULL);
8134 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ROT, mod_idx, 0, NULL);
8165 static bool gaudi2_handle_psoc_razwi_happened(struct hl_device *hdev, u32 razwi_reg,
8194 dev_err(hdev->dev,
8197 hl_handle_razwi(hdev, addr, &eng_id[0],
8208 dev_err(hdev->dev,
8211 hl_handle_razwi(hdev, addr, &eng_id[0],
8220 dev_err(hdev->dev,
8223 hl_handle_razwi(hdev, addr_lo, &eng_id[0],
8232 dev_err(hdev->dev,
8235 hl_handle_razwi(hdev, addr_lo, &eng_id[0],
8254 static int gaudi2_ack_psoc_razwi_event_handler(struct hl_device *hdev, u64 *event_mask)
8258 if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX)) {
8266 dev_err_ratelimited(hdev->dev,
8274 if (gaudi2_handle_psoc_razwi_happened(hdev, razwi_mask_info, event_mask))
8277 dev_err_ratelimited(hdev->dev,
8282 if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX))
8288 static int _gaudi2_handle_qm_sei_err(struct hl_device *hdev, u64 qman_base, u16 event_type)
8296 gaudi2_print_event(hdev, event_type, true,
8308 static int gaudi2_handle_qm_sei_err(struct hl_device *hdev, u16 event_type,
8354 error_count = _gaudi2_handle_qm_sei_err(hdev, qman_base, event_type);
8359 error_count += _gaudi2_handle_qm_sei_err(hdev,
8364 gaudi2_ack_module_razwi_event_handler(hdev, module, 0, 0, event_mask);
8365 hl_check_for_glbl_errors(hdev);
8371 static int gaudi2_handle_qman_err(struct hl_device *hdev, u16 event_type, u64 *event_mask)
8478 error_count = gaudi2_handle_qman_err_generic(hdev, event_type, qman_base, qid_base);
8482 error_count += _gaudi2_handle_qm_sei_err(hdev, qman_base, event_type);
8483 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_EDMA, index, 0, event_mask);
8486 hl_check_for_glbl_errors(hdev);
8491 static int gaudi2_handle_arc_farm_sei_err(struct hl_device *hdev, u16 event_type)
8502 gaudi2_print_event(hdev, event_type, true,
8513 hl_check_for_glbl_errors(hdev);
8518 static int gaudi2_handle_cpu_sei_err(struct hl_device *hdev, u16 event_type)
8526 gaudi2_print_event(hdev, event_type, true,
8533 hl_check_for_glbl_errors(hdev);
8540 static int gaudi2_handle_rot_err(struct hl_device *hdev, u8 rot_index, u16 event_type,
8550 gaudi2_print_event(hdev, event_type, true,
8556 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ROT, rot_index, 0, event_mask);
8557 hl_check_for_glbl_errors(hdev);
8562 static int gaudi2_tpc_ack_interrupts(struct hl_device *hdev, u8 tpc_index, u16 event_type,
8572 gaudi2_print_event(hdev, event_type, true,
8578 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_TPC, tpc_index, 0, event_mask);
8579 hl_check_for_glbl_errors(hdev);
8584 static int gaudi2_handle_dec_err(struct hl_device *hdev, u8 dec_index, u16 event_type,
8604 gaudi2_print_event(hdev, event_type, true,
8612 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_DEC, dec_index, 0, event_mask);
8613 hl_check_for_glbl_errors(hdev);
8621 static int gaudi2_handle_mme_err(struct hl_device *hdev, u8 mme_index, u16 event_type,
8634 gaudi2_print_event(hdev, event_type, true,
8643 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, i, event_mask);
8645 hl_check_for_glbl_errors(hdev);
8652 static int gaudi2_handle_mme_sbte_err(struct hl_device *hdev, u16 event_type,
8659 gaudi2_print_event(hdev, event_type, true,
8664 hl_check_for_glbl_errors(hdev);
8669 static int gaudi2_handle_mme_wap_err(struct hl_device *hdev, u8 mme_index, u16 event_type,
8682 gaudi2_print_event(hdev, event_type, true,
8690 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, MME_WAP0, event_mask);
8691 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, MME_WAP1, event_mask);
8692 hl_check_for_glbl_errors(hdev);
8699 static int gaudi2_handle_kdma_core_event(struct hl_device *hdev, u16 event_type,
8712 gaudi2_print_event(hdev, event_type, true,
8717 hl_check_for_glbl_errors(hdev);
8722 static int gaudi2_handle_dma_core_event(struct hl_device *hdev, u16 event_type, u64 intr_cause)
8729 gaudi2_print_event(hdev, event_type, true,
8734 hl_check_for_glbl_errors(hdev);
8739 static void gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(struct hl_device *hdev, u64 *event_mask)
8745 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, mstr_if_base_addr, true, "PCIE",
8752 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, mstr_if_base_addr, false, "PCIE",
8759 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, mstr_if_base_addr, true, "PCIE",
8766 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, mstr_if_base_addr, false, "PCIE",
8772 static int gaudi2_print_pcie_addr_dec_info(struct hl_device *hdev, u16 event_type,
8778 gaudi2_print_event(hdev, event_type, true,
8785 gaudi2_print_event(hdev, event_type, true,
8795 hl_check_for_glbl_errors(hdev);
8796 gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(hdev, event_mask);
8801 static int gaudi2_handle_pif_fatal(struct hl_device *hdev, u16 event_type,
8810 gaudi2_print_event(hdev, event_type, true,
8819 static int gaudi2_handle_hif_fatal(struct hl_device *hdev, u16 event_type, u64 intr_cause_data)
8826 gaudi2_print_event(hdev, event_type, true,
8835 static void gaudi2_handle_page_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu,
8852 dev_err_ratelimited(hdev->dev, "PMMU page fault on va 0x%llx\n", addr);
8855 addr = gaudi2_mmu_descramble_addr(hdev, addr);
8857 dev_err_ratelimited(hdev->dev, "HMMU page fault on va range 0x%llx - 0x%llx\n",
8861 hl_handle_page_fault(hdev, addr, 0, is_pmmu, event_mask);
8866 static void gaudi2_handle_access_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu)
8882 addr = gaudi2_mmu_descramble_addr(hdev, addr);
8884 dev_err_ratelimited(hdev->dev, "%s access error on va 0x%llx\n",
8889 static int gaudi2_handle_mmu_spi_sei_generic(struct hl_device *hdev, u16 event_type,
8899 gaudi2_print_event(hdev, event_type, true,
8903 gaudi2_handle_page_error(hdev, mmu_base, is_pmmu, event_mask);
8905 gaudi2_handle_access_error(hdev, mmu_base, is_pmmu);
8923 static int gaudi2_handle_sm_err(struct hl_device *hdev, u16 event_type, u8 sm_index)
8946 gaudi2_print_event(hdev, event_type, true,
8965 dev_err_ratelimited(hdev->dev, "SM%u err. err cause: CQ_INTR. queue index: %u\n",
8973 hl_check_for_glbl_errors(hdev);
9070 static int gaudi2_handle_mmu_spi_sei_err(struct hl_device *hdev, u16 event_type, u64 *event_mask)
9094 error_count = gaudi2_handle_mmu_spi_sei_generic(hdev, event_type, mmu_base,
9096 hl_check_for_glbl_errors(hdev);
9103 static bool gaudi2_hbm_sei_handle_read_err(struct hl_device *hdev,
9109 dev_err_ratelimited(hdev->dev,
9116 dev_err_ratelimited(hdev->dev,
9128 dev_err_ratelimited(hdev->dev, "Beat%d ECC SERR: DM: %#x, Syndrome: %#x\n",
9135 dev_err_ratelimited(hdev->dev, "Beat%d ECC DERR: DM: %#x, Syndrome: %#x\n",
9145 dev_err_ratelimited(hdev->dev,
9155 dev_err_ratelimited(hdev->dev, "Beat%d DQ data:\n", beat);
9156 dev_err_ratelimited(hdev->dev, "\t0x%08x\n",
9158 dev_err_ratelimited(hdev->dev, "\t0x%08x\n",
9165 static void gaudi2_hbm_sei_print_wr_par_info(struct hl_device *hdev,
9171 dev_err_ratelimited(hdev->dev, "WRITE PARITY ERROR count: %d\n", err_cnt);
9173 dev_err_ratelimited(hdev->dev, "CK-0 DERR: 0x%02x, CK-1 DERR: 0x%02x\n",
9177 dev_err_ratelimited(hdev->dev, "Last latched write commands addresses:\n");
9180 dev_err_ratelimited(hdev->dev,
9190 static void gaudi2_hbm_sei_print_ca_par_info(struct hl_device *hdev,
9197 dev_err_ratelimited(hdev->dev, "CA ERROR count: %d\n", err_cnt);
9199 dev_err_ratelimited(hdev->dev, "Last latched C&R bus commands:\n");
9201 dev_err_ratelimited(hdev->dev, "cmd%u: ROW(0x%04x) COL(0x%05x)\n", i,
9207 static bool gaudi2_handle_hbm_mc_sei_err(struct hl_device *hdev, u16 event_type,
9218 gaudi2_print_event(hdev, event_type, true,
9224 gaudi2_print_event(hdev, event_type, !sei_data->hdr.is_critical,
9237 gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_even_info,
9243 gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_odd_info,
9249 gaudi2_hbm_sei_print_wr_par_info(hdev, &sei_data->wr_parity_info,
9258 require_hard_reset = gaudi2_hbm_sei_handle_read_err(hdev,
9272 static int gaudi2_handle_hbm_cattrip(struct hl_device *hdev, u16 event_type,
9276 gaudi2_print_event(hdev, event_type, true,
9284 static int gaudi2_handle_hbm_mc_spi(struct hl_device *hdev, u64 intr_cause_data)
9290 dev_dbg(hdev->dev, "HBM spi event: notification cause(%s)\n",
9298 static void gaudi2_print_clk_change_info(struct hl_device *hdev, u16 event_type, u64 *event_mask)
9302 mutex_lock(&hdev->clk_throttling.lock);
9306 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_POWER;
9307 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_POWER;
9308 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].start = ktime_get();
9309 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = zero_time;
9310 dev_dbg_ratelimited(hdev->dev, "Clock throttling due to power consumption\n");
9314 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_POWER;
9315 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = ktime_get();
9316 dev_dbg_ratelimited(hdev->dev, "Power envelop is safe, back to optimal clock\n");
9320 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_THERMAL;
9321 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_THERMAL;
9322 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].start = ktime_get();
9323 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = zero_time;
9325 dev_info_ratelimited(hdev->dev, "Clock throttling due to overheating\n");
9329 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_THERMAL;
9330 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = ktime_get();
9332 dev_info_ratelimited(hdev->dev, "Thermal envelop is safe, back to optimal clock\n");
9336 dev_err(hdev->dev, "Received invalid clock change event %d\n", event_type);
9340 mutex_unlock(&hdev->clk_throttling.lock);
9343 static void gaudi2_print_out_of_sync_info(struct hl_device *hdev, u16 event_type,
9346 struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ];
9348 gaudi2_print_event(hdev, event_type, false,
9354 static int gaudi2_handle_pcie_p2p_msix(struct hl_device *hdev, u16 event_type)
9362 gaudi2_print_event(hdev, event_type, true,
9371 gaudi2_print_event(hdev, event_type, true,
9382 static int gaudi2_handle_pcie_drain(struct hl_device *hdev,
9394 dev_err_ratelimited(hdev->dev,
9401 dev_err_ratelimited(hdev->dev,
9410 static int gaudi2_handle_psoc_drain(struct hl_device *hdev, u64 intr_cause_data)
9417 dev_err_ratelimited(hdev->dev, "PSOC %s completed\n",
9423 hl_check_for_glbl_errors(hdev);
9428 static void gaudi2_print_cpu_pkt_failure_info(struct hl_device *hdev, u16 event_type,
9431 struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ];
9433 gaudi2_print_event(hdev, event_type, false,
9438 static int hl_arc_event_handle(struct hl_device *hdev, u16 event_type,
9453 gaudi2_print_event(hdev, event_type, true,
9458 gaudi2_print_event(hdev, event_type, true, "Unknown ARC event type");
9463 static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry)
9465 struct gaudi2_device *gaudi2 = hdev->asic_specific;
9475 dev_err(hdev->dev, "Event type %u exceeds maximum of %u",
9489 reset_required = gaudi2_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data);
9499 error_count = gaudi2_handle_qman_err(hdev, event_type, &event_mask);
9504 error_count = gaudi2_handle_arc_farm_sei_err(hdev, event_type);
9509 error_count = gaudi2_handle_cpu_sei_err(hdev, event_type);
9516 error_count = gaudi2_handle_qm_sei_err(hdev, event_type, true, &event_mask);
9523 error_count = gaudi2_handle_rot_err(hdev, index, event_type,
9525 error_count += gaudi2_handle_qm_sei_err(hdev, event_type, false, &event_mask);
9531 error_count = gaudi2_tpc_ack_interrupts(hdev, index, event_type,
9533 error_count += gaudi2_handle_qm_sei_err(hdev, event_type, false, &event_mask);
9539 error_count = gaudi2_handle_dec_err(hdev, index, event_type, &event_mask);
9570 error_count = gaudi2_tpc_ack_interrupts(hdev, index, event_type,
9587 error_count = gaudi2_handle_dec_err(hdev, index, event_type, &event_mask);
9598 error_count = gaudi2_handle_mme_err(hdev, index, event_type, &event_mask);
9599 error_count += gaudi2_handle_qm_sei_err(hdev, event_type, false, &event_mask);
9610 error_count = gaudi2_handle_mme_err(hdev, index, event_type, &event_mask);
9621 error_count = gaudi2_handle_mme_wap_err(hdev, index, event_type, &event_mask);
9627 error_count = gaudi2_handle_kdma_core_event(hdev, event_type,
9633 error_count = gaudi2_handle_dma_core_event(hdev, event_type,
9639 error_count = gaudi2_handle_dma_core_event(hdev, event_type,
9645 error_count = gaudi2_print_pcie_addr_dec_info(hdev, event_type,
9655 error_count = gaudi2_handle_mmu_spi_sei_err(hdev, event_type, &event_mask);
9661 error_count = gaudi2_handle_hif_fatal(hdev, event_type,
9668 error_count = gaudi2_handle_pif_fatal(hdev, event_type,
9675 error_count = gaudi2_ack_psoc_razwi_event_handler(hdev, &event_mask);
9681 if (gaudi2_handle_hbm_mc_sei_err(hdev, event_type, &eq_entry->sei_data)) {
9689 error_count = gaudi2_handle_hbm_cattrip(hdev, event_type,
9695 error_count = gaudi2_handle_hbm_mc_spi(hdev,
9701 error_count = gaudi2_handle_pcie_drain(hdev, &eq_entry->pcie_drain_ind_data);
9707 error_count = gaudi2_handle_psoc_drain(hdev,
9727 error_count = gaudi2_handle_mme_sbte_err(hdev, event_type,
9813 gaudi2_print_clk_change_info(hdev, event_type, &event_mask);
9818 gaudi2_print_out_of_sync_info(hdev, event_type, &eq_entry->pkt_sync_err);
9831 error_count = gaudi2_handle_pcie_p2p_msix(hdev, event_type);
9837 error_count = gaudi2_handle_sm_err(hdev, event_type, index);
9847 dev_info(hdev->dev, "CPLD shutdown cause, reset reason: 0x%llx\n",
9853 dev_err(hdev->dev, "CPLD shutdown event, reset reason: 0x%llx\n",
9860 gaudi2_print_cpu_pkt_failure_info(hdev, event_type, &eq_entry->pkt_sync_err);
9867 error_count = hl_arc_event_handle(hdev, event_type, &eq_entry->arc_data);
9880 dev_err_ratelimited(hdev->dev, "Cannot find handler for event %d\n",
9891 gaudi2_print_event(hdev, event_type, true, "%d", event_type);
9893 gaudi2_print_event(hdev, event_type, true,
9902 if (hdev->hard_reset_on_fw_events ||
9903 (hdev->asic_prop.fw_security_enabled && is_critical))
9909 hl_fw_unmask_irq(hdev, event_type);
9912 hl_notifier_event_send_all(hdev, event_mask);
9917 if (hdev->asic_prop.fw_security_enabled && is_critical) {
9925 hl_handle_critical_hw_err(hdev, event_type, &event_mask);
9928 hl_device_cond_reset(hdev, reset_flags, event_mask);
9931 static int gaudi2_memset_memory_chunk_using_edma_qm(struct hl_device *hdev,
9950 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, pkt_size, pkt_dma_addr);
9952 dev_err(hdev->dev, "Failed to send lin dma packet to H/W queue %d\n",
9958 static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val)
9967 struct asic_fixed_properties *prop = &hdev->asic_prop;
9973 dev_info(hdev->dev, "non of the EDMA engines is enabled - skip dram scrubbing\n");
9977 sob_offset = hdev->asic_prop.first_available_user_sob[0] * 4;
9989 lin_dma_pkts_arr = hl_asic_dma_alloc_coherent(hdev, pkt_size * num_of_pkts,
10017 gaudi2_qman_set_test_mode(hdev,
10034 rc = gaudi2_memset_memory_chunk_using_edma_qm(hdev,
10050 rc = hl_poll_timeout(hdev, sob_addr, busy, (busy == dma_num), 1000, 1000000);
10052 dev_err(hdev->dev, "DMA Timeout during HBM scrubbing\n");
10068 gaudi2_qman_set_test_mode(hdev,
10074 hl_asic_dma_free_coherent(hdev, pkt_size * num_of_pkts, lin_dma_pkts_arr, pkt_dma_addr);
10079 static int gaudi2_scrub_device_dram(struct hl_device *hdev, u64 val)
10082 struct asic_fixed_properties *prop = &hdev->asic_prop;
10085 rc = gaudi2_memset_device_memory(hdev, prop->dram_user_base_address, size, val);
10088 dev_err(hdev->dev, "Failed to scrub dram, address: 0x%llx size: %llu\n",
10093 static int gaudi2_scrub_device_mem(struct hl_device *hdev)
10096 struct asic_fixed_properties *prop = &hdev->asic_prop;
10097 u64 val = hdev->memory_scrub_val;
10100 if (!hdev->memory_scrub)
10105 size = hdev->pldm ? 0x10000 : (prop->sram_size - SRAM_USER_BASE_OFFSET);
10106 dev_dbg(hdev->dev, "Scrubbing SRAM: 0x%09llx - 0x%09llx, val: 0x%llx\n",
10108 rc = gaudi2_memset_device_memory(hdev, addr, size, val);
10110 dev_err(hdev->dev, "scrubbing SRAM failed (%d)\n", rc);
10115 rc = gaudi2_scrub_device_dram(hdev, val);
10117 dev_err(hdev->dev, "scrubbing DRAM failed (%d)\n", rc);
10123 static void gaudi2_restore_user_sm_registers(struct hl_device *hdev)
10130 offset = hdev->asic_prop.first_available_cq[0] * 4;
10141 gaudi2_memset_device_lbw(hdev, cq_lbw_l_addr, size, 0);
10142 gaudi2_memset_device_lbw(hdev, cq_lbw_h_addr, size, 0);
10143 gaudi2_memset_device_lbw(hdev, cq_lbw_data_addr, size, 0);
10144 gaudi2_memset_device_lbw(hdev, cq_base_l_addr, size, 0);
10145 gaudi2_memset_device_lbw(hdev, cq_base_h_addr, size, 0);
10146 gaudi2_memset_device_lbw(hdev, cq_size_addr, size, 0);
10157 gaudi2_memset_device_lbw(hdev, cq_lbw_l_addr, size, 0);
10158 gaudi2_memset_device_lbw(hdev, cq_lbw_h_addr, size, 0);
10159 gaudi2_memset_device_lbw(hdev, cq_lbw_data_addr, size, 0);
10160 gaudi2_memset_device_lbw(hdev, cq_base_l_addr, size, 0);
10161 gaudi2_memset_device_lbw(hdev, cq_base_h_addr, size, 0);
10162 gaudi2_memset_device_lbw(hdev, cq_size_addr, size, 0);
10172 offset = hdev->asic_prop.first_available_user_mon[0] * 4;
10178 gaudi2_memset_device_lbw(hdev, addr, size, val);
10181 gaudi2_memset_device_lbw(hdev, addr, size, 0);
10188 gaudi2_memset_device_lbw(hdev, mon_sts_addr, size, val);
10189 gaudi2_memset_device_lbw(hdev, mon_cfg_addr, size, 0);
10194 offset = hdev->asic_prop.first_available_user_sob[0] * 4;
10201 gaudi2_memset_device_lbw(hdev, addr, size, val);
10207 gaudi2_memset_device_lbw(hdev, addr, size, val);
10215 static void gaudi2_restore_user_qm_registers(struct hl_device *hdev)
10221 if (!gaudi2_is_queue_enabled(hdev, hw_queue_id))
10224 gaudi2_clear_qm_fence_counters_common(hdev, hw_queue_id, false);
10234 static void gaudi2_restore_nic_qm_registers(struct hl_device *hdev)
10240 if (!gaudi2_is_queue_enabled(hdev, hw_queue_id))
10243 gaudi2_clear_qm_fence_counters_common(hdev, hw_queue_id, false);
10253 static int gaudi2_context_switch(struct hl_device *hdev, u32 asid)
10258 static void gaudi2_restore_phase_topology(struct hl_device *hdev)
10262 static void gaudi2_init_block_instances(struct hl_device *hdev, u32 block_idx,
10276 cfg_ctx->instance_cfg_fn(hdev, block_base + i * cfg_ctx->instance_off,
10281 static void gaudi2_init_blocks_with_mask(struct hl_device *hdev, struct dup_block_ctx *cfg_ctx,
10289 gaudi2_init_block_instances(hdev, i, cfg_ctx);
10292 void gaudi2_init_blocks(struct hl_device *hdev, struct dup_block_ctx *cfg_ctx)
10294 gaudi2_init_blocks_with_mask(hdev, cfg_ctx, U64_MAX);
10297 static int gaudi2_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size, void *blob_addr)
10307 ctx = hl_get_compute_ctx(hdev);
10309 dev_err(hdev->dev, "No ctx available\n");
10314 host_mem_virtual_addr = hl_asic_dma_alloc_coherent(hdev, SZ_2M, &host_mem_dma_addr,
10317 dev_err(hdev->dev, "Failed to allocate memory for KDMA read\n");
10323 reserved_va_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST, SZ_2M,
10326 dev_err(hdev->dev, "Failed to reserve vmem on asic\n");
10332 mutex_lock(&hdev->mmu_lock);
10336 dev_err(hdev->dev, "Failed to create mapping on asic mmu\n");
10340 rc = hl_mmu_invalidate_cache_range(hdev, false,
10348 mutex_unlock(&hdev->mmu_lock);
10351 gaudi2_kdma_set_mmbp_asid(hdev, false, ctx->asid);
10361 rc = gaudi2_send_job_to_kdma(hdev, addr, reserved_va_base, size_to_dma, false);
10375 gaudi2_kdma_set_mmbp_asid(hdev, true, HL_KERNEL_ASID_ID);
10377 mutex_lock(&hdev->mmu_lock);
10383 rc = hl_mmu_invalidate_cache_range(hdev, false, MMU_OP_USERPTR,
10387 mutex_unlock(&hdev->mmu_lock);
10388 hl_unreserve_va_block(hdev, ctx, reserved_va_base, SZ_2M);
10390 hl_asic_dma_free_coherent(hdev, SZ_2M, host_mem_virtual_addr, host_mem_dma_addr);
10397 static int gaudi2_internal_cb_pool_init(struct hl_device *hdev, struct hl_ctx *ctx)
10399 struct gaudi2_device *gaudi2 = hdev->asic_specific;
10405 hdev->internal_cb_pool_virt_addr = hl_asic_dma_alloc_coherent(hdev,
10407 &hdev->internal_cb_pool_dma_addr,
10410 if (!hdev->internal_cb_pool_virt_addr)
10413 min_alloc_order = ilog2(min(gaudi2_get_signal_cb_size(hdev),
10414 gaudi2_get_wait_cb_size(hdev)));
10416 hdev->internal_cb_pool = gen_pool_create(min_alloc_order, -1);
10417 if (!hdev->internal_cb_pool) {
10418 dev_err(hdev->dev, "Failed to create internal CB pool\n");
10423 rc = gen_pool_add(hdev->internal_cb_pool, (uintptr_t) hdev->internal_cb_pool_virt_addr,
10426 dev_err(hdev->dev, "Failed to add memory to internal CB pool\n");
10431 hdev->internal_cb_va_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST,
10434 if (!hdev->internal_cb_va_base) {
10439 mutex_lock(&hdev->mmu_lock);
10441 rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base, hdev->internal_cb_pool_dma_addr,
10446 rc = hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR);
10450 mutex_unlock(&hdev->mmu_lock);
10455 hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ);
10457 mutex_unlock(&hdev->mmu_lock);
10458 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ);
10460 gen_pool_destroy(hdev->internal_cb_pool);
10462 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr,
10463 hdev->internal_cb_pool_dma_addr);
10468 static void gaudi2_internal_cb_pool_fini(struct hl_device *hdev, struct hl_ctx *ctx)
10470 struct gaudi2_device *gaudi2 = hdev->asic_specific;
10475 mutex_lock(&hdev->mmu_lock);
10476 hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ);
10477 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ);
10478 hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR);
10479 mutex_unlock(&hdev->mmu_lock);
10481 gen_pool_destroy(hdev->internal_cb_pool);
10483 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr,
10484 hdev->internal_cb_pool_dma_addr);
10487 static void gaudi2_restore_user_registers(struct hl_device *hdev)
10489 gaudi2_restore_user_sm_registers(hdev);
10490 gaudi2_restore_user_qm_registers(hdev);
10495 struct hl_device *hdev = ctx->hdev;
10496 struct asic_fixed_properties *prop = &hdev->asic_prop;
10497 struct gaudi2_device *gaudi2 = hdev->asic_specific;
10503 dev_err(hdev->dev, "Failed to map VA %#llx for virtual MSI-X doorbell memory\n",
10511 struct hl_device *hdev = ctx->hdev;
10512 struct asic_fixed_properties *prop = &hdev->asic_prop;
10518 dev_err(hdev->dev, "Failed to unmap VA %#llx of virtual MSI-X doorbell memory\n",
10526 rc = gaudi2_mmu_prepare(ctx->hdev, ctx->asid);
10533 if (ctx->hdev->reset_upon_device_release)
10534 gaudi2_restore_nic_qm_registers(ctx->hdev);
10536 gaudi2_restore_user_registers(ctx->hdev);
10538 rc = gaudi2_internal_cb_pool_init(ctx->hdev, ctx);
10544 gaudi2_internal_cb_pool_fini(ctx->hdev, ctx);
10554 gaudi2_internal_cb_pool_fini(ctx->hdev, ctx);
10561 struct hl_device *hdev = cs->ctx->hdev;
10562 int index = cs->sequence & (hdev->asic_prop.max_pending_cs - 1);
10580 gaudi2_arm_cq_monitor(hdev, sob_id, mon_id, GAUDI2_RESERVED_CQ_CS_COMPLETION, mon_payload,
10586 static u32 gaudi2_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx)
10591 static u32 gaudi2_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id, u32 size, bool eb)
10634 static u32 gaudi2_add_arm_monitor_pkt(struct hl_device *hdev, struct packet_msg_short *pkt,
10641 dev_err(hdev->dev, "sob_base %u (mask %#x) is not valid\n", sob_base, sob_mask);
10684 static u32 gaudi2_gen_wait_cb(struct hl_device *hdev, struct hl_gen_wait_properties *prop)
10727 size += gaudi2_add_arm_monitor_pkt(hdev, buf + size, prop->sob_base, prop->sob_mask,
10736 static void gaudi2_reset_sob(struct hl_device *hdev, void *data)
10740 dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx, hw_sob->sob_id);
10747 static void gaudi2_reset_sob_group(struct hl_device *hdev, u16 sob_group)
10751 static u64 gaudi2_get_device_time(struct hl_device *hdev)
10763 static int gaudi2_collective_wait_create_jobs(struct hl_device *hdev, struct hl_ctx *ctx,
10788 static u64 gaudi2_mmu_scramble_addr(struct hl_device *hdev, u64 raw_addr)
10790 struct asic_fixed_properties *prop = &hdev->asic_prop;
10808 static u64 gaudi2_mmu_descramble_addr(struct hl_device *hdev, u64 scrambled_addr)
10810 struct asic_fixed_properties *prop = &hdev->asic_prop;
10829 static u32 gaudi2_get_dec_base_addr(struct hl_device *hdev, u32 core_id)
10834 dev_err(hdev->dev, "Unexpected core number %d for DEC\n", core_id);
10852 static int gaudi2_get_hw_block_id(struct hl_device *hdev, u64 block_addr,
10855 struct gaudi2_device *gaudi2 = hdev->asic_specific;
10867 dev_err(hdev->dev, "Invalid block address %#llx", block_addr);
10872 static int gaudi2_block_mmap(struct hl_device *hdev, struct vm_area_struct *vma,
10875 struct gaudi2_device *gaudi2 = hdev->asic_specific;
10881 dev_err(hdev->dev, "Invalid block id %u", block_id);
10887 dev_err(hdev->dev, "Invalid block size %u", block_size);
10893 address = pci_resource_start(hdev->pdev, SRAM_CFG_BAR_ID) + offset_in_bar;
10901 dev_err(hdev->dev, "remap_pfn_range error %d", rc);
10906 static void gaudi2_enable_events_from_fw(struct hl_device *hdev)
10908 struct gaudi2_device *gaudi2 = hdev->asic_specific;
10910 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
10918 static int gaudi2_get_mmu_base(struct hl_device *hdev, u64 mmu_id, u32 *mmu_base)
10979 static void gaudi2_ack_mmu_error(struct hl_device *hdev, u64 mmu_id)
10982 struct gaudi2_device *gaudi2 = hdev->asic_specific;
10988 if (gaudi2_get_mmu_base(hdev, mmu_id, &mmu_base))
10991 gaudi2_handle_page_error(hdev, mmu_base, is_pmmu, NULL);
10992 gaudi2_handle_access_error(hdev, mmu_base, is_pmmu);
10995 static int gaudi2_ack_mmu_page_fault_or_access_error(struct hl_device *hdev, u64 mmu_cap_mask)
11004 gaudi2_ack_mmu_error(hdev, mmu_id);
11009 gaudi2_ack_mmu_error(hdev, HW_CAP_PMMU);
11038 static int gaudi2_gen_sync_to_engine_map(struct hl_device *hdev, struct hl_sync_to_engine_map *map)
11051 struct hl_device *hdev, struct hl_mon_state_dump *mon)
11058 static int gaudi2_print_fences_single_engine(struct hl_device *hdev, u64 base_offset,
11074 static void gaudi2_state_dump_init(struct hl_device *hdev)
11077 hdev->state_dump_specs.props = gaudi2_state_dump_specs_props;
11078 hdev->state_dump_specs.funcs = gaudi2_state_dump_funcs;
11081 static u32 gaudi2_get_sob_addr(struct hl_device *hdev, u32 sob_id)
11091 static void gaudi2_add_device_attr(struct hl_device *hdev, struct attribute_group *dev_clk_attr_grp,
11094 hl_sysfs_add_dev_clk_attr(hdev, dev_clk_attr_grp);
11095 hl_sysfs_add_dev_vrm_attr(hdev, dev_vrm_attr_grp);
11098 static int gaudi2_mmu_get_real_page_size(struct hl_device *hdev, struct hl_mmu_properties *mmu_prop,
11101 struct asic_fixed_properties *prop = &hdev->asic_prop;
11128 dev_err(hdev->dev, "page size of %u is not %uKB aligned, can't map\n",
11133 static int gaudi2_get_monitor_dump(struct hl_device *hdev, void *data)
11138 int gaudi2_send_device_activity(struct hl_device *hdev, bool open)
11140 struct gaudi2_device *gaudi2 = hdev->asic_specific;
11145 return hl_fw_send_device_activity(hdev, open);
11250 void gaudi2_set_asic_funcs(struct hl_device *hdev)
11252 hdev->asic_funcs = &gaudi2_funcs;