Lines Matching defs:hdev
355 static int goya_mmu_clear_pgt_range(struct hl_device *hdev);
356 static int goya_mmu_set_dram_default_page(struct hl_device *hdev);
357 static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev);
358 static void goya_mmu_prepare(struct hl_device *hdev, u32 asid);
360 int goya_get_fixed_properties(struct hl_device *hdev)
362 struct asic_fixed_properties *prop = &hdev->asic_prop;
407 if (hdev->pldm)
465 * @hdev: pointer to hl_device structure
471 static int goya_pci_bars_map(struct hl_device *hdev)
477 rc = hl_pci_bars_map(hdev, name, is_wc);
481 hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] +
487 static u64 goya_set_ddr_bar_base(struct hl_device *hdev, u64 addr)
489 struct goya_device *goya = hdev->asic_specific;
501 rc = hl_pci_set_inbound_region(hdev, 1, &pci_region);
516 * @hdev: pointer to hl_device structure
521 static int goya_init_iatu(struct hl_device *hdev)
531 rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region);
539 rc = hl_pci_set_inbound_region(hdev, 1, &inbound_region);
543 hdev->asic_funcs->set_dma_mask_from_fw(hdev);
548 rc = hl_pci_set_outbound_region(hdev, &outbound_region);
557 * @hdev: pointer to hl_device structure
565 static int goya_early_init(struct hl_device *hdev)
567 struct asic_fixed_properties *prop = &hdev->asic_prop;
568 struct pci_dev *pdev = hdev->pdev;
572 rc = goya_get_fixed_properties(hdev);
574 dev_err(hdev->dev, "Failed to get fixed properties\n");
580 dev_err(hdev->dev,
591 dev_err(hdev->dev,
603 rc = hl_pci_init(hdev, mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS,
610 dev_info(hdev->dev, "firmware-level security is disabled\n");
612 if (!hdev->pldm) {
615 dev_warn(hdev->dev,
622 kfree(hdev->asic_prop.hw_queues_props);
629 * @hdev: pointer to hl_device structure
634 static int goya_early_fini(struct hl_device *hdev)
636 kfree(hdev->asic_prop.hw_queues_props);
637 hl_pci_fini(hdev);
642 static void goya_mmu_prepare_reg(struct hl_device *hdev, u64 reg, u32 asid)
649 static void goya_qman0_set_security(struct hl_device *hdev, bool secure)
651 struct goya_device *goya = hdev->asic_specific;
667 * @hdev: pointer to hl_device structure
670 static void goya_fetch_psoc_frequency(struct hl_device *hdev)
672 struct asic_fixed_properties *prop = &hdev->asic_prop;
694 dev_warn(hdev->dev,
705 int goya_late_init(struct hl_device *hdev)
707 struct asic_fixed_properties *prop = &hdev->asic_prop;
710 goya_fetch_psoc_frequency(hdev);
712 rc = goya_mmu_clear_pgt_range(hdev);
714 dev_err(hdev->dev,
719 rc = goya_mmu_set_dram_default_page(hdev);
721 dev_err(hdev->dev, "Failed to set DRAM default page %d\n", rc);
725 rc = goya_mmu_add_mappings_for_device_cpu(hdev);
729 rc = goya_init_cpu_queues(hdev);
733 rc = goya_test_cpu_queue(hdev);
737 rc = goya_cpucp_info_get(hdev);
739 dev_err(hdev->dev, "Failed to get cpucp info %d\n", rc);
749 rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_ENABLE_PCI_ACCESS);
751 dev_err(hdev->dev,
765 * @hdev: pointer to hl_device structure
769 void goya_late_fini(struct hl_device *hdev)
774 if (!hdev->hl_chip_info->info)
777 channel_info_arr = hdev->hl_chip_info->info;
787 hdev->hl_chip_info->info = NULL;
793 * @hdev: pointer to hl_device structure
796 static int goya_sw_init(struct hl_device *hdev)
813 hdev->asic_specific = goya;
816 hdev->dma_pool = dma_pool_create(dev_name(hdev->dev),
817 &hdev->pdev->dev, GOYA_DMA_POOL_BLK_SIZE, 8, 0);
818 if (!hdev->dma_pool) {
819 dev_err(hdev->dev, "failed to create DMA pool\n");
824 hdev->cpu_accessible_dma_mem =
825 hdev->asic_funcs->asic_dma_alloc_coherent(hdev,
827 &hdev->cpu_accessible_dma_address,
830 if (!hdev->cpu_accessible_dma_mem) {
835 dev_dbg(hdev->dev, "cpu accessible memory at bus address %pad\n",
836 &hdev->cpu_accessible_dma_address);
838 hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1);
839 if (!hdev->cpu_accessible_dma_pool) {
840 dev_err(hdev->dev,
846 rc = gen_pool_add(hdev->cpu_accessible_dma_pool,
847 (uintptr_t) hdev->cpu_accessible_dma_mem,
850 dev_err(hdev->dev,
857 hdev->supports_coresight = true;
858 hdev->supports_soft_reset = true;
863 gen_pool_destroy(hdev->cpu_accessible_dma_pool);
865 hdev->asic_funcs->asic_dma_free_coherent(hdev,
867 hdev->cpu_accessible_dma_mem,
868 hdev->cpu_accessible_dma_address);
870 dma_pool_destroy(hdev->dma_pool);
880 * @hdev: pointer to hl_device structure
883 static int goya_sw_fini(struct hl_device *hdev)
885 struct goya_device *goya = hdev->asic_specific;
887 gen_pool_destroy(hdev->cpu_accessible_dma_pool);
889 hdev->asic_funcs->asic_dma_free_coherent(hdev,
891 hdev->cpu_accessible_dma_mem,
892 hdev->cpu_accessible_dma_address);
894 dma_pool_destroy(hdev->dma_pool);
901 static void goya_init_dma_qman(struct hl_device *hdev, int dma_id,
904 struct goya_device *goya = hdev->asic_specific;
946 if (hdev->stop_on_err)
953 static void goya_init_dma_ch(struct hl_device *hdev, int dma_id)
982 * @hdev: pointer to hl_device structure
987 void goya_init_dma_qmans(struct hl_device *hdev)
989 struct goya_device *goya = hdev->asic_specific;
996 q = &hdev->kernel_queues[0];
1000 goya_init_dma_qman(hdev, i, q->bus_address);
1001 goya_init_dma_ch(hdev, i);
1010 * @hdev: pointer to hl_device structure
1013 static void goya_disable_external_queues(struct hl_device *hdev)
1015 struct goya_device *goya = hdev->asic_specific;
1027 static int goya_stop_queue(struct hl_device *hdev, u32 cfg_reg,
1040 hdev,
1053 hdev,
1061 dev_err(hdev->dev,
1072 * @hdev: pointer to hl_device structure
1077 static int goya_stop_external_queues(struct hl_device *hdev)
1081 struct goya_device *goya = hdev->asic_specific;
1086 rc = goya_stop_queue(hdev,
1092 dev_err(hdev->dev, "failed to stop DMA QMAN 0\n");
1096 rc = goya_stop_queue(hdev,
1102 dev_err(hdev->dev, "failed to stop DMA QMAN 1\n");
1106 rc = goya_stop_queue(hdev,
1112 dev_err(hdev->dev, "failed to stop DMA QMAN 2\n");
1116 rc = goya_stop_queue(hdev,
1122 dev_err(hdev->dev, "failed to stop DMA QMAN 3\n");
1126 rc = goya_stop_queue(hdev,
1132 dev_err(hdev->dev, "failed to stop DMA QMAN 4\n");
1142 * @hdev: pointer to hl_device structure
1147 int goya_init_cpu_queues(struct hl_device *hdev)
1149 struct goya_device *goya = hdev->asic_specific;
1152 struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GOYA_QUEUE_ID_CPU_PQ];
1155 if (!hdev->cpu_queues_enable)
1161 eq = &hdev->event_queue;
1189 hdev,
1197 dev_err(hdev->dev,
1206 static void goya_set_pll_refclk(struct hl_device *hdev)
1244 static void goya_disable_clk_rlx(struct hl_device *hdev)
1250 static void _goya_tpc_mbist_workaround(struct hl_device *hdev, u8 tpc_id)
1268 dev_warn(hdev->dev, "TPC%d MBIST ACTIVE is not cleared\n",
1288 hdev,
1296 dev_err(hdev->dev,
1315 static void goya_tpc_mbist_workaround(struct hl_device *hdev)
1317 struct goya_device *goya = hdev->asic_specific;
1320 if (hdev->pldm)
1329 _goya_tpc_mbist_workaround(hdev, i);
1337 * @hdev: pointer to hl_device structure
1342 static void goya_init_golden_registers(struct hl_device *hdev)
1344 struct goya_device *goya = hdev->asic_specific;
1637 static void goya_init_mme_qman(struct hl_device *hdev)
1654 qman_base_addr = hdev->asic_prop.sram_base_address +
1687 static void goya_init_mme_cmdq(struct hl_device *hdev)
1723 void goya_init_mme_qmans(struct hl_device *hdev)
1725 struct goya_device *goya = hdev->asic_specific;
1737 goya_init_mme_qman(hdev);
1738 goya_init_mme_cmdq(hdev);
1743 static void goya_init_tpc_qman(struct hl_device *hdev, u32 base_off, int tpc_id)
1761 qman_base_addr = hdev->asic_prop.sram_base_address + base_off;
1793 static void goya_init_tpc_cmdq(struct hl_device *hdev, int tpc_id)
1830 void goya_init_tpc_qmans(struct hl_device *hdev)
1832 struct goya_device *goya = hdev->asic_specific;
1851 goya_init_tpc_qman(hdev, TPC0_QMAN_BASE_OFFSET, 0);
1852 goya_init_tpc_qman(hdev, TPC1_QMAN_BASE_OFFSET, 1);
1853 goya_init_tpc_qman(hdev, TPC2_QMAN_BASE_OFFSET, 2);
1854 goya_init_tpc_qman(hdev, TPC3_QMAN_BASE_OFFSET, 3);
1855 goya_init_tpc_qman(hdev, TPC4_QMAN_BASE_OFFSET, 4);
1856 goya_init_tpc_qman(hdev, TPC5_QMAN_BASE_OFFSET, 5);
1857 goya_init_tpc_qman(hdev, TPC6_QMAN_BASE_OFFSET, 6);
1858 goya_init_tpc_qman(hdev, TPC7_QMAN_BASE_OFFSET, 7);
1861 goya_init_tpc_cmdq(hdev, i);
1869 * @hdev: pointer to hl_device structure
1872 static void goya_disable_internal_queues(struct hl_device *hdev)
1874 struct goya_device *goya = hdev->asic_specific;
1914 * @hdev: pointer to hl_device structure
1919 static int goya_stop_internal_queues(struct hl_device *hdev)
1921 struct goya_device *goya = hdev->asic_specific;
1933 rc = goya_stop_queue(hdev,
1939 dev_err(hdev->dev, "failed to stop MME QMAN\n");
1943 rc = goya_stop_queue(hdev,
1949 dev_err(hdev->dev, "failed to stop MME CMDQ\n");
1957 rc = goya_stop_queue(hdev,
1963 dev_err(hdev->dev, "failed to stop TPC 0 QMAN\n");
1967 rc = goya_stop_queue(hdev,
1973 dev_err(hdev->dev, "failed to stop TPC 0 CMDQ\n");
1977 rc = goya_stop_queue(hdev,
1983 dev_err(hdev->dev, "failed to stop TPC 1 QMAN\n");
1987 rc = goya_stop_queue(hdev,
1993 dev_err(hdev->dev, "failed to stop TPC 1 CMDQ\n");
1997 rc = goya_stop_queue(hdev,
2003 dev_err(hdev->dev, "failed to stop TPC 2 QMAN\n");
2007 rc = goya_stop_queue(hdev,
2013 dev_err(hdev->dev, "failed to stop TPC 2 CMDQ\n");
2017 rc = goya_stop_queue(hdev,
2023 dev_err(hdev->dev, "failed to stop TPC 3 QMAN\n");
2027 rc = goya_stop_queue(hdev,
2033 dev_err(hdev->dev, "failed to stop TPC 3 CMDQ\n");
2037 rc = goya_stop_queue(hdev,
2043 dev_err(hdev->dev, "failed to stop TPC 4 QMAN\n");
2047 rc = goya_stop_queue(hdev,
2053 dev_err(hdev->dev, "failed to stop TPC 4 CMDQ\n");
2057 rc = goya_stop_queue(hdev,
2063 dev_err(hdev->dev, "failed to stop TPC 5 QMAN\n");
2067 rc = goya_stop_queue(hdev,
2073 dev_err(hdev->dev, "failed to stop TPC 5 CMDQ\n");
2077 rc = goya_stop_queue(hdev,
2083 dev_err(hdev->dev, "failed to stop TPC 6 QMAN\n");
2087 rc = goya_stop_queue(hdev,
2093 dev_err(hdev->dev, "failed to stop TPC 6 CMDQ\n");
2097 rc = goya_stop_queue(hdev,
2103 dev_err(hdev->dev, "failed to stop TPC 7 QMAN\n");
2107 rc = goya_stop_queue(hdev,
2113 dev_err(hdev->dev, "failed to stop TPC 7 CMDQ\n");
2120 static void goya_dma_stall(struct hl_device *hdev)
2122 struct goya_device *goya = hdev->asic_specific;
2134 static void goya_tpc_stall(struct hl_device *hdev)
2136 struct goya_device *goya = hdev->asic_specific;
2151 static void goya_mme_stall(struct hl_device *hdev)
2153 struct goya_device *goya = hdev->asic_specific;
2161 static int goya_enable_msix(struct hl_device *hdev)
2163 struct goya_device *goya = hdev->asic_specific;
2164 int cq_cnt = hdev->asic_prop.completion_queues_count;
2170 rc = pci_alloc_irq_vectors(hdev->pdev, GOYA_MSIX_ENTRIES,
2173 dev_err(hdev->dev,
2180 irq = pci_irq_vector(hdev->pdev, i);
2182 &hdev->completion_queue[i]);
2184 dev_err(hdev->dev, "Failed to request IRQ %d", irq);
2189 irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX);
2193 &hdev->event_queue);
2195 dev_err(hdev->dev, "Failed to request IRQ %d", irq);
2204 free_irq(pci_irq_vector(hdev->pdev, i),
2205 &hdev->completion_queue[i]);
2207 pci_free_irq_vectors(hdev->pdev);
2211 static void goya_sync_irqs(struct hl_device *hdev)
2213 struct goya_device *goya = hdev->asic_specific;
2220 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
2221 synchronize_irq(pci_irq_vector(hdev->pdev, i));
2223 synchronize_irq(pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX));
2226 static void goya_disable_msix(struct hl_device *hdev)
2228 struct goya_device *goya = hdev->asic_specific;
2234 goya_sync_irqs(hdev);
2236 irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX);
2237 free_irq(irq, &hdev->event_queue);
2239 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) {
2240 irq = pci_irq_vector(hdev->pdev, i);
2241 free_irq(irq, &hdev->completion_queue[i]);
2244 pci_free_irq_vectors(hdev->pdev);
2249 static void goya_enable_timestamp(struct hl_device *hdev)
2262 static void goya_disable_timestamp(struct hl_device *hdev)
2268 static void goya_halt_engines(struct hl_device *hdev, bool hard_reset)
2272 dev_info(hdev->dev,
2275 if (hdev->pldm)
2280 goya_stop_external_queues(hdev);
2281 goya_stop_internal_queues(hdev);
2285 goya_dma_stall(hdev);
2286 goya_tpc_stall(hdev);
2287 goya_mme_stall(hdev);
2291 goya_disable_external_queues(hdev);
2292 goya_disable_internal_queues(hdev);
2294 goya_disable_timestamp(hdev);
2297 goya_disable_msix(hdev);
2298 goya_mmu_remove_device_cpu_mappings(hdev);
2300 goya_sync_irqs(hdev);
2306 * @hdev: Pointer to hl_device structure.
2312 static int goya_load_firmware_to_device(struct hl_device *hdev)
2316 dst = hdev->pcie_bar[DDR_BAR_ID] + LINUX_FW_OFFSET;
2318 return hl_fw_load_fw_to_device(hdev, GOYA_LINUX_FW_FILE, dst);
2323 * @hdev: Pointer to hl_device structure.
2329 static int goya_load_boot_fit_to_device(struct hl_device *hdev)
2333 dst = hdev->pcie_bar[SRAM_CFG_BAR_ID] + BOOT_FIT_SRAM_OFFSET;
2335 return hl_fw_load_fw_to_device(hdev, GOYA_BOOT_FIT_FILE, dst);
2342 static void goya_read_device_fw_version(struct hl_device *hdev,
2352 dest = hdev->asic_prop.uboot_ver;
2357 dest = hdev->asic_prop.preboot_ver;
2361 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
2368 memcpy_fromio(dest, hdev->pcie_bar[SRAM_CFG_BAR_ID] + ver_off,
2371 dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n",
2377 static int goya_init_cpu(struct hl_device *hdev)
2379 struct goya_device *goya = hdev->asic_specific;
2382 if (!hdev->cpu_enable)
2392 if (goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) {
2393 dev_err(hdev->dev,
2398 rc = hl_fw_init_cpu(hdev, mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS,
2412 static int goya_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 asid,
2418 if (hdev->pldm)
2428 hdev,
2436 dev_err(hdev->dev,
2444 int goya_mmu_init(struct hl_device *hdev)
2446 struct asic_fixed_properties *prop = &hdev->asic_prop;
2447 struct goya_device *goya = hdev->asic_specific;
2451 if (!hdev->mmu_enable)
2457 hdev->dram_supports_virtual_memory = true;
2458 hdev->dram_default_page_mapping = true;
2464 rc = goya_mmu_update_asid_hop0_addr(hdev, i, hop0_addr);
2466 dev_err(hdev->dev,
2483 hdev->asic_funcs->mmu_invalidate_cache(hdev, true,
2498 * @hdev: pointer to hl_device structure
2503 static int goya_hw_init(struct hl_device *hdev)
2505 struct asic_fixed_properties *prop = &hdev->asic_prop;
2508 dev_info(hdev->dev, "Starting initialization of H/W\n");
2521 rc = goya_init_cpu(hdev);
2523 dev_err(hdev->dev, "failed to initialize CPU\n");
2527 goya_tpc_mbist_workaround(hdev);
2529 goya_init_golden_registers(hdev);
2535 if (goya_set_ddr_bar_base(hdev, (MMU_PAGE_TABLES_ADDR &
2537 dev_err(hdev->dev,
2542 rc = goya_mmu_init(hdev);
2546 goya_init_security(hdev);
2548 goya_init_dma_qmans(hdev);
2550 goya_init_mme_qmans(hdev);
2552 goya_init_tpc_qmans(hdev);
2554 goya_enable_timestamp(hdev);
2557 rc = goya_enable_msix(hdev);
2567 goya_disable_internal_queues(hdev);
2568 goya_disable_external_queues(hdev);
2576 * @hdev: pointer to hl_device structure
2580 static void goya_hw_fini(struct hl_device *hdev, bool hard_reset)
2582 struct goya_device *goya = hdev->asic_specific;
2585 if (hdev->pldm) {
2603 goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE);
2604 goya_disable_clk_rlx(hdev);
2605 goya_set_pll_refclk(hdev);
2608 dev_info(hdev->dev,
2613 dev_info(hdev->dev,
2627 dev_err(hdev->dev,
2654 int goya_suspend(struct hl_device *hdev)
2658 rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS);
2660 dev_err(hdev->dev, "Failed to disable PCI access from CPU\n");
2665 int goya_resume(struct hl_device *hdev)
2667 return goya_init_iatu(hdev);
2670 static int goya_cb_mmap(struct hl_device *hdev, struct vm_area_struct *vma,
2678 rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr,
2681 dev_err(hdev->dev, "dma_mmap_coherent error %d", rc);
2686 void goya_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi)
2753 dev_err(hdev->dev, "H/W queue %d is invalid. Can't set pi\n",
2768 void goya_pqe_write(struct hl_device *hdev, __le64 *pqe, struct hl_bd *bd)
2774 static void *goya_dma_alloc_coherent(struct hl_device *hdev, size_t size,
2777 void *kernel_addr = dma_alloc_coherent(&hdev->pdev->dev, size,
2787 static void goya_dma_free_coherent(struct hl_device *hdev, size_t size,
2793 dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, fixed_dma_handle);
2796 void *goya_get_int_queue_base(struct hl_device *hdev, u32 queue_id,
2802 *dma_handle = hdev->asic_prop.sram_base_address;
2804 base = (void *) hdev->pcie_bar[SRAM_CFG_BAR_ID];
2844 dev_err(hdev->dev, "Got invalid queue id %d\n", queue_id);
2854 static int goya_send_job_on_qman0(struct hl_device *hdev, struct hl_cs_job *job)
2863 if (hdev->pldm)
2868 if (!hdev->asic_funcs->is_device_idle(hdev, NULL, NULL)) {
2869 dev_err_ratelimited(hdev->dev,
2874 fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL,
2877 dev_err(hdev->dev,
2882 goya_qman0_set_security(hdev, true);
2896 rc = hl_hw_queue_send_cb_no_cmpl(hdev, GOYA_QUEUE_ID_DMA_0,
2899 dev_err(hdev->dev, "Failed to send CB on QMAN0, %d\n", rc);
2903 rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp,
2907 hl_hw_queue_inc_ci_kernel(hdev, GOYA_QUEUE_ID_DMA_0);
2910 dev_err(hdev->dev, "QMAN0 Job timeout (0x%x)\n", tmp);
2915 hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr,
2918 goya_qman0_set_security(hdev, false);
2923 int goya_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len,
2926 struct goya_device *goya = hdev->asic_specific;
2937 return hl_fw_send_cpu_message(hdev, GOYA_QUEUE_ID_CPU_PQ, msg, len,
2941 int goya_test_queue(struct hl_device *hdev, u32 hw_queue_id)
2952 fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL,
2955 dev_err(hdev->dev,
2963 fence_pkt = hdev->asic_funcs->asic_dma_pool_zalloc(hdev,
2967 dev_err(hdev->dev,
2981 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id,
2985 dev_err(hdev->dev,
2991 rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp, (tmp == fence_val),
2994 hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
2997 dev_err(hdev->dev,
3004 hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_pkt,
3007 hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr,
3012 int goya_test_cpu_queue(struct hl_device *hdev)
3014 struct goya_device *goya = hdev->asic_specific;
3023 return hl_fw_test_cpu_queue(hdev);
3026 int goya_test_queues(struct hl_device *hdev)
3031 rc = goya_test_queue(hdev, i);
3039 static void *goya_dma_pool_zalloc(struct hl_device *hdev, size_t size,
3047 kernel_addr = dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle);
3056 static void goya_dma_pool_free(struct hl_device *hdev, void *vaddr,
3062 dma_pool_free(hdev->dma_pool, vaddr, fixed_dma_addr);
3065 void *goya_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
3070 vaddr = hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle);
3071 *dma_handle = (*dma_handle) - hdev->cpu_accessible_dma_address +
3077 void goya_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
3080 hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr);
3083 static int goya_dma_map_sg(struct hl_device *hdev, struct scatterlist *sgl,
3089 if (!dma_map_sg(&hdev->pdev->dev, sgl, nents, dir))
3099 static void goya_dma_unmap_sg(struct hl_device *hdev, struct scatterlist *sgl,
3109 dma_unmap_sg(&hdev->pdev->dev, sgl, nents, dir);
3112 u32 goya_get_dma_desc_list_size(struct hl_device *hdev, struct sg_table *sgt)
3153 static int goya_pin_memory_before_cs(struct hl_device *hdev,
3161 if (hl_userptr_is_pinned(hdev, addr, le32_to_cpu(user_dma_pkt->tsize),
3169 rc = hl_pin_host_memory(hdev, addr, le32_to_cpu(user_dma_pkt->tsize),
3176 rc = hdev->asic_funcs->asic_dma_map_sg(hdev, userptr->sgt->sgl,
3179 dev_err(hdev->dev, "failed to map sgt with DMA region\n");
3188 goya_get_dma_desc_list_size(hdev, userptr->sgt);
3194 hl_unpin_host_memory(hdev, userptr);
3200 static int goya_validate_dma_pkt_host(struct hl_device *hdev,
3223 dev_dbg(hdev->dev, "DMA direction is HOST --> DRAM\n");
3233 dev_dbg(hdev->dev, "DMA direction is DRAM --> HOST\n");
3241 dev_dbg(hdev->dev, "DMA direction is HOST --> SRAM\n");
3250 dev_dbg(hdev->dev, "DMA direction is SRAM --> HOST\n");
3256 dev_err(hdev->dev, "DMA direction is undefined\n");
3263 hdev->asic_prop.sram_user_base_address,
3264 hdev->asic_prop.sram_end_address)) {
3266 dev_err(hdev->dev,
3275 hdev->asic_prop.dram_user_base_address,
3276 hdev->asic_prop.dram_end_address)) {
3278 dev_err(hdev->dev,
3291 dev_err(hdev->dev,
3296 rc = goya_pin_memory_before_cs(hdev, parser, user_dma_pkt,
3303 static int goya_validate_dma_pkt_no_host(struct hl_device *hdev,
3316 dev_dbg(hdev->dev, "DMA direction is DRAM --> SRAM\n");
3320 dev_dbg(hdev->dev, "DMA direction is SRAM --> DRAM\n");
3327 hdev->asic_prop.sram_user_base_address,
3328 hdev->asic_prop.sram_end_address)) {
3329 dev_err(hdev->dev, "SRAM address 0x%llx + 0x%x is invalid\n",
3336 hdev->asic_prop.dram_user_base_address,
3337 hdev->asic_prop.dram_end_address)) {
3338 dev_err(hdev->dev, "DRAM address 0x%llx + 0x%x is invalid\n",
3348 static int goya_validate_dma_pkt_no_mmu(struct hl_device *hdev,
3356 dev_dbg(hdev->dev, "DMA packet details:\n");
3357 dev_dbg(hdev->dev, "source == 0x%llx\n",
3359 dev_dbg(hdev->dev, "destination == 0x%llx\n",
3361 dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize));
3372 dev_err(hdev->dev,
3378 rc = goya_validate_dma_pkt_no_host(hdev, parser, user_dma_pkt);
3380 rc = goya_validate_dma_pkt_host(hdev, parser, user_dma_pkt);
3385 static int goya_validate_dma_pkt_mmu(struct hl_device *hdev,
3389 dev_dbg(hdev->dev, "DMA packet details:\n");
3390 dev_dbg(hdev->dev, "source == 0x%llx\n",
3392 dev_dbg(hdev->dev, "destination == 0x%llx\n",
3394 dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize));
3404 hdev->asic_prop.pmmu.start_addr,
3405 hdev->asic_prop.pmmu.end_addr)) {
3406 dev_err(hdev->dev,
3412 dev_err(hdev->dev,
3422 static int goya_validate_wreg32(struct hl_device *hdev,
3426 struct goya_device *goya = hdev->asic_specific;
3433 dev_dbg(hdev->dev, "WREG32 packet details:\n");
3434 dev_dbg(hdev->dev, "reg_offset == 0x%x\n", reg_offset);
3435 dev_dbg(hdev->dev, "value == 0x%x\n",
3439 dev_err(hdev->dev, "WREG32 packet with illegal address 0x%x\n",
3458 dev_err(hdev->dev, "WREG32 packet with illegal value 0x%x\n",
3466 static int goya_validate_cb(struct hl_device *hdev,
3488 dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id);
3496 dev_err(hdev->dev,
3509 rc = goya_validate_wreg32(hdev,
3515 dev_err(hdev->dev,
3521 dev_err(hdev->dev,
3527 dev_err(hdev->dev, "User not allowed to use CP_DMA\n");
3532 dev_err(hdev->dev, "User not allowed to use STOP\n");
3538 rc = goya_validate_dma_pkt_mmu(hdev, parser,
3541 rc = goya_validate_dma_pkt_no_mmu(hdev, parser,
3553 dev_err(hdev->dev, "Invalid packet header 0x%x\n",
3573 static int goya_patch_dma_packet(struct hl_device *hdev,
3620 (hl_userptr_is_pinned(hdev, addr,
3623 dev_err(hdev->dev, "Userptr 0x%llx + 0x%x NOT mapped\n",
3689 dev_err(hdev->dev,
3703 static int goya_patch_cb(struct hl_device *hdev,
3727 dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id);
3735 dev_err(hdev->dev,
3743 rc = goya_patch_dma_packet(hdev, parser,
3753 rc = goya_validate_wreg32(hdev, parser,
3758 dev_err(hdev->dev,
3764 dev_err(hdev->dev,
3770 dev_err(hdev->dev, "User not allowed to use CP_DMA\n");
3775 dev_err(hdev->dev, "User not allowed to use STOP\n");
3788 dev_err(hdev->dev, "Invalid packet header 0x%x\n",
3801 static int goya_parse_cb_mmu(struct hl_device *hdev,
3817 rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx,
3822 dev_err(hdev->dev,
3829 parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr,
3852 rc = goya_validate_cb(hdev, parser, true);
3861 dev_err(hdev->dev, "user CB size mismatch\n");
3874 hl_cb_destroy(hdev, &hdev->kernel_cb_mgr,
3880 static int goya_parse_cb_no_mmu(struct hl_device *hdev,
3886 rc = goya_validate_cb(hdev, parser, false);
3891 rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx,
3895 dev_err(hdev->dev,
3901 parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr,
3911 rc = goya_patch_cb(hdev, parser);
3923 hl_cb_destroy(hdev, &hdev->kernel_cb_mgr,
3928 hl_userptr_delete_list(hdev, parser->job_userptr_list);
3932 static int goya_parse_cb_no_ext_queue(struct hl_device *hdev,
3935 struct asic_fixed_properties *asic_prop = &hdev->asic_prop;
3936 struct goya_device *goya = hdev->asic_specific;
3956 dev_err(hdev->dev,
3963 int goya_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser)
3965 struct goya_device *goya = hdev->asic_specific;
3968 return goya_parse_cb_no_ext_queue(hdev, parser);
3971 return goya_parse_cb_mmu(hdev, parser);
3973 return goya_parse_cb_no_mmu(hdev, parser);
3976 void goya_add_end_of_cb_packets(struct hl_device *hdev, void *kernel_address,
4001 void goya_update_eq_ci(struct hl_device *hdev, u32 val)
4006 void goya_restore_phase_topology(struct hl_device *hdev)
4011 static void goya_clear_sm_regs(struct hl_device *hdev)
4035 * @hdev: pointer to hl_device structure
4046 static int goya_debugfs_read32(struct hl_device *hdev, u64 addr, u32 *val)
4048 struct asic_fixed_properties *prop = &hdev->asic_prop;
4058 *val = readl(hdev->pcie_bar[SRAM_CFG_BAR_ID] +
4061 } else if (addr < DRAM_PHYS_BASE + hdev->asic_prop.dram_size) {
4066 ddr_bar_addr = goya_set_ddr_bar_base(hdev, bar_base_addr);
4068 *val = readl(hdev->pcie_bar[DDR_BAR_ID] +
4071 ddr_bar_addr = goya_set_ddr_bar_base(hdev,
4091 * @hdev: pointer to hl_device structure
4102 static int goya_debugfs_write32(struct hl_device *hdev, u64 addr, u32 val)
4104 struct asic_fixed_properties *prop = &hdev->asic_prop;
4114 writel(val, hdev->pcie_bar[SRAM_CFG_BAR_ID] +
4117 } else if (addr < DRAM_PHYS_BASE + hdev->asic_prop.dram_size) {
4122 ddr_bar_addr = goya_set_ddr_bar_base(hdev, bar_base_addr);
4124 writel(val, hdev->pcie_bar[DDR_BAR_ID] +
4127 ddr_bar_addr = goya_set_ddr_bar_base(hdev,
4143 static int goya_debugfs_read64(struct hl_device *hdev, u64 addr, u64 *val)
4145 struct asic_fixed_properties *prop = &hdev->asic_prop;
4158 *val = readq(hdev->pcie_bar[SRAM_CFG_BAR_ID] +
4162 DRAM_PHYS_BASE + hdev->asic_prop.dram_size - sizeof(u64)) {
4167 ddr_bar_addr = goya_set_ddr_bar_base(hdev, bar_base_addr);
4169 *val = readq(hdev->pcie_bar[DDR_BAR_ID] +
4172 ddr_bar_addr = goya_set_ddr_bar_base(hdev,
4188 static int goya_debugfs_write64(struct hl_device *hdev, u64 addr, u64 val)
4190 struct asic_fixed_properties *prop = &hdev->asic_prop;
4201 writeq(val, hdev->pcie_bar[SRAM_CFG_BAR_ID] +
4205 DRAM_PHYS_BASE + hdev->asic_prop.dram_size - sizeof(u64)) {
4210 ddr_bar_addr = goya_set_ddr_bar_base(hdev, bar_base_addr);
4212 writeq(val, hdev->pcie_bar[DDR_BAR_ID] +
4215 ddr_bar_addr = goya_set_ddr_bar_base(hdev,
4231 static u64 goya_read_pte(struct hl_device *hdev, u64 addr)
4233 struct goya_device *goya = hdev->asic_specific;
4235 if (hdev->hard_reset_pending)
4238 return readq(hdev->pcie_bar[DDR_BAR_ID] +
4242 static void goya_write_pte(struct hl_device *hdev, u64 addr, u64 val)
4244 struct goya_device *goya = hdev->asic_specific;
4246 if (hdev->hard_reset_pending)
4249 writeq(val, hdev->pcie_bar[DDR_BAR_ID] +
4446 static void goya_print_razwi_info(struct hl_device *hdev)
4449 dev_err_ratelimited(hdev->dev, "Illegal write to LBW\n");
4454 dev_err_ratelimited(hdev->dev, "Illegal read from LBW\n");
4459 dev_err_ratelimited(hdev->dev, "Illegal write to HBW\n");
4464 dev_err_ratelimited(hdev->dev, "Illegal read from HBW\n");
4469 static void goya_print_mmu_error_info(struct hl_device *hdev)
4471 struct goya_device *goya = hdev->asic_specific;
4484 dev_err_ratelimited(hdev->dev, "MMU page fault on va 0x%llx\n",
4491 static void goya_print_irq_info(struct hl_device *hdev, u16 event_type,
4497 dev_err_ratelimited(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n",
4501 goya_print_razwi_info(hdev);
4502 goya_print_mmu_error_info(hdev);
4506 static int goya_unmask_irq_arr(struct hl_device *hdev, u32 *irq_arr,
4524 dev_err(hdev->dev, "too many elements in IRQ array\n");
4546 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt,
4550 dev_err(hdev->dev, "failed to unmask IRQ array\n");
4557 static int goya_soft_reset_late_init(struct hl_device *hdev)
4563 return goya_unmask_irq_arr(hdev, goya_all_events,
4567 static int goya_unmask_irq(struct hl_device *hdev, u16 event_type)
4579 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
4583 dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type);
4588 static void goya_print_clk_change_info(struct hl_device *hdev, u16 event_type)
4592 hdev->clk_throttling_reason |= HL_CLK_THROTTLE_POWER;
4593 dev_info_ratelimited(hdev->dev,
4597 hdev->clk_throttling_reason &= ~HL_CLK_THROTTLE_POWER;
4598 dev_info_ratelimited(hdev->dev,
4602 hdev->clk_throttling_reason |= HL_CLK_THROTTLE_THERMAL;
4603 dev_info_ratelimited(hdev->dev,
4607 hdev->clk_throttling_reason &= ~HL_CLK_THROTTLE_THERMAL;
4608 dev_info_ratelimited(hdev->dev,
4613 dev_err(hdev->dev, "Received invalid clock change event %d\n",
4619 void goya_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry)
4624 struct goya_device *goya = hdev->asic_specific;
4627 dev_err(hdev->dev, "Event type %u exceeds maximum of %u",
4659 goya_print_irq_info(hdev, event_type, false);
4660 if (hdev->hard_reset_on_fw_events)
4661 hl_device_reset(hdev, true, false);
4691 goya_print_irq_info(hdev, event_type, true);
4692 goya_unmask_irq(hdev, event_type);
4705 goya_print_irq_info(hdev, event_type, false);
4706 goya_unmask_irq(hdev, event_type);
4713 goya_print_clk_change_info(hdev, event_type);
4714 goya_unmask_irq(hdev, event_type);
4718 dev_err(hdev->dev, "Received invalid H/W interrupt %d\n",
4724 void *goya_get_events_stat(struct hl_device *hdev, bool aggregate, u32 *size)
4726 struct goya_device *goya = hdev->asic_specific;
4737 static int goya_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size,
4749 cb = hl_cb_kernel_create(hdev, cb_size, false);
4779 job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true);
4781 dev_err(hdev->dev, "Failed to allocate a new job\n");
4794 hl_debugfs_add_job(hdev, job);
4796 rc = goya_send_job_on_qman0(hdev, job);
4798 hl_debugfs_remove_job(hdev, job);
4804 hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT);
4809 int goya_context_switch(struct hl_device *hdev, u32 asid)
4811 struct asic_fixed_properties *prop = &hdev->asic_prop;
4813 u32 size = hdev->pldm ? 0x10000 : prop->sram_size;
4819 rc = goya_memset_device_memory(hdev, addr, size, val, false);
4821 dev_err(hdev->dev, "Failed to clear SRAM in context switch\n");
4838 goya_mmu_prepare(hdev, asid);
4840 goya_clear_sm_regs(hdev);
4845 static int goya_mmu_clear_pgt_range(struct hl_device *hdev)
4847 struct asic_fixed_properties *prop = &hdev->asic_prop;
4848 struct goya_device *goya = hdev->asic_specific;
4856 return goya_memset_device_memory(hdev, addr, size, 0, true);
4859 static int goya_mmu_set_dram_default_page(struct hl_device *hdev)
4861 struct goya_device *goya = hdev->asic_specific;
4862 u64 addr = hdev->asic_prop.mmu_dram_default_page_addr;
4869 return goya_memset_device_memory(hdev, addr, size, val, true);
4872 static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev)
4874 struct asic_fixed_properties *prop = &hdev->asic_prop;
4875 struct goya_device *goya = hdev->asic_specific;
4883 rc = hl_mmu_map(hdev->kernel_ctx, prop->dram_base_address + off,
4887 dev_err(hdev->dev, "Map failed for address 0x%llx\n",
4893 if (!(hdev->cpu_accessible_dma_address & (PAGE_SIZE_2MB - 1))) {
4894 rc = hl_mmu_map(hdev->kernel_ctx, VA_CPU_ACCESSIBLE_MEM_ADDR,
4895 hdev->cpu_accessible_dma_address, PAGE_SIZE_2MB, true);
4898 dev_err(hdev->dev,
4905 rc = hl_mmu_map(hdev->kernel_ctx,
4907 hdev->cpu_accessible_dma_address + cpu_off,
4910 dev_err(hdev->dev,
4918 goya_mmu_prepare_reg(hdev, mmCPU_IF_ARUSER_OVR, HL_KERNEL_ASID_ID);
4919 goya_mmu_prepare_reg(hdev, mmCPU_IF_AWUSER_OVR, HL_KERNEL_ASID_ID);
4932 if (hl_mmu_unmap(hdev->kernel_ctx,
4935 dev_warn_ratelimited(hdev->dev,
4940 if (hl_mmu_unmap(hdev->kernel_ctx,
4943 dev_warn_ratelimited(hdev->dev,
4950 void goya_mmu_remove_device_cpu_mappings(struct hl_device *hdev)
4952 struct asic_fixed_properties *prop = &hdev->asic_prop;
4953 struct goya_device *goya = hdev->asic_specific;
4965 if (!(hdev->cpu_accessible_dma_address & (PAGE_SIZE_2MB - 1))) {
4966 if (hl_mmu_unmap(hdev->kernel_ctx, VA_CPU_ACCESSIBLE_MEM_ADDR,
4968 dev_warn(hdev->dev,
4972 if (hl_mmu_unmap(hdev->kernel_ctx,
4976 dev_warn_ratelimited(hdev->dev,
4982 if (hl_mmu_unmap(hdev->kernel_ctx,
4985 dev_warn_ratelimited(hdev->dev,
4992 static void goya_mmu_prepare(struct hl_device *hdev, u32 asid)
4994 struct goya_device *goya = hdev->asic_specific;
5007 goya_mmu_prepare_reg(hdev, goya_mmu_regs[i], asid);
5010 static int goya_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard,
5013 struct goya_device *goya = hdev->asic_specific;
5018 hdev->hard_reset_pending)
5025 if (hdev->pldm)
5030 mutex_lock(&hdev->mmu_cache_lock);
5036 hdev,
5043 mutex_unlock(&hdev->mmu_cache_lock);
5046 dev_err_ratelimited(hdev->dev,
5048 hl_device_reset(hdev, true, false);
5054 static int goya_mmu_invalidate_cache_range(struct hl_device *hdev,
5057 struct goya_device *goya = hdev->asic_specific;
5062 hdev->hard_reset_pending)
5069 if (hdev->pldm)
5074 mutex_lock(&hdev->mmu_cache_lock);
5091 hdev,
5098 mutex_unlock(&hdev->mmu_cache_lock);
5101 dev_err_ratelimited(hdev->dev,
5103 hl_device_reset(hdev, true, false);
5109 int goya_send_heartbeat(struct hl_device *hdev)
5111 struct goya_device *goya = hdev->asic_specific;
5116 return hl_fw_send_heartbeat(hdev);
5119 int goya_cpucp_info_get(struct hl_device *hdev)
5121 struct goya_device *goya = hdev->asic_specific;
5122 struct asic_fixed_properties *prop = &hdev->asic_prop;
5129 rc = hl_fw_cpucp_info_get(hdev);
5137 dev_err(hdev->dev,
5154 static void goya_set_clock_gating(struct hl_device *hdev)
5159 static void goya_disable_clock_gating(struct hl_device *hdev)
5164 static bool goya_is_device_idle(struct hl_device *hdev, u64 *mask,
5244 static void goya_hw_queues_lock(struct hl_device *hdev)
5247 struct goya_device *goya = hdev->asic_specific;
5252 static void goya_hw_queues_unlock(struct hl_device *hdev)
5255 struct goya_device *goya = hdev->asic_specific;
5260 static u32 goya_get_pci_id(struct hl_device *hdev)
5262 return hdev->pdev->device;
5265 static int goya_get_eeprom_data(struct hl_device *hdev, void *data,
5268 struct goya_device *goya = hdev->asic_specific;
5273 return hl_fw_get_eeprom_data(hdev, data, max_size);
5276 static enum hl_device_hw_state goya_get_hw_state(struct hl_device *hdev)
5286 u32 goya_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx)
5291 static u32 goya_get_signal_cb_size(struct hl_device *hdev)
5296 static u32 goya_get_wait_cb_size(struct hl_device *hdev)
5301 static void goya_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id)
5306 static void goya_gen_wait_cb(struct hl_device *hdev, void *data, u16 sob_id,
5312 static void goya_reset_sob(struct hl_device *hdev, void *data)
5317 static void goya_set_dma_mask_from_fw(struct hl_device *hdev)
5321 dev_dbg(hdev->dev, "Working in 64-bit DMA mode\n");
5322 hdev->power9_64bit_dma_enable = 1;
5323 hdev->dma_mask = 64;
5325 dev_dbg(hdev->dev, "Working in 48-bit DMA mode\n");
5326 hdev->power9_64bit_dma_enable = 0;
5327 hdev->dma_mask = 48;
5331 u64 goya_get_device_time(struct hl_device *hdev)
5416 * @*hdev: pointer to hl_device structure
5419 void goya_set_asic_funcs(struct hl_device *hdev)
5421 hdev->asic_funcs = &goya_funcs;