Lines Matching defs:hdev
55 struct hl_device *hdev = ctx->hdev;
56 struct hl_vm *vm = &hdev->vm;
64 page_size = hdev->asic_prop.dram_page_size;
70 dev_err(hdev->dev, "Cannot allocate 0 bytes\n");
79 dev_err(hdev->dev,
115 dev_err(hdev->dev,
131 dev_err(hdev->dev, "Failed to get handle for page\n");
142 atomic64_add(phys_pg_pack->total_size, &hdev->dram_used_mem);
167 * @hdev: habanalabs device structure
177 static int dma_map_host_va(struct hl_device *hdev, u64 addr, u64 size,
189 rc = hl_pin_host_memory(hdev, addr, size, userptr);
191 dev_err(hdev->dev, "Failed to pin host memory\n");
195 rc = hdev->asic_funcs->asic_dma_map_sg(hdev, userptr->sgt->sgl,
198 dev_err(hdev->dev, "failed to map sgt with DMA region\n");
211 hl_unpin_host_memory(hdev, userptr);
221 * @hdev: habanalabs device structure
228 static void dma_unmap_host_va(struct hl_device *hdev,
231 hl_unpin_host_memory(hdev, userptr);
259 * @hdev: habanalabs device structure
267 static void free_phys_pg_pack(struct hl_device *hdev,
270 struct hl_vm *vm = &hdev->vm;
307 struct hl_device *hdev = ctx->hdev;
308 struct hl_vm *vm = &hdev->vm;
315 dev_err(hdev->dev, "handle %u is mapped, cannot free\n",
330 atomic64_sub(phys_pg_pack->total_size, &hdev->dram_used_mem);
332 free_phys_pg_pack(hdev, phys_pg_pack);
335 dev_err(hdev->dev,
347 * @hdev : habanalabs device structure
355 static void clear_va_list_locked(struct hl_device *hdev,
369 * @hdev : habanalabs device structure
377 static void print_va_list_locked(struct hl_device *hdev,
383 dev_dbg(hdev->dev, "print va list:\n");
386 dev_dbg(hdev->dev,
395 * @hdev : pointer to the habanalabs device structure
405 static void merge_va_blocks_locked(struct hl_device *hdev,
431 * @hdev : pointer to the habanalabs device structure
442 static int add_va_block_locked(struct hl_device *hdev,
448 print_va_list_locked(hdev, va_list);
454 dev_err(hdev->dev,
477 merge_va_blocks_locked(hdev, va_list, va_block);
479 print_va_list_locked(hdev, va_list);
487 * @hdev : pointer to the habanalabs device structure
495 static inline int add_va_block(struct hl_device *hdev,
501 rc = add_va_block_locked(hdev, &va_range->list, start, end);
509 * @hdev: pointer to the habanalabs device structure.
521 static u64 get_va_block(struct hl_device *hdev, struct hl_va_range *va_range,
537 print_va_list_locked(hdev, &va_range->list);
569 dev_err(hdev->dev, "no available va block for size %llu\n",
593 add_va_block_locked(hdev, &va_range->list, prev_start,
596 print_va_list_locked(hdev, &va_range->list);
642 huge_page_size = ctx->hdev->asic_prop.pmmu_huge.page_size;
737 struct hl_device *hdev = ctx->hdev;
748 dev_err(hdev->dev,
766 dev_warn_ratelimited(hdev->dev,
786 struct hl_device *hdev = ctx->hdev;
796 dev_warn_ratelimited(hdev->dev,
803 if (hdev->pldm)
811 struct hl_device *hdev = ctx->hdev;
812 struct hl_vm *vm = &hdev->vm;
821 dev_err(hdev->dev, "no match for handle %u\n", handle);
849 struct hl_device *hdev = ctx->hdev;
850 struct hl_vm *vm = &hdev->vm;
867 u32 page_size = hdev->asic_prop.pmmu.page_size,
868 huge_page_size = hdev->asic_prop.pmmu_huge.page_size;
870 rc = dma_map_host_va(hdev, addr, size, &userptr);
872 dev_err(hdev->dev, "failed to get userptr from va\n");
879 dev_err(hdev->dev,
916 dev_err(hdev->dev,
932 va_block_align = hdev->asic_prop.dmmu.page_size;
941 dev_err(hdev->dev,
954 ret_vaddr = get_va_block(hdev, va_range, phys_pg_pack->total_size,
957 dev_err(hdev->dev, "no available va block for handle %u\n",
968 dev_err(hdev->dev, "mapping page pack failed for handle %u\n",
973 rc = hdev->asic_funcs->mmu_invalidate_cache(hdev, false, *vm_type);
978 dev_err(hdev->dev,
996 free_phys_pg_pack(hdev, phys_pg_pack);
1001 if (add_va_block(hdev, va_range, ret_vaddr,
1003 dev_warn(hdev->dev,
1013 free_phys_pg_pack(hdev, phys_pg_pack);
1016 dma_unmap_host_va(hdev, userptr);
1034 struct hl_device *hdev = ctx->hdev;
1051 dev_err(hdev->dev,
1068 dev_err(hdev->dev,
1075 hdev->asic_prop.pmmu.page_size)
1084 dev_warn(hdev->dev,
1092 dev_err(hdev->dev, "vaddr 0x%llx is not mapped\n", vaddr);
1109 rc = hdev->asic_funcs->mmu_invalidate_cache(hdev, true,
1124 dev_err(hdev->dev,
1128 tmp_rc = add_va_block(hdev, va_range, vaddr,
1131 dev_warn(hdev->dev,
1143 free_phys_pg_pack(hdev, phys_pg_pack);
1144 dma_unmap_host_va(hdev, userptr);
1151 free_phys_pg_pack(hdev, phys_pg_pack);
1162 struct hl_device *hdev = hpriv->hdev;
1171 dev_err(hdev->dev,
1209 dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n");
1221 struct hl_device *hdev = hpriv->hdev;
1227 if (hl_device_disabled_or_in_reset(hdev)) {
1228 dev_warn_ratelimited(hdev->dev,
1230 atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");
1234 if (!hdev->mmu_enable)
1239 if (!hdev->dram_supports_virtual_memory) {
1240 dev_err(hdev->dev, "DRAM alloc is not supported\n");
1246 dev_err(hdev->dev,
1274 dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n");
1283 static int get_user_memory(struct hl_device *hdev, u64 addr, u64 size,
1290 dev_err(hdev->dev, "user pointer is invalid - 0x%llx\n", addr);
1296 dev_err(hdev->dev, "Failed to create frame vector\n");
1304 dev_err(hdev->dev,
1313 dev_err(hdev->dev,
1323 dev_err(hdev->dev, "failed to create SG table from pages\n");
1338 * @hdev: pointer to the habanalabs device structure
1347 int hl_pin_host_memory(struct hl_device *hdev, u64 addr, u64 size,
1355 dev_err(hdev->dev, "size to pin is invalid - %llu\n", size);
1365 dev_err(hdev->dev,
1389 rc = get_user_memory(hdev, addr, size, npages, start, offset,
1392 dev_err(hdev->dev,
1398 hl_debugfs_add_userptr(hdev, userptr);
1409 * @hdev: pointer to the habanalabs device structure
1416 void hl_unpin_host_memory(struct hl_device *hdev, struct hl_userptr *userptr)
1420 hl_debugfs_remove_userptr(hdev, userptr);
1423 hdev->asic_funcs->hl_dma_unmap_sg(hdev, userptr->sgt->sgl,
1446 * @hdev : pointer to the habanalabs device structure
1453 void hl_userptr_delete_list(struct hl_device *hdev,
1459 hl_unpin_host_memory(hdev, userptr);
1469 * @hdev : pointer to the habanalabs device structure
1477 bool hl_userptr_is_pinned(struct hl_device *hdev, u64 addr,
1491 * @hdev: pointer to the habanalabs device structure
1500 static int va_range_init(struct hl_device *hdev, struct hl_va_range *va_range,
1518 dev_err(hdev->dev, "too small vm range for va list\n");
1522 rc = add_va_block(hdev, va_range, start, end);
1525 dev_err(hdev->dev, "Failed to init host va list\n");
1537 * @hdev: pointer to the habanalabs structure
1543 static void va_range_fini(struct hl_device *hdev,
1547 clear_va_list_locked(hdev, &va_range->list);
1579 struct hl_device *hdev = ctx->hdev;
1601 dev_err(hdev->dev, "failed to init context %d\n", ctx->asid);
1610 rc = va_range_init(hdev, ctx->host_va_range, host_range_start,
1613 dev_err(hdev->dev, "failed to init host vm range\n");
1617 if (hdev->pmmu_huge_range) {
1620 rc = va_range_init(hdev, ctx->host_huge_va_range,
1624 dev_err(hdev->dev,
1635 rc = va_range_init(hdev, ctx->dram_va_range, dram_range_start,
1638 dev_err(hdev->dev, "failed to init dram vm range\n");
1642 hl_debugfs_add_ctx_mem_hash(hdev, ctx);
1649 if (hdev->pmmu_huge_range) {
1651 clear_va_list_locked(hdev, &ctx->host_huge_va_range->list);
1655 if (hdev->pmmu_huge_range)
1658 clear_va_list_locked(hdev, &ctx->host_va_range->list);
1676 struct asic_fixed_properties *prop = &ctx->hdev->asic_prop;
1689 if (ctx->hdev->mmu_enable) {
1734 struct hl_device *hdev = ctx->hdev;
1735 struct hl_vm *vm = &hdev->vm;
1741 hl_debugfs_remove_ctx_mem_hash(hdev, ctx);
1747 if (!hdev->hard_reset_pending && !hash_empty(ctx->mem_hash))
1748 dev_notice(hdev->dev,
1752 dev_dbg(hdev->dev,
1759 hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_USERPTR);
1760 hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_PHYS_PACK);
1765 dev_dbg(hdev->dev,
1769 &hdev->dram_used_mem);
1770 free_phys_pg_pack(hdev, phys_pg_list);
1775 va_range_fini(hdev, ctx->dram_va_range);
1776 if (hdev->pmmu_huge_range)
1777 va_range_fini(hdev, ctx->host_huge_va_range);
1778 va_range_fini(hdev, ctx->host_va_range);
1787 * @hdev : pointer to the habanalabs device structure
1794 int hl_vm_init(struct hl_device *hdev)
1796 struct asic_fixed_properties *prop = &hdev->asic_prop;
1797 struct hl_vm *vm = &hdev->vm;
1802 dev_err(hdev->dev, "Failed to create dram page pool\n");
1813 dev_err(hdev->dev,
1821 atomic64_set(&hdev->dram_used_mem, 0);
1836 * @hdev : pointer to the habanalabs device structure
1843 void hl_vm_fini(struct hl_device *hdev)
1845 struct hl_vm *vm = &hdev->vm;
1855 dev_warn(hdev->dev, "dram_pg_pool was not destroyed on %s\n",