1// SPDX-License-Identifier: GPL-2.0 2 3/* 4 * Copyright 2016-2019 HabanaLabs, Ltd. 5 * All Rights Reserved. 6 */ 7 8#include <uapi/drm/habanalabs_accel.h> 9#include "habanalabs.h" 10 11#include <linux/mm.h> 12#include <linux/slab.h> 13#include <linux/uaccess.h> 14 15#define CB_VA_POOL_SIZE (4UL * SZ_1G) 16 17static int cb_map_mem(struct hl_ctx *ctx, struct hl_cb *cb) 18{ 19 struct hl_device *hdev = ctx->hdev; 20 struct asic_fixed_properties *prop = &hdev->asic_prop; 21 u32 page_size = prop->pmmu.page_size; 22 int rc; 23 24 if (!hdev->supports_cb_mapping) { 25 dev_err_ratelimited(hdev->dev, 26 "Mapping a CB to the device's MMU is not supported\n"); 27 return -EINVAL; 28 } 29 30 if (cb->is_mmu_mapped) 31 return 0; 32 33 cb->roundup_size = roundup(cb->size, page_size); 34 35 cb->virtual_addr = (u64) gen_pool_alloc(ctx->cb_va_pool, cb->roundup_size); 36 if (!cb->virtual_addr) { 37 dev_err(hdev->dev, "Failed to allocate device virtual address for CB\n"); 38 return -ENOMEM; 39 } 40 41 mutex_lock(&hdev->mmu_lock); 42 43 rc = hl_mmu_map_contiguous(ctx, cb->virtual_addr, cb->bus_address, cb->roundup_size); 44 if (rc) { 45 dev_err(hdev->dev, "Failed to map VA %#llx to CB\n", cb->virtual_addr); 46 goto err_va_pool_free; 47 } 48 49 rc = hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR | MMU_OP_SKIP_LOW_CACHE_INV); 50 if (rc) 51 goto err_mmu_unmap; 52 53 mutex_unlock(&hdev->mmu_lock); 54 55 cb->is_mmu_mapped = true; 56 57 return 0; 58 59err_mmu_unmap: 60 hl_mmu_unmap_contiguous(ctx, cb->virtual_addr, cb->roundup_size); 61err_va_pool_free: 62 mutex_unlock(&hdev->mmu_lock); 63 gen_pool_free(ctx->cb_va_pool, cb->virtual_addr, cb->roundup_size); 64 65 return rc; 66} 67 68static void cb_unmap_mem(struct hl_ctx *ctx, struct hl_cb *cb) 69{ 70 struct hl_device *hdev = ctx->hdev; 71 72 mutex_lock(&hdev->mmu_lock); 73 hl_mmu_unmap_contiguous(ctx, cb->virtual_addr, cb->roundup_size); 74 hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR); 75 mutex_unlock(&hdev->mmu_lock); 76 77 gen_pool_free(ctx->cb_va_pool, cb->virtual_addr, cb->roundup_size); 78} 79 80static void cb_fini(struct hl_device *hdev, struct hl_cb *cb) 81{ 82 if (cb->is_internal) 83 gen_pool_free(hdev->internal_cb_pool, 84 (uintptr_t)cb->kernel_address, cb->size); 85 else 86 hl_asic_dma_free_coherent(hdev, cb->size, cb->kernel_address, cb->bus_address); 87 88 kfree(cb); 89} 90 91static void cb_do_release(struct hl_device *hdev, struct hl_cb *cb) 92{ 93 if (cb->is_pool) { 94 atomic_set(&cb->is_handle_destroyed, 0); 95 spin_lock(&hdev->cb_pool_lock); 96 list_add(&cb->pool_list, &hdev->cb_pool); 97 spin_unlock(&hdev->cb_pool_lock); 98 } else { 99 cb_fini(hdev, cb); 100 } 101} 102 103static struct hl_cb *hl_cb_alloc(struct hl_device *hdev, u32 cb_size, 104 int ctx_id, bool internal_cb) 105{ 106 struct hl_cb *cb = NULL; 107 u32 cb_offset; 108 void *p; 109 110 /* 111 * We use of GFP_ATOMIC here because this function can be called from 112 * the latency-sensitive code path for command submission. Due to H/W 113 * limitations in some of the ASICs, the kernel must copy the user CB 114 * that is designated for an external queue and actually enqueue 115 * the kernel's copy. Hence, we must never sleep in this code section 116 * and must use GFP_ATOMIC for all memory allocations. 117 */ 118 if (ctx_id == HL_KERNEL_ASID_ID && !hdev->disabled) 119 cb = kzalloc(sizeof(*cb), GFP_ATOMIC); 120 121 if (!cb) 122 cb = kzalloc(sizeof(*cb), GFP_KERNEL); 123 124 if (!cb) 125 return NULL; 126 127 if (internal_cb) { 128 p = (void *) gen_pool_alloc(hdev->internal_cb_pool, cb_size); 129 if (!p) { 130 kfree(cb); 131 return NULL; 132 } 133 134 cb_offset = p - hdev->internal_cb_pool_virt_addr; 135 cb->is_internal = true; 136 cb->bus_address = hdev->internal_cb_va_base + cb_offset; 137 } else if (ctx_id == HL_KERNEL_ASID_ID) { 138 p = hl_asic_dma_alloc_coherent(hdev, cb_size, &cb->bus_address, GFP_ATOMIC); 139 if (!p) 140 p = hl_asic_dma_alloc_coherent(hdev, cb_size, &cb->bus_address, GFP_KERNEL); 141 } else { 142 p = hl_asic_dma_alloc_coherent(hdev, cb_size, &cb->bus_address, 143 GFP_USER | __GFP_ZERO); 144 } 145 146 if (!p) { 147 dev_err(hdev->dev, 148 "failed to allocate %d of dma memory for CB\n", 149 cb_size); 150 kfree(cb); 151 return NULL; 152 } 153 154 cb->kernel_address = p; 155 cb->size = cb_size; 156 157 return cb; 158} 159 160struct hl_cb_mmap_mem_alloc_args { 161 struct hl_device *hdev; 162 struct hl_ctx *ctx; 163 u32 cb_size; 164 bool internal_cb; 165 bool map_cb; 166}; 167 168static void hl_cb_mmap_mem_release(struct hl_mmap_mem_buf *buf) 169{ 170 struct hl_cb *cb = buf->private; 171 172 hl_debugfs_remove_cb(cb); 173 174 if (cb->is_mmu_mapped) 175 cb_unmap_mem(cb->ctx, cb); 176 177 hl_ctx_put(cb->ctx); 178 179 cb_do_release(cb->hdev, cb); 180} 181 182static int hl_cb_mmap_mem_alloc(struct hl_mmap_mem_buf *buf, gfp_t gfp, void *args) 183{ 184 struct hl_cb_mmap_mem_alloc_args *cb_args = args; 185 struct hl_cb *cb; 186 int rc, ctx_id = cb_args->ctx->asid; 187 bool alloc_new_cb = true; 188 189 if (!cb_args->internal_cb) { 190 /* Minimum allocation must be PAGE SIZE */ 191 if (cb_args->cb_size < PAGE_SIZE) 192 cb_args->cb_size = PAGE_SIZE; 193 194 if (ctx_id == HL_KERNEL_ASID_ID && 195 cb_args->cb_size <= cb_args->hdev->asic_prop.cb_pool_cb_size) { 196 197 spin_lock(&cb_args->hdev->cb_pool_lock); 198 if (!list_empty(&cb_args->hdev->cb_pool)) { 199 cb = list_first_entry(&cb_args->hdev->cb_pool, 200 typeof(*cb), pool_list); 201 list_del(&cb->pool_list); 202 spin_unlock(&cb_args->hdev->cb_pool_lock); 203 alloc_new_cb = false; 204 } else { 205 spin_unlock(&cb_args->hdev->cb_pool_lock); 206 dev_dbg(cb_args->hdev->dev, "CB pool is empty\n"); 207 } 208 } 209 } 210 211 if (alloc_new_cb) { 212 cb = hl_cb_alloc(cb_args->hdev, cb_args->cb_size, ctx_id, cb_args->internal_cb); 213 if (!cb) 214 return -ENOMEM; 215 } 216 217 cb->hdev = cb_args->hdev; 218 cb->ctx = cb_args->ctx; 219 cb->buf = buf; 220 cb->buf->mappable_size = cb->size; 221 cb->buf->private = cb; 222 223 hl_ctx_get(cb->ctx); 224 225 if (cb_args->map_cb) { 226 if (ctx_id == HL_KERNEL_ASID_ID) { 227 dev_err(cb_args->hdev->dev, 228 "CB mapping is not supported for kernel context\n"); 229 rc = -EINVAL; 230 goto release_cb; 231 } 232 233 rc = cb_map_mem(cb_args->ctx, cb); 234 if (rc) 235 goto release_cb; 236 } 237 238 hl_debugfs_add_cb(cb); 239 240 return 0; 241 242release_cb: 243 hl_ctx_put(cb->ctx); 244 cb_do_release(cb_args->hdev, cb); 245 246 return rc; 247} 248 249static int hl_cb_mmap(struct hl_mmap_mem_buf *buf, 250 struct vm_area_struct *vma, void *args) 251{ 252 struct hl_cb *cb = buf->private; 253 254 return cb->hdev->asic_funcs->mmap(cb->hdev, vma, cb->kernel_address, 255 cb->bus_address, cb->size); 256} 257 258static struct hl_mmap_mem_buf_behavior cb_behavior = { 259 .topic = "CB", 260 .mem_id = HL_MMAP_TYPE_CB, 261 .alloc = hl_cb_mmap_mem_alloc, 262 .release = hl_cb_mmap_mem_release, 263 .mmap = hl_cb_mmap, 264}; 265 266int hl_cb_create(struct hl_device *hdev, struct hl_mem_mgr *mmg, 267 struct hl_ctx *ctx, u32 cb_size, bool internal_cb, 268 bool map_cb, u64 *handle) 269{ 270 struct hl_cb_mmap_mem_alloc_args args = { 271 .hdev = hdev, 272 .ctx = ctx, 273 .cb_size = cb_size, 274 .internal_cb = internal_cb, 275 .map_cb = map_cb, 276 }; 277 struct hl_mmap_mem_buf *buf; 278 int ctx_id = ctx->asid; 279 280 if ((hdev->disabled) || (hdev->reset_info.in_reset && (ctx_id != HL_KERNEL_ASID_ID))) { 281 dev_warn_ratelimited(hdev->dev, 282 "Device is disabled or in reset. Can't create new CBs\n"); 283 return -EBUSY; 284 } 285 286 if (cb_size > SZ_2M) { 287 dev_err(hdev->dev, "CB size %d must be less than %d\n", 288 cb_size, SZ_2M); 289 return -EINVAL; 290 } 291 292 buf = hl_mmap_mem_buf_alloc( 293 mmg, &cb_behavior, 294 ctx_id == HL_KERNEL_ASID_ID ? GFP_ATOMIC : GFP_KERNEL, &args); 295 if (!buf) 296 return -ENOMEM; 297 298 *handle = buf->handle; 299 300 return 0; 301} 302 303int hl_cb_destroy(struct hl_mem_mgr *mmg, u64 cb_handle) 304{ 305 struct hl_cb *cb; 306 int rc; 307 308 cb = hl_cb_get(mmg, cb_handle); 309 if (!cb) { 310 dev_dbg(mmg->dev, "CB destroy failed, no CB was found for handle %#llx\n", 311 cb_handle); 312 return -EINVAL; 313 } 314 315 /* Make sure that CB handle isn't destroyed more than once */ 316 rc = atomic_cmpxchg(&cb->is_handle_destroyed, 0, 1); 317 hl_cb_put(cb); 318 if (rc) { 319 dev_dbg(mmg->dev, "CB destroy failed, handle %#llx was already destroyed\n", 320 cb_handle); 321 return -EINVAL; 322 } 323 324 rc = hl_mmap_mem_buf_put_handle(mmg, cb_handle); 325 if (rc < 0) 326 return rc; /* Invalid handle */ 327 328 if (rc == 0) 329 dev_dbg(mmg->dev, "CB 0x%llx is destroyed while still in use\n", cb_handle); 330 331 return 0; 332} 333 334static int hl_cb_info(struct hl_mem_mgr *mmg, 335 u64 handle, u32 flags, u32 *usage_cnt, u64 *device_va) 336{ 337 struct hl_cb *cb; 338 int rc = 0; 339 340 cb = hl_cb_get(mmg, handle); 341 if (!cb) { 342 dev_err(mmg->dev, 343 "CB info failed, no match to handle 0x%llx\n", handle); 344 return -EINVAL; 345 } 346 347 if (flags & HL_CB_FLAGS_GET_DEVICE_VA) { 348 if (cb->is_mmu_mapped) { 349 *device_va = cb->virtual_addr; 350 } else { 351 dev_err(mmg->dev, "CB is not mapped to the device's MMU\n"); 352 rc = -EINVAL; 353 goto out; 354 } 355 } else { 356 *usage_cnt = atomic_read(&cb->cs_cnt); 357 } 358 359out: 360 hl_cb_put(cb); 361 return rc; 362} 363 364int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data) 365{ 366 union hl_cb_args *args = data; 367 struct hl_device *hdev = hpriv->hdev; 368 u64 handle = 0, device_va = 0; 369 enum hl_device_status status; 370 u32 usage_cnt = 0; 371 int rc; 372 373 if (!hl_device_operational(hdev, &status)) { 374 dev_dbg_ratelimited(hdev->dev, 375 "Device is %s. Can't execute CB IOCTL\n", 376 hdev->status[status]); 377 return -EBUSY; 378 } 379 380 switch (args->in.op) { 381 case HL_CB_OP_CREATE: 382 if (args->in.cb_size > HL_MAX_CB_SIZE) { 383 dev_err(hdev->dev, 384 "User requested CB size %d must be less than %d\n", 385 args->in.cb_size, HL_MAX_CB_SIZE); 386 rc = -EINVAL; 387 } else { 388 rc = hl_cb_create(hdev, &hpriv->mem_mgr, hpriv->ctx, 389 args->in.cb_size, false, 390 !!(args->in.flags & HL_CB_FLAGS_MAP), 391 &handle); 392 } 393 394 memset(args, 0, sizeof(*args)); 395 args->out.cb_handle = handle; 396 break; 397 398 case HL_CB_OP_DESTROY: 399 rc = hl_cb_destroy(&hpriv->mem_mgr, 400 args->in.cb_handle); 401 break; 402 403 case HL_CB_OP_INFO: 404 rc = hl_cb_info(&hpriv->mem_mgr, args->in.cb_handle, 405 args->in.flags, 406 &usage_cnt, 407 &device_va); 408 if (rc) 409 break; 410 411 memset(&args->out, 0, sizeof(args->out)); 412 413 if (args->in.flags & HL_CB_FLAGS_GET_DEVICE_VA) 414 args->out.device_va = device_va; 415 else 416 args->out.usage_cnt = usage_cnt; 417 break; 418 419 default: 420 rc = -EINVAL; 421 break; 422 } 423 424 return rc; 425} 426 427struct hl_cb *hl_cb_get(struct hl_mem_mgr *mmg, u64 handle) 428{ 429 struct hl_mmap_mem_buf *buf; 430 431 buf = hl_mmap_mem_buf_get(mmg, handle); 432 if (!buf) 433 return NULL; 434 return buf->private; 435 436} 437 438void hl_cb_put(struct hl_cb *cb) 439{ 440 hl_mmap_mem_buf_put(cb->buf); 441} 442 443struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size, 444 bool internal_cb) 445{ 446 u64 cb_handle; 447 struct hl_cb *cb; 448 int rc; 449 450 rc = hl_cb_create(hdev, &hdev->kernel_mem_mgr, hdev->kernel_ctx, cb_size, 451 internal_cb, false, &cb_handle); 452 if (rc) { 453 dev_err(hdev->dev, 454 "Failed to allocate CB for the kernel driver %d\n", rc); 455 return NULL; 456 } 457 458 cb = hl_cb_get(&hdev->kernel_mem_mgr, cb_handle); 459 /* hl_cb_get should never fail here */ 460 if (!cb) { 461 dev_crit(hdev->dev, "Kernel CB handle invalid 0x%x\n", 462 (u32) cb_handle); 463 goto destroy_cb; 464 } 465 466 return cb; 467 468destroy_cb: 469 hl_cb_destroy(&hdev->kernel_mem_mgr, cb_handle); 470 471 return NULL; 472} 473 474int hl_cb_pool_init(struct hl_device *hdev) 475{ 476 struct hl_cb *cb; 477 int i; 478 479 INIT_LIST_HEAD(&hdev->cb_pool); 480 spin_lock_init(&hdev->cb_pool_lock); 481 482 for (i = 0 ; i < hdev->asic_prop.cb_pool_cb_cnt ; i++) { 483 cb = hl_cb_alloc(hdev, hdev->asic_prop.cb_pool_cb_size, 484 HL_KERNEL_ASID_ID, false); 485 if (cb) { 486 cb->is_pool = true; 487 list_add(&cb->pool_list, &hdev->cb_pool); 488 } else { 489 hl_cb_pool_fini(hdev); 490 return -ENOMEM; 491 } 492 } 493 494 return 0; 495} 496 497int hl_cb_pool_fini(struct hl_device *hdev) 498{ 499 struct hl_cb *cb, *tmp; 500 501 list_for_each_entry_safe(cb, tmp, &hdev->cb_pool, pool_list) { 502 list_del(&cb->pool_list); 503 cb_fini(hdev, cb); 504 } 505 506 return 0; 507} 508 509int hl_cb_va_pool_init(struct hl_ctx *ctx) 510{ 511 struct hl_device *hdev = ctx->hdev; 512 struct asic_fixed_properties *prop = &hdev->asic_prop; 513 int rc; 514 515 if (!hdev->supports_cb_mapping) 516 return 0; 517 518 ctx->cb_va_pool = gen_pool_create(__ffs(prop->pmmu.page_size), -1); 519 if (!ctx->cb_va_pool) { 520 dev_err(hdev->dev, 521 "Failed to create VA gen pool for CB mapping\n"); 522 return -ENOMEM; 523 } 524 525 ctx->cb_va_pool_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST, 526 CB_VA_POOL_SIZE, HL_MMU_VA_ALIGNMENT_NOT_NEEDED); 527 if (!ctx->cb_va_pool_base) { 528 rc = -ENOMEM; 529 goto err_pool_destroy; 530 } 531 rc = gen_pool_add(ctx->cb_va_pool, ctx->cb_va_pool_base, CB_VA_POOL_SIZE, -1); 532 if (rc) { 533 dev_err(hdev->dev, 534 "Failed to add memory to VA gen pool for CB mapping\n"); 535 goto err_unreserve_va_block; 536 } 537 538 return 0; 539 540err_unreserve_va_block: 541 hl_unreserve_va_block(hdev, ctx, ctx->cb_va_pool_base, CB_VA_POOL_SIZE); 542err_pool_destroy: 543 gen_pool_destroy(ctx->cb_va_pool); 544 545 return rc; 546} 547 548void hl_cb_va_pool_fini(struct hl_ctx *ctx) 549{ 550 struct hl_device *hdev = ctx->hdev; 551 552 if (!hdev->supports_cb_mapping) 553 return; 554 555 gen_pool_destroy(ctx->cb_va_pool); 556 hl_unreserve_va_block(hdev, ctx, ctx->cb_va_pool_base, CB_VA_POOL_SIZE); 557} 558