1/* 2 * Copyright 2008 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 22 * DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: 25 * Jerome Glisse <glisse@freedesktop.org> 26 */ 27 28#include <linux/file.h> 29#include <linux/pagemap.h> 30#include <linux/sync_file.h> 31#include <linux/dma-buf.h> 32 33#include <drm/amdgpu_drm.h> 34#include <drm/drm_syncobj.h> 35#include <drm/ttm/ttm_tt.h> 36 37#include "amdgpu_cs.h" 38#include "amdgpu.h" 39#include "amdgpu_trace.h" 40#include "amdgpu_gmc.h" 41#include "amdgpu_gem.h" 42#include "amdgpu_ras.h" 43 44static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, 45 struct amdgpu_device *adev, 46 struct drm_file *filp, 47 union drm_amdgpu_cs *cs) 48{ 49 struct amdgpu_fpriv *fpriv = filp->driver_priv; 50 51 if (cs->in.num_chunks == 0) 52 return -EINVAL; 53 54 memset(p, 0, sizeof(*p)); 55 p->adev = adev; 56 p->filp = filp; 57 58 p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id); 59 if (!p->ctx) 60 return -EINVAL; 61 62 if (atomic_read(&p->ctx->guilty)) { 63 amdgpu_ctx_put(p->ctx); 64 return -ECANCELED; 65 } 66 67 amdgpu_sync_create(&p->sync); 68 drm_exec_init(&p->exec, DRM_EXEC_INTERRUPTIBLE_WAIT | 69 DRM_EXEC_IGNORE_DUPLICATES); 70 return 0; 71} 72 73static int amdgpu_cs_job_idx(struct amdgpu_cs_parser *p, 74 struct drm_amdgpu_cs_chunk_ib *chunk_ib) 75{ 76 struct drm_sched_entity *entity; 77 unsigned int i; 78 int r; 79 80 r = amdgpu_ctx_get_entity(p->ctx, chunk_ib->ip_type, 81 chunk_ib->ip_instance, 82 chunk_ib->ring, &entity); 83 if (r) 84 return r; 85 86 /* 87 * Abort if there is no run queue associated with this entity. 88 * Possibly because of disabled HW IP. 89 */ 90 if (entity->rq == NULL) 91 return -EINVAL; 92 93 /* Check if we can add this IB to some existing job */ 94 for (i = 0; i < p->gang_size; ++i) 95 if (p->entities[i] == entity) 96 return i; 97 98 /* If not increase the gang size if possible */ 99 if (i == AMDGPU_CS_GANG_SIZE) 100 return -EINVAL; 101 102 p->entities[i] = entity; 103 p->gang_size = i + 1; 104 return i; 105} 106 107static int amdgpu_cs_p1_ib(struct amdgpu_cs_parser *p, 108 struct drm_amdgpu_cs_chunk_ib *chunk_ib, 109 unsigned int *num_ibs) 110{ 111 int r; 112 113 r = amdgpu_cs_job_idx(p, chunk_ib); 114 if (r < 0) 115 return r; 116 117 if (num_ibs[r] >= amdgpu_ring_max_ibs(chunk_ib->ip_type)) 118 return -EINVAL; 119 120 ++(num_ibs[r]); 121 p->gang_leader_idx = r; 122 return 0; 123} 124 125static int amdgpu_cs_p1_user_fence(struct amdgpu_cs_parser *p, 126 struct drm_amdgpu_cs_chunk_fence *data, 127 uint32_t *offset) 128{ 129 struct drm_gem_object *gobj; 130 unsigned long size; 131 132 gobj = drm_gem_object_lookup(p->filp, data->handle); 133 if (gobj == NULL) 134 return -EINVAL; 135 136 p->uf_bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj)); 137 drm_gem_object_put(gobj); 138 139 size = amdgpu_bo_size(p->uf_bo); 140 if (size != PAGE_SIZE || data->offset > (size - 8)) 141 return -EINVAL; 142 143 if (amdgpu_ttm_tt_get_usermm(p->uf_bo->tbo.ttm)) 144 return -EINVAL; 145 146 *offset = data->offset; 147 return 0; 148} 149 150static int amdgpu_cs_p1_bo_handles(struct amdgpu_cs_parser *p, 151 struct drm_amdgpu_bo_list_in *data) 152{ 153 struct drm_amdgpu_bo_list_entry *info; 154 int r; 155 156 r = amdgpu_bo_create_list_entry_array(data, &info); 157 if (r) 158 return r; 159 160 r = amdgpu_bo_list_create(p->adev, p->filp, info, data->bo_number, 161 &p->bo_list); 162 if (r) 163 goto error_free; 164 165 kvfree(info); 166 return 0; 167 168error_free: 169 kvfree(info); 170 171 return r; 172} 173 174/* Copy the data from userspace and go over it the first time */ 175static int amdgpu_cs_pass1(struct amdgpu_cs_parser *p, 176 union drm_amdgpu_cs *cs) 177{ 178 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 179 unsigned int num_ibs[AMDGPU_CS_GANG_SIZE] = { }; 180 struct amdgpu_vm *vm = &fpriv->vm; 181 uint64_t *chunk_array_user; 182 uint64_t *chunk_array; 183 uint32_t uf_offset = 0; 184 size_t size; 185 int ret; 186 int i; 187 188 chunk_array = kvmalloc_array(cs->in.num_chunks, sizeof(uint64_t), 189 GFP_KERNEL); 190 if (!chunk_array) 191 return -ENOMEM; 192 193 /* get chunks */ 194 chunk_array_user = u64_to_user_ptr(cs->in.chunks); 195 if (copy_from_user(chunk_array, chunk_array_user, 196 sizeof(uint64_t)*cs->in.num_chunks)) { 197 ret = -EFAULT; 198 goto free_chunk; 199 } 200 201 p->nchunks = cs->in.num_chunks; 202 p->chunks = kvmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk), 203 GFP_KERNEL); 204 if (!p->chunks) { 205 ret = -ENOMEM; 206 goto free_chunk; 207 } 208 209 for (i = 0; i < p->nchunks; i++) { 210 struct drm_amdgpu_cs_chunk __user *chunk_ptr = NULL; 211 struct drm_amdgpu_cs_chunk user_chunk; 212 uint32_t __user *cdata; 213 214 chunk_ptr = u64_to_user_ptr(chunk_array[i]); 215 if (copy_from_user(&user_chunk, chunk_ptr, 216 sizeof(struct drm_amdgpu_cs_chunk))) { 217 ret = -EFAULT; 218 i--; 219 goto free_partial_kdata; 220 } 221 p->chunks[i].chunk_id = user_chunk.chunk_id; 222 p->chunks[i].length_dw = user_chunk.length_dw; 223 224 size = p->chunks[i].length_dw; 225 cdata = u64_to_user_ptr(user_chunk.chunk_data); 226 227 p->chunks[i].kdata = kvmalloc_array(size, sizeof(uint32_t), 228 GFP_KERNEL); 229 if (p->chunks[i].kdata == NULL) { 230 ret = -ENOMEM; 231 i--; 232 goto free_partial_kdata; 233 } 234 size *= sizeof(uint32_t); 235 if (copy_from_user(p->chunks[i].kdata, cdata, size)) { 236 ret = -EFAULT; 237 goto free_partial_kdata; 238 } 239 240 /* Assume the worst on the following checks */ 241 ret = -EINVAL; 242 switch (p->chunks[i].chunk_id) { 243 case AMDGPU_CHUNK_ID_IB: 244 if (size < sizeof(struct drm_amdgpu_cs_chunk_ib)) 245 goto free_partial_kdata; 246 247 ret = amdgpu_cs_p1_ib(p, p->chunks[i].kdata, num_ibs); 248 if (ret) 249 goto free_partial_kdata; 250 break; 251 252 case AMDGPU_CHUNK_ID_FENCE: 253 if (size < sizeof(struct drm_amdgpu_cs_chunk_fence)) 254 goto free_partial_kdata; 255 256 ret = amdgpu_cs_p1_user_fence(p, p->chunks[i].kdata, 257 &uf_offset); 258 if (ret) 259 goto free_partial_kdata; 260 break; 261 262 case AMDGPU_CHUNK_ID_BO_HANDLES: 263 if (size < sizeof(struct drm_amdgpu_bo_list_in)) 264 goto free_partial_kdata; 265 266 ret = amdgpu_cs_p1_bo_handles(p, p->chunks[i].kdata); 267 if (ret) 268 goto free_partial_kdata; 269 break; 270 271 case AMDGPU_CHUNK_ID_DEPENDENCIES: 272 case AMDGPU_CHUNK_ID_SYNCOBJ_IN: 273 case AMDGPU_CHUNK_ID_SYNCOBJ_OUT: 274 case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES: 275 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT: 276 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL: 277 case AMDGPU_CHUNK_ID_CP_GFX_SHADOW: 278 break; 279 280 default: 281 goto free_partial_kdata; 282 } 283 } 284 285 if (!p->gang_size) { 286 ret = -EINVAL; 287 goto free_all_kdata; 288 } 289 290 for (i = 0; i < p->gang_size; ++i) { 291 ret = amdgpu_job_alloc(p->adev, vm, p->entities[i], vm, 292 num_ibs[i], &p->jobs[i]); 293 if (ret) 294 goto free_all_kdata; 295 } 296 p->gang_leader = p->jobs[p->gang_leader_idx]; 297 298 if (p->ctx->generation != p->gang_leader->generation) { 299 ret = -ECANCELED; 300 goto free_all_kdata; 301 } 302 303 if (p->uf_bo) 304 p->gang_leader->uf_addr = uf_offset; 305 kvfree(chunk_array); 306 307 /* Use this opportunity to fill in task info for the vm */ 308 amdgpu_vm_set_task_info(vm); 309 310 return 0; 311 312free_all_kdata: 313 i = p->nchunks - 1; 314free_partial_kdata: 315 for (; i >= 0; i--) 316 kvfree(p->chunks[i].kdata); 317 kvfree(p->chunks); 318 p->chunks = NULL; 319 p->nchunks = 0; 320free_chunk: 321 kvfree(chunk_array); 322 323 return ret; 324} 325 326static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser *p, 327 struct amdgpu_cs_chunk *chunk, 328 unsigned int *ce_preempt, 329 unsigned int *de_preempt) 330{ 331 struct drm_amdgpu_cs_chunk_ib *chunk_ib = chunk->kdata; 332 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 333 struct amdgpu_vm *vm = &fpriv->vm; 334 struct amdgpu_ring *ring; 335 struct amdgpu_job *job; 336 struct amdgpu_ib *ib; 337 int r; 338 339 r = amdgpu_cs_job_idx(p, chunk_ib); 340 if (r < 0) 341 return r; 342 343 job = p->jobs[r]; 344 ring = amdgpu_job_ring(job); 345 ib = &job->ibs[job->num_ibs++]; 346 347 /* MM engine doesn't support user fences */ 348 if (p->uf_bo && ring->funcs->no_user_fence) 349 return -EINVAL; 350 351 if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX && 352 chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) { 353 if (chunk_ib->flags & AMDGPU_IB_FLAG_CE) 354 (*ce_preempt)++; 355 else 356 (*de_preempt)++; 357 358 /* Each GFX command submit allows only 1 IB max 359 * preemptible for CE & DE */ 360 if (*ce_preempt > 1 || *de_preempt > 1) 361 return -EINVAL; 362 } 363 364 if (chunk_ib->flags & AMDGPU_IB_FLAG_PREAMBLE) 365 job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT; 366 367 r = amdgpu_ib_get(p->adev, vm, ring->funcs->parse_cs ? 368 chunk_ib->ib_bytes : 0, 369 AMDGPU_IB_POOL_DELAYED, ib); 370 if (r) { 371 DRM_ERROR("Failed to get ib !\n"); 372 return r; 373 } 374 375 ib->gpu_addr = chunk_ib->va_start; 376 ib->length_dw = chunk_ib->ib_bytes / 4; 377 ib->flags = chunk_ib->flags; 378 return 0; 379} 380 381static int amdgpu_cs_p2_dependencies(struct amdgpu_cs_parser *p, 382 struct amdgpu_cs_chunk *chunk) 383{ 384 struct drm_amdgpu_cs_chunk_dep *deps = chunk->kdata; 385 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 386 unsigned int num_deps; 387 int i, r; 388 389 num_deps = chunk->length_dw * 4 / 390 sizeof(struct drm_amdgpu_cs_chunk_dep); 391 392 for (i = 0; i < num_deps; ++i) { 393 struct amdgpu_ctx *ctx; 394 struct drm_sched_entity *entity; 395 struct dma_fence *fence; 396 397 ctx = amdgpu_ctx_get(fpriv, deps[i].ctx_id); 398 if (ctx == NULL) 399 return -EINVAL; 400 401 r = amdgpu_ctx_get_entity(ctx, deps[i].ip_type, 402 deps[i].ip_instance, 403 deps[i].ring, &entity); 404 if (r) { 405 amdgpu_ctx_put(ctx); 406 return r; 407 } 408 409 fence = amdgpu_ctx_get_fence(ctx, entity, deps[i].handle); 410 amdgpu_ctx_put(ctx); 411 412 if (IS_ERR(fence)) 413 return PTR_ERR(fence); 414 else if (!fence) 415 continue; 416 417 if (chunk->chunk_id == AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES) { 418 struct drm_sched_fence *s_fence; 419 struct dma_fence *old = fence; 420 421 s_fence = to_drm_sched_fence(fence); 422 fence = dma_fence_get(&s_fence->scheduled); 423 dma_fence_put(old); 424 } 425 426 r = amdgpu_sync_fence(&p->sync, fence); 427 dma_fence_put(fence); 428 if (r) 429 return r; 430 } 431 return 0; 432} 433 434static int amdgpu_syncobj_lookup_and_add(struct amdgpu_cs_parser *p, 435 uint32_t handle, u64 point, 436 u64 flags) 437{ 438 struct dma_fence *fence; 439 int r; 440 441 r = drm_syncobj_find_fence(p->filp, handle, point, flags, &fence); 442 if (r) { 443 DRM_ERROR("syncobj %u failed to find fence @ %llu (%d)!\n", 444 handle, point, r); 445 return r; 446 } 447 448 r = amdgpu_sync_fence(&p->sync, fence); 449 dma_fence_put(fence); 450 return r; 451} 452 453static int amdgpu_cs_p2_syncobj_in(struct amdgpu_cs_parser *p, 454 struct amdgpu_cs_chunk *chunk) 455{ 456 struct drm_amdgpu_cs_chunk_sem *deps = chunk->kdata; 457 unsigned int num_deps; 458 int i, r; 459 460 num_deps = chunk->length_dw * 4 / 461 sizeof(struct drm_amdgpu_cs_chunk_sem); 462 for (i = 0; i < num_deps; ++i) { 463 r = amdgpu_syncobj_lookup_and_add(p, deps[i].handle, 0, 0); 464 if (r) 465 return r; 466 } 467 468 return 0; 469} 470 471static int amdgpu_cs_p2_syncobj_timeline_wait(struct amdgpu_cs_parser *p, 472 struct amdgpu_cs_chunk *chunk) 473{ 474 struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps = chunk->kdata; 475 unsigned int num_deps; 476 int i, r; 477 478 num_deps = chunk->length_dw * 4 / 479 sizeof(struct drm_amdgpu_cs_chunk_syncobj); 480 for (i = 0; i < num_deps; ++i) { 481 r = amdgpu_syncobj_lookup_and_add(p, syncobj_deps[i].handle, 482 syncobj_deps[i].point, 483 syncobj_deps[i].flags); 484 if (r) 485 return r; 486 } 487 488 return 0; 489} 490 491static int amdgpu_cs_p2_syncobj_out(struct amdgpu_cs_parser *p, 492 struct amdgpu_cs_chunk *chunk) 493{ 494 struct drm_amdgpu_cs_chunk_sem *deps = chunk->kdata; 495 unsigned int num_deps; 496 int i; 497 498 num_deps = chunk->length_dw * 4 / 499 sizeof(struct drm_amdgpu_cs_chunk_sem); 500 501 if (p->post_deps) 502 return -EINVAL; 503 504 p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps), 505 GFP_KERNEL); 506 p->num_post_deps = 0; 507 508 if (!p->post_deps) 509 return -ENOMEM; 510 511 512 for (i = 0; i < num_deps; ++i) { 513 p->post_deps[i].syncobj = 514 drm_syncobj_find(p->filp, deps[i].handle); 515 if (!p->post_deps[i].syncobj) 516 return -EINVAL; 517 p->post_deps[i].chain = NULL; 518 p->post_deps[i].point = 0; 519 p->num_post_deps++; 520 } 521 522 return 0; 523} 524 525static int amdgpu_cs_p2_syncobj_timeline_signal(struct amdgpu_cs_parser *p, 526 struct amdgpu_cs_chunk *chunk) 527{ 528 struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps = chunk->kdata; 529 unsigned int num_deps; 530 int i; 531 532 num_deps = chunk->length_dw * 4 / 533 sizeof(struct drm_amdgpu_cs_chunk_syncobj); 534 535 if (p->post_deps) 536 return -EINVAL; 537 538 p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps), 539 GFP_KERNEL); 540 p->num_post_deps = 0; 541 542 if (!p->post_deps) 543 return -ENOMEM; 544 545 for (i = 0; i < num_deps; ++i) { 546 struct amdgpu_cs_post_dep *dep = &p->post_deps[i]; 547 548 dep->chain = NULL; 549 if (syncobj_deps[i].point) { 550 dep->chain = dma_fence_chain_alloc(); 551 if (!dep->chain) 552 return -ENOMEM; 553 } 554 555 dep->syncobj = drm_syncobj_find(p->filp, 556 syncobj_deps[i].handle); 557 if (!dep->syncobj) { 558 dma_fence_chain_free(dep->chain); 559 return -EINVAL; 560 } 561 dep->point = syncobj_deps[i].point; 562 p->num_post_deps++; 563 } 564 565 return 0; 566} 567 568static int amdgpu_cs_p2_shadow(struct amdgpu_cs_parser *p, 569 struct amdgpu_cs_chunk *chunk) 570{ 571 struct drm_amdgpu_cs_chunk_cp_gfx_shadow *shadow = chunk->kdata; 572 int i; 573 574 if (shadow->flags & ~AMDGPU_CS_CHUNK_CP_GFX_SHADOW_FLAGS_INIT_SHADOW) 575 return -EINVAL; 576 577 for (i = 0; i < p->gang_size; ++i) { 578 p->jobs[i]->shadow_va = shadow->shadow_va; 579 p->jobs[i]->csa_va = shadow->csa_va; 580 p->jobs[i]->gds_va = shadow->gds_va; 581 p->jobs[i]->init_shadow = 582 shadow->flags & AMDGPU_CS_CHUNK_CP_GFX_SHADOW_FLAGS_INIT_SHADOW; 583 } 584 585 return 0; 586} 587 588static int amdgpu_cs_pass2(struct amdgpu_cs_parser *p) 589{ 590 unsigned int ce_preempt = 0, de_preempt = 0; 591 int i, r; 592 593 for (i = 0; i < p->nchunks; ++i) { 594 struct amdgpu_cs_chunk *chunk; 595 596 chunk = &p->chunks[i]; 597 598 switch (chunk->chunk_id) { 599 case AMDGPU_CHUNK_ID_IB: 600 r = amdgpu_cs_p2_ib(p, chunk, &ce_preempt, &de_preempt); 601 if (r) 602 return r; 603 break; 604 case AMDGPU_CHUNK_ID_DEPENDENCIES: 605 case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES: 606 r = amdgpu_cs_p2_dependencies(p, chunk); 607 if (r) 608 return r; 609 break; 610 case AMDGPU_CHUNK_ID_SYNCOBJ_IN: 611 r = amdgpu_cs_p2_syncobj_in(p, chunk); 612 if (r) 613 return r; 614 break; 615 case AMDGPU_CHUNK_ID_SYNCOBJ_OUT: 616 r = amdgpu_cs_p2_syncobj_out(p, chunk); 617 if (r) 618 return r; 619 break; 620 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT: 621 r = amdgpu_cs_p2_syncobj_timeline_wait(p, chunk); 622 if (r) 623 return r; 624 break; 625 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL: 626 r = amdgpu_cs_p2_syncobj_timeline_signal(p, chunk); 627 if (r) 628 return r; 629 break; 630 case AMDGPU_CHUNK_ID_CP_GFX_SHADOW: 631 r = amdgpu_cs_p2_shadow(p, chunk); 632 if (r) 633 return r; 634 break; 635 } 636 } 637 638 return 0; 639} 640 641/* Convert microseconds to bytes. */ 642static u64 us_to_bytes(struct amdgpu_device *adev, s64 us) 643{ 644 if (us <= 0 || !adev->mm_stats.log2_max_MBps) 645 return 0; 646 647 /* Since accum_us is incremented by a million per second, just 648 * multiply it by the number of MB/s to get the number of bytes. 649 */ 650 return us << adev->mm_stats.log2_max_MBps; 651} 652 653static s64 bytes_to_us(struct amdgpu_device *adev, u64 bytes) 654{ 655 if (!adev->mm_stats.log2_max_MBps) 656 return 0; 657 658 return bytes >> adev->mm_stats.log2_max_MBps; 659} 660 661/* Returns how many bytes TTM can move right now. If no bytes can be moved, 662 * it returns 0. If it returns non-zero, it's OK to move at least one buffer, 663 * which means it can go over the threshold once. If that happens, the driver 664 * will be in debt and no other buffer migrations can be done until that debt 665 * is repaid. 666 * 667 * This approach allows moving a buffer of any size (it's important to allow 668 * that). 669 * 670 * The currency is simply time in microseconds and it increases as the clock 671 * ticks. The accumulated microseconds (us) are converted to bytes and 672 * returned. 673 */ 674static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev, 675 u64 *max_bytes, 676 u64 *max_vis_bytes) 677{ 678 s64 time_us, increment_us; 679 u64 free_vram, total_vram, used_vram; 680 /* Allow a maximum of 200 accumulated ms. This is basically per-IB 681 * throttling. 682 * 683 * It means that in order to get full max MBps, at least 5 IBs per 684 * second must be submitted and not more than 200ms apart from each 685 * other. 686 */ 687 const s64 us_upper_bound = 200000; 688 689 if (!adev->mm_stats.log2_max_MBps) { 690 *max_bytes = 0; 691 *max_vis_bytes = 0; 692 return; 693 } 694 695 total_vram = adev->gmc.real_vram_size - atomic64_read(&adev->vram_pin_size); 696 used_vram = ttm_resource_manager_usage(&adev->mman.vram_mgr.manager); 697 free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram; 698 699 spin_lock(&adev->mm_stats.lock); 700 701 /* Increase the amount of accumulated us. */ 702 time_us = ktime_to_us(ktime_get()); 703 increment_us = time_us - adev->mm_stats.last_update_us; 704 adev->mm_stats.last_update_us = time_us; 705 adev->mm_stats.accum_us = min(adev->mm_stats.accum_us + increment_us, 706 us_upper_bound); 707 708 /* This prevents the short period of low performance when the VRAM 709 * usage is low and the driver is in debt or doesn't have enough 710 * accumulated us to fill VRAM quickly. 711 * 712 * The situation can occur in these cases: 713 * - a lot of VRAM is freed by userspace 714 * - the presence of a big buffer causes a lot of evictions 715 * (solution: split buffers into smaller ones) 716 * 717 * If 128 MB or 1/8th of VRAM is free, start filling it now by setting 718 * accum_us to a positive number. 719 */ 720 if (free_vram >= 128 * 1024 * 1024 || free_vram >= total_vram / 8) { 721 s64 min_us; 722 723 /* Be more aggressive on dGPUs. Try to fill a portion of free 724 * VRAM now. 725 */ 726 if (!(adev->flags & AMD_IS_APU)) 727 min_us = bytes_to_us(adev, free_vram / 4); 728 else 729 min_us = 0; /* Reset accum_us on APUs. */ 730 731 adev->mm_stats.accum_us = max(min_us, adev->mm_stats.accum_us); 732 } 733 734 /* This is set to 0 if the driver is in debt to disallow (optional) 735 * buffer moves. 736 */ 737 *max_bytes = us_to_bytes(adev, adev->mm_stats.accum_us); 738 739 /* Do the same for visible VRAM if half of it is free */ 740 if (!amdgpu_gmc_vram_full_visible(&adev->gmc)) { 741 u64 total_vis_vram = adev->gmc.visible_vram_size; 742 u64 used_vis_vram = 743 amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr); 744 745 if (used_vis_vram < total_vis_vram) { 746 u64 free_vis_vram = total_vis_vram - used_vis_vram; 747 748 adev->mm_stats.accum_us_vis = min(adev->mm_stats.accum_us_vis + 749 increment_us, us_upper_bound); 750 751 if (free_vis_vram >= total_vis_vram / 2) 752 adev->mm_stats.accum_us_vis = 753 max(bytes_to_us(adev, free_vis_vram / 2), 754 adev->mm_stats.accum_us_vis); 755 } 756 757 *max_vis_bytes = us_to_bytes(adev, adev->mm_stats.accum_us_vis); 758 } else { 759 *max_vis_bytes = 0; 760 } 761 762 spin_unlock(&adev->mm_stats.lock); 763} 764 765/* Report how many bytes have really been moved for the last command 766 * submission. This can result in a debt that can stop buffer migrations 767 * temporarily. 768 */ 769void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes, 770 u64 num_vis_bytes) 771{ 772 spin_lock(&adev->mm_stats.lock); 773 adev->mm_stats.accum_us -= bytes_to_us(adev, num_bytes); 774 adev->mm_stats.accum_us_vis -= bytes_to_us(adev, num_vis_bytes); 775 spin_unlock(&adev->mm_stats.lock); 776} 777 778static int amdgpu_cs_bo_validate(void *param, struct amdgpu_bo *bo) 779{ 780 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 781 struct amdgpu_cs_parser *p = param; 782 struct ttm_operation_ctx ctx = { 783 .interruptible = true, 784 .no_wait_gpu = false, 785 .resv = bo->tbo.base.resv 786 }; 787 uint32_t domain; 788 int r; 789 790 if (bo->tbo.pin_count) 791 return 0; 792 793 /* Don't move this buffer if we have depleted our allowance 794 * to move it. Don't move anything if the threshold is zero. 795 */ 796 if (p->bytes_moved < p->bytes_moved_threshold && 797 (!bo->tbo.base.dma_buf || 798 list_empty(&bo->tbo.base.dma_buf->attachments))) { 799 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && 800 (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)) { 801 /* And don't move a CPU_ACCESS_REQUIRED BO to limited 802 * visible VRAM if we've depleted our allowance to do 803 * that. 804 */ 805 if (p->bytes_moved_vis < p->bytes_moved_vis_threshold) 806 domain = bo->preferred_domains; 807 else 808 domain = bo->allowed_domains; 809 } else { 810 domain = bo->preferred_domains; 811 } 812 } else { 813 domain = bo->allowed_domains; 814 } 815 816retry: 817 amdgpu_bo_placement_from_domain(bo, domain); 818 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 819 820 p->bytes_moved += ctx.bytes_moved; 821 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && 822 amdgpu_bo_in_cpu_visible_vram(bo)) 823 p->bytes_moved_vis += ctx.bytes_moved; 824 825 if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) { 826 domain = bo->allowed_domains; 827 goto retry; 828 } 829 830 return r; 831} 832 833static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, 834 union drm_amdgpu_cs *cs) 835{ 836 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 837 struct ttm_operation_ctx ctx = { true, false }; 838 struct amdgpu_vm *vm = &fpriv->vm; 839 struct amdgpu_bo_list_entry *e; 840 struct drm_gem_object *obj; 841 unsigned long index; 842 unsigned int i; 843 int r; 844 845 /* p->bo_list could already be assigned if AMDGPU_CHUNK_ID_BO_HANDLES is present */ 846 if (cs->in.bo_list_handle) { 847 if (p->bo_list) 848 return -EINVAL; 849 850 r = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle, 851 &p->bo_list); 852 if (r) 853 return r; 854 } else if (!p->bo_list) { 855 /* Create a empty bo_list when no handle is provided */ 856 r = amdgpu_bo_list_create(p->adev, p->filp, NULL, 0, 857 &p->bo_list); 858 if (r) 859 return r; 860 } 861 862 mutex_lock(&p->bo_list->bo_list_mutex); 863 864 /* Get userptr backing pages. If pages are updated after registered 865 * in amdgpu_gem_userptr_ioctl(), amdgpu_cs_list_validate() will do 866 * amdgpu_ttm_backend_bind() to flush and invalidate new pages 867 */ 868 amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) { 869 bool userpage_invalidated = false; 870 struct amdgpu_bo *bo = e->bo; 871 int i; 872 873 e->user_pages = kvmalloc_array(bo->tbo.ttm->num_pages, 874 sizeof(struct page *), 875 GFP_KERNEL | __GFP_ZERO); 876 if (!e->user_pages) { 877 DRM_ERROR("kvmalloc_array failure\n"); 878 r = -ENOMEM; 879 goto out_free_user_pages; 880 } 881 882 r = amdgpu_ttm_tt_get_user_pages(bo, e->user_pages, &e->range); 883 if (r) { 884 kvfree(e->user_pages); 885 e->user_pages = NULL; 886 goto out_free_user_pages; 887 } 888 889 for (i = 0; i < bo->tbo.ttm->num_pages; i++) { 890 if (bo->tbo.ttm->pages[i] != e->user_pages[i]) { 891 userpage_invalidated = true; 892 break; 893 } 894 } 895 e->user_invalidated = userpage_invalidated; 896 } 897 898 drm_exec_until_all_locked(&p->exec) { 899 r = amdgpu_vm_lock_pd(&fpriv->vm, &p->exec, 1 + p->gang_size); 900 drm_exec_retry_on_contention(&p->exec); 901 if (unlikely(r)) 902 goto out_free_user_pages; 903 904 amdgpu_bo_list_for_each_entry(e, p->bo_list) { 905 /* One fence for TTM and one for each CS job */ 906 r = drm_exec_prepare_obj(&p->exec, &e->bo->tbo.base, 907 1 + p->gang_size); 908 drm_exec_retry_on_contention(&p->exec); 909 if (unlikely(r)) 910 goto out_free_user_pages; 911 912 e->bo_va = amdgpu_vm_bo_find(vm, e->bo); 913 } 914 915 if (p->uf_bo) { 916 r = drm_exec_prepare_obj(&p->exec, &p->uf_bo->tbo.base, 917 1 + p->gang_size); 918 drm_exec_retry_on_contention(&p->exec); 919 if (unlikely(r)) 920 goto out_free_user_pages; 921 } 922 } 923 924 amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) { 925 struct mm_struct *usermm; 926 927 usermm = amdgpu_ttm_tt_get_usermm(e->bo->tbo.ttm); 928 if (usermm && usermm != current->mm) { 929 r = -EPERM; 930 goto out_free_user_pages; 931 } 932 933 if (amdgpu_ttm_tt_is_userptr(e->bo->tbo.ttm) && 934 e->user_invalidated && e->user_pages) { 935 amdgpu_bo_placement_from_domain(e->bo, 936 AMDGPU_GEM_DOMAIN_CPU); 937 r = ttm_bo_validate(&e->bo->tbo, &e->bo->placement, 938 &ctx); 939 if (r) 940 goto out_free_user_pages; 941 942 amdgpu_ttm_tt_set_user_pages(e->bo->tbo.ttm, 943 e->user_pages); 944 } 945 946 kvfree(e->user_pages); 947 e->user_pages = NULL; 948 } 949 950 amdgpu_cs_get_threshold_for_moves(p->adev, &p->bytes_moved_threshold, 951 &p->bytes_moved_vis_threshold); 952 p->bytes_moved = 0; 953 p->bytes_moved_vis = 0; 954 955 r = amdgpu_vm_validate_pt_bos(p->adev, &fpriv->vm, 956 amdgpu_cs_bo_validate, p); 957 if (r) { 958 DRM_ERROR("amdgpu_vm_validate_pt_bos() failed.\n"); 959 goto out_free_user_pages; 960 } 961 962 drm_exec_for_each_locked_object(&p->exec, index, obj) { 963 r = amdgpu_cs_bo_validate(p, gem_to_amdgpu_bo(obj)); 964 if (unlikely(r)) 965 goto out_free_user_pages; 966 } 967 968 if (p->uf_bo) { 969 r = amdgpu_ttm_alloc_gart(&p->uf_bo->tbo); 970 if (unlikely(r)) 971 goto out_free_user_pages; 972 973 p->gang_leader->uf_addr += amdgpu_bo_gpu_offset(p->uf_bo); 974 } 975 976 amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved, 977 p->bytes_moved_vis); 978 979 for (i = 0; i < p->gang_size; ++i) 980 amdgpu_job_set_resources(p->jobs[i], p->bo_list->gds_obj, 981 p->bo_list->gws_obj, 982 p->bo_list->oa_obj); 983 return 0; 984 985out_free_user_pages: 986 amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) { 987 struct amdgpu_bo *bo = e->bo; 988 989 if (!e->user_pages) 990 continue; 991 amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm, e->range); 992 kvfree(e->user_pages); 993 e->user_pages = NULL; 994 e->range = NULL; 995 } 996 mutex_unlock(&p->bo_list->bo_list_mutex); 997 return r; 998} 999 1000static void trace_amdgpu_cs_ibs(struct amdgpu_cs_parser *p) 1001{ 1002 int i, j; 1003 1004 if (!trace_amdgpu_cs_enabled()) 1005 return; 1006 1007 for (i = 0; i < p->gang_size; ++i) { 1008 struct amdgpu_job *job = p->jobs[i]; 1009 1010 for (j = 0; j < job->num_ibs; ++j) 1011 trace_amdgpu_cs(p, job, &job->ibs[j]); 1012 } 1013} 1014 1015static int amdgpu_cs_patch_ibs(struct amdgpu_cs_parser *p, 1016 struct amdgpu_job *job) 1017{ 1018 struct amdgpu_ring *ring = amdgpu_job_ring(job); 1019 unsigned int i; 1020 int r; 1021 1022 /* Only for UVD/VCE VM emulation */ 1023 if (!ring->funcs->parse_cs && !ring->funcs->patch_cs_in_place) 1024 return 0; 1025 1026 for (i = 0; i < job->num_ibs; ++i) { 1027 struct amdgpu_ib *ib = &job->ibs[i]; 1028 struct amdgpu_bo_va_mapping *m; 1029 struct amdgpu_bo *aobj; 1030 uint64_t va_start; 1031 uint8_t *kptr; 1032 1033 va_start = ib->gpu_addr & AMDGPU_GMC_HOLE_MASK; 1034 r = amdgpu_cs_find_mapping(p, va_start, &aobj, &m); 1035 if (r) { 1036 DRM_ERROR("IB va_start is invalid\n"); 1037 return r; 1038 } 1039 1040 if ((va_start + ib->length_dw * 4) > 1041 (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) { 1042 DRM_ERROR("IB va_start+ib_bytes is invalid\n"); 1043 return -EINVAL; 1044 } 1045 1046 /* the IB should be reserved at this point */ 1047 r = amdgpu_bo_kmap(aobj, (void **)&kptr); 1048 if (r) 1049 return r; 1050 1051 kptr += va_start - (m->start * AMDGPU_GPU_PAGE_SIZE); 1052 1053 if (ring->funcs->parse_cs) { 1054 memcpy(ib->ptr, kptr, ib->length_dw * 4); 1055 amdgpu_bo_kunmap(aobj); 1056 1057 r = amdgpu_ring_parse_cs(ring, p, job, ib); 1058 if (r) 1059 return r; 1060 } else { 1061 ib->ptr = (uint32_t *)kptr; 1062 r = amdgpu_ring_patch_cs_in_place(ring, p, job, ib); 1063 amdgpu_bo_kunmap(aobj); 1064 if (r) 1065 return r; 1066 } 1067 } 1068 1069 return 0; 1070} 1071 1072static int amdgpu_cs_patch_jobs(struct amdgpu_cs_parser *p) 1073{ 1074 unsigned int i; 1075 int r; 1076 1077 for (i = 0; i < p->gang_size; ++i) { 1078 r = amdgpu_cs_patch_ibs(p, p->jobs[i]); 1079 if (r) 1080 return r; 1081 } 1082 return 0; 1083} 1084 1085static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p) 1086{ 1087 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 1088 struct amdgpu_job *job = p->gang_leader; 1089 struct amdgpu_device *adev = p->adev; 1090 struct amdgpu_vm *vm = &fpriv->vm; 1091 struct amdgpu_bo_list_entry *e; 1092 struct amdgpu_bo_va *bo_va; 1093 unsigned int i; 1094 int r; 1095 1096 r = amdgpu_vm_clear_freed(adev, vm, NULL); 1097 if (r) 1098 return r; 1099 1100 r = amdgpu_vm_bo_update(adev, fpriv->prt_va, false); 1101 if (r) 1102 return r; 1103 1104 r = amdgpu_sync_fence(&p->sync, fpriv->prt_va->last_pt_update); 1105 if (r) 1106 return r; 1107 1108 if (fpriv->csa_va) { 1109 bo_va = fpriv->csa_va; 1110 BUG_ON(!bo_va); 1111 r = amdgpu_vm_bo_update(adev, bo_va, false); 1112 if (r) 1113 return r; 1114 1115 r = amdgpu_sync_fence(&p->sync, bo_va->last_pt_update); 1116 if (r) 1117 return r; 1118 } 1119 1120 amdgpu_bo_list_for_each_entry(e, p->bo_list) { 1121 bo_va = e->bo_va; 1122 if (bo_va == NULL) 1123 continue; 1124 1125 r = amdgpu_vm_bo_update(adev, bo_va, false); 1126 if (r) 1127 return r; 1128 1129 r = amdgpu_sync_fence(&p->sync, bo_va->last_pt_update); 1130 if (r) 1131 return r; 1132 } 1133 1134 r = amdgpu_vm_handle_moved(adev, vm); 1135 if (r) 1136 return r; 1137 1138 r = amdgpu_vm_update_pdes(adev, vm, false); 1139 if (r) 1140 return r; 1141 1142 r = amdgpu_sync_fence(&p->sync, vm->last_update); 1143 if (r) 1144 return r; 1145 1146 for (i = 0; i < p->gang_size; ++i) { 1147 job = p->jobs[i]; 1148 1149 if (!job->vm) 1150 continue; 1151 1152 job->vm_pd_addr = amdgpu_gmc_pd_addr(vm->root.bo); 1153 } 1154 1155 if (amdgpu_vm_debug) { 1156 /* Invalidate all BOs to test for userspace bugs */ 1157 amdgpu_bo_list_for_each_entry(e, p->bo_list) { 1158 struct amdgpu_bo *bo = e->bo; 1159 1160 /* ignore duplicates */ 1161 if (!bo) 1162 continue; 1163 1164 amdgpu_vm_bo_invalidate(adev, bo, false); 1165 } 1166 } 1167 1168 return 0; 1169} 1170 1171static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p) 1172{ 1173 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 1174 struct drm_gpu_scheduler *sched; 1175 struct drm_gem_object *obj; 1176 struct dma_fence *fence; 1177 unsigned long index; 1178 unsigned int i; 1179 int r; 1180 1181 r = amdgpu_ctx_wait_prev_fence(p->ctx, p->entities[p->gang_leader_idx]); 1182 if (r) { 1183 if (r != -ERESTARTSYS) 1184 DRM_ERROR("amdgpu_ctx_wait_prev_fence failed.\n"); 1185 return r; 1186 } 1187 1188 drm_exec_for_each_locked_object(&p->exec, index, obj) { 1189 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 1190 1191 struct dma_resv *resv = bo->tbo.base.resv; 1192 enum amdgpu_sync_mode sync_mode; 1193 1194 sync_mode = amdgpu_bo_explicit_sync(bo) ? 1195 AMDGPU_SYNC_EXPLICIT : AMDGPU_SYNC_NE_OWNER; 1196 r = amdgpu_sync_resv(p->adev, &p->sync, resv, sync_mode, 1197 &fpriv->vm); 1198 if (r) 1199 return r; 1200 } 1201 1202 for (i = 0; i < p->gang_size; ++i) { 1203 r = amdgpu_sync_push_to_job(&p->sync, p->jobs[i]); 1204 if (r) 1205 return r; 1206 } 1207 1208 sched = p->gang_leader->base.entity->rq->sched; 1209 while ((fence = amdgpu_sync_get_fence(&p->sync))) { 1210 struct drm_sched_fence *s_fence = to_drm_sched_fence(fence); 1211 1212 /* 1213 * When we have an dependency it might be necessary to insert a 1214 * pipeline sync to make sure that all caches etc are flushed and the 1215 * next job actually sees the results from the previous one 1216 * before we start executing on the same scheduler ring. 1217 */ 1218 if (!s_fence || s_fence->sched != sched) { 1219 dma_fence_put(fence); 1220 continue; 1221 } 1222 1223 r = amdgpu_sync_fence(&p->gang_leader->explicit_sync, fence); 1224 dma_fence_put(fence); 1225 if (r) 1226 return r; 1227 } 1228 return 0; 1229} 1230 1231static void amdgpu_cs_post_dependencies(struct amdgpu_cs_parser *p) 1232{ 1233 int i; 1234 1235 for (i = 0; i < p->num_post_deps; ++i) { 1236 if (p->post_deps[i].chain && p->post_deps[i].point) { 1237 drm_syncobj_add_point(p->post_deps[i].syncobj, 1238 p->post_deps[i].chain, 1239 p->fence, p->post_deps[i].point); 1240 p->post_deps[i].chain = NULL; 1241 } else { 1242 drm_syncobj_replace_fence(p->post_deps[i].syncobj, 1243 p->fence); 1244 } 1245 } 1246} 1247 1248static int amdgpu_cs_submit(struct amdgpu_cs_parser *p, 1249 union drm_amdgpu_cs *cs) 1250{ 1251 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 1252 struct amdgpu_job *leader = p->gang_leader; 1253 struct amdgpu_bo_list_entry *e; 1254 struct drm_gem_object *gobj; 1255 unsigned long index; 1256 unsigned int i; 1257 uint64_t seq; 1258 int r; 1259 1260 for (i = 0; i < p->gang_size; ++i) 1261 drm_sched_job_arm(&p->jobs[i]->base); 1262 1263 for (i = 0; i < p->gang_size; ++i) { 1264 struct dma_fence *fence; 1265 1266 if (p->jobs[i] == leader) 1267 continue; 1268 1269 fence = &p->jobs[i]->base.s_fence->scheduled; 1270 dma_fence_get(fence); 1271 r = drm_sched_job_add_dependency(&leader->base, fence); 1272 if (r) { 1273 dma_fence_put(fence); 1274 return r; 1275 } 1276 } 1277 1278 if (p->gang_size > 1) { 1279 for (i = 0; i < p->gang_size; ++i) 1280 amdgpu_job_set_gang_leader(p->jobs[i], leader); 1281 } 1282 1283 /* No memory allocation is allowed while holding the notifier lock. 1284 * The lock is held until amdgpu_cs_submit is finished and fence is 1285 * added to BOs. 1286 */ 1287 mutex_lock(&p->adev->notifier_lock); 1288 1289 /* If userptr are invalidated after amdgpu_cs_parser_bos(), return 1290 * -EAGAIN, drmIoctl in libdrm will restart the amdgpu_cs_ioctl. 1291 */ 1292 r = 0; 1293 amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) { 1294 r |= !amdgpu_ttm_tt_get_user_pages_done(e->bo->tbo.ttm, 1295 e->range); 1296 e->range = NULL; 1297 } 1298 if (r) { 1299 r = -EAGAIN; 1300 mutex_unlock(&p->adev->notifier_lock); 1301 return r; 1302 } 1303 1304 p->fence = dma_fence_get(&leader->base.s_fence->finished); 1305 drm_exec_for_each_locked_object(&p->exec, index, gobj) { 1306 1307 ttm_bo_move_to_lru_tail_unlocked(&gem_to_amdgpu_bo(gobj)->tbo); 1308 1309 /* Everybody except for the gang leader uses READ */ 1310 for (i = 0; i < p->gang_size; ++i) { 1311 if (p->jobs[i] == leader) 1312 continue; 1313 1314 dma_resv_add_fence(gobj->resv, 1315 &p->jobs[i]->base.s_fence->finished, 1316 DMA_RESV_USAGE_READ); 1317 } 1318 1319 /* The gang leader as remembered as writer */ 1320 dma_resv_add_fence(gobj->resv, p->fence, DMA_RESV_USAGE_WRITE); 1321 } 1322 1323 seq = amdgpu_ctx_add_fence(p->ctx, p->entities[p->gang_leader_idx], 1324 p->fence); 1325 amdgpu_cs_post_dependencies(p); 1326 1327 if ((leader->preamble_status & AMDGPU_PREAMBLE_IB_PRESENT) && 1328 !p->ctx->preamble_presented) { 1329 leader->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT_FIRST; 1330 p->ctx->preamble_presented = true; 1331 } 1332 1333 cs->out.handle = seq; 1334 leader->uf_sequence = seq; 1335 1336 amdgpu_vm_bo_trace_cs(&fpriv->vm, &p->exec.ticket); 1337 for (i = 0; i < p->gang_size; ++i) { 1338 amdgpu_job_free_resources(p->jobs[i]); 1339 trace_amdgpu_cs_ioctl(p->jobs[i]); 1340 drm_sched_entity_push_job(&p->jobs[i]->base); 1341 p->jobs[i] = NULL; 1342 } 1343 1344 amdgpu_vm_move_to_lru_tail(p->adev, &fpriv->vm); 1345 1346 mutex_unlock(&p->adev->notifier_lock); 1347 mutex_unlock(&p->bo_list->bo_list_mutex); 1348 return 0; 1349} 1350 1351/* Cleanup the parser structure */ 1352static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser) 1353{ 1354 unsigned int i; 1355 1356 amdgpu_sync_free(&parser->sync); 1357 drm_exec_fini(&parser->exec); 1358 1359 for (i = 0; i < parser->num_post_deps; i++) { 1360 drm_syncobj_put(parser->post_deps[i].syncobj); 1361 kfree(parser->post_deps[i].chain); 1362 } 1363 kfree(parser->post_deps); 1364 1365 dma_fence_put(parser->fence); 1366 1367 if (parser->ctx) 1368 amdgpu_ctx_put(parser->ctx); 1369 if (parser->bo_list) 1370 amdgpu_bo_list_put(parser->bo_list); 1371 1372 for (i = 0; i < parser->nchunks; i++) 1373 kvfree(parser->chunks[i].kdata); 1374 kvfree(parser->chunks); 1375 for (i = 0; i < parser->gang_size; ++i) { 1376 if (parser->jobs[i]) 1377 amdgpu_job_free(parser->jobs[i]); 1378 } 1379 amdgpu_bo_unref(&parser->uf_bo); 1380} 1381 1382int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 1383{ 1384 struct amdgpu_device *adev = drm_to_adev(dev); 1385 struct amdgpu_cs_parser parser; 1386 int r; 1387 1388 if (amdgpu_ras_intr_triggered()) 1389 return -EHWPOISON; 1390 1391 if (!adev->accel_working) 1392 return -EBUSY; 1393 1394 r = amdgpu_cs_parser_init(&parser, adev, filp, data); 1395 if (r) { 1396 if (printk_ratelimit()) 1397 DRM_ERROR("Failed to initialize parser %d!\n", r); 1398 return r; 1399 } 1400 1401 r = amdgpu_cs_pass1(&parser, data); 1402 if (r) 1403 goto error_fini; 1404 1405 r = amdgpu_cs_pass2(&parser); 1406 if (r) 1407 goto error_fini; 1408 1409 r = amdgpu_cs_parser_bos(&parser, data); 1410 if (r) { 1411 if (r == -ENOMEM) 1412 DRM_ERROR("Not enough memory for command submission!\n"); 1413 else if (r != -ERESTARTSYS && r != -EAGAIN) 1414 DRM_DEBUG("Failed to process the buffer list %d!\n", r); 1415 goto error_fini; 1416 } 1417 1418 r = amdgpu_cs_patch_jobs(&parser); 1419 if (r) 1420 goto error_backoff; 1421 1422 r = amdgpu_cs_vm_handling(&parser); 1423 if (r) 1424 goto error_backoff; 1425 1426 r = amdgpu_cs_sync_rings(&parser); 1427 if (r) 1428 goto error_backoff; 1429 1430 trace_amdgpu_cs_ibs(&parser); 1431 1432 r = amdgpu_cs_submit(&parser, data); 1433 if (r) 1434 goto error_backoff; 1435 1436 amdgpu_cs_parser_fini(&parser); 1437 return 0; 1438 1439error_backoff: 1440 mutex_unlock(&parser.bo_list->bo_list_mutex); 1441 1442error_fini: 1443 amdgpu_cs_parser_fini(&parser); 1444 return r; 1445} 1446 1447/** 1448 * amdgpu_cs_wait_ioctl - wait for a command submission to finish 1449 * 1450 * @dev: drm device 1451 * @data: data from userspace 1452 * @filp: file private 1453 * 1454 * Wait for the command submission identified by handle to finish. 1455 */ 1456int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data, 1457 struct drm_file *filp) 1458{ 1459 union drm_amdgpu_wait_cs *wait = data; 1460 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout); 1461 struct drm_sched_entity *entity; 1462 struct amdgpu_ctx *ctx; 1463 struct dma_fence *fence; 1464 long r; 1465 1466 ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id); 1467 if (ctx == NULL) 1468 return -EINVAL; 1469 1470 r = amdgpu_ctx_get_entity(ctx, wait->in.ip_type, wait->in.ip_instance, 1471 wait->in.ring, &entity); 1472 if (r) { 1473 amdgpu_ctx_put(ctx); 1474 return r; 1475 } 1476 1477 fence = amdgpu_ctx_get_fence(ctx, entity, wait->in.handle); 1478 if (IS_ERR(fence)) 1479 r = PTR_ERR(fence); 1480 else if (fence) { 1481 r = dma_fence_wait_timeout(fence, true, timeout); 1482 if (r > 0 && fence->error) 1483 r = fence->error; 1484 dma_fence_put(fence); 1485 } else 1486 r = 1; 1487 1488 amdgpu_ctx_put(ctx); 1489 if (r < 0) 1490 return r; 1491 1492 memset(wait, 0, sizeof(*wait)); 1493 wait->out.status = (r == 0); 1494 1495 return 0; 1496} 1497 1498/** 1499 * amdgpu_cs_get_fence - helper to get fence from drm_amdgpu_fence 1500 * 1501 * @adev: amdgpu device 1502 * @filp: file private 1503 * @user: drm_amdgpu_fence copied from user space 1504 */ 1505static struct dma_fence *amdgpu_cs_get_fence(struct amdgpu_device *adev, 1506 struct drm_file *filp, 1507 struct drm_amdgpu_fence *user) 1508{ 1509 struct drm_sched_entity *entity; 1510 struct amdgpu_ctx *ctx; 1511 struct dma_fence *fence; 1512 int r; 1513 1514 ctx = amdgpu_ctx_get(filp->driver_priv, user->ctx_id); 1515 if (ctx == NULL) 1516 return ERR_PTR(-EINVAL); 1517 1518 r = amdgpu_ctx_get_entity(ctx, user->ip_type, user->ip_instance, 1519 user->ring, &entity); 1520 if (r) { 1521 amdgpu_ctx_put(ctx); 1522 return ERR_PTR(r); 1523 } 1524 1525 fence = amdgpu_ctx_get_fence(ctx, entity, user->seq_no); 1526 amdgpu_ctx_put(ctx); 1527 1528 return fence; 1529} 1530 1531int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data, 1532 struct drm_file *filp) 1533{ 1534 struct amdgpu_device *adev = drm_to_adev(dev); 1535 union drm_amdgpu_fence_to_handle *info = data; 1536 struct dma_fence *fence; 1537 struct drm_syncobj *syncobj; 1538 struct sync_file *sync_file; 1539 int fd, r; 1540 1541 fence = amdgpu_cs_get_fence(adev, filp, &info->in.fence); 1542 if (IS_ERR(fence)) 1543 return PTR_ERR(fence); 1544 1545 if (!fence) 1546 fence = dma_fence_get_stub(); 1547 1548 switch (info->in.what) { 1549 case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ: 1550 r = drm_syncobj_create(&syncobj, 0, fence); 1551 dma_fence_put(fence); 1552 if (r) 1553 return r; 1554 r = drm_syncobj_get_handle(filp, syncobj, &info->out.handle); 1555 drm_syncobj_put(syncobj); 1556 return r; 1557 1558 case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ_FD: 1559 r = drm_syncobj_create(&syncobj, 0, fence); 1560 dma_fence_put(fence); 1561 if (r) 1562 return r; 1563 r = drm_syncobj_get_fd(syncobj, (int *)&info->out.handle); 1564 drm_syncobj_put(syncobj); 1565 return r; 1566 1567 case AMDGPU_FENCE_TO_HANDLE_GET_SYNC_FILE_FD: 1568 fd = get_unused_fd_flags(O_CLOEXEC); 1569 if (fd < 0) { 1570 dma_fence_put(fence); 1571 return fd; 1572 } 1573 1574 sync_file = sync_file_create(fence); 1575 dma_fence_put(fence); 1576 if (!sync_file) { 1577 put_unused_fd(fd); 1578 return -ENOMEM; 1579 } 1580 1581 fd_install(fd, sync_file->file); 1582 info->out.handle = fd; 1583 return 0; 1584 1585 default: 1586 dma_fence_put(fence); 1587 return -EINVAL; 1588 } 1589} 1590 1591/** 1592 * amdgpu_cs_wait_all_fences - wait on all fences to signal 1593 * 1594 * @adev: amdgpu device 1595 * @filp: file private 1596 * @wait: wait parameters 1597 * @fences: array of drm_amdgpu_fence 1598 */ 1599static int amdgpu_cs_wait_all_fences(struct amdgpu_device *adev, 1600 struct drm_file *filp, 1601 union drm_amdgpu_wait_fences *wait, 1602 struct drm_amdgpu_fence *fences) 1603{ 1604 uint32_t fence_count = wait->in.fence_count; 1605 unsigned int i; 1606 long r = 1; 1607 1608 for (i = 0; i < fence_count; i++) { 1609 struct dma_fence *fence; 1610 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns); 1611 1612 fence = amdgpu_cs_get_fence(adev, filp, &fences[i]); 1613 if (IS_ERR(fence)) 1614 return PTR_ERR(fence); 1615 else if (!fence) 1616 continue; 1617 1618 r = dma_fence_wait_timeout(fence, true, timeout); 1619 if (r > 0 && fence->error) 1620 r = fence->error; 1621 1622 dma_fence_put(fence); 1623 if (r < 0) 1624 return r; 1625 1626 if (r == 0) 1627 break; 1628 } 1629 1630 memset(wait, 0, sizeof(*wait)); 1631 wait->out.status = (r > 0); 1632 1633 return 0; 1634} 1635 1636/** 1637 * amdgpu_cs_wait_any_fence - wait on any fence to signal 1638 * 1639 * @adev: amdgpu device 1640 * @filp: file private 1641 * @wait: wait parameters 1642 * @fences: array of drm_amdgpu_fence 1643 */ 1644static int amdgpu_cs_wait_any_fence(struct amdgpu_device *adev, 1645 struct drm_file *filp, 1646 union drm_amdgpu_wait_fences *wait, 1647 struct drm_amdgpu_fence *fences) 1648{ 1649 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns); 1650 uint32_t fence_count = wait->in.fence_count; 1651 uint32_t first = ~0; 1652 struct dma_fence **array; 1653 unsigned int i; 1654 long r; 1655 1656 /* Prepare the fence array */ 1657 array = kcalloc(fence_count, sizeof(struct dma_fence *), GFP_KERNEL); 1658 1659 if (array == NULL) 1660 return -ENOMEM; 1661 1662 for (i = 0; i < fence_count; i++) { 1663 struct dma_fence *fence; 1664 1665 fence = amdgpu_cs_get_fence(adev, filp, &fences[i]); 1666 if (IS_ERR(fence)) { 1667 r = PTR_ERR(fence); 1668 goto err_free_fence_array; 1669 } else if (fence) { 1670 array[i] = fence; 1671 } else { /* NULL, the fence has been already signaled */ 1672 r = 1; 1673 first = i; 1674 goto out; 1675 } 1676 } 1677 1678 r = dma_fence_wait_any_timeout(array, fence_count, true, timeout, 1679 &first); 1680 if (r < 0) 1681 goto err_free_fence_array; 1682 1683out: 1684 memset(wait, 0, sizeof(*wait)); 1685 wait->out.status = (r > 0); 1686 wait->out.first_signaled = first; 1687 1688 if (first < fence_count && array[first]) 1689 r = array[first]->error; 1690 else 1691 r = 0; 1692 1693err_free_fence_array: 1694 for (i = 0; i < fence_count; i++) 1695 dma_fence_put(array[i]); 1696 kfree(array); 1697 1698 return r; 1699} 1700 1701/** 1702 * amdgpu_cs_wait_fences_ioctl - wait for multiple command submissions to finish 1703 * 1704 * @dev: drm device 1705 * @data: data from userspace 1706 * @filp: file private 1707 */ 1708int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data, 1709 struct drm_file *filp) 1710{ 1711 struct amdgpu_device *adev = drm_to_adev(dev); 1712 union drm_amdgpu_wait_fences *wait = data; 1713 uint32_t fence_count = wait->in.fence_count; 1714 struct drm_amdgpu_fence *fences_user; 1715 struct drm_amdgpu_fence *fences; 1716 int r; 1717 1718 /* Get the fences from userspace */ 1719 fences = kmalloc_array(fence_count, sizeof(struct drm_amdgpu_fence), 1720 GFP_KERNEL); 1721 if (fences == NULL) 1722 return -ENOMEM; 1723 1724 fences_user = u64_to_user_ptr(wait->in.fences); 1725 if (copy_from_user(fences, fences_user, 1726 sizeof(struct drm_amdgpu_fence) * fence_count)) { 1727 r = -EFAULT; 1728 goto err_free_fences; 1729 } 1730 1731 if (wait->in.wait_all) 1732 r = amdgpu_cs_wait_all_fences(adev, filp, wait, fences); 1733 else 1734 r = amdgpu_cs_wait_any_fence(adev, filp, wait, fences); 1735 1736err_free_fences: 1737 kfree(fences); 1738 1739 return r; 1740} 1741 1742/** 1743 * amdgpu_cs_find_mapping - find bo_va for VM address 1744 * 1745 * @parser: command submission parser context 1746 * @addr: VM address 1747 * @bo: resulting BO of the mapping found 1748 * @map: Placeholder to return found BO mapping 1749 * 1750 * Search the buffer objects in the command submission context for a certain 1751 * virtual memory address. Returns allocation structure when found, NULL 1752 * otherwise. 1753 */ 1754int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser, 1755 uint64_t addr, struct amdgpu_bo **bo, 1756 struct amdgpu_bo_va_mapping **map) 1757{ 1758 struct amdgpu_fpriv *fpriv = parser->filp->driver_priv; 1759 struct ttm_operation_ctx ctx = { false, false }; 1760 struct amdgpu_vm *vm = &fpriv->vm; 1761 struct amdgpu_bo_va_mapping *mapping; 1762 int r; 1763 1764 addr /= AMDGPU_GPU_PAGE_SIZE; 1765 1766 mapping = amdgpu_vm_bo_lookup_mapping(vm, addr); 1767 if (!mapping || !mapping->bo_va || !mapping->bo_va->base.bo) 1768 return -EINVAL; 1769 1770 *bo = mapping->bo_va->base.bo; 1771 *map = mapping; 1772 1773 /* Double check that the BO is reserved by this CS */ 1774 if (dma_resv_locking_ctx((*bo)->tbo.base.resv) != &parser->exec.ticket) 1775 return -EINVAL; 1776 1777 if (!((*bo)->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)) { 1778 (*bo)->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 1779 amdgpu_bo_placement_from_domain(*bo, (*bo)->allowed_domains); 1780 r = ttm_bo_validate(&(*bo)->tbo, &(*bo)->placement, &ctx); 1781 if (r) 1782 return r; 1783 } 1784 1785 return amdgpu_ttm_alloc_gart(&(*bo)->tbo); 1786} 1787