1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (C) 2020-2023 Intel Corporation 4 */ 5 6#include <drm/drm_file.h> 7 8#include <linux/bitfield.h> 9#include <linux/highmem.h> 10#include <linux/kthread.h> 11#include <linux/pci.h> 12#include <linux/module.h> 13#include <uapi/drm/ivpu_accel.h> 14 15#include "ivpu_drv.h" 16#include "ivpu_hw.h" 17#include "ivpu_ipc.h" 18#include "ivpu_job.h" 19#include "ivpu_jsm_msg.h" 20#include "ivpu_pm.h" 21 22#define CMD_BUF_IDX 0 23#define JOB_ID_JOB_MASK GENMASK(7, 0) 24#define JOB_ID_CONTEXT_MASK GENMASK(31, 8) 25#define JOB_MAX_BUFFER_COUNT 65535 26 27static unsigned int ivpu_tdr_timeout_ms; 28module_param_named(tdr_timeout_ms, ivpu_tdr_timeout_ms, uint, 0644); 29MODULE_PARM_DESC(tdr_timeout_ms, "Timeout for device hang detection, in milliseconds, 0 - default"); 30 31static void ivpu_cmdq_ring_db(struct ivpu_device *vdev, struct ivpu_cmdq *cmdq) 32{ 33 ivpu_hw_reg_db_set(vdev, cmdq->db_id); 34} 35 36static struct ivpu_cmdq *ivpu_cmdq_alloc(struct ivpu_file_priv *file_priv, u16 engine) 37{ 38 struct ivpu_device *vdev = file_priv->vdev; 39 struct vpu_job_queue_header *jobq_header; 40 struct ivpu_cmdq *cmdq; 41 42 cmdq = kzalloc(sizeof(*cmdq), GFP_KERNEL); 43 if (!cmdq) 44 return NULL; 45 46 cmdq->mem = ivpu_bo_alloc_internal(vdev, 0, SZ_4K, DRM_IVPU_BO_WC); 47 if (!cmdq->mem) 48 goto cmdq_free; 49 50 cmdq->db_id = file_priv->ctx.id + engine * ivpu_get_context_count(vdev); 51 cmdq->entry_count = (u32)((cmdq->mem->base.size - sizeof(struct vpu_job_queue_header)) / 52 sizeof(struct vpu_job_queue_entry)); 53 54 cmdq->jobq = (struct vpu_job_queue *)cmdq->mem->kvaddr; 55 jobq_header = &cmdq->jobq->header; 56 jobq_header->engine_idx = engine; 57 jobq_header->head = 0; 58 jobq_header->tail = 0; 59 wmb(); /* Flush WC buffer for jobq->header */ 60 61 return cmdq; 62 63cmdq_free: 64 kfree(cmdq); 65 return NULL; 66} 67 68static void ivpu_cmdq_free(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq) 69{ 70 if (!cmdq) 71 return; 72 73 ivpu_bo_free_internal(cmdq->mem); 74 kfree(cmdq); 75} 76 77static struct ivpu_cmdq *ivpu_cmdq_acquire(struct ivpu_file_priv *file_priv, u16 engine) 78{ 79 struct ivpu_device *vdev = file_priv->vdev; 80 struct ivpu_cmdq *cmdq = file_priv->cmdq[engine]; 81 int ret; 82 83 lockdep_assert_held(&file_priv->lock); 84 85 if (!cmdq) { 86 cmdq = ivpu_cmdq_alloc(file_priv, engine); 87 if (!cmdq) 88 return NULL; 89 file_priv->cmdq[engine] = cmdq; 90 } 91 92 if (cmdq->db_registered) 93 return cmdq; 94 95 ret = ivpu_jsm_register_db(vdev, file_priv->ctx.id, cmdq->db_id, 96 cmdq->mem->vpu_addr, cmdq->mem->base.size); 97 if (ret) 98 return NULL; 99 100 cmdq->db_registered = true; 101 102 return cmdq; 103} 104 105static void ivpu_cmdq_release_locked(struct ivpu_file_priv *file_priv, u16 engine) 106{ 107 struct ivpu_cmdq *cmdq = file_priv->cmdq[engine]; 108 109 lockdep_assert_held(&file_priv->lock); 110 111 if (cmdq) { 112 file_priv->cmdq[engine] = NULL; 113 if (cmdq->db_registered) 114 ivpu_jsm_unregister_db(file_priv->vdev, cmdq->db_id); 115 116 ivpu_cmdq_free(file_priv, cmdq); 117 } 118} 119 120void ivpu_cmdq_release_all(struct ivpu_file_priv *file_priv) 121{ 122 int i; 123 124 mutex_lock(&file_priv->lock); 125 126 for (i = 0; i < IVPU_NUM_ENGINES; i++) 127 ivpu_cmdq_release_locked(file_priv, i); 128 129 mutex_unlock(&file_priv->lock); 130} 131 132/* 133 * Mark the doorbell as unregistered and reset job queue pointers. 134 * This function needs to be called when the VPU hardware is restarted 135 * and FW looses job queue state. The next time job queue is used it 136 * will be registered again. 137 */ 138static void ivpu_cmdq_reset_locked(struct ivpu_file_priv *file_priv, u16 engine) 139{ 140 struct ivpu_cmdq *cmdq = file_priv->cmdq[engine]; 141 142 lockdep_assert_held(&file_priv->lock); 143 144 if (cmdq) { 145 cmdq->db_registered = false; 146 cmdq->jobq->header.head = 0; 147 cmdq->jobq->header.tail = 0; 148 wmb(); /* Flush WC buffer for jobq header */ 149 } 150} 151 152static void ivpu_cmdq_reset_all(struct ivpu_file_priv *file_priv) 153{ 154 int i; 155 156 mutex_lock(&file_priv->lock); 157 158 for (i = 0; i < IVPU_NUM_ENGINES; i++) 159 ivpu_cmdq_reset_locked(file_priv, i); 160 161 mutex_unlock(&file_priv->lock); 162} 163 164void ivpu_cmdq_reset_all_contexts(struct ivpu_device *vdev) 165{ 166 struct ivpu_file_priv *file_priv; 167 unsigned long ctx_id; 168 169 xa_for_each(&vdev->context_xa, ctx_id, file_priv) { 170 file_priv = ivpu_file_priv_get_by_ctx_id(vdev, ctx_id); 171 if (!file_priv) 172 continue; 173 174 ivpu_cmdq_reset_all(file_priv); 175 176 ivpu_file_priv_put(&file_priv); 177 } 178} 179 180static int ivpu_cmdq_push_job(struct ivpu_cmdq *cmdq, struct ivpu_job *job) 181{ 182 struct ivpu_device *vdev = job->vdev; 183 struct vpu_job_queue_header *header = &cmdq->jobq->header; 184 struct vpu_job_queue_entry *entry; 185 u32 tail = READ_ONCE(header->tail); 186 u32 next_entry = (tail + 1) % cmdq->entry_count; 187 188 /* Check if there is space left in job queue */ 189 if (next_entry == header->head) { 190 ivpu_dbg(vdev, JOB, "Job queue full: ctx %d engine %d db %d head %d tail %d\n", 191 job->file_priv->ctx.id, job->engine_idx, cmdq->db_id, header->head, tail); 192 return -EBUSY; 193 } 194 195 entry = &cmdq->jobq->job[tail]; 196 entry->batch_buf_addr = job->cmd_buf_vpu_addr; 197 entry->job_id = job->job_id; 198 entry->flags = 0; 199 wmb(); /* Ensure that tail is updated after filling entry */ 200 header->tail = next_entry; 201 wmb(); /* Flush WC buffer for jobq header */ 202 203 return 0; 204} 205 206struct ivpu_fence { 207 struct dma_fence base; 208 spinlock_t lock; /* protects base */ 209 struct ivpu_device *vdev; 210}; 211 212static inline struct ivpu_fence *to_vpu_fence(struct dma_fence *fence) 213{ 214 return container_of(fence, struct ivpu_fence, base); 215} 216 217static const char *ivpu_fence_get_driver_name(struct dma_fence *fence) 218{ 219 return DRIVER_NAME; 220} 221 222static const char *ivpu_fence_get_timeline_name(struct dma_fence *fence) 223{ 224 struct ivpu_fence *ivpu_fence = to_vpu_fence(fence); 225 226 return dev_name(ivpu_fence->vdev->drm.dev); 227} 228 229static const struct dma_fence_ops ivpu_fence_ops = { 230 .get_driver_name = ivpu_fence_get_driver_name, 231 .get_timeline_name = ivpu_fence_get_timeline_name, 232}; 233 234static struct dma_fence *ivpu_fence_create(struct ivpu_device *vdev) 235{ 236 struct ivpu_fence *fence; 237 238 fence = kzalloc(sizeof(*fence), GFP_KERNEL); 239 if (!fence) 240 return NULL; 241 242 fence->vdev = vdev; 243 spin_lock_init(&fence->lock); 244 dma_fence_init(&fence->base, &ivpu_fence_ops, &fence->lock, dma_fence_context_alloc(1), 1); 245 246 return &fence->base; 247} 248 249static void job_get(struct ivpu_job *job, struct ivpu_job **link) 250{ 251 struct ivpu_device *vdev = job->vdev; 252 253 kref_get(&job->ref); 254 *link = job; 255 256 ivpu_dbg(vdev, KREF, "Job get: id %u refcount %u\n", job->job_id, kref_read(&job->ref)); 257} 258 259static void job_release(struct kref *ref) 260{ 261 struct ivpu_job *job = container_of(ref, struct ivpu_job, ref); 262 struct ivpu_device *vdev = job->vdev; 263 u32 i; 264 265 for (i = 0; i < job->bo_count; i++) 266 if (job->bos[i]) 267 drm_gem_object_put(&job->bos[i]->base); 268 269 dma_fence_put(job->done_fence); 270 ivpu_file_priv_put(&job->file_priv); 271 272 ivpu_dbg(vdev, KREF, "Job released: id %u\n", job->job_id); 273 kfree(job); 274 275 /* Allow the VPU to get suspended, must be called after ivpu_file_priv_put() */ 276 ivpu_rpm_put(vdev); 277} 278 279static void job_put(struct ivpu_job *job) 280{ 281 struct ivpu_device *vdev = job->vdev; 282 283 ivpu_dbg(vdev, KREF, "Job put: id %u refcount %u\n", job->job_id, kref_read(&job->ref)); 284 kref_put(&job->ref, job_release); 285} 286 287static struct ivpu_job * 288ivpu_create_job(struct ivpu_file_priv *file_priv, u32 engine_idx, u32 bo_count) 289{ 290 struct ivpu_device *vdev = file_priv->vdev; 291 struct ivpu_job *job; 292 int ret; 293 294 ret = ivpu_rpm_get(vdev); 295 if (ret < 0) 296 return NULL; 297 298 job = kzalloc(struct_size(job, bos, bo_count), GFP_KERNEL); 299 if (!job) 300 goto err_rpm_put; 301 302 kref_init(&job->ref); 303 304 job->vdev = vdev; 305 job->engine_idx = engine_idx; 306 job->bo_count = bo_count; 307 job->done_fence = ivpu_fence_create(vdev); 308 if (!job->done_fence) { 309 ivpu_warn_ratelimited(vdev, "Failed to create a fence\n"); 310 goto err_free_job; 311 } 312 313 job->file_priv = ivpu_file_priv_get(file_priv); 314 315 ivpu_dbg(vdev, JOB, "Job created: ctx %2d engine %d", file_priv->ctx.id, job->engine_idx); 316 317 return job; 318 319err_free_job: 320 kfree(job); 321err_rpm_put: 322 ivpu_rpm_put(vdev); 323 return NULL; 324} 325 326static int ivpu_job_done(struct ivpu_device *vdev, u32 job_id, u32 job_status) 327{ 328 struct ivpu_job *job; 329 330 job = xa_erase(&vdev->submitted_jobs_xa, job_id); 331 if (!job) 332 return -ENOENT; 333 334 if (job->file_priv->has_mmu_faults) 335 job_status = VPU_JSM_STATUS_ABORTED; 336 337 job->bos[CMD_BUF_IDX]->job_status = job_status; 338 dma_fence_signal(job->done_fence); 339 340 ivpu_dbg(vdev, JOB, "Job complete: id %3u ctx %2d engine %d status 0x%x\n", 341 job->job_id, job->file_priv->ctx.id, job->engine_idx, job_status); 342 343 job_put(job); 344 return 0; 345} 346 347static void ivpu_job_done_message(struct ivpu_device *vdev, void *msg) 348{ 349 struct vpu_ipc_msg_payload_job_done *payload; 350 struct vpu_jsm_msg *job_ret_msg = msg; 351 int ret; 352 353 payload = (struct vpu_ipc_msg_payload_job_done *)&job_ret_msg->payload; 354 355 ret = ivpu_job_done(vdev, payload->job_id, payload->job_status); 356 if (ret) 357 ivpu_err(vdev, "Failed to finish job %d: %d\n", payload->job_id, ret); 358} 359 360void ivpu_jobs_abort_all(struct ivpu_device *vdev) 361{ 362 struct ivpu_job *job; 363 unsigned long id; 364 365 xa_for_each(&vdev->submitted_jobs_xa, id, job) 366 ivpu_job_done(vdev, id, VPU_JSM_STATUS_ABORTED); 367} 368 369static int ivpu_direct_job_submission(struct ivpu_job *job) 370{ 371 struct ivpu_file_priv *file_priv = job->file_priv; 372 struct ivpu_device *vdev = job->vdev; 373 struct xa_limit job_id_range; 374 struct ivpu_cmdq *cmdq; 375 int ret; 376 377 mutex_lock(&file_priv->lock); 378 379 cmdq = ivpu_cmdq_acquire(job->file_priv, job->engine_idx); 380 if (!cmdq) { 381 ivpu_warn(vdev, "Failed get job queue, ctx %d engine %d\n", 382 file_priv->ctx.id, job->engine_idx); 383 ret = -EINVAL; 384 goto err_unlock; 385 } 386 387 job_id_range.min = FIELD_PREP(JOB_ID_CONTEXT_MASK, (file_priv->ctx.id - 1)); 388 job_id_range.max = job_id_range.min | JOB_ID_JOB_MASK; 389 390 job_get(job, &job); 391 ret = xa_alloc(&vdev->submitted_jobs_xa, &job->job_id, job, job_id_range, GFP_KERNEL); 392 if (ret) { 393 ivpu_warn_ratelimited(vdev, "Failed to allocate job id: %d\n", ret); 394 goto err_job_put; 395 } 396 397 ret = ivpu_cmdq_push_job(cmdq, job); 398 if (ret) 399 goto err_xa_erase; 400 401 ivpu_dbg(vdev, JOB, "Job submitted: id %3u addr 0x%llx ctx %2d engine %d next %d\n", 402 job->job_id, job->cmd_buf_vpu_addr, file_priv->ctx.id, 403 job->engine_idx, cmdq->jobq->header.tail); 404 405 if (ivpu_test_mode == IVPU_TEST_MODE_NULL_HW) { 406 ivpu_job_done(vdev, job->job_id, VPU_JSM_STATUS_SUCCESS); 407 cmdq->jobq->header.head = cmdq->jobq->header.tail; 408 wmb(); /* Flush WC buffer for jobq header */ 409 } else { 410 ivpu_cmdq_ring_db(vdev, cmdq); 411 } 412 413 mutex_unlock(&file_priv->lock); 414 return 0; 415 416err_xa_erase: 417 xa_erase(&vdev->submitted_jobs_xa, job->job_id); 418err_job_put: 419 job_put(job); 420err_unlock: 421 mutex_unlock(&file_priv->lock); 422 return ret; 423} 424 425static int 426ivpu_job_prepare_bos_for_submit(struct drm_file *file, struct ivpu_job *job, u32 *buf_handles, 427 u32 buf_count, u32 commands_offset) 428{ 429 struct ivpu_file_priv *file_priv = file->driver_priv; 430 struct ivpu_device *vdev = file_priv->vdev; 431 struct ww_acquire_ctx acquire_ctx; 432 enum dma_resv_usage usage; 433 struct ivpu_bo *bo; 434 int ret; 435 u32 i; 436 437 for (i = 0; i < buf_count; i++) { 438 struct drm_gem_object *obj = drm_gem_object_lookup(file, buf_handles[i]); 439 440 if (!obj) 441 return -ENOENT; 442 443 job->bos[i] = to_ivpu_bo(obj); 444 445 ret = ivpu_bo_pin(job->bos[i]); 446 if (ret) 447 return ret; 448 } 449 450 bo = job->bos[CMD_BUF_IDX]; 451 if (!dma_resv_test_signaled(bo->base.resv, DMA_RESV_USAGE_READ)) { 452 ivpu_warn(vdev, "Buffer is already in use\n"); 453 return -EBUSY; 454 } 455 456 if (commands_offset >= bo->base.size) { 457 ivpu_warn(vdev, "Invalid command buffer offset %u\n", commands_offset); 458 return -EINVAL; 459 } 460 461 job->cmd_buf_vpu_addr = bo->vpu_addr + commands_offset; 462 463 ret = drm_gem_lock_reservations((struct drm_gem_object **)job->bos, buf_count, 464 &acquire_ctx); 465 if (ret) { 466 ivpu_warn(vdev, "Failed to lock reservations: %d\n", ret); 467 return ret; 468 } 469 470 for (i = 0; i < buf_count; i++) { 471 ret = dma_resv_reserve_fences(job->bos[i]->base.resv, 1); 472 if (ret) { 473 ivpu_warn(vdev, "Failed to reserve fences: %d\n", ret); 474 goto unlock_reservations; 475 } 476 } 477 478 for (i = 0; i < buf_count; i++) { 479 usage = (i == CMD_BUF_IDX) ? DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_BOOKKEEP; 480 dma_resv_add_fence(job->bos[i]->base.resv, job->done_fence, usage); 481 } 482 483unlock_reservations: 484 drm_gem_unlock_reservations((struct drm_gem_object **)job->bos, buf_count, &acquire_ctx); 485 486 wmb(); /* Flush write combining buffers */ 487 488 return ret; 489} 490 491int ivpu_submit_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 492{ 493 struct ivpu_file_priv *file_priv = file->driver_priv; 494 struct ivpu_device *vdev = file_priv->vdev; 495 struct drm_ivpu_submit *params = data; 496 struct ivpu_job *job; 497 u32 *buf_handles; 498 int idx, ret; 499 500 if (params->engine > DRM_IVPU_ENGINE_COPY) 501 return -EINVAL; 502 503 if (params->buffer_count == 0 || params->buffer_count > JOB_MAX_BUFFER_COUNT) 504 return -EINVAL; 505 506 if (!IS_ALIGNED(params->commands_offset, 8)) 507 return -EINVAL; 508 509 if (!file_priv->ctx.id) 510 return -EINVAL; 511 512 if (file_priv->has_mmu_faults) 513 return -EBADFD; 514 515 buf_handles = kcalloc(params->buffer_count, sizeof(u32), GFP_KERNEL); 516 if (!buf_handles) 517 return -ENOMEM; 518 519 ret = copy_from_user(buf_handles, 520 (void __user *)params->buffers_ptr, 521 params->buffer_count * sizeof(u32)); 522 if (ret) { 523 ret = -EFAULT; 524 goto free_handles; 525 } 526 527 if (!drm_dev_enter(&vdev->drm, &idx)) { 528 ret = -ENODEV; 529 goto free_handles; 530 } 531 532 ivpu_dbg(vdev, JOB, "Submit ioctl: ctx %u buf_count %u\n", 533 file_priv->ctx.id, params->buffer_count); 534 535 job = ivpu_create_job(file_priv, params->engine, params->buffer_count); 536 if (!job) { 537 ivpu_err(vdev, "Failed to create job\n"); 538 ret = -ENOMEM; 539 goto dev_exit; 540 } 541 542 ret = ivpu_job_prepare_bos_for_submit(file, job, buf_handles, params->buffer_count, 543 params->commands_offset); 544 if (ret) { 545 ivpu_err(vdev, "Failed to prepare job, ret %d\n", ret); 546 goto job_put; 547 } 548 549 ret = ivpu_direct_job_submission(job); 550 if (ret) { 551 dma_fence_signal(job->done_fence); 552 ivpu_err(vdev, "Failed to submit job to the HW, ret %d\n", ret); 553 } 554 555job_put: 556 job_put(job); 557dev_exit: 558 drm_dev_exit(idx); 559free_handles: 560 kfree(buf_handles); 561 562 return ret; 563} 564 565static int ivpu_job_done_thread(void *arg) 566{ 567 struct ivpu_device *vdev = (struct ivpu_device *)arg; 568 struct ivpu_ipc_consumer cons; 569 struct vpu_jsm_msg jsm_msg; 570 bool jobs_submitted; 571 unsigned int timeout; 572 int ret; 573 574 ivpu_dbg(vdev, JOB, "Started %s\n", __func__); 575 576 ivpu_ipc_consumer_add(vdev, &cons, VPU_IPC_CHAN_JOB_RET); 577 578 while (!kthread_should_stop()) { 579 timeout = ivpu_tdr_timeout_ms ? ivpu_tdr_timeout_ms : vdev->timeout.tdr; 580 jobs_submitted = !xa_empty(&vdev->submitted_jobs_xa); 581 ret = ivpu_ipc_receive(vdev, &cons, NULL, &jsm_msg, timeout); 582 if (!ret) { 583 ivpu_job_done_message(vdev, &jsm_msg); 584 } else if (ret == -ETIMEDOUT) { 585 if (jobs_submitted && !xa_empty(&vdev->submitted_jobs_xa)) { 586 ivpu_err(vdev, "TDR detected, timeout %d ms", timeout); 587 ivpu_hw_diagnose_failure(vdev); 588 ivpu_pm_schedule_recovery(vdev); 589 } 590 } 591 } 592 593 ivpu_ipc_consumer_del(vdev, &cons); 594 595 ivpu_jobs_abort_all(vdev); 596 597 ivpu_dbg(vdev, JOB, "Stopped %s\n", __func__); 598 return 0; 599} 600 601int ivpu_job_done_thread_init(struct ivpu_device *vdev) 602{ 603 struct task_struct *thread; 604 605 thread = kthread_run(&ivpu_job_done_thread, (void *)vdev, "ivpu_job_done_thread"); 606 if (IS_ERR(thread)) { 607 ivpu_err(vdev, "Failed to start job completion thread\n"); 608 return -EIO; 609 } 610 611 get_task_struct(thread); 612 wake_up_process(thread); 613 614 vdev->job_done_thread = thread; 615 616 return 0; 617} 618 619void ivpu_job_done_thread_fini(struct ivpu_device *vdev) 620{ 621 kthread_stop(vdev->job_done_thread); 622 put_task_struct(vdev->job_done_thread); 623} 624