1/* 2 * Copyright © 2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25#include <linux/prime_numbers.h> 26#include <linux/pm_qos.h> 27#include <linux/sort.h> 28 29#include "gem/i915_gem_pm.h" 30#include "gem/selftests/mock_context.h" 31 32#include "gt/intel_engine_heartbeat.h" 33#include "gt/intel_engine_pm.h" 34#include "gt/intel_engine_user.h" 35#include "gt/intel_gt.h" 36#include "gt/intel_gt_requests.h" 37#include "gt/selftest_engine_heartbeat.h" 38 39#include "i915_random.h" 40#include "i915_selftest.h" 41#include "igt_flush_test.h" 42#include "igt_live_test.h" 43#include "igt_spinner.h" 44#include "lib_sw_fence.h" 45 46#include "mock_drm.h" 47#include "mock_gem_device.h" 48 49static unsigned int num_uabi_engines(struct drm_i915_private *i915) 50{ 51 struct intel_engine_cs *engine; 52 unsigned int count; 53 54 count = 0; 55 for_each_uabi_engine(engine, i915) 56 count++; 57 58 return count; 59} 60 61static struct intel_engine_cs *rcs0(struct drm_i915_private *i915) 62{ 63 return intel_engine_lookup_user(i915, I915_ENGINE_CLASS_RENDER, 0); 64} 65 66static int igt_add_request(void *arg) 67{ 68 struct drm_i915_private *i915 = arg; 69 struct i915_request *request; 70 71 /* Basic preliminary test to create a request and let it loose! */ 72 73 request = mock_request(rcs0(i915)->kernel_context, HZ / 10); 74 if (!request) 75 return -ENOMEM; 76 77 i915_request_add(request); 78 79 return 0; 80} 81 82static int igt_wait_request(void *arg) 83{ 84 const long T = HZ / 4; 85 struct drm_i915_private *i915 = arg; 86 struct i915_request *request; 87 int err = -EINVAL; 88 89 /* Submit a request, then wait upon it */ 90 91 request = mock_request(rcs0(i915)->kernel_context, T); 92 if (!request) 93 return -ENOMEM; 94 95 i915_request_get(request); 96 97 if (i915_request_wait(request, 0, 0) != -ETIME) { 98 pr_err("request wait (busy query) succeeded (expected timeout before submit!)\n"); 99 goto out_request; 100 } 101 102 if (i915_request_wait(request, 0, T) != -ETIME) { 103 pr_err("request wait succeeded (expected timeout before submit!)\n"); 104 goto out_request; 105 } 106 107 if (i915_request_completed(request)) { 108 pr_err("request completed before submit!!\n"); 109 goto out_request; 110 } 111 112 i915_request_add(request); 113 114 if (i915_request_wait(request, 0, 0) != -ETIME) { 115 pr_err("request wait (busy query) succeeded (expected timeout after submit!)\n"); 116 goto out_request; 117 } 118 119 if (i915_request_completed(request)) { 120 pr_err("request completed immediately!\n"); 121 goto out_request; 122 } 123 124 if (i915_request_wait(request, 0, T / 2) != -ETIME) { 125 pr_err("request wait succeeded (expected timeout!)\n"); 126 goto out_request; 127 } 128 129 if (i915_request_wait(request, 0, T) == -ETIME) { 130 pr_err("request wait timed out!\n"); 131 goto out_request; 132 } 133 134 if (!i915_request_completed(request)) { 135 pr_err("request not complete after waiting!\n"); 136 goto out_request; 137 } 138 139 if (i915_request_wait(request, 0, T) == -ETIME) { 140 pr_err("request wait timed out when already complete!\n"); 141 goto out_request; 142 } 143 144 err = 0; 145out_request: 146 i915_request_put(request); 147 mock_device_flush(i915); 148 return err; 149} 150 151static int igt_fence_wait(void *arg) 152{ 153 const long T = HZ / 4; 154 struct drm_i915_private *i915 = arg; 155 struct i915_request *request; 156 int err = -EINVAL; 157 158 /* Submit a request, treat it as a fence and wait upon it */ 159 160 request = mock_request(rcs0(i915)->kernel_context, T); 161 if (!request) 162 return -ENOMEM; 163 164 if (dma_fence_wait_timeout(&request->fence, false, T) != -ETIME) { 165 pr_err("fence wait success before submit (expected timeout)!\n"); 166 goto out; 167 } 168 169 i915_request_add(request); 170 171 if (dma_fence_is_signaled(&request->fence)) { 172 pr_err("fence signaled immediately!\n"); 173 goto out; 174 } 175 176 if (dma_fence_wait_timeout(&request->fence, false, T / 2) != -ETIME) { 177 pr_err("fence wait success after submit (expected timeout)!\n"); 178 goto out; 179 } 180 181 if (dma_fence_wait_timeout(&request->fence, false, T) <= 0) { 182 pr_err("fence wait timed out (expected success)!\n"); 183 goto out; 184 } 185 186 if (!dma_fence_is_signaled(&request->fence)) { 187 pr_err("fence unsignaled after waiting!\n"); 188 goto out; 189 } 190 191 if (dma_fence_wait_timeout(&request->fence, false, T) <= 0) { 192 pr_err("fence wait timed out when complete (expected success)!\n"); 193 goto out; 194 } 195 196 err = 0; 197out: 198 mock_device_flush(i915); 199 return err; 200} 201 202static int igt_request_rewind(void *arg) 203{ 204 struct drm_i915_private *i915 = arg; 205 struct i915_request *request, *vip; 206 struct i915_gem_context *ctx[2]; 207 struct intel_context *ce; 208 int err = -EINVAL; 209 210 ctx[0] = mock_context(i915, "A"); 211 212 ce = i915_gem_context_get_engine(ctx[0], RCS0); 213 GEM_BUG_ON(IS_ERR(ce)); 214 request = mock_request(ce, 2 * HZ); 215 intel_context_put(ce); 216 if (!request) { 217 err = -ENOMEM; 218 goto err_context_0; 219 } 220 221 i915_request_get(request); 222 i915_request_add(request); 223 224 ctx[1] = mock_context(i915, "B"); 225 226 ce = i915_gem_context_get_engine(ctx[1], RCS0); 227 GEM_BUG_ON(IS_ERR(ce)); 228 vip = mock_request(ce, 0); 229 intel_context_put(ce); 230 if (!vip) { 231 err = -ENOMEM; 232 goto err_context_1; 233 } 234 235 /* Simulate preemption by manual reordering */ 236 if (!mock_cancel_request(request)) { 237 pr_err("failed to cancel request (already executed)!\n"); 238 i915_request_add(vip); 239 goto err_context_1; 240 } 241 i915_request_get(vip); 242 i915_request_add(vip); 243 rcu_read_lock(); 244 request->engine->submit_request(request); 245 rcu_read_unlock(); 246 247 248 if (i915_request_wait(vip, 0, HZ) == -ETIME) { 249 pr_err("timed out waiting for high priority request\n"); 250 goto err; 251 } 252 253 if (i915_request_completed(request)) { 254 pr_err("low priority request already completed\n"); 255 goto err; 256 } 257 258 err = 0; 259err: 260 i915_request_put(vip); 261err_context_1: 262 mock_context_close(ctx[1]); 263 i915_request_put(request); 264err_context_0: 265 mock_context_close(ctx[0]); 266 mock_device_flush(i915); 267 return err; 268} 269 270struct smoketest { 271 struct intel_engine_cs *engine; 272 struct i915_gem_context **contexts; 273 atomic_long_t num_waits, num_fences; 274 int ncontexts, max_batch; 275 struct i915_request *(*request_alloc)(struct intel_context *ce); 276}; 277 278static struct i915_request * 279__mock_request_alloc(struct intel_context *ce) 280{ 281 return mock_request(ce, 0); 282} 283 284static struct i915_request * 285__live_request_alloc(struct intel_context *ce) 286{ 287 return intel_context_create_request(ce); 288} 289 290static int __igt_breadcrumbs_smoketest(void *arg) 291{ 292 struct smoketest *t = arg; 293 const unsigned int max_batch = min(t->ncontexts, t->max_batch) - 1; 294 const unsigned int total = 4 * t->ncontexts + 1; 295 unsigned int num_waits = 0, num_fences = 0; 296 struct i915_request **requests; 297 I915_RND_STATE(prng); 298 unsigned int *order; 299 int err = 0; 300 301 /* 302 * A very simple test to catch the most egregious of list handling bugs. 303 * 304 * At its heart, we simply create oodles of requests running across 305 * multiple kthreads and enable signaling on them, for the sole purpose 306 * of stressing our breadcrumb handling. The only inspection we do is 307 * that the fences were marked as signaled. 308 */ 309 310 requests = kcalloc(total, sizeof(*requests), GFP_KERNEL); 311 if (!requests) 312 return -ENOMEM; 313 314 order = i915_random_order(total, &prng); 315 if (!order) { 316 err = -ENOMEM; 317 goto out_requests; 318 } 319 320 while (!kthread_should_stop()) { 321 struct i915_sw_fence *submit, *wait; 322 unsigned int n, count; 323 324 submit = heap_fence_create(GFP_KERNEL); 325 if (!submit) { 326 err = -ENOMEM; 327 break; 328 } 329 330 wait = heap_fence_create(GFP_KERNEL); 331 if (!wait) { 332 i915_sw_fence_commit(submit); 333 heap_fence_put(submit); 334 err = -ENOMEM; 335 break; 336 } 337 338 i915_random_reorder(order, total, &prng); 339 count = 1 + i915_prandom_u32_max_state(max_batch, &prng); 340 341 for (n = 0; n < count; n++) { 342 struct i915_gem_context *ctx = 343 t->contexts[order[n] % t->ncontexts]; 344 struct i915_request *rq; 345 struct intel_context *ce; 346 347 ce = i915_gem_context_get_engine(ctx, t->engine->legacy_idx); 348 GEM_BUG_ON(IS_ERR(ce)); 349 rq = t->request_alloc(ce); 350 intel_context_put(ce); 351 if (IS_ERR(rq)) { 352 err = PTR_ERR(rq); 353 count = n; 354 break; 355 } 356 357 err = i915_sw_fence_await_sw_fence_gfp(&rq->submit, 358 submit, 359 GFP_KERNEL); 360 361 requests[n] = i915_request_get(rq); 362 i915_request_add(rq); 363 364 if (err >= 0) 365 err = i915_sw_fence_await_dma_fence(wait, 366 &rq->fence, 367 0, 368 GFP_KERNEL); 369 370 if (err < 0) { 371 i915_request_put(rq); 372 count = n; 373 break; 374 } 375 } 376 377 i915_sw_fence_commit(submit); 378 i915_sw_fence_commit(wait); 379 380 if (!wait_event_timeout(wait->wait, 381 i915_sw_fence_done(wait), 382 5 * HZ)) { 383 struct i915_request *rq = requests[count - 1]; 384 385 pr_err("waiting for %d/%d fences (last %llx:%lld) on %s timed out!\n", 386 atomic_read(&wait->pending), count, 387 rq->fence.context, rq->fence.seqno, 388 t->engine->name); 389 GEM_TRACE_DUMP(); 390 391 intel_gt_set_wedged(t->engine->gt); 392 GEM_BUG_ON(!i915_request_completed(rq)); 393 i915_sw_fence_wait(wait); 394 err = -EIO; 395 } 396 397 for (n = 0; n < count; n++) { 398 struct i915_request *rq = requests[n]; 399 400 if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, 401 &rq->fence.flags)) { 402 pr_err("%llu:%llu was not signaled!\n", 403 rq->fence.context, rq->fence.seqno); 404 err = -EINVAL; 405 } 406 407 i915_request_put(rq); 408 } 409 410 heap_fence_put(wait); 411 heap_fence_put(submit); 412 413 if (err < 0) 414 break; 415 416 num_fences += count; 417 num_waits++; 418 419 cond_resched(); 420 } 421 422 atomic_long_add(num_fences, &t->num_fences); 423 atomic_long_add(num_waits, &t->num_waits); 424 425 kfree(order); 426out_requests: 427 kfree(requests); 428 return err; 429} 430 431static int mock_breadcrumbs_smoketest(void *arg) 432{ 433 struct drm_i915_private *i915 = arg; 434 struct smoketest t = { 435 .engine = rcs0(i915), 436 .ncontexts = 1024, 437 .max_batch = 1024, 438 .request_alloc = __mock_request_alloc 439 }; 440 unsigned int ncpus = num_online_cpus(); 441 struct task_struct **threads; 442 unsigned int n; 443 int ret = 0; 444 445 /* 446 * Smoketest our breadcrumb/signal handling for requests across multiple 447 * threads. A very simple test to only catch the most egregious of bugs. 448 * See __igt_breadcrumbs_smoketest(); 449 */ 450 451 threads = kcalloc(ncpus, sizeof(*threads), GFP_KERNEL); 452 if (!threads) 453 return -ENOMEM; 454 455 t.contexts = kcalloc(t.ncontexts, sizeof(*t.contexts), GFP_KERNEL); 456 if (!t.contexts) { 457 ret = -ENOMEM; 458 goto out_threads; 459 } 460 461 for (n = 0; n < t.ncontexts; n++) { 462 t.contexts[n] = mock_context(t.engine->i915, "mock"); 463 if (!t.contexts[n]) { 464 ret = -ENOMEM; 465 goto out_contexts; 466 } 467 } 468 469 for (n = 0; n < ncpus; n++) { 470 threads[n] = kthread_run(__igt_breadcrumbs_smoketest, 471 &t, "igt/%d", n); 472 if (IS_ERR(threads[n])) { 473 ret = PTR_ERR(threads[n]); 474 ncpus = n; 475 break; 476 } 477 478 get_task_struct(threads[n]); 479 } 480 481 yield(); /* start all threads before we begin */ 482 msleep(jiffies_to_msecs(i915_selftest.timeout_jiffies)); 483 484 for (n = 0; n < ncpus; n++) { 485 int err; 486 487 err = kthread_stop(threads[n]); 488 if (err < 0 && !ret) 489 ret = err; 490 491 put_task_struct(threads[n]); 492 } 493 pr_info("Completed %lu waits for %lu fence across %d cpus\n", 494 atomic_long_read(&t.num_waits), 495 atomic_long_read(&t.num_fences), 496 ncpus); 497 498out_contexts: 499 for (n = 0; n < t.ncontexts; n++) { 500 if (!t.contexts[n]) 501 break; 502 mock_context_close(t.contexts[n]); 503 } 504 kfree(t.contexts); 505out_threads: 506 kfree(threads); 507 return ret; 508} 509 510int i915_request_mock_selftests(void) 511{ 512 static const struct i915_subtest tests[] = { 513 SUBTEST(igt_add_request), 514 SUBTEST(igt_wait_request), 515 SUBTEST(igt_fence_wait), 516 SUBTEST(igt_request_rewind), 517 SUBTEST(mock_breadcrumbs_smoketest), 518 }; 519 struct drm_i915_private *i915; 520 intel_wakeref_t wakeref; 521 int err = 0; 522 523 i915 = mock_gem_device(); 524 if (!i915) 525 return -ENOMEM; 526 527 with_intel_runtime_pm(&i915->runtime_pm, wakeref) 528 err = i915_subtests(tests, i915); 529 530 mock_destroy_device(i915); 531 532 return err; 533} 534 535static int live_nop_request(void *arg) 536{ 537 struct drm_i915_private *i915 = arg; 538 struct intel_engine_cs *engine; 539 struct igt_live_test t; 540 int err = -ENODEV; 541 542 /* 543 * Submit various sized batches of empty requests, to each engine 544 * (individually), and wait for the batch to complete. We can check 545 * the overhead of submitting requests to the hardware. 546 */ 547 548 for_each_uabi_engine(engine, i915) { 549 unsigned long n, prime; 550 IGT_TIMEOUT(end_time); 551 ktime_t times[2] = {}; 552 553 err = igt_live_test_begin(&t, i915, __func__, engine->name); 554 if (err) 555 return err; 556 557 intel_engine_pm_get(engine); 558 for_each_prime_number_from(prime, 1, 8192) { 559 struct i915_request *request = NULL; 560 561 times[1] = ktime_get_raw(); 562 563 for (n = 0; n < prime; n++) { 564 i915_request_put(request); 565 request = i915_request_create(engine->kernel_context); 566 if (IS_ERR(request)) 567 return PTR_ERR(request); 568 569 /* 570 * This space is left intentionally blank. 571 * 572 * We do not actually want to perform any 573 * action with this request, we just want 574 * to measure the latency in allocation 575 * and submission of our breadcrumbs - 576 * ensuring that the bare request is sufficient 577 * for the system to work (i.e. proper HEAD 578 * tracking of the rings, interrupt handling, 579 * etc). It also gives us the lowest bounds 580 * for latency. 581 */ 582 583 i915_request_get(request); 584 i915_request_add(request); 585 } 586 i915_request_wait(request, 0, MAX_SCHEDULE_TIMEOUT); 587 i915_request_put(request); 588 589 times[1] = ktime_sub(ktime_get_raw(), times[1]); 590 if (prime == 1) 591 times[0] = times[1]; 592 593 if (__igt_timeout(end_time, NULL)) 594 break; 595 } 596 intel_engine_pm_put(engine); 597 598 err = igt_live_test_end(&t); 599 if (err) 600 return err; 601 602 pr_info("Request latencies on %s: 1 = %lluns, %lu = %lluns\n", 603 engine->name, 604 ktime_to_ns(times[0]), 605 prime, div64_u64(ktime_to_ns(times[1]), prime)); 606 } 607 608 return err; 609} 610 611static struct i915_vma *empty_batch(struct drm_i915_private *i915) 612{ 613 struct drm_i915_gem_object *obj; 614 struct i915_vma *vma; 615 u32 *cmd; 616 int err; 617 618 obj = i915_gem_object_create_internal(i915, PAGE_SIZE); 619 if (IS_ERR(obj)) 620 return ERR_CAST(obj); 621 622 cmd = i915_gem_object_pin_map(obj, I915_MAP_WB); 623 if (IS_ERR(cmd)) { 624 err = PTR_ERR(cmd); 625 goto err; 626 } 627 628 *cmd = MI_BATCH_BUFFER_END; 629 630 __i915_gem_object_flush_map(obj, 0, 64); 631 i915_gem_object_unpin_map(obj); 632 633 intel_gt_chipset_flush(&i915->gt); 634 635 vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); 636 if (IS_ERR(vma)) { 637 err = PTR_ERR(vma); 638 goto err; 639 } 640 641 err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_GLOBAL); 642 if (err) 643 goto err; 644 645 /* Force the wait wait now to avoid including it in the benchmark */ 646 err = i915_vma_sync(vma); 647 if (err) 648 goto err_pin; 649 650 return vma; 651 652err_pin: 653 i915_vma_unpin(vma); 654err: 655 i915_gem_object_put(obj); 656 return ERR_PTR(err); 657} 658 659static struct i915_request * 660empty_request(struct intel_engine_cs *engine, 661 struct i915_vma *batch) 662{ 663 struct i915_request *request; 664 int err; 665 666 request = i915_request_create(engine->kernel_context); 667 if (IS_ERR(request)) 668 return request; 669 670 err = engine->emit_bb_start(request, 671 batch->node.start, 672 batch->node.size, 673 I915_DISPATCH_SECURE); 674 if (err) 675 goto out_request; 676 677 i915_request_get(request); 678out_request: 679 i915_request_add(request); 680 return err ? ERR_PTR(err) : request; 681} 682 683static int live_empty_request(void *arg) 684{ 685 struct drm_i915_private *i915 = arg; 686 struct intel_engine_cs *engine; 687 struct igt_live_test t; 688 struct i915_vma *batch; 689 int err = 0; 690 691 /* 692 * Submit various sized batches of empty requests, to each engine 693 * (individually), and wait for the batch to complete. We can check 694 * the overhead of submitting requests to the hardware. 695 */ 696 697 batch = empty_batch(i915); 698 if (IS_ERR(batch)) 699 return PTR_ERR(batch); 700 701 for_each_uabi_engine(engine, i915) { 702 IGT_TIMEOUT(end_time); 703 struct i915_request *request; 704 unsigned long n, prime; 705 ktime_t times[2] = {}; 706 707 err = igt_live_test_begin(&t, i915, __func__, engine->name); 708 if (err) 709 goto out_batch; 710 711 intel_engine_pm_get(engine); 712 713 /* Warmup / preload */ 714 request = empty_request(engine, batch); 715 if (IS_ERR(request)) { 716 err = PTR_ERR(request); 717 intel_engine_pm_put(engine); 718 goto out_batch; 719 } 720 i915_request_wait(request, 0, MAX_SCHEDULE_TIMEOUT); 721 722 for_each_prime_number_from(prime, 1, 8192) { 723 times[1] = ktime_get_raw(); 724 725 for (n = 0; n < prime; n++) { 726 i915_request_put(request); 727 request = empty_request(engine, batch); 728 if (IS_ERR(request)) { 729 err = PTR_ERR(request); 730 intel_engine_pm_put(engine); 731 goto out_batch; 732 } 733 } 734 i915_request_wait(request, 0, MAX_SCHEDULE_TIMEOUT); 735 736 times[1] = ktime_sub(ktime_get_raw(), times[1]); 737 if (prime == 1) 738 times[0] = times[1]; 739 740 if (__igt_timeout(end_time, NULL)) 741 break; 742 } 743 i915_request_put(request); 744 intel_engine_pm_put(engine); 745 746 err = igt_live_test_end(&t); 747 if (err) 748 goto out_batch; 749 750 pr_info("Batch latencies on %s: 1 = %lluns, %lu = %lluns\n", 751 engine->name, 752 ktime_to_ns(times[0]), 753 prime, div64_u64(ktime_to_ns(times[1]), prime)); 754 } 755 756out_batch: 757 i915_vma_unpin(batch); 758 i915_vma_put(batch); 759 return err; 760} 761 762static struct i915_vma *recursive_batch(struct drm_i915_private *i915) 763{ 764 struct drm_i915_gem_object *obj; 765 const int gen = INTEL_GEN(i915); 766 struct i915_vma *vma; 767 u32 *cmd; 768 int err; 769 770 obj = i915_gem_object_create_internal(i915, PAGE_SIZE); 771 if (IS_ERR(obj)) 772 return ERR_CAST(obj); 773 774 vma = i915_vma_instance(obj, i915->gt.vm, NULL); 775 if (IS_ERR(vma)) { 776 err = PTR_ERR(vma); 777 goto err; 778 } 779 780 err = i915_vma_pin(vma, 0, 0, PIN_USER); 781 if (err) 782 goto err; 783 784 cmd = i915_gem_object_pin_map(obj, I915_MAP_WC); 785 if (IS_ERR(cmd)) { 786 err = PTR_ERR(cmd); 787 goto err; 788 } 789 790 if (gen >= 8) { 791 *cmd++ = MI_BATCH_BUFFER_START | 1 << 8 | 1; 792 *cmd++ = lower_32_bits(vma->node.start); 793 *cmd++ = upper_32_bits(vma->node.start); 794 } else if (gen >= 6) { 795 *cmd++ = MI_BATCH_BUFFER_START | 1 << 8; 796 *cmd++ = lower_32_bits(vma->node.start); 797 } else { 798 *cmd++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT; 799 *cmd++ = lower_32_bits(vma->node.start); 800 } 801 *cmd++ = MI_BATCH_BUFFER_END; /* terminate early in case of error */ 802 803 __i915_gem_object_flush_map(obj, 0, 64); 804 i915_gem_object_unpin_map(obj); 805 806 intel_gt_chipset_flush(&i915->gt); 807 808 return vma; 809 810err: 811 i915_gem_object_put(obj); 812 return ERR_PTR(err); 813} 814 815static int recursive_batch_resolve(struct i915_vma *batch) 816{ 817 u32 *cmd; 818 819 cmd = i915_gem_object_pin_map(batch->obj, I915_MAP_WC); 820 if (IS_ERR(cmd)) 821 return PTR_ERR(cmd); 822 823 *cmd = MI_BATCH_BUFFER_END; 824 825 __i915_gem_object_flush_map(batch->obj, 0, sizeof(*cmd)); 826 i915_gem_object_unpin_map(batch->obj); 827 828 intel_gt_chipset_flush(batch->vm->gt); 829 830 return 0; 831} 832 833static int live_all_engines(void *arg) 834{ 835 struct drm_i915_private *i915 = arg; 836 const unsigned int nengines = num_uabi_engines(i915); 837 struct intel_engine_cs *engine; 838 struct i915_request **request; 839 struct igt_live_test t; 840 struct i915_vma *batch; 841 unsigned int idx; 842 int err; 843 844 /* 845 * Check we can submit requests to all engines simultaneously. We 846 * send a recursive batch to each engine - checking that we don't 847 * block doing so, and that they don't complete too soon. 848 */ 849 850 request = kcalloc(nengines, sizeof(*request), GFP_KERNEL); 851 if (!request) 852 return -ENOMEM; 853 854 err = igt_live_test_begin(&t, i915, __func__, ""); 855 if (err) 856 goto out_free; 857 858 batch = recursive_batch(i915); 859 if (IS_ERR(batch)) { 860 err = PTR_ERR(batch); 861 pr_err("%s: Unable to create batch, err=%d\n", __func__, err); 862 goto out_free; 863 } 864 865 i915_vma_lock(batch); 866 867 idx = 0; 868 for_each_uabi_engine(engine, i915) { 869 request[idx] = intel_engine_create_kernel_request(engine); 870 if (IS_ERR(request[idx])) { 871 err = PTR_ERR(request[idx]); 872 pr_err("%s: Request allocation failed with err=%d\n", 873 __func__, err); 874 goto out_request; 875 } 876 877 err = i915_request_await_object(request[idx], batch->obj, 0); 878 if (err == 0) 879 err = i915_vma_move_to_active(batch, request[idx], 0); 880 GEM_BUG_ON(err); 881 882 err = engine->emit_bb_start(request[idx], 883 batch->node.start, 884 batch->node.size, 885 0); 886 GEM_BUG_ON(err); 887 request[idx]->batch = batch; 888 889 i915_request_get(request[idx]); 890 i915_request_add(request[idx]); 891 idx++; 892 } 893 894 i915_vma_unlock(batch); 895 896 idx = 0; 897 for_each_uabi_engine(engine, i915) { 898 if (i915_request_completed(request[idx])) { 899 pr_err("%s(%s): request completed too early!\n", 900 __func__, engine->name); 901 err = -EINVAL; 902 goto out_request; 903 } 904 idx++; 905 } 906 907 err = recursive_batch_resolve(batch); 908 if (err) { 909 pr_err("%s: failed to resolve batch, err=%d\n", __func__, err); 910 goto out_request; 911 } 912 913 idx = 0; 914 for_each_uabi_engine(engine, i915) { 915 long timeout; 916 917 timeout = i915_request_wait(request[idx], 0, 918 MAX_SCHEDULE_TIMEOUT); 919 if (timeout < 0) { 920 err = timeout; 921 pr_err("%s: error waiting for request on %s, err=%d\n", 922 __func__, engine->name, err); 923 goto out_request; 924 } 925 926 GEM_BUG_ON(!i915_request_completed(request[idx])); 927 i915_request_put(request[idx]); 928 request[idx] = NULL; 929 idx++; 930 } 931 932 err = igt_live_test_end(&t); 933 934out_request: 935 idx = 0; 936 for_each_uabi_engine(engine, i915) { 937 if (request[idx]) 938 i915_request_put(request[idx]); 939 idx++; 940 } 941 i915_vma_unpin(batch); 942 i915_vma_put(batch); 943out_free: 944 kfree(request); 945 return err; 946} 947 948static int live_sequential_engines(void *arg) 949{ 950 struct drm_i915_private *i915 = arg; 951 const unsigned int nengines = num_uabi_engines(i915); 952 struct i915_request **request; 953 struct i915_request *prev = NULL; 954 struct intel_engine_cs *engine; 955 struct igt_live_test t; 956 unsigned int idx; 957 int err; 958 959 /* 960 * Check we can submit requests to all engines sequentially, such 961 * that each successive request waits for the earlier ones. This 962 * tests that we don't execute requests out of order, even though 963 * they are running on independent engines. 964 */ 965 966 request = kcalloc(nengines, sizeof(*request), GFP_KERNEL); 967 if (!request) 968 return -ENOMEM; 969 970 err = igt_live_test_begin(&t, i915, __func__, ""); 971 if (err) 972 goto out_free; 973 974 idx = 0; 975 for_each_uabi_engine(engine, i915) { 976 struct i915_vma *batch; 977 978 batch = recursive_batch(i915); 979 if (IS_ERR(batch)) { 980 err = PTR_ERR(batch); 981 pr_err("%s: Unable to create batch for %s, err=%d\n", 982 __func__, engine->name, err); 983 goto out_free; 984 } 985 986 i915_vma_lock(batch); 987 request[idx] = intel_engine_create_kernel_request(engine); 988 if (IS_ERR(request[idx])) { 989 err = PTR_ERR(request[idx]); 990 pr_err("%s: Request allocation failed for %s with err=%d\n", 991 __func__, engine->name, err); 992 goto out_unlock; 993 } 994 995 if (prev) { 996 err = i915_request_await_dma_fence(request[idx], 997 &prev->fence); 998 if (err) { 999 i915_request_add(request[idx]); 1000 pr_err("%s: Request await failed for %s with err=%d\n", 1001 __func__, engine->name, err); 1002 goto out_unlock; 1003 } 1004 } 1005 1006 err = i915_request_await_object(request[idx], 1007 batch->obj, false); 1008 if (err == 0) 1009 err = i915_vma_move_to_active(batch, request[idx], 0); 1010 GEM_BUG_ON(err); 1011 1012 err = engine->emit_bb_start(request[idx], 1013 batch->node.start, 1014 batch->node.size, 1015 0); 1016 GEM_BUG_ON(err); 1017 request[idx]->batch = batch; 1018 1019 i915_request_get(request[idx]); 1020 i915_request_add(request[idx]); 1021 1022 prev = request[idx]; 1023 idx++; 1024 1025out_unlock: 1026 i915_vma_unlock(batch); 1027 if (err) 1028 goto out_request; 1029 } 1030 1031 idx = 0; 1032 for_each_uabi_engine(engine, i915) { 1033 long timeout; 1034 1035 if (i915_request_completed(request[idx])) { 1036 pr_err("%s(%s): request completed too early!\n", 1037 __func__, engine->name); 1038 err = -EINVAL; 1039 goto out_request; 1040 } 1041 1042 err = recursive_batch_resolve(request[idx]->batch); 1043 if (err) { 1044 pr_err("%s: failed to resolve batch, err=%d\n", 1045 __func__, err); 1046 goto out_request; 1047 } 1048 1049 timeout = i915_request_wait(request[idx], 0, 1050 MAX_SCHEDULE_TIMEOUT); 1051 if (timeout < 0) { 1052 err = timeout; 1053 pr_err("%s: error waiting for request on %s, err=%d\n", 1054 __func__, engine->name, err); 1055 goto out_request; 1056 } 1057 1058 GEM_BUG_ON(!i915_request_completed(request[idx])); 1059 idx++; 1060 } 1061 1062 err = igt_live_test_end(&t); 1063 1064out_request: 1065 idx = 0; 1066 for_each_uabi_engine(engine, i915) { 1067 u32 *cmd; 1068 1069 if (!request[idx]) 1070 break; 1071 1072 cmd = i915_gem_object_pin_map(request[idx]->batch->obj, 1073 I915_MAP_WC); 1074 if (!IS_ERR(cmd)) { 1075 *cmd = MI_BATCH_BUFFER_END; 1076 1077 __i915_gem_object_flush_map(request[idx]->batch->obj, 1078 0, sizeof(*cmd)); 1079 i915_gem_object_unpin_map(request[idx]->batch->obj); 1080 1081 intel_gt_chipset_flush(engine->gt); 1082 } 1083 1084 i915_vma_put(request[idx]->batch); 1085 i915_request_put(request[idx]); 1086 idx++; 1087 } 1088out_free: 1089 kfree(request); 1090 return err; 1091} 1092 1093static int __live_parallel_engine1(void *arg) 1094{ 1095 struct intel_engine_cs *engine = arg; 1096 IGT_TIMEOUT(end_time); 1097 unsigned long count; 1098 int err = 0; 1099 1100 count = 0; 1101 intel_engine_pm_get(engine); 1102 do { 1103 struct i915_request *rq; 1104 1105 rq = i915_request_create(engine->kernel_context); 1106 if (IS_ERR(rq)) { 1107 err = PTR_ERR(rq); 1108 break; 1109 } 1110 1111 i915_request_get(rq); 1112 i915_request_add(rq); 1113 1114 err = 0; 1115 if (i915_request_wait(rq, 0, HZ / 5) < 0) 1116 err = -ETIME; 1117 i915_request_put(rq); 1118 if (err) 1119 break; 1120 1121 count++; 1122 } while (!__igt_timeout(end_time, NULL)); 1123 intel_engine_pm_put(engine); 1124 1125 pr_info("%s: %lu request + sync\n", engine->name, count); 1126 return err; 1127} 1128 1129static int __live_parallel_engineN(void *arg) 1130{ 1131 struct intel_engine_cs *engine = arg; 1132 IGT_TIMEOUT(end_time); 1133 unsigned long count; 1134 int err = 0; 1135 1136 count = 0; 1137 intel_engine_pm_get(engine); 1138 do { 1139 struct i915_request *rq; 1140 1141 rq = i915_request_create(engine->kernel_context); 1142 if (IS_ERR(rq)) { 1143 err = PTR_ERR(rq); 1144 break; 1145 } 1146 1147 i915_request_add(rq); 1148 count++; 1149 } while (!__igt_timeout(end_time, NULL)); 1150 intel_engine_pm_put(engine); 1151 1152 pr_info("%s: %lu requests\n", engine->name, count); 1153 return err; 1154} 1155 1156static bool wake_all(struct drm_i915_private *i915) 1157{ 1158 if (atomic_dec_and_test(&i915->selftest.counter)) { 1159 wake_up_var(&i915->selftest.counter); 1160 return true; 1161 } 1162 1163 return false; 1164} 1165 1166static int wait_for_all(struct drm_i915_private *i915) 1167{ 1168 if (wake_all(i915)) 1169 return 0; 1170 1171 if (wait_var_event_timeout(&i915->selftest.counter, 1172 !atomic_read(&i915->selftest.counter), 1173 i915_selftest.timeout_jiffies)) 1174 return 0; 1175 1176 return -ETIME; 1177} 1178 1179static int __live_parallel_spin(void *arg) 1180{ 1181 struct intel_engine_cs *engine = arg; 1182 struct igt_spinner spin; 1183 struct i915_request *rq; 1184 int err = 0; 1185 1186 /* 1187 * Create a spinner running for eternity on each engine. If a second 1188 * spinner is incorrectly placed on the same engine, it will not be 1189 * able to start in time. 1190 */ 1191 1192 if (igt_spinner_init(&spin, engine->gt)) { 1193 wake_all(engine->i915); 1194 return -ENOMEM; 1195 } 1196 1197 intel_engine_pm_get(engine); 1198 rq = igt_spinner_create_request(&spin, 1199 engine->kernel_context, 1200 MI_NOOP); /* no preemption */ 1201 intel_engine_pm_put(engine); 1202 if (IS_ERR(rq)) { 1203 err = PTR_ERR(rq); 1204 if (err == -ENODEV) 1205 err = 0; 1206 wake_all(engine->i915); 1207 goto out_spin; 1208 } 1209 1210 i915_request_get(rq); 1211 i915_request_add(rq); 1212 if (igt_wait_for_spinner(&spin, rq)) { 1213 /* Occupy this engine for the whole test */ 1214 err = wait_for_all(engine->i915); 1215 } else { 1216 pr_err("Failed to start spinner on %s\n", engine->name); 1217 err = -EINVAL; 1218 } 1219 igt_spinner_end(&spin); 1220 1221 if (err == 0 && i915_request_wait(rq, 0, HZ / 5) < 0) 1222 err = -EIO; 1223 i915_request_put(rq); 1224 1225out_spin: 1226 igt_spinner_fini(&spin); 1227 return err; 1228} 1229 1230static int live_parallel_engines(void *arg) 1231{ 1232 struct drm_i915_private *i915 = arg; 1233 static int (* const func[])(void *arg) = { 1234 __live_parallel_engine1, 1235 __live_parallel_engineN, 1236 __live_parallel_spin, 1237 NULL, 1238 }; 1239 const unsigned int nengines = num_uabi_engines(i915); 1240 struct intel_engine_cs *engine; 1241 int (* const *fn)(void *arg); 1242 struct task_struct **tsk; 1243 int err = 0; 1244 1245 /* 1246 * Check we can submit requests to all engines concurrently. This 1247 * tests that we load up the system maximally. 1248 */ 1249 1250 tsk = kcalloc(nengines, sizeof(*tsk), GFP_KERNEL); 1251 if (!tsk) 1252 return -ENOMEM; 1253 1254 for (fn = func; !err && *fn; fn++) { 1255 char name[KSYM_NAME_LEN]; 1256 struct igt_live_test t; 1257 unsigned int idx; 1258 1259 snprintf(name, sizeof(name), "%ps", *fn); 1260 err = igt_live_test_begin(&t, i915, __func__, name); 1261 if (err) 1262 break; 1263 1264 atomic_set(&i915->selftest.counter, nengines); 1265 1266 idx = 0; 1267 for_each_uabi_engine(engine, i915) { 1268 tsk[idx] = kthread_run(*fn, engine, 1269 "igt/parallel:%s", 1270 engine->name); 1271 if (IS_ERR(tsk[idx])) { 1272 err = PTR_ERR(tsk[idx]); 1273 break; 1274 } 1275 get_task_struct(tsk[idx++]); 1276 } 1277 1278 yield(); /* start all threads before we kthread_stop() */ 1279 1280 idx = 0; 1281 for_each_uabi_engine(engine, i915) { 1282 int status; 1283 1284 if (IS_ERR(tsk[idx])) 1285 break; 1286 1287 status = kthread_stop(tsk[idx]); 1288 if (status && !err) 1289 err = status; 1290 1291 put_task_struct(tsk[idx++]); 1292 } 1293 1294 if (igt_live_test_end(&t)) 1295 err = -EIO; 1296 } 1297 1298 kfree(tsk); 1299 return err; 1300} 1301 1302static int 1303max_batches(struct i915_gem_context *ctx, struct intel_engine_cs *engine) 1304{ 1305 struct i915_request *rq; 1306 int ret; 1307 1308 /* 1309 * Before execlists, all contexts share the same ringbuffer. With 1310 * execlists, each context/engine has a separate ringbuffer and 1311 * for the purposes of this test, inexhaustible. 1312 * 1313 * For the global ringbuffer though, we have to be very careful 1314 * that we do not wrap while preventing the execution of requests 1315 * with a unsignaled fence. 1316 */ 1317 if (HAS_EXECLISTS(ctx->i915)) 1318 return INT_MAX; 1319 1320 rq = igt_request_alloc(ctx, engine); 1321 if (IS_ERR(rq)) { 1322 ret = PTR_ERR(rq); 1323 } else { 1324 int sz; 1325 1326 ret = rq->ring->size - rq->reserved_space; 1327 i915_request_add(rq); 1328 1329 sz = rq->ring->emit - rq->head; 1330 if (sz < 0) 1331 sz += rq->ring->size; 1332 ret /= sz; 1333 ret /= 2; /* leave half spare, in case of emergency! */ 1334 } 1335 1336 return ret; 1337} 1338 1339static int live_breadcrumbs_smoketest(void *arg) 1340{ 1341 struct drm_i915_private *i915 = arg; 1342 const unsigned int nengines = num_uabi_engines(i915); 1343 const unsigned int ncpus = num_online_cpus(); 1344 unsigned long num_waits, num_fences; 1345 struct intel_engine_cs *engine; 1346 struct task_struct **threads; 1347 struct igt_live_test live; 1348 intel_wakeref_t wakeref; 1349 struct smoketest *smoke; 1350 unsigned int n, idx; 1351 struct file *file; 1352 int ret = 0; 1353 1354 /* 1355 * Smoketest our breadcrumb/signal handling for requests across multiple 1356 * threads. A very simple test to only catch the most egregious of bugs. 1357 * See __igt_breadcrumbs_smoketest(); 1358 * 1359 * On real hardware this time. 1360 */ 1361 1362 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 1363 1364 file = mock_file(i915); 1365 if (IS_ERR(file)) { 1366 ret = PTR_ERR(file); 1367 goto out_rpm; 1368 } 1369 1370 smoke = kcalloc(nengines, sizeof(*smoke), GFP_KERNEL); 1371 if (!smoke) { 1372 ret = -ENOMEM; 1373 goto out_file; 1374 } 1375 1376 threads = kcalloc(ncpus * nengines, sizeof(*threads), GFP_KERNEL); 1377 if (!threads) { 1378 ret = -ENOMEM; 1379 goto out_smoke; 1380 } 1381 1382 smoke[0].request_alloc = __live_request_alloc; 1383 smoke[0].ncontexts = 64; 1384 smoke[0].contexts = kcalloc(smoke[0].ncontexts, 1385 sizeof(*smoke[0].contexts), 1386 GFP_KERNEL); 1387 if (!smoke[0].contexts) { 1388 ret = -ENOMEM; 1389 goto out_threads; 1390 } 1391 1392 for (n = 0; n < smoke[0].ncontexts; n++) { 1393 smoke[0].contexts[n] = live_context(i915, file); 1394 if (IS_ERR(smoke[0].contexts[n])) { 1395 ret = PTR_ERR(smoke[0].contexts[n]); 1396 goto out_contexts; 1397 } 1398 } 1399 1400 ret = igt_live_test_begin(&live, i915, __func__, ""); 1401 if (ret) 1402 goto out_contexts; 1403 1404 idx = 0; 1405 for_each_uabi_engine(engine, i915) { 1406 smoke[idx] = smoke[0]; 1407 smoke[idx].engine = engine; 1408 smoke[idx].max_batch = 1409 max_batches(smoke[0].contexts[0], engine); 1410 if (smoke[idx].max_batch < 0) { 1411 ret = smoke[idx].max_batch; 1412 goto out_flush; 1413 } 1414 /* One ring interleaved between requests from all cpus */ 1415 smoke[idx].max_batch /= num_online_cpus() + 1; 1416 pr_debug("Limiting batches to %d requests on %s\n", 1417 smoke[idx].max_batch, engine->name); 1418 1419 for (n = 0; n < ncpus; n++) { 1420 struct task_struct *tsk; 1421 1422 tsk = kthread_run(__igt_breadcrumbs_smoketest, 1423 &smoke[idx], "igt/%d.%d", idx, n); 1424 if (IS_ERR(tsk)) { 1425 ret = PTR_ERR(tsk); 1426 goto out_flush; 1427 } 1428 1429 get_task_struct(tsk); 1430 threads[idx * ncpus + n] = tsk; 1431 } 1432 1433 idx++; 1434 } 1435 1436 yield(); /* start all threads before we begin */ 1437 msleep(jiffies_to_msecs(i915_selftest.timeout_jiffies)); 1438 1439out_flush: 1440 idx = 0; 1441 num_waits = 0; 1442 num_fences = 0; 1443 for_each_uabi_engine(engine, i915) { 1444 for (n = 0; n < ncpus; n++) { 1445 struct task_struct *tsk = threads[idx * ncpus + n]; 1446 int err; 1447 1448 if (!tsk) 1449 continue; 1450 1451 err = kthread_stop(tsk); 1452 if (err < 0 && !ret) 1453 ret = err; 1454 1455 put_task_struct(tsk); 1456 } 1457 1458 num_waits += atomic_long_read(&smoke[idx].num_waits); 1459 num_fences += atomic_long_read(&smoke[idx].num_fences); 1460 idx++; 1461 } 1462 pr_info("Completed %lu waits for %lu fences across %d engines and %d cpus\n", 1463 num_waits, num_fences, idx, ncpus); 1464 1465 ret = igt_live_test_end(&live) ?: ret; 1466out_contexts: 1467 kfree(smoke[0].contexts); 1468out_threads: 1469 kfree(threads); 1470out_smoke: 1471 kfree(smoke); 1472out_file: 1473 fput(file); 1474out_rpm: 1475 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 1476 1477 return ret; 1478} 1479 1480int i915_request_live_selftests(struct drm_i915_private *i915) 1481{ 1482 static const struct i915_subtest tests[] = { 1483 SUBTEST(live_nop_request), 1484 SUBTEST(live_all_engines), 1485 SUBTEST(live_sequential_engines), 1486 SUBTEST(live_parallel_engines), 1487 SUBTEST(live_empty_request), 1488 SUBTEST(live_breadcrumbs_smoketest), 1489 }; 1490 1491 if (intel_gt_is_wedged(&i915->gt)) 1492 return 0; 1493 1494 return i915_subtests(tests, i915); 1495} 1496 1497static int switch_to_kernel_sync(struct intel_context *ce, int err) 1498{ 1499 struct i915_request *rq; 1500 struct dma_fence *fence; 1501 1502 rq = intel_engine_create_kernel_request(ce->engine); 1503 if (IS_ERR(rq)) 1504 return PTR_ERR(rq); 1505 1506 fence = i915_active_fence_get(&ce->timeline->last_request); 1507 if (fence) { 1508 i915_request_await_dma_fence(rq, fence); 1509 dma_fence_put(fence); 1510 } 1511 1512 rq = i915_request_get(rq); 1513 i915_request_add(rq); 1514 if (i915_request_wait(rq, 0, HZ / 2) < 0 && !err) 1515 err = -ETIME; 1516 i915_request_put(rq); 1517 1518 while (!err && !intel_engine_is_idle(ce->engine)) 1519 intel_engine_flush_submission(ce->engine); 1520 1521 return err; 1522} 1523 1524struct perf_stats { 1525 struct intel_engine_cs *engine; 1526 unsigned long count; 1527 ktime_t time; 1528 ktime_t busy; 1529 u64 runtime; 1530}; 1531 1532struct perf_series { 1533 struct drm_i915_private *i915; 1534 unsigned int nengines; 1535 struct intel_context *ce[]; 1536}; 1537 1538static int cmp_u32(const void *A, const void *B) 1539{ 1540 const u32 *a = A, *b = B; 1541 1542 return *a - *b; 1543} 1544 1545static u32 trifilter(u32 *a) 1546{ 1547 u64 sum; 1548 1549#define TF_COUNT 5 1550 sort(a, TF_COUNT, sizeof(*a), cmp_u32, NULL); 1551 1552 sum = mul_u32_u32(a[2], 2); 1553 sum += a[1]; 1554 sum += a[3]; 1555 1556 GEM_BUG_ON(sum > U32_MAX); 1557 return sum; 1558#define TF_BIAS 2 1559} 1560 1561static u64 cycles_to_ns(struct intel_engine_cs *engine, u32 cycles) 1562{ 1563 u64 ns = i915_cs_timestamp_ticks_to_ns(engine->i915, cycles); 1564 1565 return DIV_ROUND_CLOSEST(ns, 1 << TF_BIAS); 1566} 1567 1568static u32 *emit_timestamp_store(u32 *cs, struct intel_context *ce, u32 offset) 1569{ 1570 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; 1571 *cs++ = i915_mmio_reg_offset(RING_TIMESTAMP((ce->engine->mmio_base))); 1572 *cs++ = offset; 1573 *cs++ = 0; 1574 1575 return cs; 1576} 1577 1578static u32 *emit_store_dw(u32 *cs, u32 offset, u32 value) 1579{ 1580 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 1581 *cs++ = offset; 1582 *cs++ = 0; 1583 *cs++ = value; 1584 1585 return cs; 1586} 1587 1588static u32 *emit_semaphore_poll(u32 *cs, u32 mode, u32 value, u32 offset) 1589{ 1590 *cs++ = MI_SEMAPHORE_WAIT | 1591 MI_SEMAPHORE_GLOBAL_GTT | 1592 MI_SEMAPHORE_POLL | 1593 mode; 1594 *cs++ = value; 1595 *cs++ = offset; 1596 *cs++ = 0; 1597 1598 return cs; 1599} 1600 1601static u32 *emit_semaphore_poll_until(u32 *cs, u32 offset, u32 value) 1602{ 1603 return emit_semaphore_poll(cs, MI_SEMAPHORE_SAD_EQ_SDD, value, offset); 1604} 1605 1606static void semaphore_set(u32 *sema, u32 value) 1607{ 1608 WRITE_ONCE(*sema, value); 1609 wmb(); /* flush the update to the cache, and beyond */ 1610} 1611 1612static u32 *hwsp_scratch(const struct intel_context *ce) 1613{ 1614 return memset32(ce->engine->status_page.addr + 1000, 0, 21); 1615} 1616 1617static u32 hwsp_offset(const struct intel_context *ce, u32 *dw) 1618{ 1619 return (i915_ggtt_offset(ce->engine->status_page.vma) + 1620 offset_in_page(dw)); 1621} 1622 1623static int measure_semaphore_response(struct intel_context *ce) 1624{ 1625 u32 *sema = hwsp_scratch(ce); 1626 const u32 offset = hwsp_offset(ce, sema); 1627 u32 elapsed[TF_COUNT], cycles; 1628 struct i915_request *rq; 1629 u32 *cs; 1630 int err; 1631 int i; 1632 1633 /* 1634 * Measure how many cycles it takes for the HW to detect the change 1635 * in a semaphore value. 1636 * 1637 * A: read CS_TIMESTAMP from CPU 1638 * poke semaphore 1639 * B: read CS_TIMESTAMP on GPU 1640 * 1641 * Semaphore latency: B - A 1642 */ 1643 1644 semaphore_set(sema, -1); 1645 1646 rq = i915_request_create(ce); 1647 if (IS_ERR(rq)) 1648 return PTR_ERR(rq); 1649 1650 cs = intel_ring_begin(rq, 4 + 12 * ARRAY_SIZE(elapsed)); 1651 if (IS_ERR(cs)) { 1652 i915_request_add(rq); 1653 err = PTR_ERR(cs); 1654 goto err; 1655 } 1656 1657 cs = emit_store_dw(cs, offset, 0); 1658 for (i = 1; i <= ARRAY_SIZE(elapsed); i++) { 1659 cs = emit_semaphore_poll_until(cs, offset, i); 1660 cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32)); 1661 cs = emit_store_dw(cs, offset, 0); 1662 } 1663 1664 intel_ring_advance(rq, cs); 1665 i915_request_add(rq); 1666 1667 if (wait_for(READ_ONCE(*sema) == 0, 50)) { 1668 err = -EIO; 1669 goto err; 1670 } 1671 1672 for (i = 1; i <= ARRAY_SIZE(elapsed); i++) { 1673 preempt_disable(); 1674 cycles = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP); 1675 semaphore_set(sema, i); 1676 preempt_enable(); 1677 1678 if (wait_for(READ_ONCE(*sema) == 0, 50)) { 1679 err = -EIO; 1680 goto err; 1681 } 1682 1683 elapsed[i - 1] = sema[i] - cycles; 1684 } 1685 1686 cycles = trifilter(elapsed); 1687 pr_info("%s: semaphore response %d cycles, %lluns\n", 1688 ce->engine->name, cycles >> TF_BIAS, 1689 cycles_to_ns(ce->engine, cycles)); 1690 1691 return intel_gt_wait_for_idle(ce->engine->gt, HZ); 1692 1693err: 1694 intel_gt_set_wedged(ce->engine->gt); 1695 return err; 1696} 1697 1698static int measure_idle_dispatch(struct intel_context *ce) 1699{ 1700 u32 *sema = hwsp_scratch(ce); 1701 const u32 offset = hwsp_offset(ce, sema); 1702 u32 elapsed[TF_COUNT], cycles; 1703 u32 *cs; 1704 int err; 1705 int i; 1706 1707 /* 1708 * Measure how long it takes for us to submit a request while the 1709 * engine is idle, but is resting in our context. 1710 * 1711 * A: read CS_TIMESTAMP from CPU 1712 * submit request 1713 * B: read CS_TIMESTAMP on GPU 1714 * 1715 * Submission latency: B - A 1716 */ 1717 1718 for (i = 0; i < ARRAY_SIZE(elapsed); i++) { 1719 struct i915_request *rq; 1720 1721 err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2); 1722 if (err) 1723 return err; 1724 1725 rq = i915_request_create(ce); 1726 if (IS_ERR(rq)) { 1727 err = PTR_ERR(rq); 1728 goto err; 1729 } 1730 1731 cs = intel_ring_begin(rq, 4); 1732 if (IS_ERR(cs)) { 1733 i915_request_add(rq); 1734 err = PTR_ERR(cs); 1735 goto err; 1736 } 1737 1738 cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32)); 1739 1740 intel_ring_advance(rq, cs); 1741 1742 preempt_disable(); 1743 local_bh_disable(); 1744 elapsed[i] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP); 1745 i915_request_add(rq); 1746 local_bh_enable(); 1747 preempt_enable(); 1748 } 1749 1750 err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2); 1751 if (err) 1752 goto err; 1753 1754 for (i = 0; i < ARRAY_SIZE(elapsed); i++) 1755 elapsed[i] = sema[i] - elapsed[i]; 1756 1757 cycles = trifilter(elapsed); 1758 pr_info("%s: idle dispatch latency %d cycles, %lluns\n", 1759 ce->engine->name, cycles >> TF_BIAS, 1760 cycles_to_ns(ce->engine, cycles)); 1761 1762 return intel_gt_wait_for_idle(ce->engine->gt, HZ); 1763 1764err: 1765 intel_gt_set_wedged(ce->engine->gt); 1766 return err; 1767} 1768 1769static int measure_busy_dispatch(struct intel_context *ce) 1770{ 1771 u32 *sema = hwsp_scratch(ce); 1772 const u32 offset = hwsp_offset(ce, sema); 1773 u32 elapsed[TF_COUNT + 1], cycles; 1774 u32 *cs; 1775 int err; 1776 int i; 1777 1778 /* 1779 * Measure how long it takes for us to submit a request while the 1780 * engine is busy, polling on a semaphore in our context. With 1781 * direct submission, this will include the cost of a lite restore. 1782 * 1783 * A: read CS_TIMESTAMP from CPU 1784 * submit request 1785 * B: read CS_TIMESTAMP on GPU 1786 * 1787 * Submission latency: B - A 1788 */ 1789 1790 for (i = 1; i <= ARRAY_SIZE(elapsed); i++) { 1791 struct i915_request *rq; 1792 1793 rq = i915_request_create(ce); 1794 if (IS_ERR(rq)) { 1795 err = PTR_ERR(rq); 1796 goto err; 1797 } 1798 1799 cs = intel_ring_begin(rq, 12); 1800 if (IS_ERR(cs)) { 1801 i915_request_add(rq); 1802 err = PTR_ERR(cs); 1803 goto err; 1804 } 1805 1806 cs = emit_store_dw(cs, offset + i * sizeof(u32), -1); 1807 cs = emit_semaphore_poll_until(cs, offset, i); 1808 cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32)); 1809 1810 intel_ring_advance(rq, cs); 1811 1812 if (i > 1 && wait_for(READ_ONCE(sema[i - 1]), 500)) { 1813 err = -EIO; 1814 goto err; 1815 } 1816 1817 preempt_disable(); 1818 local_bh_disable(); 1819 elapsed[i - 1] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP); 1820 i915_request_add(rq); 1821 local_bh_enable(); 1822 semaphore_set(sema, i - 1); 1823 preempt_enable(); 1824 } 1825 1826 wait_for(READ_ONCE(sema[i - 1]), 500); 1827 semaphore_set(sema, i - 1); 1828 1829 for (i = 1; i <= TF_COUNT; i++) { 1830 GEM_BUG_ON(sema[i] == -1); 1831 elapsed[i - 1] = sema[i] - elapsed[i]; 1832 } 1833 1834 cycles = trifilter(elapsed); 1835 pr_info("%s: busy dispatch latency %d cycles, %lluns\n", 1836 ce->engine->name, cycles >> TF_BIAS, 1837 cycles_to_ns(ce->engine, cycles)); 1838 1839 return intel_gt_wait_for_idle(ce->engine->gt, HZ); 1840 1841err: 1842 intel_gt_set_wedged(ce->engine->gt); 1843 return err; 1844} 1845 1846static int plug(struct intel_engine_cs *engine, u32 *sema, u32 mode, int value) 1847{ 1848 const u32 offset = 1849 i915_ggtt_offset(engine->status_page.vma) + 1850 offset_in_page(sema); 1851 struct i915_request *rq; 1852 u32 *cs; 1853 1854 rq = i915_request_create(engine->kernel_context); 1855 if (IS_ERR(rq)) 1856 return PTR_ERR(rq); 1857 1858 cs = intel_ring_begin(rq, 4); 1859 if (IS_ERR(cs)) { 1860 i915_request_add(rq); 1861 return PTR_ERR(cs); 1862 } 1863 1864 cs = emit_semaphore_poll(cs, mode, value, offset); 1865 1866 intel_ring_advance(rq, cs); 1867 i915_request_add(rq); 1868 1869 return 0; 1870} 1871 1872static int measure_inter_request(struct intel_context *ce) 1873{ 1874 u32 *sema = hwsp_scratch(ce); 1875 const u32 offset = hwsp_offset(ce, sema); 1876 u32 elapsed[TF_COUNT + 1], cycles; 1877 struct i915_sw_fence *submit; 1878 int i, err; 1879 1880 /* 1881 * Measure how long it takes to advance from one request into the 1882 * next. Between each request we flush the GPU caches to memory, 1883 * update the breadcrumbs, and then invalidate those caches. 1884 * We queue up all the requests to be submitted in one batch so 1885 * it should be one set of contiguous measurements. 1886 * 1887 * A: read CS_TIMESTAMP on GPU 1888 * advance request 1889 * B: read CS_TIMESTAMP on GPU 1890 * 1891 * Request latency: B - A 1892 */ 1893 1894 err = plug(ce->engine, sema, MI_SEMAPHORE_SAD_NEQ_SDD, 0); 1895 if (err) 1896 return err; 1897 1898 submit = heap_fence_create(GFP_KERNEL); 1899 if (!submit) { 1900 semaphore_set(sema, 1); 1901 return -ENOMEM; 1902 } 1903 1904 intel_engine_flush_submission(ce->engine); 1905 for (i = 1; i <= ARRAY_SIZE(elapsed); i++) { 1906 struct i915_request *rq; 1907 u32 *cs; 1908 1909 rq = i915_request_create(ce); 1910 if (IS_ERR(rq)) { 1911 err = PTR_ERR(rq); 1912 goto err_submit; 1913 } 1914 1915 err = i915_sw_fence_await_sw_fence_gfp(&rq->submit, 1916 submit, 1917 GFP_KERNEL); 1918 if (err < 0) { 1919 i915_request_add(rq); 1920 goto err_submit; 1921 } 1922 1923 cs = intel_ring_begin(rq, 4); 1924 if (IS_ERR(cs)) { 1925 i915_request_add(rq); 1926 err = PTR_ERR(cs); 1927 goto err_submit; 1928 } 1929 1930 cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32)); 1931 1932 intel_ring_advance(rq, cs); 1933 i915_request_add(rq); 1934 } 1935 local_bh_disable(); 1936 i915_sw_fence_commit(submit); 1937 local_bh_enable(); 1938 intel_engine_flush_submission(ce->engine); 1939 heap_fence_put(submit); 1940 1941 semaphore_set(sema, 1); 1942 err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2); 1943 if (err) 1944 goto err; 1945 1946 for (i = 1; i <= TF_COUNT; i++) 1947 elapsed[i - 1] = sema[i + 1] - sema[i]; 1948 1949 cycles = trifilter(elapsed); 1950 pr_info("%s: inter-request latency %d cycles, %lluns\n", 1951 ce->engine->name, cycles >> TF_BIAS, 1952 cycles_to_ns(ce->engine, cycles)); 1953 1954 return intel_gt_wait_for_idle(ce->engine->gt, HZ); 1955 1956err_submit: 1957 i915_sw_fence_commit(submit); 1958 heap_fence_put(submit); 1959 semaphore_set(sema, 1); 1960err: 1961 intel_gt_set_wedged(ce->engine->gt); 1962 return err; 1963} 1964 1965static int measure_context_switch(struct intel_context *ce) 1966{ 1967 u32 *sema = hwsp_scratch(ce); 1968 const u32 offset = hwsp_offset(ce, sema); 1969 struct i915_request *fence = NULL; 1970 u32 elapsed[TF_COUNT + 1], cycles; 1971 int i, j, err; 1972 u32 *cs; 1973 1974 /* 1975 * Measure how long it takes to advance from one request in one 1976 * context to a request in another context. This allows us to 1977 * measure how long the context save/restore take, along with all 1978 * the inter-context setup we require. 1979 * 1980 * A: read CS_TIMESTAMP on GPU 1981 * switch context 1982 * B: read CS_TIMESTAMP on GPU 1983 * 1984 * Context switch latency: B - A 1985 */ 1986 1987 err = plug(ce->engine, sema, MI_SEMAPHORE_SAD_NEQ_SDD, 0); 1988 if (err) 1989 return err; 1990 1991 for (i = 1; i <= ARRAY_SIZE(elapsed); i++) { 1992 struct intel_context *arr[] = { 1993 ce, ce->engine->kernel_context 1994 }; 1995 u32 addr = offset + ARRAY_SIZE(arr) * i * sizeof(u32); 1996 1997 for (j = 0; j < ARRAY_SIZE(arr); j++) { 1998 struct i915_request *rq; 1999 2000 rq = i915_request_create(arr[j]); 2001 if (IS_ERR(rq)) { 2002 err = PTR_ERR(rq); 2003 goto err_fence; 2004 } 2005 2006 if (fence) { 2007 err = i915_request_await_dma_fence(rq, 2008 &fence->fence); 2009 if (err) { 2010 i915_request_add(rq); 2011 goto err_fence; 2012 } 2013 } 2014 2015 cs = intel_ring_begin(rq, 4); 2016 if (IS_ERR(cs)) { 2017 i915_request_add(rq); 2018 err = PTR_ERR(cs); 2019 goto err_fence; 2020 } 2021 2022 cs = emit_timestamp_store(cs, ce, addr); 2023 addr += sizeof(u32); 2024 2025 intel_ring_advance(rq, cs); 2026 2027 i915_request_put(fence); 2028 fence = i915_request_get(rq); 2029 2030 i915_request_add(rq); 2031 } 2032 } 2033 i915_request_put(fence); 2034 intel_engine_flush_submission(ce->engine); 2035 2036 semaphore_set(sema, 1); 2037 err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2); 2038 if (err) 2039 goto err; 2040 2041 for (i = 1; i <= TF_COUNT; i++) 2042 elapsed[i - 1] = sema[2 * i + 2] - sema[2 * i + 1]; 2043 2044 cycles = trifilter(elapsed); 2045 pr_info("%s: context switch latency %d cycles, %lluns\n", 2046 ce->engine->name, cycles >> TF_BIAS, 2047 cycles_to_ns(ce->engine, cycles)); 2048 2049 return intel_gt_wait_for_idle(ce->engine->gt, HZ); 2050 2051err_fence: 2052 i915_request_put(fence); 2053 semaphore_set(sema, 1); 2054err: 2055 intel_gt_set_wedged(ce->engine->gt); 2056 return err; 2057} 2058 2059static int measure_preemption(struct intel_context *ce) 2060{ 2061 u32 *sema = hwsp_scratch(ce); 2062 const u32 offset = hwsp_offset(ce, sema); 2063 u32 elapsed[TF_COUNT], cycles; 2064 u32 *cs; 2065 int err; 2066 int i; 2067 2068 /* 2069 * We measure two latencies while triggering preemption. The first 2070 * latency is how long it takes for us to submit a preempting request. 2071 * The second latency is how it takes for us to return from the 2072 * preemption back to the original context. 2073 * 2074 * A: read CS_TIMESTAMP from CPU 2075 * submit preemption 2076 * B: read CS_TIMESTAMP on GPU (in preempting context) 2077 * context switch 2078 * C: read CS_TIMESTAMP on GPU (in original context) 2079 * 2080 * Preemption dispatch latency: B - A 2081 * Preemption switch latency: C - B 2082 */ 2083 2084 if (!intel_engine_has_preemption(ce->engine)) 2085 return 0; 2086 2087 for (i = 1; i <= ARRAY_SIZE(elapsed); i++) { 2088 u32 addr = offset + 2 * i * sizeof(u32); 2089 struct i915_request *rq; 2090 2091 rq = i915_request_create(ce); 2092 if (IS_ERR(rq)) { 2093 err = PTR_ERR(rq); 2094 goto err; 2095 } 2096 2097 cs = intel_ring_begin(rq, 12); 2098 if (IS_ERR(cs)) { 2099 i915_request_add(rq); 2100 err = PTR_ERR(cs); 2101 goto err; 2102 } 2103 2104 cs = emit_store_dw(cs, addr, -1); 2105 cs = emit_semaphore_poll_until(cs, offset, i); 2106 cs = emit_timestamp_store(cs, ce, addr + sizeof(u32)); 2107 2108 intel_ring_advance(rq, cs); 2109 i915_request_add(rq); 2110 2111 if (wait_for(READ_ONCE(sema[2 * i]) == -1, 500)) { 2112 err = -EIO; 2113 goto err; 2114 } 2115 2116 rq = i915_request_create(ce->engine->kernel_context); 2117 if (IS_ERR(rq)) { 2118 err = PTR_ERR(rq); 2119 goto err; 2120 } 2121 2122 cs = intel_ring_begin(rq, 8); 2123 if (IS_ERR(cs)) { 2124 i915_request_add(rq); 2125 err = PTR_ERR(cs); 2126 goto err; 2127 } 2128 2129 cs = emit_timestamp_store(cs, ce, addr); 2130 cs = emit_store_dw(cs, offset, i); 2131 2132 intel_ring_advance(rq, cs); 2133 rq->sched.attr.priority = I915_PRIORITY_BARRIER; 2134 2135 elapsed[i - 1] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP); 2136 i915_request_add(rq); 2137 } 2138 2139 if (wait_for(READ_ONCE(sema[2 * i - 2]) != -1, 500)) { 2140 err = -EIO; 2141 goto err; 2142 } 2143 2144 for (i = 1; i <= TF_COUNT; i++) 2145 elapsed[i - 1] = sema[2 * i + 0] - elapsed[i - 1]; 2146 2147 cycles = trifilter(elapsed); 2148 pr_info("%s: preemption dispatch latency %d cycles, %lluns\n", 2149 ce->engine->name, cycles >> TF_BIAS, 2150 cycles_to_ns(ce->engine, cycles)); 2151 2152 for (i = 1; i <= TF_COUNT; i++) 2153 elapsed[i - 1] = sema[2 * i + 1] - sema[2 * i + 0]; 2154 2155 cycles = trifilter(elapsed); 2156 pr_info("%s: preemption switch latency %d cycles, %lluns\n", 2157 ce->engine->name, cycles >> TF_BIAS, 2158 cycles_to_ns(ce->engine, cycles)); 2159 2160 return intel_gt_wait_for_idle(ce->engine->gt, HZ); 2161 2162err: 2163 intel_gt_set_wedged(ce->engine->gt); 2164 return err; 2165} 2166 2167struct signal_cb { 2168 struct dma_fence_cb base; 2169 bool seen; 2170}; 2171 2172static void signal_cb(struct dma_fence *fence, struct dma_fence_cb *cb) 2173{ 2174 struct signal_cb *s = container_of(cb, typeof(*s), base); 2175 2176 smp_store_mb(s->seen, true); /* be safe, be strong */ 2177} 2178 2179static int measure_completion(struct intel_context *ce) 2180{ 2181 u32 *sema = hwsp_scratch(ce); 2182 const u32 offset = hwsp_offset(ce, sema); 2183 u32 elapsed[TF_COUNT], cycles; 2184 u32 *cs; 2185 int err; 2186 int i; 2187 2188 /* 2189 * Measure how long it takes for the signal (interrupt) to be 2190 * sent from the GPU to be processed by the CPU. 2191 * 2192 * A: read CS_TIMESTAMP on GPU 2193 * signal 2194 * B: read CS_TIMESTAMP from CPU 2195 * 2196 * Completion latency: B - A 2197 */ 2198 2199 for (i = 1; i <= ARRAY_SIZE(elapsed); i++) { 2200 struct signal_cb cb = { .seen = false }; 2201 struct i915_request *rq; 2202 2203 rq = i915_request_create(ce); 2204 if (IS_ERR(rq)) { 2205 err = PTR_ERR(rq); 2206 goto err; 2207 } 2208 2209 cs = intel_ring_begin(rq, 12); 2210 if (IS_ERR(cs)) { 2211 i915_request_add(rq); 2212 err = PTR_ERR(cs); 2213 goto err; 2214 } 2215 2216 cs = emit_store_dw(cs, offset + i * sizeof(u32), -1); 2217 cs = emit_semaphore_poll_until(cs, offset, i); 2218 cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32)); 2219 2220 intel_ring_advance(rq, cs); 2221 2222 dma_fence_add_callback(&rq->fence, &cb.base, signal_cb); 2223 2224 local_bh_disable(); 2225 i915_request_add(rq); 2226 local_bh_enable(); 2227 2228 if (wait_for(READ_ONCE(sema[i]) == -1, 50)) { 2229 err = -EIO; 2230 goto err; 2231 } 2232 2233 preempt_disable(); 2234 semaphore_set(sema, i); 2235 while (!READ_ONCE(cb.seen)) 2236 cpu_relax(); 2237 2238 elapsed[i - 1] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP); 2239 preempt_enable(); 2240 } 2241 2242 err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2); 2243 if (err) 2244 goto err; 2245 2246 for (i = 0; i < ARRAY_SIZE(elapsed); i++) { 2247 GEM_BUG_ON(sema[i + 1] == -1); 2248 elapsed[i] = elapsed[i] - sema[i + 1]; 2249 } 2250 2251 cycles = trifilter(elapsed); 2252 pr_info("%s: completion latency %d cycles, %lluns\n", 2253 ce->engine->name, cycles >> TF_BIAS, 2254 cycles_to_ns(ce->engine, cycles)); 2255 2256 return intel_gt_wait_for_idle(ce->engine->gt, HZ); 2257 2258err: 2259 intel_gt_set_wedged(ce->engine->gt); 2260 return err; 2261} 2262 2263static void rps_pin(struct intel_gt *gt) 2264{ 2265 /* Pin the frequency to max */ 2266 atomic_inc(>->rps.num_waiters); 2267 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); 2268 2269 mutex_lock(>->rps.lock); 2270 intel_rps_set(>->rps, gt->rps.max_freq); 2271 mutex_unlock(>->rps.lock); 2272} 2273 2274static void rps_unpin(struct intel_gt *gt) 2275{ 2276 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); 2277 atomic_dec(>->rps.num_waiters); 2278} 2279 2280static int perf_request_latency(void *arg) 2281{ 2282 struct drm_i915_private *i915 = arg; 2283 struct intel_engine_cs *engine; 2284 struct pm_qos_request qos; 2285 int err = 0; 2286 2287 if (INTEL_GEN(i915) < 8) /* per-engine CS timestamp, semaphores */ 2288 return 0; 2289 2290 cpu_latency_qos_add_request(&qos, 0); /* disable cstates */ 2291 2292 for_each_uabi_engine(engine, i915) { 2293 struct intel_context *ce; 2294 2295 ce = intel_context_create(engine); 2296 if (IS_ERR(ce)) { 2297 err = PTR_ERR(ce); 2298 goto out; 2299 } 2300 2301 err = intel_context_pin(ce); 2302 if (err) { 2303 intel_context_put(ce); 2304 goto out; 2305 } 2306 2307 st_engine_heartbeat_disable(engine); 2308 rps_pin(engine->gt); 2309 2310 if (err == 0) 2311 err = measure_semaphore_response(ce); 2312 if (err == 0) 2313 err = measure_idle_dispatch(ce); 2314 if (err == 0) 2315 err = measure_busy_dispatch(ce); 2316 if (err == 0) 2317 err = measure_inter_request(ce); 2318 if (err == 0) 2319 err = measure_context_switch(ce); 2320 if (err == 0) 2321 err = measure_preemption(ce); 2322 if (err == 0) 2323 err = measure_completion(ce); 2324 2325 rps_unpin(engine->gt); 2326 st_engine_heartbeat_enable(engine); 2327 2328 intel_context_unpin(ce); 2329 intel_context_put(ce); 2330 if (err) 2331 goto out; 2332 } 2333 2334out: 2335 if (igt_flush_test(i915)) 2336 err = -EIO; 2337 2338 cpu_latency_qos_remove_request(&qos); 2339 return err; 2340} 2341 2342static int s_sync0(void *arg) 2343{ 2344 struct perf_series *ps = arg; 2345 IGT_TIMEOUT(end_time); 2346 unsigned int idx = 0; 2347 int err = 0; 2348 2349 GEM_BUG_ON(!ps->nengines); 2350 do { 2351 struct i915_request *rq; 2352 2353 rq = i915_request_create(ps->ce[idx]); 2354 if (IS_ERR(rq)) { 2355 err = PTR_ERR(rq); 2356 break; 2357 } 2358 2359 i915_request_get(rq); 2360 i915_request_add(rq); 2361 2362 if (i915_request_wait(rq, 0, HZ / 5) < 0) 2363 err = -ETIME; 2364 i915_request_put(rq); 2365 if (err) 2366 break; 2367 2368 if (++idx == ps->nengines) 2369 idx = 0; 2370 } while (!__igt_timeout(end_time, NULL)); 2371 2372 return err; 2373} 2374 2375static int s_sync1(void *arg) 2376{ 2377 struct perf_series *ps = arg; 2378 struct i915_request *prev = NULL; 2379 IGT_TIMEOUT(end_time); 2380 unsigned int idx = 0; 2381 int err = 0; 2382 2383 GEM_BUG_ON(!ps->nengines); 2384 do { 2385 struct i915_request *rq; 2386 2387 rq = i915_request_create(ps->ce[idx]); 2388 if (IS_ERR(rq)) { 2389 err = PTR_ERR(rq); 2390 break; 2391 } 2392 2393 i915_request_get(rq); 2394 i915_request_add(rq); 2395 2396 if (prev && i915_request_wait(prev, 0, HZ / 5) < 0) 2397 err = -ETIME; 2398 i915_request_put(prev); 2399 prev = rq; 2400 if (err) 2401 break; 2402 2403 if (++idx == ps->nengines) 2404 idx = 0; 2405 } while (!__igt_timeout(end_time, NULL)); 2406 i915_request_put(prev); 2407 2408 return err; 2409} 2410 2411static int s_many(void *arg) 2412{ 2413 struct perf_series *ps = arg; 2414 IGT_TIMEOUT(end_time); 2415 unsigned int idx = 0; 2416 2417 GEM_BUG_ON(!ps->nengines); 2418 do { 2419 struct i915_request *rq; 2420 2421 rq = i915_request_create(ps->ce[idx]); 2422 if (IS_ERR(rq)) 2423 return PTR_ERR(rq); 2424 2425 i915_request_add(rq); 2426 2427 if (++idx == ps->nengines) 2428 idx = 0; 2429 } while (!__igt_timeout(end_time, NULL)); 2430 2431 return 0; 2432} 2433 2434static int perf_series_engines(void *arg) 2435{ 2436 struct drm_i915_private *i915 = arg; 2437 static int (* const func[])(void *arg) = { 2438 s_sync0, 2439 s_sync1, 2440 s_many, 2441 NULL, 2442 }; 2443 const unsigned int nengines = num_uabi_engines(i915); 2444 struct intel_engine_cs *engine; 2445 int (* const *fn)(void *arg); 2446 struct pm_qos_request qos; 2447 struct perf_stats *stats; 2448 struct perf_series *ps; 2449 unsigned int idx; 2450 int err = 0; 2451 2452 stats = kcalloc(nengines, sizeof(*stats), GFP_KERNEL); 2453 if (!stats) 2454 return -ENOMEM; 2455 2456 ps = kzalloc(struct_size(ps, ce, nengines), GFP_KERNEL); 2457 if (!ps) { 2458 kfree(stats); 2459 return -ENOMEM; 2460 } 2461 2462 cpu_latency_qos_add_request(&qos, 0); /* disable cstates */ 2463 2464 ps->i915 = i915; 2465 ps->nengines = nengines; 2466 2467 idx = 0; 2468 for_each_uabi_engine(engine, i915) { 2469 struct intel_context *ce; 2470 2471 ce = intel_context_create(engine); 2472 if (IS_ERR(ce)) { 2473 err = PTR_ERR(ce); 2474 goto out; 2475 } 2476 2477 err = intel_context_pin(ce); 2478 if (err) { 2479 intel_context_put(ce); 2480 goto out; 2481 } 2482 2483 ps->ce[idx++] = ce; 2484 } 2485 GEM_BUG_ON(idx != ps->nengines); 2486 2487 for (fn = func; *fn && !err; fn++) { 2488 char name[KSYM_NAME_LEN]; 2489 struct igt_live_test t; 2490 2491 snprintf(name, sizeof(name), "%ps", *fn); 2492 err = igt_live_test_begin(&t, i915, __func__, name); 2493 if (err) 2494 break; 2495 2496 for (idx = 0; idx < nengines; idx++) { 2497 struct perf_stats *p = 2498 memset(&stats[idx], 0, sizeof(stats[idx])); 2499 struct intel_context *ce = ps->ce[idx]; 2500 2501 p->engine = ps->ce[idx]->engine; 2502 intel_engine_pm_get(p->engine); 2503 2504 if (intel_engine_supports_stats(p->engine)) 2505 p->busy = intel_engine_get_busy_time(p->engine, 2506 &p->time) + 1; 2507 else 2508 p->time = ktime_get(); 2509 p->runtime = -intel_context_get_total_runtime_ns(ce); 2510 } 2511 2512 err = (*fn)(ps); 2513 if (igt_live_test_end(&t)) 2514 err = -EIO; 2515 2516 for (idx = 0; idx < nengines; idx++) { 2517 struct perf_stats *p = &stats[idx]; 2518 struct intel_context *ce = ps->ce[idx]; 2519 int integer, decimal; 2520 u64 busy, dt, now; 2521 2522 if (p->busy) 2523 p->busy = ktime_sub(intel_engine_get_busy_time(p->engine, 2524 &now), 2525 p->busy - 1); 2526 else 2527 now = ktime_get(); 2528 p->time = ktime_sub(now, p->time); 2529 2530 err = switch_to_kernel_sync(ce, err); 2531 p->runtime += intel_context_get_total_runtime_ns(ce); 2532 intel_engine_pm_put(p->engine); 2533 2534 busy = 100 * ktime_to_ns(p->busy); 2535 dt = ktime_to_ns(p->time); 2536 if (dt) { 2537 integer = div64_u64(busy, dt); 2538 busy -= integer * dt; 2539 decimal = div64_u64(100 * busy, dt); 2540 } else { 2541 integer = 0; 2542 decimal = 0; 2543 } 2544 2545 pr_info("%s %5s: { seqno:%d, busy:%d.%02d%%, runtime:%lldms, walltime:%lldms }\n", 2546 name, p->engine->name, ce->timeline->seqno, 2547 integer, decimal, 2548 div_u64(p->runtime, 1000 * 1000), 2549 div_u64(ktime_to_ns(p->time), 1000 * 1000)); 2550 } 2551 } 2552 2553out: 2554 for (idx = 0; idx < nengines; idx++) { 2555 if (IS_ERR_OR_NULL(ps->ce[idx])) 2556 break; 2557 2558 intel_context_unpin(ps->ce[idx]); 2559 intel_context_put(ps->ce[idx]); 2560 } 2561 kfree(ps); 2562 2563 cpu_latency_qos_remove_request(&qos); 2564 kfree(stats); 2565 return err; 2566} 2567 2568static int p_sync0(void *arg) 2569{ 2570 struct perf_stats *p = arg; 2571 struct intel_engine_cs *engine = p->engine; 2572 struct intel_context *ce; 2573 IGT_TIMEOUT(end_time); 2574 unsigned long count; 2575 bool busy; 2576 int err = 0; 2577 2578 ce = intel_context_create(engine); 2579 if (IS_ERR(ce)) 2580 return PTR_ERR(ce); 2581 2582 err = intel_context_pin(ce); 2583 if (err) { 2584 intel_context_put(ce); 2585 return err; 2586 } 2587 2588 if (intel_engine_supports_stats(engine)) { 2589 p->busy = intel_engine_get_busy_time(engine, &p->time); 2590 busy = true; 2591 } else { 2592 p->time = ktime_get(); 2593 busy = false; 2594 } 2595 2596 count = 0; 2597 do { 2598 struct i915_request *rq; 2599 2600 rq = i915_request_create(ce); 2601 if (IS_ERR(rq)) { 2602 err = PTR_ERR(rq); 2603 break; 2604 } 2605 2606 i915_request_get(rq); 2607 i915_request_add(rq); 2608 2609 err = 0; 2610 if (i915_request_wait(rq, 0, HZ / 5) < 0) 2611 err = -ETIME; 2612 i915_request_put(rq); 2613 if (err) 2614 break; 2615 2616 count++; 2617 } while (!__igt_timeout(end_time, NULL)); 2618 2619 if (busy) { 2620 ktime_t now; 2621 2622 p->busy = ktime_sub(intel_engine_get_busy_time(engine, &now), 2623 p->busy); 2624 p->time = ktime_sub(now, p->time); 2625 } else { 2626 p->time = ktime_sub(ktime_get(), p->time); 2627 } 2628 2629 err = switch_to_kernel_sync(ce, err); 2630 p->runtime = intel_context_get_total_runtime_ns(ce); 2631 p->count = count; 2632 2633 intel_context_unpin(ce); 2634 intel_context_put(ce); 2635 return err; 2636} 2637 2638static int p_sync1(void *arg) 2639{ 2640 struct perf_stats *p = arg; 2641 struct intel_engine_cs *engine = p->engine; 2642 struct i915_request *prev = NULL; 2643 struct intel_context *ce; 2644 IGT_TIMEOUT(end_time); 2645 unsigned long count; 2646 bool busy; 2647 int err = 0; 2648 2649 ce = intel_context_create(engine); 2650 if (IS_ERR(ce)) 2651 return PTR_ERR(ce); 2652 2653 err = intel_context_pin(ce); 2654 if (err) { 2655 intel_context_put(ce); 2656 return err; 2657 } 2658 2659 if (intel_engine_supports_stats(engine)) { 2660 p->busy = intel_engine_get_busy_time(engine, &p->time); 2661 busy = true; 2662 } else { 2663 p->time = ktime_get(); 2664 busy = false; 2665 } 2666 2667 count = 0; 2668 do { 2669 struct i915_request *rq; 2670 2671 rq = i915_request_create(ce); 2672 if (IS_ERR(rq)) { 2673 err = PTR_ERR(rq); 2674 break; 2675 } 2676 2677 i915_request_get(rq); 2678 i915_request_add(rq); 2679 2680 err = 0; 2681 if (prev && i915_request_wait(prev, 0, HZ / 5) < 0) 2682 err = -ETIME; 2683 i915_request_put(prev); 2684 prev = rq; 2685 if (err) 2686 break; 2687 2688 count++; 2689 } while (!__igt_timeout(end_time, NULL)); 2690 i915_request_put(prev); 2691 2692 if (busy) { 2693 ktime_t now; 2694 2695 p->busy = ktime_sub(intel_engine_get_busy_time(engine, &now), 2696 p->busy); 2697 p->time = ktime_sub(now, p->time); 2698 } else { 2699 p->time = ktime_sub(ktime_get(), p->time); 2700 } 2701 2702 err = switch_to_kernel_sync(ce, err); 2703 p->runtime = intel_context_get_total_runtime_ns(ce); 2704 p->count = count; 2705 2706 intel_context_unpin(ce); 2707 intel_context_put(ce); 2708 return err; 2709} 2710 2711static int p_many(void *arg) 2712{ 2713 struct perf_stats *p = arg; 2714 struct intel_engine_cs *engine = p->engine; 2715 struct intel_context *ce; 2716 IGT_TIMEOUT(end_time); 2717 unsigned long count; 2718 int err = 0; 2719 bool busy; 2720 2721 ce = intel_context_create(engine); 2722 if (IS_ERR(ce)) 2723 return PTR_ERR(ce); 2724 2725 err = intel_context_pin(ce); 2726 if (err) { 2727 intel_context_put(ce); 2728 return err; 2729 } 2730 2731 if (intel_engine_supports_stats(engine)) { 2732 p->busy = intel_engine_get_busy_time(engine, &p->time); 2733 busy = true; 2734 } else { 2735 p->time = ktime_get(); 2736 busy = false; 2737 } 2738 2739 count = 0; 2740 do { 2741 struct i915_request *rq; 2742 2743 rq = i915_request_create(ce); 2744 if (IS_ERR(rq)) { 2745 err = PTR_ERR(rq); 2746 break; 2747 } 2748 2749 i915_request_add(rq); 2750 count++; 2751 } while (!__igt_timeout(end_time, NULL)); 2752 2753 if (busy) { 2754 ktime_t now; 2755 2756 p->busy = ktime_sub(intel_engine_get_busy_time(engine, &now), 2757 p->busy); 2758 p->time = ktime_sub(now, p->time); 2759 } else { 2760 p->time = ktime_sub(ktime_get(), p->time); 2761 } 2762 2763 err = switch_to_kernel_sync(ce, err); 2764 p->runtime = intel_context_get_total_runtime_ns(ce); 2765 p->count = count; 2766 2767 intel_context_unpin(ce); 2768 intel_context_put(ce); 2769 return err; 2770} 2771 2772static int perf_parallel_engines(void *arg) 2773{ 2774 struct drm_i915_private *i915 = arg; 2775 static int (* const func[])(void *arg) = { 2776 p_sync0, 2777 p_sync1, 2778 p_many, 2779 NULL, 2780 }; 2781 const unsigned int nengines = num_uabi_engines(i915); 2782 struct intel_engine_cs *engine; 2783 int (* const *fn)(void *arg); 2784 struct pm_qos_request qos; 2785 struct { 2786 struct perf_stats p; 2787 struct task_struct *tsk; 2788 } *engines; 2789 int err = 0; 2790 2791 engines = kcalloc(nengines, sizeof(*engines), GFP_KERNEL); 2792 if (!engines) 2793 return -ENOMEM; 2794 2795 cpu_latency_qos_add_request(&qos, 0); 2796 2797 for (fn = func; *fn; fn++) { 2798 char name[KSYM_NAME_LEN]; 2799 struct igt_live_test t; 2800 unsigned int idx; 2801 2802 snprintf(name, sizeof(name), "%ps", *fn); 2803 err = igt_live_test_begin(&t, i915, __func__, name); 2804 if (err) 2805 break; 2806 2807 atomic_set(&i915->selftest.counter, nengines); 2808 2809 idx = 0; 2810 for_each_uabi_engine(engine, i915) { 2811 intel_engine_pm_get(engine); 2812 2813 memset(&engines[idx].p, 0, sizeof(engines[idx].p)); 2814 engines[idx].p.engine = engine; 2815 2816 engines[idx].tsk = kthread_run(*fn, &engines[idx].p, 2817 "igt:%s", engine->name); 2818 if (IS_ERR(engines[idx].tsk)) { 2819 err = PTR_ERR(engines[idx].tsk); 2820 intel_engine_pm_put(engine); 2821 break; 2822 } 2823 get_task_struct(engines[idx++].tsk); 2824 } 2825 2826 yield(); /* start all threads before we kthread_stop() */ 2827 2828 idx = 0; 2829 for_each_uabi_engine(engine, i915) { 2830 int status; 2831 2832 if (IS_ERR(engines[idx].tsk)) 2833 break; 2834 2835 status = kthread_stop(engines[idx].tsk); 2836 if (status && !err) 2837 err = status; 2838 2839 intel_engine_pm_put(engine); 2840 put_task_struct(engines[idx++].tsk); 2841 } 2842 2843 if (igt_live_test_end(&t)) 2844 err = -EIO; 2845 if (err) 2846 break; 2847 2848 idx = 0; 2849 for_each_uabi_engine(engine, i915) { 2850 struct perf_stats *p = &engines[idx].p; 2851 u64 busy = 100 * ktime_to_ns(p->busy); 2852 u64 dt = ktime_to_ns(p->time); 2853 int integer, decimal; 2854 2855 if (dt) { 2856 integer = div64_u64(busy, dt); 2857 busy -= integer * dt; 2858 decimal = div64_u64(100 * busy, dt); 2859 } else { 2860 integer = 0; 2861 decimal = 0; 2862 } 2863 2864 GEM_BUG_ON(engine != p->engine); 2865 pr_info("%s %5s: { count:%lu, busy:%d.%02d%%, runtime:%lldms, walltime:%lldms }\n", 2866 name, engine->name, p->count, integer, decimal, 2867 div_u64(p->runtime, 1000 * 1000), 2868 div_u64(ktime_to_ns(p->time), 1000 * 1000)); 2869 idx++; 2870 } 2871 } 2872 2873 cpu_latency_qos_remove_request(&qos); 2874 kfree(engines); 2875 return err; 2876} 2877 2878int i915_request_perf_selftests(struct drm_i915_private *i915) 2879{ 2880 static const struct i915_subtest tests[] = { 2881 SUBTEST(perf_request_latency), 2882 SUBTEST(perf_series_engines), 2883 SUBTEST(perf_parallel_engines), 2884 }; 2885 2886 if (intel_gt_is_wedged(&i915->gt)) 2887 return 0; 2888 2889 return i915_subtests(tests, i915); 2890} 2891