1// SPDX-License-Identifier: MIT 2/* 3 * Copyright © 2020 Intel Corporation 4 */ 5#include <linux/kernel.h> 6#include <linux/pm_qos.h> 7#include <linux/slab.h> 8 9#include <drm/drm_atomic_helper.h> 10#include <drm/drm_fourcc.h> 11#include <drm/drm_plane.h> 12#include <drm/drm_vblank_work.h> 13 14#include "i915_vgpu.h" 15#include "i9xx_plane.h" 16#include "icl_dsi.h" 17#include "intel_atomic.h" 18#include "intel_atomic_plane.h" 19#include "intel_color.h" 20#include "intel_crtc.h" 21#include "intel_cursor.h" 22#include "intel_display_debugfs.h" 23#include "intel_display_irq.h" 24#include "intel_display_trace.h" 25#include "intel_display_types.h" 26#include "intel_drrs.h" 27#include "intel_dsi.h" 28#include "intel_fifo_underrun.h" 29#include "intel_pipe_crc.h" 30#include "intel_psr.h" 31#include "intel_sprite.h" 32#include "intel_vblank.h" 33#include "intel_vrr.h" 34#include "skl_universal_plane.h" 35 36static void assert_vblank_disabled(struct drm_crtc *crtc) 37{ 38 struct drm_i915_private *i915 = to_i915(crtc->dev); 39 40 if (I915_STATE_WARN(i915, drm_crtc_vblank_get(crtc) == 0, 41 "[CRTC:%d:%s] vblank assertion failure (expected off, current on)\n", 42 crtc->base.id, crtc->name)) 43 drm_crtc_vblank_put(crtc); 44} 45 46struct intel_crtc *intel_first_crtc(struct drm_i915_private *i915) 47{ 48 return to_intel_crtc(drm_crtc_from_index(&i915->drm, 0)); 49} 50 51struct intel_crtc *intel_crtc_for_pipe(struct drm_i915_private *i915, 52 enum pipe pipe) 53{ 54 struct intel_crtc *crtc; 55 56 for_each_intel_crtc(&i915->drm, crtc) { 57 if (crtc->pipe == pipe) 58 return crtc; 59 } 60 61 return NULL; 62} 63 64void intel_crtc_wait_for_next_vblank(struct intel_crtc *crtc) 65{ 66 drm_crtc_wait_one_vblank(&crtc->base); 67} 68 69void intel_wait_for_vblank_if_active(struct drm_i915_private *i915, 70 enum pipe pipe) 71{ 72 struct intel_crtc *crtc = intel_crtc_for_pipe(i915, pipe); 73 74 if (crtc->active) 75 intel_crtc_wait_for_next_vblank(crtc); 76} 77 78u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc) 79{ 80 struct drm_device *dev = crtc->base.dev; 81 struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(&crtc->base)]; 82 83 if (!crtc->active) 84 return 0; 85 86 if (!vblank->max_vblank_count) 87 return (u32)drm_crtc_accurate_vblank_count(&crtc->base); 88 89 return crtc->base.funcs->get_vblank_counter(&crtc->base); 90} 91 92u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state) 93{ 94 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); 95 96 /* 97 * From Gen 11, In case of dsi cmd mode, frame counter wouldnt 98 * have updated at the beginning of TE, if we want to use 99 * the hw counter, then we would find it updated in only 100 * the next TE, hence switching to sw counter. 101 */ 102 if (crtc_state->mode_flags & (I915_MODE_FLAG_DSI_USE_TE0 | 103 I915_MODE_FLAG_DSI_USE_TE1)) 104 return 0; 105 106 /* 107 * On i965gm the hardware frame counter reads 108 * zero when the TV encoder is enabled :( 109 */ 110 if (IS_I965GM(dev_priv) && 111 (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT))) 112 return 0; 113 114 if (DISPLAY_VER(dev_priv) >= 5 || IS_G4X(dev_priv)) 115 return 0xffffffff; /* full 32 bit counter */ 116 else if (DISPLAY_VER(dev_priv) >= 3) 117 return 0xffffff; /* only 24 bits of frame count */ 118 else 119 return 0; /* Gen2 doesn't have a hardware frame counter */ 120} 121 122void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state) 123{ 124 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 125 126 assert_vblank_disabled(&crtc->base); 127 drm_crtc_set_max_vblank_count(&crtc->base, 128 intel_crtc_max_vblank_count(crtc_state)); 129 drm_crtc_vblank_on(&crtc->base); 130 131 /* 132 * Should really happen exactly when we enable the pipe 133 * but we want the frame counters in the trace, and that 134 * requires vblank support on some platforms/outputs. 135 */ 136 trace_intel_pipe_enable(crtc); 137} 138 139void intel_crtc_vblank_off(const struct intel_crtc_state *crtc_state) 140{ 141 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 142 143 /* 144 * Should really happen exactly when we disable the pipe 145 * but we want the frame counters in the trace, and that 146 * requires vblank support on some platforms/outputs. 147 */ 148 trace_intel_pipe_disable(crtc); 149 150 drm_crtc_vblank_off(&crtc->base); 151 assert_vblank_disabled(&crtc->base); 152} 153 154struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc) 155{ 156 struct intel_crtc_state *crtc_state; 157 158 crtc_state = kmalloc(sizeof(*crtc_state), GFP_KERNEL); 159 160 if (crtc_state) 161 intel_crtc_state_reset(crtc_state, crtc); 162 163 return crtc_state; 164} 165 166void intel_crtc_state_reset(struct intel_crtc_state *crtc_state, 167 struct intel_crtc *crtc) 168{ 169 memset(crtc_state, 0, sizeof(*crtc_state)); 170 171 __drm_atomic_helper_crtc_state_reset(&crtc_state->uapi, &crtc->base); 172 173 crtc_state->cpu_transcoder = INVALID_TRANSCODER; 174 crtc_state->master_transcoder = INVALID_TRANSCODER; 175 crtc_state->hsw_workaround_pipe = INVALID_PIPE; 176 crtc_state->scaler_state.scaler_id = -1; 177 crtc_state->mst_master_transcoder = INVALID_TRANSCODER; 178} 179 180static struct intel_crtc *intel_crtc_alloc(void) 181{ 182 struct intel_crtc_state *crtc_state; 183 struct intel_crtc *crtc; 184 185 crtc = kzalloc(sizeof(*crtc), GFP_KERNEL); 186 if (!crtc) 187 return ERR_PTR(-ENOMEM); 188 189 crtc_state = intel_crtc_state_alloc(crtc); 190 if (!crtc_state) { 191 kfree(crtc); 192 return ERR_PTR(-ENOMEM); 193 } 194 195 crtc->base.state = &crtc_state->uapi; 196 crtc->config = crtc_state; 197 198 return crtc; 199} 200 201static void intel_crtc_free(struct intel_crtc *crtc) 202{ 203 intel_crtc_destroy_state(&crtc->base, crtc->base.state); 204 kfree(crtc); 205} 206 207static void intel_crtc_destroy(struct drm_crtc *_crtc) 208{ 209 struct intel_crtc *crtc = to_intel_crtc(_crtc); 210 211 cpu_latency_qos_remove_request(&crtc->vblank_pm_qos); 212 213 drm_crtc_cleanup(&crtc->base); 214 kfree(crtc); 215} 216 217static int intel_crtc_late_register(struct drm_crtc *crtc) 218{ 219 intel_crtc_debugfs_add(to_intel_crtc(crtc)); 220 return 0; 221} 222 223#define INTEL_CRTC_FUNCS \ 224 .set_config = drm_atomic_helper_set_config, \ 225 .destroy = intel_crtc_destroy, \ 226 .page_flip = drm_atomic_helper_page_flip, \ 227 .atomic_duplicate_state = intel_crtc_duplicate_state, \ 228 .atomic_destroy_state = intel_crtc_destroy_state, \ 229 .set_crc_source = intel_crtc_set_crc_source, \ 230 .verify_crc_source = intel_crtc_verify_crc_source, \ 231 .get_crc_sources = intel_crtc_get_crc_sources, \ 232 .late_register = intel_crtc_late_register 233 234static const struct drm_crtc_funcs bdw_crtc_funcs = { 235 INTEL_CRTC_FUNCS, 236 237 .get_vblank_counter = g4x_get_vblank_counter, 238 .enable_vblank = bdw_enable_vblank, 239 .disable_vblank = bdw_disable_vblank, 240 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, 241}; 242 243static const struct drm_crtc_funcs ilk_crtc_funcs = { 244 INTEL_CRTC_FUNCS, 245 246 .get_vblank_counter = g4x_get_vblank_counter, 247 .enable_vblank = ilk_enable_vblank, 248 .disable_vblank = ilk_disable_vblank, 249 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, 250}; 251 252static const struct drm_crtc_funcs g4x_crtc_funcs = { 253 INTEL_CRTC_FUNCS, 254 255 .get_vblank_counter = g4x_get_vblank_counter, 256 .enable_vblank = i965_enable_vblank, 257 .disable_vblank = i965_disable_vblank, 258 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, 259}; 260 261static const struct drm_crtc_funcs i965_crtc_funcs = { 262 INTEL_CRTC_FUNCS, 263 264 .get_vblank_counter = i915_get_vblank_counter, 265 .enable_vblank = i965_enable_vblank, 266 .disable_vblank = i965_disable_vblank, 267 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, 268}; 269 270static const struct drm_crtc_funcs i915gm_crtc_funcs = { 271 INTEL_CRTC_FUNCS, 272 273 .get_vblank_counter = i915_get_vblank_counter, 274 .enable_vblank = i915gm_enable_vblank, 275 .disable_vblank = i915gm_disable_vblank, 276 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, 277}; 278 279static const struct drm_crtc_funcs i915_crtc_funcs = { 280 INTEL_CRTC_FUNCS, 281 282 .get_vblank_counter = i915_get_vblank_counter, 283 .enable_vblank = i8xx_enable_vblank, 284 .disable_vblank = i8xx_disable_vblank, 285 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, 286}; 287 288static const struct drm_crtc_funcs i8xx_crtc_funcs = { 289 INTEL_CRTC_FUNCS, 290 291 /* no hw vblank counter */ 292 .enable_vblank = i8xx_enable_vblank, 293 .disable_vblank = i8xx_disable_vblank, 294 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, 295}; 296 297int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe) 298{ 299 struct intel_plane *primary, *cursor; 300 const struct drm_crtc_funcs *funcs; 301 struct intel_crtc *crtc; 302 int sprite, ret; 303 304 crtc = intel_crtc_alloc(); 305 if (IS_ERR(crtc)) 306 return PTR_ERR(crtc); 307 308 crtc->pipe = pipe; 309 crtc->num_scalers = DISPLAY_RUNTIME_INFO(dev_priv)->num_scalers[pipe]; 310 311 if (DISPLAY_VER(dev_priv) >= 9) 312 primary = skl_universal_plane_create(dev_priv, pipe, 313 PLANE_PRIMARY); 314 else 315 primary = intel_primary_plane_create(dev_priv, pipe); 316 if (IS_ERR(primary)) { 317 ret = PTR_ERR(primary); 318 goto fail; 319 } 320 crtc->plane_ids_mask |= BIT(primary->id); 321 322 intel_init_fifo_underrun_reporting(dev_priv, crtc, false); 323 324 for_each_sprite(dev_priv, pipe, sprite) { 325 struct intel_plane *plane; 326 327 if (DISPLAY_VER(dev_priv) >= 9) 328 plane = skl_universal_plane_create(dev_priv, pipe, 329 PLANE_SPRITE0 + sprite); 330 else 331 plane = intel_sprite_plane_create(dev_priv, pipe, sprite); 332 if (IS_ERR(plane)) { 333 ret = PTR_ERR(plane); 334 goto fail; 335 } 336 crtc->plane_ids_mask |= BIT(plane->id); 337 } 338 339 cursor = intel_cursor_plane_create(dev_priv, pipe); 340 if (IS_ERR(cursor)) { 341 ret = PTR_ERR(cursor); 342 goto fail; 343 } 344 crtc->plane_ids_mask |= BIT(cursor->id); 345 346 if (HAS_GMCH(dev_priv)) { 347 if (IS_CHERRYVIEW(dev_priv) || 348 IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv)) 349 funcs = &g4x_crtc_funcs; 350 else if (DISPLAY_VER(dev_priv) == 4) 351 funcs = &i965_crtc_funcs; 352 else if (IS_I945GM(dev_priv) || IS_I915GM(dev_priv)) 353 funcs = &i915gm_crtc_funcs; 354 else if (DISPLAY_VER(dev_priv) == 3) 355 funcs = &i915_crtc_funcs; 356 else 357 funcs = &i8xx_crtc_funcs; 358 } else { 359 if (DISPLAY_VER(dev_priv) >= 8) 360 funcs = &bdw_crtc_funcs; 361 else 362 funcs = &ilk_crtc_funcs; 363 } 364 365 ret = drm_crtc_init_with_planes(&dev_priv->drm, &crtc->base, 366 &primary->base, &cursor->base, 367 funcs, "pipe %c", pipe_name(pipe)); 368 if (ret) 369 goto fail; 370 371 if (DISPLAY_VER(dev_priv) >= 11) 372 drm_crtc_create_scaling_filter_property(&crtc->base, 373 BIT(DRM_SCALING_FILTER_DEFAULT) | 374 BIT(DRM_SCALING_FILTER_NEAREST_NEIGHBOR)); 375 376 intel_color_crtc_init(crtc); 377 intel_drrs_crtc_init(crtc); 378 intel_crtc_crc_init(crtc); 379 380 cpu_latency_qos_add_request(&crtc->vblank_pm_qos, PM_QOS_DEFAULT_VALUE); 381 382 drm_WARN_ON(&dev_priv->drm, drm_crtc_index(&crtc->base) != crtc->pipe); 383 384 return 0; 385 386fail: 387 intel_crtc_free(crtc); 388 389 return ret; 390} 391 392static bool intel_crtc_needs_vblank_work(const struct intel_crtc_state *crtc_state) 393{ 394 return crtc_state->hw.active && 395 !intel_crtc_needs_modeset(crtc_state) && 396 !crtc_state->preload_luts && 397 intel_crtc_needs_color_update(crtc_state); 398} 399 400static void intel_crtc_vblank_work(struct kthread_work *base) 401{ 402 struct drm_vblank_work *work = to_drm_vblank_work(base); 403 struct intel_crtc_state *crtc_state = 404 container_of(work, typeof(*crtc_state), vblank_work); 405 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 406 407 trace_intel_crtc_vblank_work_start(crtc); 408 409 intel_color_load_luts(crtc_state); 410 411 if (crtc_state->uapi.event) { 412 spin_lock_irq(&crtc->base.dev->event_lock); 413 drm_crtc_send_vblank_event(&crtc->base, crtc_state->uapi.event); 414 crtc_state->uapi.event = NULL; 415 spin_unlock_irq(&crtc->base.dev->event_lock); 416 } 417 418 trace_intel_crtc_vblank_work_end(crtc); 419} 420 421static void intel_crtc_vblank_work_init(struct intel_crtc_state *crtc_state) 422{ 423 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 424 425 drm_vblank_work_init(&crtc_state->vblank_work, &crtc->base, 426 intel_crtc_vblank_work); 427 /* 428 * Interrupt latency is critical for getting the vblank 429 * work executed as early as possible during the vblank. 430 */ 431 cpu_latency_qos_update_request(&crtc->vblank_pm_qos, 0); 432} 433 434void intel_wait_for_vblank_workers(struct intel_atomic_state *state) 435{ 436 struct intel_crtc_state *crtc_state; 437 struct intel_crtc *crtc; 438 int i; 439 440 for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) { 441 if (!intel_crtc_needs_vblank_work(crtc_state)) 442 continue; 443 444 drm_vblank_work_flush(&crtc_state->vblank_work); 445 cpu_latency_qos_update_request(&crtc->vblank_pm_qos, 446 PM_QOS_DEFAULT_VALUE); 447 } 448} 449 450int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode, 451 int usecs) 452{ 453 /* paranoia */ 454 if (!adjusted_mode->crtc_htotal) 455 return 1; 456 457 return DIV_ROUND_UP(usecs * adjusted_mode->crtc_clock, 458 1000 * adjusted_mode->crtc_htotal); 459} 460 461static int intel_mode_vblank_start(const struct drm_display_mode *mode) 462{ 463 int vblank_start = mode->crtc_vblank_start; 464 465 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 466 vblank_start = DIV_ROUND_UP(vblank_start, 2); 467 468 return vblank_start; 469} 470 471/** 472 * intel_pipe_update_start() - start update of a set of display registers 473 * @new_crtc_state: the new crtc state 474 * 475 * Mark the start of an update to pipe registers that should be updated 476 * atomically regarding vblank. If the next vblank will happens within 477 * the next 100 us, this function waits until the vblank passes. 478 * 479 * After a successful call to this function, interrupts will be disabled 480 * until a subsequent call to intel_pipe_update_end(). That is done to 481 * avoid random delays. 482 */ 483void intel_pipe_update_start(struct intel_crtc_state *new_crtc_state) 484{ 485 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc); 486 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 487 const struct drm_display_mode *adjusted_mode = &new_crtc_state->hw.adjusted_mode; 488 long timeout = msecs_to_jiffies_timeout(1); 489 int scanline, min, max, vblank_start; 490 wait_queue_head_t *wq = drm_crtc_vblank_waitqueue(&crtc->base); 491 bool need_vlv_dsi_wa = (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) && 492 intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI); 493 DEFINE_WAIT(wait); 494 495 intel_psr_lock(new_crtc_state); 496 497 if (new_crtc_state->do_async_flip) 498 return; 499 500 if (intel_crtc_needs_vblank_work(new_crtc_state)) 501 intel_crtc_vblank_work_init(new_crtc_state); 502 503 if (new_crtc_state->vrr.enable) { 504 if (intel_vrr_is_push_sent(new_crtc_state)) 505 vblank_start = intel_vrr_vmin_vblank_start(new_crtc_state); 506 else 507 vblank_start = intel_vrr_vmax_vblank_start(new_crtc_state); 508 } else { 509 vblank_start = intel_mode_vblank_start(adjusted_mode); 510 } 511 512 /* FIXME needs to be calibrated sensibly */ 513 min = vblank_start - intel_usecs_to_scanlines(adjusted_mode, 514 VBLANK_EVASION_TIME_US); 515 max = vblank_start - 1; 516 517 /* 518 * M/N is double buffered on the transcoder's undelayed vblank, 519 * so with seamless M/N we must evade both vblanks. 520 */ 521 if (new_crtc_state->seamless_m_n && intel_crtc_needs_fastset(new_crtc_state)) 522 min -= adjusted_mode->crtc_vblank_start - adjusted_mode->crtc_vdisplay; 523 524 if (min <= 0 || max <= 0) 525 goto irq_disable; 526 527 if (drm_WARN_ON(&dev_priv->drm, drm_crtc_vblank_get(&crtc->base))) 528 goto irq_disable; 529 530 /* 531 * Wait for psr to idle out after enabling the VBL interrupts 532 * VBL interrupts will start the PSR exit and prevent a PSR 533 * re-entry as well. 534 */ 535 intel_psr_wait_for_idle_locked(new_crtc_state); 536 537 local_irq_disable(); 538 539 crtc->debug.min_vbl = min; 540 crtc->debug.max_vbl = max; 541 trace_intel_pipe_update_start(crtc); 542 543 for (;;) { 544 /* 545 * prepare_to_wait() has a memory barrier, which guarantees 546 * other CPUs can see the task state update by the time we 547 * read the scanline. 548 */ 549 prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE); 550 551 scanline = intel_get_crtc_scanline(crtc); 552 if (scanline < min || scanline > max) 553 break; 554 555 if (!timeout) { 556 drm_err(&dev_priv->drm, 557 "Potential atomic update failure on pipe %c\n", 558 pipe_name(crtc->pipe)); 559 break; 560 } 561 562 local_irq_enable(); 563 564 timeout = schedule_timeout(timeout); 565 566 local_irq_disable(); 567 } 568 569 finish_wait(wq, &wait); 570 571 drm_crtc_vblank_put(&crtc->base); 572 573 /* 574 * On VLV/CHV DSI the scanline counter would appear to 575 * increment approx. 1/3 of a scanline before start of vblank. 576 * The registers still get latched at start of vblank however. 577 * This means we must not write any registers on the first 578 * line of vblank (since not the whole line is actually in 579 * vblank). And unfortunately we can't use the interrupt to 580 * wait here since it will fire too soon. We could use the 581 * frame start interrupt instead since it will fire after the 582 * critical scanline, but that would require more changes 583 * in the interrupt code. So for now we'll just do the nasty 584 * thing and poll for the bad scanline to pass us by. 585 * 586 * FIXME figure out if BXT+ DSI suffers from this as well 587 */ 588 while (need_vlv_dsi_wa && scanline == vblank_start) 589 scanline = intel_get_crtc_scanline(crtc); 590 591 crtc->debug.scanline_start = scanline; 592 crtc->debug.start_vbl_time = ktime_get(); 593 crtc->debug.start_vbl_count = intel_crtc_get_vblank_counter(crtc); 594 595 trace_intel_pipe_update_vblank_evaded(crtc); 596 return; 597 598irq_disable: 599 local_irq_disable(); 600} 601 602#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE) 603static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end) 604{ 605 u64 delta = ktime_to_ns(ktime_sub(end, crtc->debug.start_vbl_time)); 606 unsigned int h; 607 608 h = ilog2(delta >> 9); 609 if (h >= ARRAY_SIZE(crtc->debug.vbl.times)) 610 h = ARRAY_SIZE(crtc->debug.vbl.times) - 1; 611 crtc->debug.vbl.times[h]++; 612 613 crtc->debug.vbl.sum += delta; 614 if (!crtc->debug.vbl.min || delta < crtc->debug.vbl.min) 615 crtc->debug.vbl.min = delta; 616 if (delta > crtc->debug.vbl.max) 617 crtc->debug.vbl.max = delta; 618 619 if (delta > 1000 * VBLANK_EVASION_TIME_US) { 620 drm_dbg_kms(crtc->base.dev, 621 "Atomic update on pipe (%c) took %lld us, max time under evasion is %u us\n", 622 pipe_name(crtc->pipe), 623 div_u64(delta, 1000), 624 VBLANK_EVASION_TIME_US); 625 crtc->debug.vbl.over++; 626 } 627} 628#else 629static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end) {} 630#endif 631 632/** 633 * intel_pipe_update_end() - end update of a set of display registers 634 * @new_crtc_state: the new crtc state 635 * 636 * Mark the end of an update started with intel_pipe_update_start(). This 637 * re-enables interrupts and verifies the update was actually completed 638 * before a vblank. 639 */ 640void intel_pipe_update_end(struct intel_crtc_state *new_crtc_state) 641{ 642 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc); 643 enum pipe pipe = crtc->pipe; 644 int scanline_end = intel_get_crtc_scanline(crtc); 645 u32 end_vbl_count = intel_crtc_get_vblank_counter(crtc); 646 ktime_t end_vbl_time = ktime_get(); 647 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 648 649 intel_psr_unlock(new_crtc_state); 650 651 if (new_crtc_state->do_async_flip) 652 return; 653 654 trace_intel_pipe_update_end(crtc, end_vbl_count, scanline_end); 655 656 /* 657 * Incase of mipi dsi command mode, we need to set frame update 658 * request for every commit. 659 */ 660 if (DISPLAY_VER(dev_priv) >= 11 && 661 intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI)) 662 icl_dsi_frame_update(new_crtc_state); 663 664 /* We're still in the vblank-evade critical section, this can't race. 665 * Would be slightly nice to just grab the vblank count and arm the 666 * event outside of the critical section - the spinlock might spin for a 667 * while ... */ 668 if (intel_crtc_needs_vblank_work(new_crtc_state)) { 669 drm_vblank_work_schedule(&new_crtc_state->vblank_work, 670 drm_crtc_accurate_vblank_count(&crtc->base) + 1, 671 false); 672 } else if (new_crtc_state->uapi.event) { 673 drm_WARN_ON(&dev_priv->drm, 674 drm_crtc_vblank_get(&crtc->base) != 0); 675 676 spin_lock(&crtc->base.dev->event_lock); 677 drm_crtc_arm_vblank_event(&crtc->base, 678 new_crtc_state->uapi.event); 679 spin_unlock(&crtc->base.dev->event_lock); 680 681 new_crtc_state->uapi.event = NULL; 682 } 683 684 /* 685 * Send VRR Push to terminate Vblank. If we are already in vblank 686 * this has to be done _after_ sampling the frame counter, as 687 * otherwise the push would immediately terminate the vblank and 688 * the sampled frame counter would correspond to the next frame 689 * instead of the current frame. 690 * 691 * There is a tiny race here (iff vblank evasion failed us) where 692 * we might sample the frame counter just before vmax vblank start 693 * but the push would be sent just after it. That would cause the 694 * push to affect the next frame instead of the current frame, 695 * which would cause the next frame to terminate already at vmin 696 * vblank start instead of vmax vblank start. 697 */ 698 intel_vrr_send_push(new_crtc_state); 699 700 /* 701 * Seamless M/N update may need to update frame timings. 702 * 703 * FIXME Should be synchronized with the start of vblank somehow... 704 */ 705 if (new_crtc_state->seamless_m_n && intel_crtc_needs_fastset(new_crtc_state)) 706 intel_crtc_update_active_timings(new_crtc_state, 707 new_crtc_state->vrr.enable); 708 709 local_irq_enable(); 710 711 if (intel_vgpu_active(dev_priv)) 712 return; 713 714 if (crtc->debug.start_vbl_count && 715 crtc->debug.start_vbl_count != end_vbl_count) { 716 drm_err(&dev_priv->drm, 717 "Atomic update failure on pipe %c (start=%u end=%u) time %lld us, min %d, max %d, scanline start %d, end %d\n", 718 pipe_name(pipe), crtc->debug.start_vbl_count, 719 end_vbl_count, 720 ktime_us_delta(end_vbl_time, 721 crtc->debug.start_vbl_time), 722 crtc->debug.min_vbl, crtc->debug.max_vbl, 723 crtc->debug.scanline_start, scanline_end); 724 } 725 726 dbg_vblank_evade(crtc, end_vbl_time); 727} 728