1/* 2 * drm_irq.c IRQ and vblank support 3 * 4 * \author Rickard E. (Rik) Faith <faith@valinux.com> 5 * \author Gareth Hughes <gareth@valinux.com> 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the "Software"), 9 * to deal in the Software without restriction, including without limitation 10 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 11 * and/or sell copies of the Software, and to permit persons to whom the 12 * Software is furnished to do so, subject to the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the next 15 * paragraph) shall be included in all copies or substantial portions of the 16 * Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 21 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 22 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 23 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 24 * OTHER DEALINGS IN THE SOFTWARE. 25 */ 26 27#include <linux/export.h> 28#include <linux/kthread.h> 29#include <linux/moduleparam.h> 30 31#include <drm/drm_crtc.h> 32#include <drm/drm_drv.h> 33#include <drm/drm_framebuffer.h> 34#include <drm/drm_managed.h> 35#include <drm/drm_modeset_helper_vtables.h> 36#include <drm/drm_print.h> 37#include <drm/drm_vblank.h> 38 39#include "drm_internal.h" 40#include "drm_trace.h" 41 42/** 43 * DOC: vblank handling 44 * 45 * From the computer's perspective, every time the monitor displays 46 * a new frame the scanout engine has "scanned out" the display image 47 * from top to bottom, one row of pixels at a time. The current row 48 * of pixels is referred to as the current scanline. 49 * 50 * In addition to the display's visible area, there's usually a couple of 51 * extra scanlines which aren't actually displayed on the screen. 52 * These extra scanlines don't contain image data and are occasionally used 53 * for features like audio and infoframes. The region made up of these 54 * scanlines is referred to as the vertical blanking region, or vblank for 55 * short. 56 * 57 * For historical reference, the vertical blanking period was designed to 58 * give the electron gun (on CRTs) enough time to move back to the top of 59 * the screen to start scanning out the next frame. Similar for horizontal 60 * blanking periods. They were designed to give the electron gun enough 61 * time to move back to the other side of the screen to start scanning the 62 * next scanline. 63 * 64 * :: 65 * 66 * 67 * physical → ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽ 68 * top of | | 69 * display | | 70 * | New frame | 71 * | | 72 * |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓| 73 * |~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~| ← Scanline, 74 * |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓| updates the 75 * | | frame as it 76 * | | travels down 77 * | | ("scan out") 78 * | Old frame | 79 * | | 80 * | | 81 * | | 82 * | | physical 83 * | | bottom of 84 * vertical |⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽| ← display 85 * blanking ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆ 86 * region → ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆ 87 * ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆ 88 * start of → ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽ 89 * new frame 90 * 91 * "Physical top of display" is the reference point for the high-precision/ 92 * corrected timestamp. 93 * 94 * On a lot of display hardware, programming needs to take effect during the 95 * vertical blanking period so that settings like gamma, the image buffer 96 * buffer to be scanned out, etc. can safely be changed without showing 97 * any visual artifacts on the screen. In some unforgiving hardware, some of 98 * this programming has to both start and end in the same vblank. To help 99 * with the timing of the hardware programming, an interrupt is usually 100 * available to notify the driver when it can start the updating of registers. 101 * The interrupt is in this context named the vblank interrupt. 102 * 103 * The vblank interrupt may be fired at different points depending on the 104 * hardware. Some hardware implementations will fire the interrupt when the 105 * new frame start, other implementations will fire the interrupt at different 106 * points in time. 107 * 108 * Vertical blanking plays a major role in graphics rendering. To achieve 109 * tear-free display, users must synchronize page flips and/or rendering to 110 * vertical blanking. The DRM API offers ioctls to perform page flips 111 * synchronized to vertical blanking and wait for vertical blanking. 112 * 113 * The DRM core handles most of the vertical blanking management logic, which 114 * involves filtering out spurious interrupts, keeping race-free blanking 115 * counters, coping with counter wrap-around and resets and keeping use counts. 116 * It relies on the driver to generate vertical blanking interrupts and 117 * optionally provide a hardware vertical blanking counter. 118 * 119 * Drivers must initialize the vertical blanking handling core with a call to 120 * drm_vblank_init(). Minimally, a driver needs to implement 121 * &drm_crtc_funcs.enable_vblank and &drm_crtc_funcs.disable_vblank plus call 122 * drm_crtc_handle_vblank() in its vblank interrupt handler for working vblank 123 * support. 124 * 125 * Vertical blanking interrupts can be enabled by the DRM core or by drivers 126 * themselves (for instance to handle page flipping operations). The DRM core 127 * maintains a vertical blanking use count to ensure that the interrupts are not 128 * disabled while a user still needs them. To increment the use count, drivers 129 * call drm_crtc_vblank_get() and release the vblank reference again with 130 * drm_crtc_vblank_put(). In between these two calls vblank interrupts are 131 * guaranteed to be enabled. 132 * 133 * On many hardware disabling the vblank interrupt cannot be done in a race-free 134 * manner, see &drm_driver.vblank_disable_immediate and 135 * &drm_driver.max_vblank_count. In that case the vblank core only disables the 136 * vblanks after a timer has expired, which can be configured through the 137 * ``vblankoffdelay`` module parameter. 138 * 139 * Drivers for hardware without support for vertical-blanking interrupts 140 * must not call drm_vblank_init(). For such drivers, atomic helpers will 141 * automatically generate fake vblank events as part of the display update. 142 * This functionality also can be controlled by the driver by enabling and 143 * disabling struct drm_crtc_state.no_vblank. 144 */ 145 146/* Retry timestamp calculation up to 3 times to satisfy 147 * drm_timestamp_precision before giving up. 148 */ 149#define DRM_TIMESTAMP_MAXRETRIES 3 150 151/* Threshold in nanoseconds for detection of redundant 152 * vblank irq in drm_handle_vblank(). 1 msec should be ok. 153 */ 154#define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000 155 156static bool 157drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe, 158 ktime_t *tvblank, bool in_vblank_irq); 159 160static unsigned int drm_timestamp_precision = 20; /* Default to 20 usecs. */ 161 162static int drm_vblank_offdelay = 5000; /* Default to 5000 msecs. */ 163 164module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600); 165module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600); 166MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)"); 167MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]"); 168 169static void store_vblank(struct drm_device *dev, unsigned int pipe, 170 u32 vblank_count_inc, 171 ktime_t t_vblank, u32 last) 172{ 173 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 174 175 assert_spin_locked(&dev->vblank_time_lock); 176 177 vblank->last = last; 178 179 write_seqlock(&vblank->seqlock); 180 vblank->time = t_vblank; 181 atomic64_add(vblank_count_inc, &vblank->count); 182 write_sequnlock(&vblank->seqlock); 183} 184 185static u32 drm_max_vblank_count(struct drm_device *dev, unsigned int pipe) 186{ 187 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 188 189 return vblank->max_vblank_count ?: dev->max_vblank_count; 190} 191 192/* 193 * "No hw counter" fallback implementation of .get_vblank_counter() hook, 194 * if there is no usable hardware frame counter available. 195 */ 196static u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe) 197{ 198 drm_WARN_ON_ONCE(dev, drm_max_vblank_count(dev, pipe) != 0); 199 return 0; 200} 201 202static u32 __get_vblank_counter(struct drm_device *dev, unsigned int pipe) 203{ 204 if (drm_core_check_feature(dev, DRIVER_MODESET)) { 205 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe); 206 207 if (drm_WARN_ON(dev, !crtc)) 208 return 0; 209 210 if (crtc->funcs->get_vblank_counter) 211 return crtc->funcs->get_vblank_counter(crtc); 212 } 213#ifdef CONFIG_DRM_LEGACY 214 else if (dev->driver->get_vblank_counter) { 215 return dev->driver->get_vblank_counter(dev, pipe); 216 } 217#endif 218 219 return drm_vblank_no_hw_counter(dev, pipe); 220} 221 222/* 223 * Reset the stored timestamp for the current vblank count to correspond 224 * to the last vblank occurred. 225 * 226 * Only to be called from drm_crtc_vblank_on(). 227 * 228 * Note: caller must hold &drm_device.vbl_lock since this reads & writes 229 * device vblank fields. 230 */ 231static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe) 232{ 233 u32 cur_vblank; 234 bool rc; 235 ktime_t t_vblank; 236 int count = DRM_TIMESTAMP_MAXRETRIES; 237 238 spin_lock(&dev->vblank_time_lock); 239 240 /* 241 * sample the current counter to avoid random jumps 242 * when drm_vblank_enable() applies the diff 243 */ 244 do { 245 cur_vblank = __get_vblank_counter(dev, pipe); 246 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false); 247 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0); 248 249 /* 250 * Only reinitialize corresponding vblank timestamp if high-precision query 251 * available and didn't fail. Otherwise reinitialize delayed at next vblank 252 * interrupt and assign 0 for now, to mark the vblanktimestamp as invalid. 253 */ 254 if (!rc) 255 t_vblank = 0; 256 257 /* 258 * +1 to make sure user will never see the same 259 * vblank counter value before and after a modeset 260 */ 261 store_vblank(dev, pipe, 1, t_vblank, cur_vblank); 262 263 spin_unlock(&dev->vblank_time_lock); 264} 265 266/* 267 * Call back into the driver to update the appropriate vblank counter 268 * (specified by @pipe). Deal with wraparound, if it occurred, and 269 * update the last read value so we can deal with wraparound on the next 270 * call if necessary. 271 * 272 * Only necessary when going from off->on, to account for frames we 273 * didn't get an interrupt for. 274 * 275 * Note: caller must hold &drm_device.vbl_lock since this reads & writes 276 * device vblank fields. 277 */ 278static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe, 279 bool in_vblank_irq) 280{ 281 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 282 u32 cur_vblank, diff; 283 bool rc; 284 ktime_t t_vblank; 285 int count = DRM_TIMESTAMP_MAXRETRIES; 286 int framedur_ns = vblank->framedur_ns; 287 u32 max_vblank_count = drm_max_vblank_count(dev, pipe); 288 289 /* 290 * Interrupts were disabled prior to this call, so deal with counter 291 * wrap if needed. 292 * NOTE! It's possible we lost a full dev->max_vblank_count + 1 events 293 * here if the register is small or we had vblank interrupts off for 294 * a long time. 295 * 296 * We repeat the hardware vblank counter & timestamp query until 297 * we get consistent results. This to prevent races between gpu 298 * updating its hardware counter while we are retrieving the 299 * corresponding vblank timestamp. 300 */ 301 do { 302 cur_vblank = __get_vblank_counter(dev, pipe); 303 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, in_vblank_irq); 304 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0); 305 306 if (max_vblank_count) { 307 /* trust the hw counter when it's around */ 308 diff = (cur_vblank - vblank->last) & max_vblank_count; 309 } else if (rc && framedur_ns) { 310 u64 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time)); 311 312 /* 313 * Figure out how many vblanks we've missed based 314 * on the difference in the timestamps and the 315 * frame/field duration. 316 */ 317 318 drm_dbg_vbl(dev, "crtc %u: Calculating number of vblanks." 319 " diff_ns = %lld, framedur_ns = %d)\n", 320 pipe, (long long)diff_ns, framedur_ns); 321 322 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns); 323 324 if (diff == 0 && in_vblank_irq) 325 drm_dbg_vbl(dev, "crtc %u: Redundant vblirq ignored\n", 326 pipe); 327 } else { 328 /* some kind of default for drivers w/o accurate vbl timestamping */ 329 diff = in_vblank_irq ? 1 : 0; 330 } 331 332 /* 333 * Within a drm_vblank_pre_modeset - drm_vblank_post_modeset 334 * interval? If so then vblank irqs keep running and it will likely 335 * happen that the hardware vblank counter is not trustworthy as it 336 * might reset at some point in that interval and vblank timestamps 337 * are not trustworthy either in that interval. Iow. this can result 338 * in a bogus diff >> 1 which must be avoided as it would cause 339 * random large forward jumps of the software vblank counter. 340 */ 341 if (diff > 1 && (vblank->inmodeset & 0x2)) { 342 drm_dbg_vbl(dev, 343 "clamping vblank bump to 1 on crtc %u: diffr=%u" 344 " due to pre-modeset.\n", pipe, diff); 345 diff = 1; 346 } 347 348 drm_dbg_vbl(dev, "updating vblank count on crtc %u:" 349 " current=%llu, diff=%u, hw=%u hw_last=%u\n", 350 pipe, (unsigned long long)atomic64_read(&vblank->count), 351 diff, cur_vblank, vblank->last); 352 353 if (diff == 0) { 354 drm_WARN_ON_ONCE(dev, cur_vblank != vblank->last); 355 return; 356 } 357 358 /* 359 * Only reinitialize corresponding vblank timestamp if high-precision query 360 * available and didn't fail, or we were called from the vblank interrupt. 361 * Otherwise reinitialize delayed at next vblank interrupt and assign 0 362 * for now, to mark the vblanktimestamp as invalid. 363 */ 364 if (!rc && !in_vblank_irq) 365 t_vblank = 0; 366 367 store_vblank(dev, pipe, diff, t_vblank, cur_vblank); 368} 369 370u64 drm_vblank_count(struct drm_device *dev, unsigned int pipe) 371{ 372 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 373 u64 count; 374 375 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 376 return 0; 377 378 count = atomic64_read(&vblank->count); 379 380 /* 381 * This read barrier corresponds to the implicit write barrier of the 382 * write seqlock in store_vblank(). Note that this is the only place 383 * where we need an explicit barrier, since all other access goes 384 * through drm_vblank_count_and_time(), which already has the required 385 * read barrier curtesy of the read seqlock. 386 */ 387 smp_rmb(); 388 389 return count; 390} 391 392/** 393 * drm_crtc_accurate_vblank_count - retrieve the master vblank counter 394 * @crtc: which counter to retrieve 395 * 396 * This function is similar to drm_crtc_vblank_count() but this function 397 * interpolates to handle a race with vblank interrupts using the high precision 398 * timestamping support. 399 * 400 * This is mostly useful for hardware that can obtain the scanout position, but 401 * doesn't have a hardware frame counter. 402 */ 403u64 drm_crtc_accurate_vblank_count(struct drm_crtc *crtc) 404{ 405 struct drm_device *dev = crtc->dev; 406 unsigned int pipe = drm_crtc_index(crtc); 407 u64 vblank; 408 unsigned long flags; 409 410 drm_WARN_ONCE(dev, drm_debug_enabled(DRM_UT_VBL) && 411 !crtc->funcs->get_vblank_timestamp, 412 "This function requires support for accurate vblank timestamps."); 413 414 spin_lock_irqsave(&dev->vblank_time_lock, flags); 415 416 drm_update_vblank_count(dev, pipe, false); 417 vblank = drm_vblank_count(dev, pipe); 418 419 spin_unlock_irqrestore(&dev->vblank_time_lock, flags); 420 421 return vblank; 422} 423EXPORT_SYMBOL(drm_crtc_accurate_vblank_count); 424 425static void __disable_vblank(struct drm_device *dev, unsigned int pipe) 426{ 427 if (drm_core_check_feature(dev, DRIVER_MODESET)) { 428 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe); 429 430 if (drm_WARN_ON(dev, !crtc)) 431 return; 432 433 if (crtc->funcs->disable_vblank) 434 crtc->funcs->disable_vblank(crtc); 435 } 436#ifdef CONFIG_DRM_LEGACY 437 else { 438 dev->driver->disable_vblank(dev, pipe); 439 } 440#endif 441} 442 443/* 444 * Disable vblank irq's on crtc, make sure that last vblank count 445 * of hardware and corresponding consistent software vblank counter 446 * are preserved, even if there are any spurious vblank irq's after 447 * disable. 448 */ 449void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe) 450{ 451 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 452 unsigned long irqflags; 453 454 assert_spin_locked(&dev->vbl_lock); 455 456 /* Prevent vblank irq processing while disabling vblank irqs, 457 * so no updates of timestamps or count can happen after we've 458 * disabled. Needed to prevent races in case of delayed irq's. 459 */ 460 spin_lock_irqsave(&dev->vblank_time_lock, irqflags); 461 462 /* 463 * Update vblank count and disable vblank interrupts only if the 464 * interrupts were enabled. This avoids calling the ->disable_vblank() 465 * operation in atomic context with the hardware potentially runtime 466 * suspended. 467 */ 468 if (!vblank->enabled) 469 goto out; 470 471 /* 472 * Update the count and timestamp to maintain the 473 * appearance that the counter has been ticking all along until 474 * this time. This makes the count account for the entire time 475 * between drm_crtc_vblank_on() and drm_crtc_vblank_off(). 476 */ 477 drm_update_vblank_count(dev, pipe, false); 478 __disable_vblank(dev, pipe); 479 vblank->enabled = false; 480 481out: 482 spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags); 483} 484 485static void vblank_disable_fn(struct timer_list *t) 486{ 487 struct drm_vblank_crtc *vblank = from_timer(vblank, t, disable_timer); 488 struct drm_device *dev = vblank->dev; 489 unsigned int pipe = vblank->pipe; 490 unsigned long irqflags; 491 492 spin_lock_irqsave(&dev->vbl_lock, irqflags); 493 if (atomic_read(&vblank->refcount) == 0 && vblank->enabled) { 494 drm_dbg_core(dev, "disabling vblank on crtc %u\n", pipe); 495 drm_vblank_disable_and_save(dev, pipe); 496 } 497 spin_unlock_irqrestore(&dev->vbl_lock, irqflags); 498} 499 500static void drm_vblank_init_release(struct drm_device *dev, void *ptr) 501{ 502 struct drm_vblank_crtc *vblank = ptr; 503 504 drm_WARN_ON(dev, READ_ONCE(vblank->enabled) && 505 drm_core_check_feature(dev, DRIVER_MODESET)); 506 507 drm_vblank_destroy_worker(vblank); 508 del_timer_sync(&vblank->disable_timer); 509} 510 511/** 512 * drm_vblank_init - initialize vblank support 513 * @dev: DRM device 514 * @num_crtcs: number of CRTCs supported by @dev 515 * 516 * This function initializes vblank support for @num_crtcs display pipelines. 517 * Cleanup is handled automatically through a cleanup function added with 518 * drmm_add_action_or_reset(). 519 * 520 * Returns: 521 * Zero on success or a negative error code on failure. 522 */ 523int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs) 524{ 525 int ret; 526 unsigned int i; 527 528 spin_lock_init(&dev->vbl_lock); 529 spin_lock_init(&dev->vblank_time_lock); 530 531 dev->vblank = drmm_kcalloc(dev, num_crtcs, sizeof(*dev->vblank), GFP_KERNEL); 532 if (!dev->vblank) 533 return -ENOMEM; 534 535 dev->num_crtcs = num_crtcs; 536 537 for (i = 0; i < num_crtcs; i++) { 538 struct drm_vblank_crtc *vblank = &dev->vblank[i]; 539 540 vblank->dev = dev; 541 vblank->pipe = i; 542 init_waitqueue_head(&vblank->queue); 543 timer_setup(&vblank->disable_timer, vblank_disable_fn, 0); 544 seqlock_init(&vblank->seqlock); 545 546 ret = drmm_add_action_or_reset(dev, drm_vblank_init_release, 547 vblank); 548 if (ret) 549 return ret; 550 551 ret = drm_vblank_worker_init(vblank); 552 if (ret) 553 return ret; 554 } 555 556 return 0; 557} 558EXPORT_SYMBOL(drm_vblank_init); 559 560/** 561 * drm_dev_has_vblank - test if vblanking has been initialized for 562 * a device 563 * @dev: the device 564 * 565 * Drivers may call this function to test if vblank support is 566 * initialized for a device. For most hardware this means that vblanking 567 * can also be enabled. 568 * 569 * Atomic helpers use this function to initialize 570 * &drm_crtc_state.no_vblank. See also drm_atomic_helper_check_modeset(). 571 * 572 * Returns: 573 * True if vblanking has been initialized for the given device, false 574 * otherwise. 575 */ 576bool drm_dev_has_vblank(const struct drm_device *dev) 577{ 578 return dev->num_crtcs != 0; 579} 580EXPORT_SYMBOL(drm_dev_has_vblank); 581 582/** 583 * drm_crtc_vblank_waitqueue - get vblank waitqueue for the CRTC 584 * @crtc: which CRTC's vblank waitqueue to retrieve 585 * 586 * This function returns a pointer to the vblank waitqueue for the CRTC. 587 * Drivers can use this to implement vblank waits using wait_event() and related 588 * functions. 589 */ 590wait_queue_head_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc) 591{ 592 return &crtc->dev->vblank[drm_crtc_index(crtc)].queue; 593} 594EXPORT_SYMBOL(drm_crtc_vblank_waitqueue); 595 596 597/** 598 * drm_calc_timestamping_constants - calculate vblank timestamp constants 599 * @crtc: drm_crtc whose timestamp constants should be updated. 600 * @mode: display mode containing the scanout timings 601 * 602 * Calculate and store various constants which are later needed by vblank and 603 * swap-completion timestamping, e.g, by 604 * drm_crtc_vblank_helper_get_vblank_timestamp(). They are derived from 605 * CRTC's true scanout timing, so they take things like panel scaling or 606 * other adjustments into account. 607 */ 608void drm_calc_timestamping_constants(struct drm_crtc *crtc, 609 const struct drm_display_mode *mode) 610{ 611 struct drm_device *dev = crtc->dev; 612 unsigned int pipe = drm_crtc_index(crtc); 613 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 614 int linedur_ns = 0, framedur_ns = 0; 615 int dotclock = mode->crtc_clock; 616 617 if (!drm_dev_has_vblank(dev)) 618 return; 619 620 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 621 return; 622 623 /* Valid dotclock? */ 624 if (dotclock > 0) { 625 int frame_size = mode->crtc_htotal * mode->crtc_vtotal; 626 627 /* 628 * Convert scanline length in pixels and video 629 * dot clock to line duration and frame duration 630 * in nanoseconds: 631 */ 632 linedur_ns = div_u64((u64) mode->crtc_htotal * 1000000, dotclock); 633 framedur_ns = div_u64((u64) frame_size * 1000000, dotclock); 634 635 /* 636 * Fields of interlaced scanout modes are only half a frame duration. 637 */ 638 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 639 framedur_ns /= 2; 640 } else { 641 drm_err(dev, "crtc %u: Can't calculate constants, dotclock = 0!\n", 642 crtc->base.id); 643 } 644 645 vblank->linedur_ns = linedur_ns; 646 vblank->framedur_ns = framedur_ns; 647 drm_mode_copy(&vblank->hwmode, mode); 648 649 drm_dbg_core(dev, 650 "crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n", 651 crtc->base.id, mode->crtc_htotal, 652 mode->crtc_vtotal, mode->crtc_vdisplay); 653 drm_dbg_core(dev, "crtc %u: clock %d kHz framedur %d linedur %d\n", 654 crtc->base.id, dotclock, framedur_ns, linedur_ns); 655} 656EXPORT_SYMBOL(drm_calc_timestamping_constants); 657 658/** 659 * drm_crtc_vblank_helper_get_vblank_timestamp_internal - precise vblank 660 * timestamp helper 661 * @crtc: CRTC whose vblank timestamp to retrieve 662 * @max_error: Desired maximum allowable error in timestamps (nanosecs) 663 * On return contains true maximum error of timestamp 664 * @vblank_time: Pointer to time which should receive the timestamp 665 * @in_vblank_irq: 666 * True when called from drm_crtc_handle_vblank(). Some drivers 667 * need to apply some workarounds for gpu-specific vblank irq quirks 668 * if flag is set. 669 * @get_scanout_position: 670 * Callback function to retrieve the scanout position. See 671 * @struct drm_crtc_helper_funcs.get_scanout_position. 672 * 673 * Implements calculation of exact vblank timestamps from given drm_display_mode 674 * timings and current video scanout position of a CRTC. 675 * 676 * The current implementation only handles standard video modes. For double scan 677 * and interlaced modes the driver is supposed to adjust the hardware mode 678 * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to 679 * match the scanout position reported. 680 * 681 * Note that atomic drivers must call drm_calc_timestamping_constants() before 682 * enabling a CRTC. The atomic helpers already take care of that in 683 * drm_atomic_helper_calc_timestamping_constants(). 684 * 685 * Returns: 686 * 687 * Returns true on success, and false on failure, i.e. when no accurate 688 * timestamp could be acquired. 689 */ 690bool 691drm_crtc_vblank_helper_get_vblank_timestamp_internal( 692 struct drm_crtc *crtc, int *max_error, ktime_t *vblank_time, 693 bool in_vblank_irq, 694 drm_vblank_get_scanout_position_func get_scanout_position) 695{ 696 struct drm_device *dev = crtc->dev; 697 unsigned int pipe = crtc->index; 698 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 699 struct timespec64 ts_etime, ts_vblank_time; 700 ktime_t stime, etime; 701 bool vbl_status; 702 const struct drm_display_mode *mode; 703 int vpos, hpos, i; 704 int delta_ns, duration_ns; 705 706 if (pipe >= dev->num_crtcs) { 707 drm_err(dev, "Invalid crtc %u\n", pipe); 708 return false; 709 } 710 711 /* Scanout position query not supported? Should not happen. */ 712 if (!get_scanout_position) { 713 drm_err(dev, "Called from CRTC w/o get_scanout_position()!?\n"); 714 return false; 715 } 716 717 if (drm_drv_uses_atomic_modeset(dev)) 718 mode = &vblank->hwmode; 719 else 720 mode = &crtc->hwmode; 721 722 /* If mode timing undefined, just return as no-op: 723 * Happens during initial modesetting of a crtc. 724 */ 725 if (mode->crtc_clock == 0) { 726 drm_dbg_core(dev, "crtc %u: Noop due to uninitialized mode.\n", 727 pipe); 728 drm_WARN_ON_ONCE(dev, drm_drv_uses_atomic_modeset(dev)); 729 return false; 730 } 731 732 /* Get current scanout position with system timestamp. 733 * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times 734 * if single query takes longer than max_error nanoseconds. 735 * 736 * This guarantees a tight bound on maximum error if 737 * code gets preempted or delayed for some reason. 738 */ 739 for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) { 740 /* 741 * Get vertical and horizontal scanout position vpos, hpos, 742 * and bounding timestamps stime, etime, pre/post query. 743 */ 744 vbl_status = get_scanout_position(crtc, in_vblank_irq, 745 &vpos, &hpos, 746 &stime, &etime, 747 mode); 748 749 /* Return as no-op if scanout query unsupported or failed. */ 750 if (!vbl_status) { 751 drm_dbg_core(dev, 752 "crtc %u : scanoutpos query failed.\n", 753 pipe); 754 return false; 755 } 756 757 /* Compute uncertainty in timestamp of scanout position query. */ 758 duration_ns = ktime_to_ns(etime) - ktime_to_ns(stime); 759 760 /* Accept result with < max_error nsecs timing uncertainty. */ 761 if (duration_ns <= *max_error) 762 break; 763 } 764 765 /* Noisy system timing? */ 766 if (i == DRM_TIMESTAMP_MAXRETRIES) { 767 drm_dbg_core(dev, 768 "crtc %u: Noisy timestamp %d us > %d us [%d reps].\n", 769 pipe, duration_ns / 1000, *max_error / 1000, i); 770 } 771 772 /* Return upper bound of timestamp precision error. */ 773 *max_error = duration_ns; 774 775 /* Convert scanout position into elapsed time at raw_time query 776 * since start of scanout at first display scanline. delta_ns 777 * can be negative if start of scanout hasn't happened yet. 778 */ 779 delta_ns = div_s64(1000000LL * (vpos * mode->crtc_htotal + hpos), 780 mode->crtc_clock); 781 782 /* Subtract time delta from raw timestamp to get final 783 * vblank_time timestamp for end of vblank. 784 */ 785 *vblank_time = ktime_sub_ns(etime, delta_ns); 786 787 if (!drm_debug_enabled(DRM_UT_VBL)) 788 return true; 789 790 ts_etime = ktime_to_timespec64(etime); 791 ts_vblank_time = ktime_to_timespec64(*vblank_time); 792 793 drm_dbg_vbl(dev, 794 "crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n", 795 pipe, hpos, vpos, 796 (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000, 797 (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000, 798 duration_ns / 1000, i); 799 800 return true; 801} 802EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp_internal); 803 804/** 805 * drm_crtc_vblank_helper_get_vblank_timestamp - precise vblank timestamp 806 * helper 807 * @crtc: CRTC whose vblank timestamp to retrieve 808 * @max_error: Desired maximum allowable error in timestamps (nanosecs) 809 * On return contains true maximum error of timestamp 810 * @vblank_time: Pointer to time which should receive the timestamp 811 * @in_vblank_irq: 812 * True when called from drm_crtc_handle_vblank(). Some drivers 813 * need to apply some workarounds for gpu-specific vblank irq quirks 814 * if flag is set. 815 * 816 * Implements calculation of exact vblank timestamps from given drm_display_mode 817 * timings and current video scanout position of a CRTC. This can be directly 818 * used as the &drm_crtc_funcs.get_vblank_timestamp implementation of a kms 819 * driver if &drm_crtc_helper_funcs.get_scanout_position is implemented. 820 * 821 * The current implementation only handles standard video modes. For double scan 822 * and interlaced modes the driver is supposed to adjust the hardware mode 823 * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to 824 * match the scanout position reported. 825 * 826 * Note that atomic drivers must call drm_calc_timestamping_constants() before 827 * enabling a CRTC. The atomic helpers already take care of that in 828 * drm_atomic_helper_calc_timestamping_constants(). 829 * 830 * Returns: 831 * 832 * Returns true on success, and false on failure, i.e. when no accurate 833 * timestamp could be acquired. 834 */ 835bool drm_crtc_vblank_helper_get_vblank_timestamp(struct drm_crtc *crtc, 836 int *max_error, 837 ktime_t *vblank_time, 838 bool in_vblank_irq) 839{ 840 return drm_crtc_vblank_helper_get_vblank_timestamp_internal( 841 crtc, max_error, vblank_time, in_vblank_irq, 842 crtc->helper_private->get_scanout_position); 843} 844EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp); 845 846/** 847 * drm_crtc_get_last_vbltimestamp - retrieve raw timestamp for the most 848 * recent vblank interval 849 * @crtc: CRTC whose vblank timestamp to retrieve 850 * @tvblank: Pointer to target time which should receive the timestamp 851 * @in_vblank_irq: 852 * True when called from drm_crtc_handle_vblank(). Some drivers 853 * need to apply some workarounds for gpu-specific vblank irq quirks 854 * if flag is set. 855 * 856 * Fetches the system timestamp corresponding to the time of the most recent 857 * vblank interval on specified CRTC. May call into kms-driver to 858 * compute the timestamp with a high-precision GPU specific method. 859 * 860 * Returns zero if timestamp originates from uncorrected do_gettimeofday() 861 * call, i.e., it isn't very precisely locked to the true vblank. 862 * 863 * Returns: 864 * True if timestamp is considered to be very precise, false otherwise. 865 */ 866static bool 867drm_crtc_get_last_vbltimestamp(struct drm_crtc *crtc, ktime_t *tvblank, 868 bool in_vblank_irq) 869{ 870 bool ret = false; 871 872 /* Define requested maximum error on timestamps (nanoseconds). */ 873 int max_error = (int) drm_timestamp_precision * 1000; 874 875 /* Query driver if possible and precision timestamping enabled. */ 876 if (crtc && crtc->funcs->get_vblank_timestamp && max_error > 0) { 877 ret = crtc->funcs->get_vblank_timestamp(crtc, &max_error, 878 tvblank, in_vblank_irq); 879 } 880 881 /* GPU high precision timestamp query unsupported or failed. 882 * Return current monotonic/gettimeofday timestamp as best estimate. 883 */ 884 if (!ret) 885 *tvblank = ktime_get(); 886 887 return ret; 888} 889 890static bool 891drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe, 892 ktime_t *tvblank, bool in_vblank_irq) 893{ 894 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe); 895 896 return drm_crtc_get_last_vbltimestamp(crtc, tvblank, in_vblank_irq); 897} 898 899/** 900 * drm_crtc_vblank_count - retrieve "cooked" vblank counter value 901 * @crtc: which counter to retrieve 902 * 903 * Fetches the "cooked" vblank count value that represents the number of 904 * vblank events since the system was booted, including lost events due to 905 * modesetting activity. Note that this timer isn't correct against a racing 906 * vblank interrupt (since it only reports the software vblank counter), see 907 * drm_crtc_accurate_vblank_count() for such use-cases. 908 * 909 * Note that for a given vblank counter value drm_crtc_handle_vblank() 910 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time() 911 * provide a barrier: Any writes done before calling 912 * drm_crtc_handle_vblank() will be visible to callers of the later 913 * functions, if the vblank count is the same or a later one. 914 * 915 * See also &drm_vblank_crtc.count. 916 * 917 * Returns: 918 * The software vblank counter. 919 */ 920u64 drm_crtc_vblank_count(struct drm_crtc *crtc) 921{ 922 return drm_vblank_count(crtc->dev, drm_crtc_index(crtc)); 923} 924EXPORT_SYMBOL(drm_crtc_vblank_count); 925 926/** 927 * drm_vblank_count_and_time - retrieve "cooked" vblank counter value and the 928 * system timestamp corresponding to that vblank counter value. 929 * @dev: DRM device 930 * @pipe: index of CRTC whose counter to retrieve 931 * @vblanktime: Pointer to ktime_t to receive the vblank timestamp. 932 * 933 * Fetches the "cooked" vblank count value that represents the number of 934 * vblank events since the system was booted, including lost events due to 935 * modesetting activity. Returns corresponding system timestamp of the time 936 * of the vblank interval that corresponds to the current vblank counter value. 937 * 938 * This is the legacy version of drm_crtc_vblank_count_and_time(). 939 */ 940static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe, 941 ktime_t *vblanktime) 942{ 943 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 944 u64 vblank_count; 945 unsigned int seq; 946 947 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) { 948 *vblanktime = 0; 949 return 0; 950 } 951 952 do { 953 seq = read_seqbegin(&vblank->seqlock); 954 vblank_count = atomic64_read(&vblank->count); 955 *vblanktime = vblank->time; 956 } while (read_seqretry(&vblank->seqlock, seq)); 957 958 return vblank_count; 959} 960 961/** 962 * drm_crtc_vblank_count_and_time - retrieve "cooked" vblank counter value 963 * and the system timestamp corresponding to that vblank counter value 964 * @crtc: which counter to retrieve 965 * @vblanktime: Pointer to time to receive the vblank timestamp. 966 * 967 * Fetches the "cooked" vblank count value that represents the number of 968 * vblank events since the system was booted, including lost events due to 969 * modesetting activity. Returns corresponding system timestamp of the time 970 * of the vblank interval that corresponds to the current vblank counter value. 971 * 972 * Note that for a given vblank counter value drm_crtc_handle_vblank() 973 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time() 974 * provide a barrier: Any writes done before calling 975 * drm_crtc_handle_vblank() will be visible to callers of the later 976 * functions, if the vblank count is the same or a later one. 977 * 978 * See also &drm_vblank_crtc.count. 979 */ 980u64 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc, 981 ktime_t *vblanktime) 982{ 983 return drm_vblank_count_and_time(crtc->dev, drm_crtc_index(crtc), 984 vblanktime); 985} 986EXPORT_SYMBOL(drm_crtc_vblank_count_and_time); 987 988/** 989 * drm_crtc_next_vblank_start - calculate the time of the next vblank 990 * @crtc: the crtc for which to calculate next vblank time 991 * @vblanktime: pointer to time to receive the next vblank timestamp. 992 * 993 * Calculate the expected time of the start of the next vblank period, 994 * based on time of previous vblank and frame duration 995 */ 996int drm_crtc_next_vblank_start(struct drm_crtc *crtc, ktime_t *vblanktime) 997{ 998 unsigned int pipe = drm_crtc_index(crtc); 999 struct drm_vblank_crtc *vblank; 1000 struct drm_display_mode *mode; 1001 u64 vblank_start; 1002 1003 if (!drm_dev_has_vblank(crtc->dev)) 1004 return -EINVAL; 1005 1006 vblank = &crtc->dev->vblank[pipe]; 1007 mode = &vblank->hwmode; 1008 1009 if (!vblank->framedur_ns || !vblank->linedur_ns) 1010 return -EINVAL; 1011 1012 if (!drm_crtc_get_last_vbltimestamp(crtc, vblanktime, false)) 1013 return -EINVAL; 1014 1015 vblank_start = DIV_ROUND_DOWN_ULL( 1016 (u64)vblank->framedur_ns * mode->crtc_vblank_start, 1017 mode->crtc_vtotal); 1018 *vblanktime = ktime_add(*vblanktime, ns_to_ktime(vblank_start)); 1019 1020 return 0; 1021} 1022EXPORT_SYMBOL(drm_crtc_next_vblank_start); 1023 1024static void send_vblank_event(struct drm_device *dev, 1025 struct drm_pending_vblank_event *e, 1026 u64 seq, ktime_t now) 1027{ 1028 struct timespec64 tv; 1029 1030 switch (e->event.base.type) { 1031 case DRM_EVENT_VBLANK: 1032 case DRM_EVENT_FLIP_COMPLETE: 1033 tv = ktime_to_timespec64(now); 1034 e->event.vbl.sequence = seq; 1035 /* 1036 * e->event is a user space structure, with hardcoded unsigned 1037 * 32-bit seconds/microseconds. This is safe as we always use 1038 * monotonic timestamps since linux-4.15 1039 */ 1040 e->event.vbl.tv_sec = tv.tv_sec; 1041 e->event.vbl.tv_usec = tv.tv_nsec / 1000; 1042 break; 1043 case DRM_EVENT_CRTC_SEQUENCE: 1044 if (seq) 1045 e->event.seq.sequence = seq; 1046 e->event.seq.time_ns = ktime_to_ns(now); 1047 break; 1048 } 1049 trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq); 1050 /* 1051 * Use the same timestamp for any associated fence signal to avoid 1052 * mismatch in timestamps for vsync & fence events triggered by the 1053 * same HW event. Frameworks like SurfaceFlinger in Android expects the 1054 * retire-fence timestamp to match exactly with HW vsync as it uses it 1055 * for its software vsync modeling. 1056 */ 1057 drm_send_event_timestamp_locked(dev, &e->base, now); 1058} 1059 1060/** 1061 * drm_crtc_arm_vblank_event - arm vblank event after pageflip 1062 * @crtc: the source CRTC of the vblank event 1063 * @e: the event to send 1064 * 1065 * A lot of drivers need to generate vblank events for the very next vblank 1066 * interrupt. For example when the page flip interrupt happens when the page 1067 * flip gets armed, but not when it actually executes within the next vblank 1068 * period. This helper function implements exactly the required vblank arming 1069 * behaviour. 1070 * 1071 * NOTE: Drivers using this to send out the &drm_crtc_state.event as part of an 1072 * atomic commit must ensure that the next vblank happens at exactly the same 1073 * time as the atomic commit is committed to the hardware. This function itself 1074 * does **not** protect against the next vblank interrupt racing with either this 1075 * function call or the atomic commit operation. A possible sequence could be: 1076 * 1077 * 1. Driver commits new hardware state into vblank-synchronized registers. 1078 * 2. A vblank happens, committing the hardware state. Also the corresponding 1079 * vblank interrupt is fired off and fully processed by the interrupt 1080 * handler. 1081 * 3. The atomic commit operation proceeds to call drm_crtc_arm_vblank_event(). 1082 * 4. The event is only send out for the next vblank, which is wrong. 1083 * 1084 * An equivalent race can happen when the driver calls 1085 * drm_crtc_arm_vblank_event() before writing out the new hardware state. 1086 * 1087 * The only way to make this work safely is to prevent the vblank from firing 1088 * (and the hardware from committing anything else) until the entire atomic 1089 * commit sequence has run to completion. If the hardware does not have such a 1090 * feature (e.g. using a "go" bit), then it is unsafe to use this functions. 1091 * Instead drivers need to manually send out the event from their interrupt 1092 * handler by calling drm_crtc_send_vblank_event() and make sure that there's no 1093 * possible race with the hardware committing the atomic update. 1094 * 1095 * Caller must hold a vblank reference for the event @e acquired by a 1096 * drm_crtc_vblank_get(), which will be dropped when the next vblank arrives. 1097 */ 1098void drm_crtc_arm_vblank_event(struct drm_crtc *crtc, 1099 struct drm_pending_vblank_event *e) 1100{ 1101 struct drm_device *dev = crtc->dev; 1102 unsigned int pipe = drm_crtc_index(crtc); 1103 1104 assert_spin_locked(&dev->event_lock); 1105 1106 e->pipe = pipe; 1107 e->sequence = drm_crtc_accurate_vblank_count(crtc) + 1; 1108 list_add_tail(&e->base.link, &dev->vblank_event_list); 1109} 1110EXPORT_SYMBOL(drm_crtc_arm_vblank_event); 1111 1112/** 1113 * drm_crtc_send_vblank_event - helper to send vblank event after pageflip 1114 * @crtc: the source CRTC of the vblank event 1115 * @e: the event to send 1116 * 1117 * Updates sequence # and timestamp on event for the most recently processed 1118 * vblank, and sends it to userspace. Caller must hold event lock. 1119 * 1120 * See drm_crtc_arm_vblank_event() for a helper which can be used in certain 1121 * situation, especially to send out events for atomic commit operations. 1122 */ 1123void drm_crtc_send_vblank_event(struct drm_crtc *crtc, 1124 struct drm_pending_vblank_event *e) 1125{ 1126 struct drm_device *dev = crtc->dev; 1127 u64 seq; 1128 unsigned int pipe = drm_crtc_index(crtc); 1129 ktime_t now; 1130 1131 if (drm_dev_has_vblank(dev)) { 1132 seq = drm_vblank_count_and_time(dev, pipe, &now); 1133 } else { 1134 seq = 0; 1135 1136 now = ktime_get(); 1137 } 1138 e->pipe = pipe; 1139 send_vblank_event(dev, e, seq, now); 1140} 1141EXPORT_SYMBOL(drm_crtc_send_vblank_event); 1142 1143static int __enable_vblank(struct drm_device *dev, unsigned int pipe) 1144{ 1145 if (drm_core_check_feature(dev, DRIVER_MODESET)) { 1146 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe); 1147 1148 if (drm_WARN_ON(dev, !crtc)) 1149 return 0; 1150 1151 if (crtc->funcs->enable_vblank) 1152 return crtc->funcs->enable_vblank(crtc); 1153 } 1154#ifdef CONFIG_DRM_LEGACY 1155 else if (dev->driver->enable_vblank) { 1156 return dev->driver->enable_vblank(dev, pipe); 1157 } 1158#endif 1159 1160 return -EINVAL; 1161} 1162 1163static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe) 1164{ 1165 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1166 int ret = 0; 1167 1168 assert_spin_locked(&dev->vbl_lock); 1169 1170 spin_lock(&dev->vblank_time_lock); 1171 1172 if (!vblank->enabled) { 1173 /* 1174 * Enable vblank irqs under vblank_time_lock protection. 1175 * All vblank count & timestamp updates are held off 1176 * until we are done reinitializing master counter and 1177 * timestamps. Filtercode in drm_handle_vblank() will 1178 * prevent double-accounting of same vblank interval. 1179 */ 1180 ret = __enable_vblank(dev, pipe); 1181 drm_dbg_core(dev, "enabling vblank on crtc %u, ret: %d\n", 1182 pipe, ret); 1183 if (ret) { 1184 atomic_dec(&vblank->refcount); 1185 } else { 1186 drm_update_vblank_count(dev, pipe, 0); 1187 /* drm_update_vblank_count() includes a wmb so we just 1188 * need to ensure that the compiler emits the write 1189 * to mark the vblank as enabled after the call 1190 * to drm_update_vblank_count(). 1191 */ 1192 WRITE_ONCE(vblank->enabled, true); 1193 } 1194 } 1195 1196 spin_unlock(&dev->vblank_time_lock); 1197 1198 return ret; 1199} 1200 1201int drm_vblank_get(struct drm_device *dev, unsigned int pipe) 1202{ 1203 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1204 unsigned long irqflags; 1205 int ret = 0; 1206 1207 if (!drm_dev_has_vblank(dev)) 1208 return -EINVAL; 1209 1210 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 1211 return -EINVAL; 1212 1213 spin_lock_irqsave(&dev->vbl_lock, irqflags); 1214 /* Going from 0->1 means we have to enable interrupts again */ 1215 if (atomic_add_return(1, &vblank->refcount) == 1) { 1216 ret = drm_vblank_enable(dev, pipe); 1217 } else { 1218 if (!vblank->enabled) { 1219 atomic_dec(&vblank->refcount); 1220 ret = -EINVAL; 1221 } 1222 } 1223 spin_unlock_irqrestore(&dev->vbl_lock, irqflags); 1224 1225 return ret; 1226} 1227 1228/** 1229 * drm_crtc_vblank_get - get a reference count on vblank events 1230 * @crtc: which CRTC to own 1231 * 1232 * Acquire a reference count on vblank events to avoid having them disabled 1233 * while in use. 1234 * 1235 * Returns: 1236 * Zero on success or a negative error code on failure. 1237 */ 1238int drm_crtc_vblank_get(struct drm_crtc *crtc) 1239{ 1240 return drm_vblank_get(crtc->dev, drm_crtc_index(crtc)); 1241} 1242EXPORT_SYMBOL(drm_crtc_vblank_get); 1243 1244void drm_vblank_put(struct drm_device *dev, unsigned int pipe) 1245{ 1246 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1247 1248 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 1249 return; 1250 1251 if (drm_WARN_ON(dev, atomic_read(&vblank->refcount) == 0)) 1252 return; 1253 1254 /* Last user schedules interrupt disable */ 1255 if (atomic_dec_and_test(&vblank->refcount)) { 1256 if (drm_vblank_offdelay == 0) 1257 return; 1258 else if (drm_vblank_offdelay < 0) 1259 vblank_disable_fn(&vblank->disable_timer); 1260 else if (!dev->vblank_disable_immediate) 1261 mod_timer(&vblank->disable_timer, 1262 jiffies + ((drm_vblank_offdelay * HZ)/1000)); 1263 } 1264} 1265 1266/** 1267 * drm_crtc_vblank_put - give up ownership of vblank events 1268 * @crtc: which counter to give up 1269 * 1270 * Release ownership of a given vblank counter, turning off interrupts 1271 * if possible. Disable interrupts after drm_vblank_offdelay milliseconds. 1272 */ 1273void drm_crtc_vblank_put(struct drm_crtc *crtc) 1274{ 1275 drm_vblank_put(crtc->dev, drm_crtc_index(crtc)); 1276} 1277EXPORT_SYMBOL(drm_crtc_vblank_put); 1278 1279/** 1280 * drm_wait_one_vblank - wait for one vblank 1281 * @dev: DRM device 1282 * @pipe: CRTC index 1283 * 1284 * This waits for one vblank to pass on @pipe, using the irq driver interfaces. 1285 * It is a failure to call this when the vblank irq for @pipe is disabled, e.g. 1286 * due to lack of driver support or because the crtc is off. 1287 * 1288 * This is the legacy version of drm_crtc_wait_one_vblank(). 1289 */ 1290void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe) 1291{ 1292 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1293 int ret; 1294 u64 last; 1295 1296 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 1297 return; 1298 1299 ret = drm_vblank_get(dev, pipe); 1300 if (drm_WARN(dev, ret, "vblank not available on crtc %i, ret=%i\n", 1301 pipe, ret)) 1302 return; 1303 1304 last = drm_vblank_count(dev, pipe); 1305 1306 ret = wait_event_timeout(vblank->queue, 1307 last != drm_vblank_count(dev, pipe), 1308 msecs_to_jiffies(100)); 1309 1310 drm_WARN(dev, ret == 0, "vblank wait timed out on crtc %i\n", pipe); 1311 1312 drm_vblank_put(dev, pipe); 1313} 1314EXPORT_SYMBOL(drm_wait_one_vblank); 1315 1316/** 1317 * drm_crtc_wait_one_vblank - wait for one vblank 1318 * @crtc: DRM crtc 1319 * 1320 * This waits for one vblank to pass on @crtc, using the irq driver interfaces. 1321 * It is a failure to call this when the vblank irq for @crtc is disabled, e.g. 1322 * due to lack of driver support or because the crtc is off. 1323 */ 1324void drm_crtc_wait_one_vblank(struct drm_crtc *crtc) 1325{ 1326 drm_wait_one_vblank(crtc->dev, drm_crtc_index(crtc)); 1327} 1328EXPORT_SYMBOL(drm_crtc_wait_one_vblank); 1329 1330/** 1331 * drm_crtc_vblank_off - disable vblank events on a CRTC 1332 * @crtc: CRTC in question 1333 * 1334 * Drivers can use this function to shut down the vblank interrupt handling when 1335 * disabling a crtc. This function ensures that the latest vblank frame count is 1336 * stored so that drm_vblank_on can restore it again. 1337 * 1338 * Drivers must use this function when the hardware vblank counter can get 1339 * reset, e.g. when suspending or disabling the @crtc in general. 1340 */ 1341void drm_crtc_vblank_off(struct drm_crtc *crtc) 1342{ 1343 struct drm_device *dev = crtc->dev; 1344 unsigned int pipe = drm_crtc_index(crtc); 1345 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1346 struct drm_pending_vblank_event *e, *t; 1347 ktime_t now; 1348 u64 seq; 1349 1350 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 1351 return; 1352 1353 /* 1354 * Grab event_lock early to prevent vblank work from being scheduled 1355 * while we're in the middle of shutting down vblank interrupts 1356 */ 1357 spin_lock_irq(&dev->event_lock); 1358 1359 spin_lock(&dev->vbl_lock); 1360 drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n", 1361 pipe, vblank->enabled, vblank->inmodeset); 1362 1363 /* Avoid redundant vblank disables without previous 1364 * drm_crtc_vblank_on(). */ 1365 if (drm_core_check_feature(dev, DRIVER_ATOMIC) || !vblank->inmodeset) 1366 drm_vblank_disable_and_save(dev, pipe); 1367 1368 wake_up(&vblank->queue); 1369 1370 /* 1371 * Prevent subsequent drm_vblank_get() from re-enabling 1372 * the vblank interrupt by bumping the refcount. 1373 */ 1374 if (!vblank->inmodeset) { 1375 atomic_inc(&vblank->refcount); 1376 vblank->inmodeset = 1; 1377 } 1378 spin_unlock(&dev->vbl_lock); 1379 1380 /* Send any queued vblank events, lest the natives grow disquiet */ 1381 seq = drm_vblank_count_and_time(dev, pipe, &now); 1382 1383 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) { 1384 if (e->pipe != pipe) 1385 continue; 1386 drm_dbg_core(dev, "Sending premature vblank event on disable: " 1387 "wanted %llu, current %llu\n", 1388 e->sequence, seq); 1389 list_del(&e->base.link); 1390 drm_vblank_put(dev, pipe); 1391 send_vblank_event(dev, e, seq, now); 1392 } 1393 1394 /* Cancel any leftover pending vblank work */ 1395 drm_vblank_cancel_pending_works(vblank); 1396 1397 spin_unlock_irq(&dev->event_lock); 1398 1399 /* Will be reset by the modeset helpers when re-enabling the crtc by 1400 * calling drm_calc_timestamping_constants(). */ 1401 vblank->hwmode.crtc_clock = 0; 1402 1403 /* Wait for any vblank work that's still executing to finish */ 1404 drm_vblank_flush_worker(vblank); 1405} 1406EXPORT_SYMBOL(drm_crtc_vblank_off); 1407 1408/** 1409 * drm_crtc_vblank_reset - reset vblank state to off on a CRTC 1410 * @crtc: CRTC in question 1411 * 1412 * Drivers can use this function to reset the vblank state to off at load time. 1413 * Drivers should use this together with the drm_crtc_vblank_off() and 1414 * drm_crtc_vblank_on() functions. The difference compared to 1415 * drm_crtc_vblank_off() is that this function doesn't save the vblank counter 1416 * and hence doesn't need to call any driver hooks. 1417 * 1418 * This is useful for recovering driver state e.g. on driver load, or on resume. 1419 */ 1420void drm_crtc_vblank_reset(struct drm_crtc *crtc) 1421{ 1422 struct drm_device *dev = crtc->dev; 1423 unsigned int pipe = drm_crtc_index(crtc); 1424 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1425 1426 spin_lock_irq(&dev->vbl_lock); 1427 /* 1428 * Prevent subsequent drm_vblank_get() from enabling the vblank 1429 * interrupt by bumping the refcount. 1430 */ 1431 if (!vblank->inmodeset) { 1432 atomic_inc(&vblank->refcount); 1433 vblank->inmodeset = 1; 1434 } 1435 spin_unlock_irq(&dev->vbl_lock); 1436 1437 drm_WARN_ON(dev, !list_empty(&dev->vblank_event_list)); 1438 drm_WARN_ON(dev, !list_empty(&vblank->pending_work)); 1439} 1440EXPORT_SYMBOL(drm_crtc_vblank_reset); 1441 1442/** 1443 * drm_crtc_set_max_vblank_count - configure the hw max vblank counter value 1444 * @crtc: CRTC in question 1445 * @max_vblank_count: max hardware vblank counter value 1446 * 1447 * Update the maximum hardware vblank counter value for @crtc 1448 * at runtime. Useful for hardware where the operation of the 1449 * hardware vblank counter depends on the currently active 1450 * display configuration. 1451 * 1452 * For example, if the hardware vblank counter does not work 1453 * when a specific connector is active the maximum can be set 1454 * to zero. And when that specific connector isn't active the 1455 * maximum can again be set to the appropriate non-zero value. 1456 * 1457 * If used, must be called before drm_vblank_on(). 1458 */ 1459void drm_crtc_set_max_vblank_count(struct drm_crtc *crtc, 1460 u32 max_vblank_count) 1461{ 1462 struct drm_device *dev = crtc->dev; 1463 unsigned int pipe = drm_crtc_index(crtc); 1464 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1465 1466 drm_WARN_ON(dev, dev->max_vblank_count); 1467 drm_WARN_ON(dev, !READ_ONCE(vblank->inmodeset)); 1468 1469 vblank->max_vblank_count = max_vblank_count; 1470} 1471EXPORT_SYMBOL(drm_crtc_set_max_vblank_count); 1472 1473/** 1474 * drm_crtc_vblank_on - enable vblank events on a CRTC 1475 * @crtc: CRTC in question 1476 * 1477 * This functions restores the vblank interrupt state captured with 1478 * drm_crtc_vblank_off() again and is generally called when enabling @crtc. Note 1479 * that calls to drm_crtc_vblank_on() and drm_crtc_vblank_off() can be 1480 * unbalanced and so can also be unconditionally called in driver load code to 1481 * reflect the current hardware state of the crtc. 1482 */ 1483void drm_crtc_vblank_on(struct drm_crtc *crtc) 1484{ 1485 struct drm_device *dev = crtc->dev; 1486 unsigned int pipe = drm_crtc_index(crtc); 1487 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1488 1489 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 1490 return; 1491 1492 spin_lock_irq(&dev->vbl_lock); 1493 drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n", 1494 pipe, vblank->enabled, vblank->inmodeset); 1495 1496 /* Drop our private "prevent drm_vblank_get" refcount */ 1497 if (vblank->inmodeset) { 1498 atomic_dec(&vblank->refcount); 1499 vblank->inmodeset = 0; 1500 } 1501 1502 drm_reset_vblank_timestamp(dev, pipe); 1503 1504 /* 1505 * re-enable interrupts if there are users left, or the 1506 * user wishes vblank interrupts to be enabled all the time. 1507 */ 1508 if (atomic_read(&vblank->refcount) != 0 || drm_vblank_offdelay == 0) 1509 drm_WARN_ON(dev, drm_vblank_enable(dev, pipe)); 1510 spin_unlock_irq(&dev->vbl_lock); 1511} 1512EXPORT_SYMBOL(drm_crtc_vblank_on); 1513 1514static void drm_vblank_restore(struct drm_device *dev, unsigned int pipe) 1515{ 1516 ktime_t t_vblank; 1517 struct drm_vblank_crtc *vblank; 1518 int framedur_ns; 1519 u64 diff_ns; 1520 u32 cur_vblank, diff = 1; 1521 int count = DRM_TIMESTAMP_MAXRETRIES; 1522 u32 max_vblank_count = drm_max_vblank_count(dev, pipe); 1523 1524 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 1525 return; 1526 1527 assert_spin_locked(&dev->vbl_lock); 1528 assert_spin_locked(&dev->vblank_time_lock); 1529 1530 vblank = &dev->vblank[pipe]; 1531 drm_WARN_ONCE(dev, 1532 drm_debug_enabled(DRM_UT_VBL) && !vblank->framedur_ns, 1533 "Cannot compute missed vblanks without frame duration\n"); 1534 framedur_ns = vblank->framedur_ns; 1535 1536 do { 1537 cur_vblank = __get_vblank_counter(dev, pipe); 1538 drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false); 1539 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0); 1540 1541 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time)); 1542 if (framedur_ns) 1543 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns); 1544 1545 1546 drm_dbg_vbl(dev, 1547 "missed %d vblanks in %lld ns, frame duration=%d ns, hw_diff=%d\n", 1548 diff, diff_ns, framedur_ns, cur_vblank - vblank->last); 1549 vblank->last = (cur_vblank - diff) & max_vblank_count; 1550} 1551 1552/** 1553 * drm_crtc_vblank_restore - estimate missed vblanks and update vblank count. 1554 * @crtc: CRTC in question 1555 * 1556 * Power manamement features can cause frame counter resets between vblank 1557 * disable and enable. Drivers can use this function in their 1558 * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since 1559 * the last &drm_crtc_funcs.disable_vblank using timestamps and update the 1560 * vblank counter. 1561 * 1562 * Note that drivers must have race-free high-precision timestamping support, 1563 * i.e. &drm_crtc_funcs.get_vblank_timestamp must be hooked up and 1564 * &drm_driver.vblank_disable_immediate must be set to indicate the 1565 * time-stamping functions are race-free against vblank hardware counter 1566 * increments. 1567 */ 1568void drm_crtc_vblank_restore(struct drm_crtc *crtc) 1569{ 1570 WARN_ON_ONCE(!crtc->funcs->get_vblank_timestamp); 1571 WARN_ON_ONCE(!crtc->dev->vblank_disable_immediate); 1572 1573 drm_vblank_restore(crtc->dev, drm_crtc_index(crtc)); 1574} 1575EXPORT_SYMBOL(drm_crtc_vblank_restore); 1576 1577static void drm_legacy_vblank_pre_modeset(struct drm_device *dev, 1578 unsigned int pipe) 1579{ 1580 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1581 1582 /* vblank is not initialized (IRQ not installed ?), or has been freed */ 1583 if (!drm_dev_has_vblank(dev)) 1584 return; 1585 1586 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 1587 return; 1588 1589 /* 1590 * To avoid all the problems that might happen if interrupts 1591 * were enabled/disabled around or between these calls, we just 1592 * have the kernel take a reference on the CRTC (just once though 1593 * to avoid corrupting the count if multiple, mismatch calls occur), 1594 * so that interrupts remain enabled in the interim. 1595 */ 1596 if (!vblank->inmodeset) { 1597 vblank->inmodeset = 0x1; 1598 if (drm_vblank_get(dev, pipe) == 0) 1599 vblank->inmodeset |= 0x2; 1600 } 1601} 1602 1603static void drm_legacy_vblank_post_modeset(struct drm_device *dev, 1604 unsigned int pipe) 1605{ 1606 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1607 1608 /* vblank is not initialized (IRQ not installed ?), or has been freed */ 1609 if (!drm_dev_has_vblank(dev)) 1610 return; 1611 1612 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 1613 return; 1614 1615 if (vblank->inmodeset) { 1616 spin_lock_irq(&dev->vbl_lock); 1617 drm_reset_vblank_timestamp(dev, pipe); 1618 spin_unlock_irq(&dev->vbl_lock); 1619 1620 if (vblank->inmodeset & 0x2) 1621 drm_vblank_put(dev, pipe); 1622 1623 vblank->inmodeset = 0; 1624 } 1625} 1626 1627int drm_legacy_modeset_ctl_ioctl(struct drm_device *dev, void *data, 1628 struct drm_file *file_priv) 1629{ 1630 struct drm_modeset_ctl *modeset = data; 1631 unsigned int pipe; 1632 1633 /* If drm_vblank_init() hasn't been called yet, just no-op */ 1634 if (!drm_dev_has_vblank(dev)) 1635 return 0; 1636 1637 /* KMS drivers handle this internally */ 1638 if (!drm_core_check_feature(dev, DRIVER_LEGACY)) 1639 return 0; 1640 1641 pipe = modeset->crtc; 1642 if (pipe >= dev->num_crtcs) 1643 return -EINVAL; 1644 1645 switch (modeset->cmd) { 1646 case _DRM_PRE_MODESET: 1647 drm_legacy_vblank_pre_modeset(dev, pipe); 1648 break; 1649 case _DRM_POST_MODESET: 1650 drm_legacy_vblank_post_modeset(dev, pipe); 1651 break; 1652 default: 1653 return -EINVAL; 1654 } 1655 1656 return 0; 1657} 1658 1659static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe, 1660 u64 req_seq, 1661 union drm_wait_vblank *vblwait, 1662 struct drm_file *file_priv) 1663{ 1664 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1665 struct drm_pending_vblank_event *e; 1666 ktime_t now; 1667 u64 seq; 1668 int ret; 1669 1670 e = kzalloc(sizeof(*e), GFP_KERNEL); 1671 if (e == NULL) { 1672 ret = -ENOMEM; 1673 goto err_put; 1674 } 1675 1676 e->pipe = pipe; 1677 e->event.base.type = DRM_EVENT_VBLANK; 1678 e->event.base.length = sizeof(e->event.vbl); 1679 e->event.vbl.user_data = vblwait->request.signal; 1680 e->event.vbl.crtc_id = 0; 1681 if (drm_core_check_feature(dev, DRIVER_MODESET)) { 1682 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe); 1683 1684 if (crtc) 1685 e->event.vbl.crtc_id = crtc->base.id; 1686 } 1687 1688 spin_lock_irq(&dev->event_lock); 1689 1690 /* 1691 * drm_crtc_vblank_off() might have been called after we called 1692 * drm_vblank_get(). drm_crtc_vblank_off() holds event_lock around the 1693 * vblank disable, so no need for further locking. The reference from 1694 * drm_vblank_get() protects against vblank disable from another source. 1695 */ 1696 if (!READ_ONCE(vblank->enabled)) { 1697 ret = -EINVAL; 1698 goto err_unlock; 1699 } 1700 1701 ret = drm_event_reserve_init_locked(dev, file_priv, &e->base, 1702 &e->event.base); 1703 1704 if (ret) 1705 goto err_unlock; 1706 1707 seq = drm_vblank_count_and_time(dev, pipe, &now); 1708 1709 drm_dbg_core(dev, "event on vblank count %llu, current %llu, crtc %u\n", 1710 req_seq, seq, pipe); 1711 1712 trace_drm_vblank_event_queued(file_priv, pipe, req_seq); 1713 1714 e->sequence = req_seq; 1715 if (drm_vblank_passed(seq, req_seq)) { 1716 drm_vblank_put(dev, pipe); 1717 send_vblank_event(dev, e, seq, now); 1718 vblwait->reply.sequence = seq; 1719 } else { 1720 /* drm_handle_vblank_events will call drm_vblank_put */ 1721 list_add_tail(&e->base.link, &dev->vblank_event_list); 1722 vblwait->reply.sequence = req_seq; 1723 } 1724 1725 spin_unlock_irq(&dev->event_lock); 1726 1727 return 0; 1728 1729err_unlock: 1730 spin_unlock_irq(&dev->event_lock); 1731 kfree(e); 1732err_put: 1733 drm_vblank_put(dev, pipe); 1734 return ret; 1735} 1736 1737static bool drm_wait_vblank_is_query(union drm_wait_vblank *vblwait) 1738{ 1739 if (vblwait->request.sequence) 1740 return false; 1741 1742 return _DRM_VBLANK_RELATIVE == 1743 (vblwait->request.type & (_DRM_VBLANK_TYPES_MASK | 1744 _DRM_VBLANK_EVENT | 1745 _DRM_VBLANK_NEXTONMISS)); 1746} 1747 1748/* 1749 * Widen a 32-bit param to 64-bits. 1750 * 1751 * \param narrow 32-bit value (missing upper 32 bits) 1752 * \param near 64-bit value that should be 'close' to near 1753 * 1754 * This function returns a 64-bit value using the lower 32-bits from 1755 * 'narrow' and constructing the upper 32-bits so that the result is 1756 * as close as possible to 'near'. 1757 */ 1758 1759static u64 widen_32_to_64(u32 narrow, u64 near) 1760{ 1761 return near + (s32) (narrow - near); 1762} 1763 1764static void drm_wait_vblank_reply(struct drm_device *dev, unsigned int pipe, 1765 struct drm_wait_vblank_reply *reply) 1766{ 1767 ktime_t now; 1768 struct timespec64 ts; 1769 1770 /* 1771 * drm_wait_vblank_reply is a UAPI structure that uses 'long' 1772 * to store the seconds. This is safe as we always use monotonic 1773 * timestamps since linux-4.15. 1774 */ 1775 reply->sequence = drm_vblank_count_and_time(dev, pipe, &now); 1776 ts = ktime_to_timespec64(now); 1777 reply->tval_sec = (u32)ts.tv_sec; 1778 reply->tval_usec = ts.tv_nsec / 1000; 1779} 1780 1781static bool drm_wait_vblank_supported(struct drm_device *dev) 1782{ 1783#if IS_ENABLED(CONFIG_DRM_LEGACY) 1784 if (unlikely(drm_core_check_feature(dev, DRIVER_LEGACY))) 1785 return dev->irq_enabled; 1786#endif 1787 return drm_dev_has_vblank(dev); 1788} 1789 1790int drm_wait_vblank_ioctl(struct drm_device *dev, void *data, 1791 struct drm_file *file_priv) 1792{ 1793 struct drm_crtc *crtc; 1794 struct drm_vblank_crtc *vblank; 1795 union drm_wait_vblank *vblwait = data; 1796 int ret; 1797 u64 req_seq, seq; 1798 unsigned int pipe_index; 1799 unsigned int flags, pipe, high_pipe; 1800 1801 if (!drm_wait_vblank_supported(dev)) 1802 return -EOPNOTSUPP; 1803 1804 if (vblwait->request.type & _DRM_VBLANK_SIGNAL) 1805 return -EINVAL; 1806 1807 if (vblwait->request.type & 1808 ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK | 1809 _DRM_VBLANK_HIGH_CRTC_MASK)) { 1810 drm_dbg_core(dev, 1811 "Unsupported type value 0x%x, supported mask 0x%x\n", 1812 vblwait->request.type, 1813 (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK | 1814 _DRM_VBLANK_HIGH_CRTC_MASK)); 1815 return -EINVAL; 1816 } 1817 1818 flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK; 1819 high_pipe = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK); 1820 if (high_pipe) 1821 pipe_index = high_pipe >> _DRM_VBLANK_HIGH_CRTC_SHIFT; 1822 else 1823 pipe_index = flags & _DRM_VBLANK_SECONDARY ? 1 : 0; 1824 1825 /* Convert lease-relative crtc index into global crtc index */ 1826 if (drm_core_check_feature(dev, DRIVER_MODESET)) { 1827 pipe = 0; 1828 drm_for_each_crtc(crtc, dev) { 1829 if (drm_lease_held(file_priv, crtc->base.id)) { 1830 if (pipe_index == 0) 1831 break; 1832 pipe_index--; 1833 } 1834 pipe++; 1835 } 1836 } else { 1837 pipe = pipe_index; 1838 } 1839 1840 if (pipe >= dev->num_crtcs) 1841 return -EINVAL; 1842 1843 vblank = &dev->vblank[pipe]; 1844 1845 /* If the counter is currently enabled and accurate, short-circuit 1846 * queries to return the cached timestamp of the last vblank. 1847 */ 1848 if (dev->vblank_disable_immediate && 1849 drm_wait_vblank_is_query(vblwait) && 1850 READ_ONCE(vblank->enabled)) { 1851 drm_wait_vblank_reply(dev, pipe, &vblwait->reply); 1852 return 0; 1853 } 1854 1855 ret = drm_vblank_get(dev, pipe); 1856 if (ret) { 1857 drm_dbg_core(dev, 1858 "crtc %d failed to acquire vblank counter, %d\n", 1859 pipe, ret); 1860 return ret; 1861 } 1862 seq = drm_vblank_count(dev, pipe); 1863 1864 switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) { 1865 case _DRM_VBLANK_RELATIVE: 1866 req_seq = seq + vblwait->request.sequence; 1867 vblwait->request.sequence = req_seq; 1868 vblwait->request.type &= ~_DRM_VBLANK_RELATIVE; 1869 break; 1870 case _DRM_VBLANK_ABSOLUTE: 1871 req_seq = widen_32_to_64(vblwait->request.sequence, seq); 1872 break; 1873 default: 1874 ret = -EINVAL; 1875 goto done; 1876 } 1877 1878 if ((flags & _DRM_VBLANK_NEXTONMISS) && 1879 drm_vblank_passed(seq, req_seq)) { 1880 req_seq = seq + 1; 1881 vblwait->request.type &= ~_DRM_VBLANK_NEXTONMISS; 1882 vblwait->request.sequence = req_seq; 1883 } 1884 1885 if (flags & _DRM_VBLANK_EVENT) { 1886 /* must hold on to the vblank ref until the event fires 1887 * drm_vblank_put will be called asynchronously 1888 */ 1889 return drm_queue_vblank_event(dev, pipe, req_seq, vblwait, file_priv); 1890 } 1891 1892 if (req_seq != seq) { 1893 int wait; 1894 1895 drm_dbg_core(dev, "waiting on vblank count %llu, crtc %u\n", 1896 req_seq, pipe); 1897 wait = wait_event_interruptible_timeout(vblank->queue, 1898 drm_vblank_passed(drm_vblank_count(dev, pipe), req_seq) || 1899 !READ_ONCE(vblank->enabled), 1900 msecs_to_jiffies(3000)); 1901 1902 switch (wait) { 1903 case 0: 1904 /* timeout */ 1905 ret = -EBUSY; 1906 break; 1907 case -ERESTARTSYS: 1908 /* interrupted by signal */ 1909 ret = -EINTR; 1910 break; 1911 default: 1912 ret = 0; 1913 break; 1914 } 1915 } 1916 1917 if (ret != -EINTR) { 1918 drm_wait_vblank_reply(dev, pipe, &vblwait->reply); 1919 1920 drm_dbg_core(dev, "crtc %d returning %u to client\n", 1921 pipe, vblwait->reply.sequence); 1922 } else { 1923 drm_dbg_core(dev, "crtc %d vblank wait interrupted by signal\n", 1924 pipe); 1925 } 1926 1927done: 1928 drm_vblank_put(dev, pipe); 1929 return ret; 1930} 1931 1932static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe) 1933{ 1934 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe); 1935 bool high_prec = false; 1936 struct drm_pending_vblank_event *e, *t; 1937 ktime_t now; 1938 u64 seq; 1939 1940 assert_spin_locked(&dev->event_lock); 1941 1942 seq = drm_vblank_count_and_time(dev, pipe, &now); 1943 1944 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) { 1945 if (e->pipe != pipe) 1946 continue; 1947 if (!drm_vblank_passed(seq, e->sequence)) 1948 continue; 1949 1950 drm_dbg_core(dev, "vblank event on %llu, current %llu\n", 1951 e->sequence, seq); 1952 1953 list_del(&e->base.link); 1954 drm_vblank_put(dev, pipe); 1955 send_vblank_event(dev, e, seq, now); 1956 } 1957 1958 if (crtc && crtc->funcs->get_vblank_timestamp) 1959 high_prec = true; 1960 1961 trace_drm_vblank_event(pipe, seq, now, high_prec); 1962} 1963 1964/** 1965 * drm_handle_vblank - handle a vblank event 1966 * @dev: DRM device 1967 * @pipe: index of CRTC where this event occurred 1968 * 1969 * Drivers should call this routine in their vblank interrupt handlers to 1970 * update the vblank counter and send any signals that may be pending. 1971 * 1972 * This is the legacy version of drm_crtc_handle_vblank(). 1973 */ 1974bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe) 1975{ 1976 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1977 unsigned long irqflags; 1978 bool disable_irq; 1979 1980 if (drm_WARN_ON_ONCE(dev, !drm_dev_has_vblank(dev))) 1981 return false; 1982 1983 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 1984 return false; 1985 1986 spin_lock_irqsave(&dev->event_lock, irqflags); 1987 1988 /* Need timestamp lock to prevent concurrent execution with 1989 * vblank enable/disable, as this would cause inconsistent 1990 * or corrupted timestamps and vblank counts. 1991 */ 1992 spin_lock(&dev->vblank_time_lock); 1993 1994 /* Vblank irq handling disabled. Nothing to do. */ 1995 if (!vblank->enabled) { 1996 spin_unlock(&dev->vblank_time_lock); 1997 spin_unlock_irqrestore(&dev->event_lock, irqflags); 1998 return false; 1999 } 2000 2001 drm_update_vblank_count(dev, pipe, true); 2002 2003 spin_unlock(&dev->vblank_time_lock); 2004 2005 wake_up(&vblank->queue); 2006 2007 /* With instant-off, we defer disabling the interrupt until after 2008 * we finish processing the following vblank after all events have 2009 * been signaled. The disable has to be last (after 2010 * drm_handle_vblank_events) so that the timestamp is always accurate. 2011 */ 2012 disable_irq = (dev->vblank_disable_immediate && 2013 drm_vblank_offdelay > 0 && 2014 !atomic_read(&vblank->refcount)); 2015 2016 drm_handle_vblank_events(dev, pipe); 2017 drm_handle_vblank_works(vblank); 2018 2019 spin_unlock_irqrestore(&dev->event_lock, irqflags); 2020 2021 if (disable_irq) 2022 vblank_disable_fn(&vblank->disable_timer); 2023 2024 return true; 2025} 2026EXPORT_SYMBOL(drm_handle_vblank); 2027 2028/** 2029 * drm_crtc_handle_vblank - handle a vblank event 2030 * @crtc: where this event occurred 2031 * 2032 * Drivers should call this routine in their vblank interrupt handlers to 2033 * update the vblank counter and send any signals that may be pending. 2034 * 2035 * This is the native KMS version of drm_handle_vblank(). 2036 * 2037 * Note that for a given vblank counter value drm_crtc_handle_vblank() 2038 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time() 2039 * provide a barrier: Any writes done before calling 2040 * drm_crtc_handle_vblank() will be visible to callers of the later 2041 * functions, if the vblank count is the same or a later one. 2042 * 2043 * See also &drm_vblank_crtc.count. 2044 * 2045 * Returns: 2046 * True if the event was successfully handled, false on failure. 2047 */ 2048bool drm_crtc_handle_vblank(struct drm_crtc *crtc) 2049{ 2050 return drm_handle_vblank(crtc->dev, drm_crtc_index(crtc)); 2051} 2052EXPORT_SYMBOL(drm_crtc_handle_vblank); 2053 2054/* 2055 * Get crtc VBLANK count. 2056 * 2057 * \param dev DRM device 2058 * \param data user argument, pointing to a drm_crtc_get_sequence structure. 2059 * \param file_priv drm file private for the user's open file descriptor 2060 */ 2061 2062int drm_crtc_get_sequence_ioctl(struct drm_device *dev, void *data, 2063 struct drm_file *file_priv) 2064{ 2065 struct drm_crtc *crtc; 2066 struct drm_vblank_crtc *vblank; 2067 int pipe; 2068 struct drm_crtc_get_sequence *get_seq = data; 2069 ktime_t now; 2070 bool vblank_enabled; 2071 int ret; 2072 2073 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 2074 return -EOPNOTSUPP; 2075 2076 if (!drm_dev_has_vblank(dev)) 2077 return -EOPNOTSUPP; 2078 2079 crtc = drm_crtc_find(dev, file_priv, get_seq->crtc_id); 2080 if (!crtc) 2081 return -ENOENT; 2082 2083 pipe = drm_crtc_index(crtc); 2084 2085 vblank = &dev->vblank[pipe]; 2086 vblank_enabled = dev->vblank_disable_immediate && READ_ONCE(vblank->enabled); 2087 2088 if (!vblank_enabled) { 2089 ret = drm_crtc_vblank_get(crtc); 2090 if (ret) { 2091 drm_dbg_core(dev, 2092 "crtc %d failed to acquire vblank counter, %d\n", 2093 pipe, ret); 2094 return ret; 2095 } 2096 } 2097 drm_modeset_lock(&crtc->mutex, NULL); 2098 if (crtc->state) 2099 get_seq->active = crtc->state->enable; 2100 else 2101 get_seq->active = crtc->enabled; 2102 drm_modeset_unlock(&crtc->mutex); 2103 get_seq->sequence = drm_vblank_count_and_time(dev, pipe, &now); 2104 get_seq->sequence_ns = ktime_to_ns(now); 2105 if (!vblank_enabled) 2106 drm_crtc_vblank_put(crtc); 2107 return 0; 2108} 2109 2110/* 2111 * Queue a event for VBLANK sequence 2112 * 2113 * \param dev DRM device 2114 * \param data user argument, pointing to a drm_crtc_queue_sequence structure. 2115 * \param file_priv drm file private for the user's open file descriptor 2116 */ 2117 2118int drm_crtc_queue_sequence_ioctl(struct drm_device *dev, void *data, 2119 struct drm_file *file_priv) 2120{ 2121 struct drm_crtc *crtc; 2122 struct drm_vblank_crtc *vblank; 2123 int pipe; 2124 struct drm_crtc_queue_sequence *queue_seq = data; 2125 ktime_t now; 2126 struct drm_pending_vblank_event *e; 2127 u32 flags; 2128 u64 seq; 2129 u64 req_seq; 2130 int ret; 2131 2132 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 2133 return -EOPNOTSUPP; 2134 2135 if (!drm_dev_has_vblank(dev)) 2136 return -EOPNOTSUPP; 2137 2138 crtc = drm_crtc_find(dev, file_priv, queue_seq->crtc_id); 2139 if (!crtc) 2140 return -ENOENT; 2141 2142 flags = queue_seq->flags; 2143 /* Check valid flag bits */ 2144 if (flags & ~(DRM_CRTC_SEQUENCE_RELATIVE| 2145 DRM_CRTC_SEQUENCE_NEXT_ON_MISS)) 2146 return -EINVAL; 2147 2148 pipe = drm_crtc_index(crtc); 2149 2150 vblank = &dev->vblank[pipe]; 2151 2152 e = kzalloc(sizeof(*e), GFP_KERNEL); 2153 if (e == NULL) 2154 return -ENOMEM; 2155 2156 ret = drm_crtc_vblank_get(crtc); 2157 if (ret) { 2158 drm_dbg_core(dev, 2159 "crtc %d failed to acquire vblank counter, %d\n", 2160 pipe, ret); 2161 goto err_free; 2162 } 2163 2164 seq = drm_vblank_count_and_time(dev, pipe, &now); 2165 req_seq = queue_seq->sequence; 2166 2167 if (flags & DRM_CRTC_SEQUENCE_RELATIVE) 2168 req_seq += seq; 2169 2170 if ((flags & DRM_CRTC_SEQUENCE_NEXT_ON_MISS) && drm_vblank_passed(seq, req_seq)) 2171 req_seq = seq + 1; 2172 2173 e->pipe = pipe; 2174 e->event.base.type = DRM_EVENT_CRTC_SEQUENCE; 2175 e->event.base.length = sizeof(e->event.seq); 2176 e->event.seq.user_data = queue_seq->user_data; 2177 2178 spin_lock_irq(&dev->event_lock); 2179 2180 /* 2181 * drm_crtc_vblank_off() might have been called after we called 2182 * drm_crtc_vblank_get(). drm_crtc_vblank_off() holds event_lock around the 2183 * vblank disable, so no need for further locking. The reference from 2184 * drm_crtc_vblank_get() protects against vblank disable from another source. 2185 */ 2186 if (!READ_ONCE(vblank->enabled)) { 2187 ret = -EINVAL; 2188 goto err_unlock; 2189 } 2190 2191 ret = drm_event_reserve_init_locked(dev, file_priv, &e->base, 2192 &e->event.base); 2193 2194 if (ret) 2195 goto err_unlock; 2196 2197 e->sequence = req_seq; 2198 2199 if (drm_vblank_passed(seq, req_seq)) { 2200 drm_crtc_vblank_put(crtc); 2201 send_vblank_event(dev, e, seq, now); 2202 queue_seq->sequence = seq; 2203 } else { 2204 /* drm_handle_vblank_events will call drm_vblank_put */ 2205 list_add_tail(&e->base.link, &dev->vblank_event_list); 2206 queue_seq->sequence = req_seq; 2207 } 2208 2209 spin_unlock_irq(&dev->event_lock); 2210 return 0; 2211 2212err_unlock: 2213 spin_unlock_irq(&dev->event_lock); 2214 drm_crtc_vblank_put(crtc); 2215err_free: 2216 kfree(e); 2217 return ret; 2218} 2219 2220