Lines Matching refs:stream
36 * descriptor representing a stream of GPU metrics which can then be read() as
37 * a stream of sample records.
56 * i915 perf file descriptors represent a "stream" instead of an "event"; where
57 * a perf event primarily corresponds to a single 64bit value, while a stream
61 * of related counters. Samples for an i915 perf stream capturing OA metrics
64 * selected by the user opening the stream. Perf has support for grouping
68 * i915 perf stream configurations are provided as an array of u64 (key,value)
327 * struct perf_open_properties - for validated properties given to open a stream
346 * to open a stream of metrics the configuration is built up in the structure
414 static u32 gen12_oa_hw_tail_read(struct i915_perf_stream *stream)
416 struct intel_uncore *uncore = stream->uncore;
422 static u32 gen8_oa_hw_tail_read(struct i915_perf_stream *stream)
424 struct intel_uncore *uncore = stream->uncore;
429 static u32 gen7_oa_hw_tail_read(struct i915_perf_stream *stream)
431 struct intel_uncore *uncore = stream->uncore;
439 * @stream: i915 stream instance
456 * only called while the stream is enabled, while the global OA configuration
461 static bool oa_buffer_check_unlocked(struct i915_perf_stream *stream)
463 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);
464 int report_size = stream->oa_buffer.format_size;
474 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
476 hw_tail = stream->perf->ops.oa_hw_tail_read(stream);
485 if (hw_tail == stream->oa_buffer.aging_tail &&
486 (now - stream->oa_buffer.aging_timestamp) > OA_TAIL_MARGIN_NSEC) {
491 stream->oa_buffer.tail = stream->oa_buffer.aging_tail;
497 * anywhere between this head and stream->oa_buffer.tail.
499 head = stream->oa_buffer.head - gtt_offset;
500 aged_tail = stream->oa_buffer.tail - gtt_offset;
505 /* Walk the stream backward until we find a report with dword 0
517 u32 *report32 = (void *)(stream->oa_buffer.vaddr + tail);
526 __ratelimit(&stream->perf->tail_pointer_race))
531 stream->oa_buffer.tail = gtt_offset + tail;
532 stream->oa_buffer.aging_tail = gtt_offset + hw_tail;
533 stream->oa_buffer.aging_timestamp = now;
536 pollin = OA_TAKEN(stream->oa_buffer.tail - gtt_offset,
537 stream->oa_buffer.head - gtt_offset) >= report_size;
539 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
546 * @stream: An i915-perf stream opened for OA metrics
559 static int append_oa_status(struct i915_perf_stream *stream,
580 * @stream: An i915-perf stream opened for OA metrics
587 * properties when opening a stream, tracked as `stream->sample_flags`. This
595 static int append_oa_sample(struct i915_perf_stream *stream,
601 int report_size = stream->oa_buffer.format_size;
606 header.size = stream->sample_size;
626 * @stream: An i915-perf stream opened for OA metrics
644 static int gen8_append_oa_reports(struct i915_perf_stream *stream,
649 struct intel_uncore *uncore = stream->uncore;
650 int report_size = stream->oa_buffer.format_size;
651 u8 *oa_buf_base = stream->oa_buffer.vaddr;
652 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);
660 if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled))
663 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
665 head = stream->oa_buffer.head;
666 tail = stream->oa_buffer.tail;
668 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
726 (IS_GEN(stream->perf->i915, 12) ?
730 if (__ratelimit(&stream->perf->spurious_report_rs))
735 ctx_id = report32[2] & stream->specific_ctx_id_mask;
745 if (!(report32[0] & stream->perf->gen8_valid_ctx_bit) &&
746 INTEL_GEN(stream->perf->i915) <= 11)
780 if (!stream->perf->exclusive_stream->ctx ||
781 stream->specific_ctx_id == ctx_id ||
782 stream->oa_buffer.last_ctx_id == stream->specific_ctx_id ||
789 if (stream->perf->exclusive_stream->ctx &&
790 stream->specific_ctx_id != ctx_id) {
794 ret = append_oa_sample(stream, buf, count, offset,
799 stream->oa_buffer.last_ctx_id = ctx_id;
813 oaheadptr = IS_GEN(stream->perf->i915, 12) ?
816 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
825 stream->oa_buffer.head = head;
827 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
835 * @stream: An i915-perf stream opened for OA metrics
853 static int gen8_oa_read(struct i915_perf_stream *stream,
858 struct intel_uncore *uncore = stream->uncore;
863 if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr))
866 oastatus_reg = IS_GEN(stream->perf->i915, 12) ?
886 ret = append_oa_status(stream, buf, count, offset,
892 stream->period_exponent);
894 stream->perf->ops.oa_disable(stream);
895 stream->perf->ops.oa_enable(stream);
905 ret = append_oa_status(stream, buf, count, offset,
918 return gen8_append_oa_reports(stream, buf, count, offset);
923 * @stream: An i915-perf stream opened for OA metrics
941 static int gen7_append_oa_reports(struct i915_perf_stream *stream,
946 struct intel_uncore *uncore = stream->uncore;
947 int report_size = stream->oa_buffer.format_size;
948 u8 *oa_buf_base = stream->oa_buffer.vaddr;
949 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);
957 if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled))
960 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
962 head = stream->oa_buffer.head;
963 tail = stream->oa_buffer.tail;
965 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
1015 if (__ratelimit(&stream->perf->spurious_report_rs))
1020 ret = append_oa_sample(stream, buf, count, offset, report);
1032 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1042 stream->oa_buffer.head = head;
1044 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
1052 * @stream: An i915-perf stream opened for OA metrics
1066 static int gen7_oa_read(struct i915_perf_stream *stream,
1071 struct intel_uncore *uncore = stream->uncore;
1075 if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr))
1085 oastatus1 &= ~stream->perf->gen7_latched_oastatus1;
1108 ret = append_oa_status(stream, buf, count, offset,
1114 stream->period_exponent);
1116 stream->perf->ops.oa_disable(stream);
1117 stream->perf->ops.oa_enable(stream);
1123 ret = append_oa_status(stream, buf, count, offset,
1127 stream->perf->gen7_latched_oastatus1 |=
1131 return gen7_append_oa_reports(stream, buf, count, offset);
1136 * @stream: An i915-perf stream opened for OA metrics
1138 * Called when userspace tries to read() from a blocking stream FD opened
1148 static int i915_oa_wait_unlocked(struct i915_perf_stream *stream)
1151 if (!stream->periodic)
1154 return wait_event_interruptible(stream->poll_wq,
1155 oa_buffer_check_unlocked(stream));
1159 * i915_oa_poll_wait - call poll_wait() for an OA stream poll()
1160 * @stream: An i915-perf stream opened for OA metrics
1161 * @file: An i915 perf stream file
1164 * For handling userspace polling on an i915 perf stream opened for OA metrics,
1168 static void i915_oa_poll_wait(struct i915_perf_stream *stream,
1172 poll_wait(file, &stream->poll_wq, wait);
1177 * @stream: An i915-perf stream opened for OA metrics
1187 static int i915_oa_read(struct i915_perf_stream *stream,
1192 return stream->perf->ops.read(stream, buf, count, offset);
1195 static struct intel_context *oa_pin_context(struct i915_perf_stream *stream)
1198 struct i915_gem_context *ctx = stream->ctx;
1204 if (ce->engine != stream->engine) /* first match! */
1232 stream->pinned_ctx = ce;
1233 return stream->pinned_ctx;
1238 * @stream: An i915-perf stream opened for OA metrics
1241 * lifetime of the stream. This ensures that we don't have to worry about
1246 static int oa_get_render_ctx_id(struct i915_perf_stream *stream)
1250 ce = oa_pin_context(stream);
1260 stream->specific_ctx_id = i915_ggtt_offset(ce->state);
1261 stream->specific_ctx_id_mask = 0;
1269 stream->specific_ctx_id_mask =
1271 stream->specific_ctx_id = stream->specific_ctx_id_mask;
1283 stream->specific_ctx_id = ce->lrc.lrca >> 12;
1289 stream->specific_ctx_id_mask =
1296 stream->specific_ctx_id_mask =
1303 stream->specific_ctx_id = (GEN12_MAX_CONTEXT_HW_ID - 1) << (GEN11_SW_CTX_ID_SHIFT - 32);
1311 ce->tag = stream->specific_ctx_id;
1313 drm_dbg(&stream->perf->i915->drm,
1315 stream->specific_ctx_id,
1316 stream->specific_ctx_id_mask);
1323 * @stream: An i915-perf stream opened for OA metrics
1326 * for the lifetime of the stream, then that can be undone here.
1328 static void oa_put_render_ctx_id(struct i915_perf_stream *stream)
1332 ce = fetch_and_zero(&stream->pinned_ctx);
1338 stream->specific_ctx_id = INVALID_CTX_ID;
1339 stream->specific_ctx_id_mask = 0;
1343 free_oa_buffer(struct i915_perf_stream *stream)
1345 i915_vma_unpin_and_release(&stream->oa_buffer.vma,
1348 stream->oa_buffer.vaddr = NULL;
1352 free_oa_configs(struct i915_perf_stream *stream)
1356 i915_oa_config_put(stream->oa_config);
1357 llist_for_each_entry_safe(oa_bo, tmp, stream->oa_config_bos.first, node)
1362 free_noa_wait(struct i915_perf_stream *stream)
1364 i915_vma_unpin_and_release(&stream->noa_wait, 0);
1367 static void i915_oa_stream_destroy(struct i915_perf_stream *stream)
1369 struct i915_perf *perf = stream->perf;
1371 BUG_ON(stream != perf->exclusive_stream);
1380 perf->ops.disable_metric_set(stream);
1382 free_oa_buffer(stream);
1384 intel_uncore_forcewake_put(stream->uncore, FORCEWAKE_ALL);
1385 intel_engine_pm_put(stream->engine);
1387 if (stream->ctx)
1388 oa_put_render_ctx_id(stream);
1390 free_oa_configs(stream);
1391 free_noa_wait(stream);
1399 static void gen7_init_oa_buffer(struct i915_perf_stream *stream)
1401 struct intel_uncore *uncore = stream->uncore;
1402 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);
1405 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1412 stream->oa_buffer.head = gtt_offset;
1420 stream->oa_buffer.aging_tail = INVALID_TAIL_PTR;
1421 stream->oa_buffer.tail = gtt_offset;
1423 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
1429 stream->perf->gen7_latched_oastatus1 = 0;
1434 * when re-enabling a stream or in error/reset paths.
1442 memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE);
1445 static void gen8_init_oa_buffer(struct i915_perf_stream *stream)
1447 struct intel_uncore *uncore = stream->uncore;
1448 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);
1451 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1455 stream->oa_buffer.head = gtt_offset;
1472 stream->oa_buffer.aging_tail = INVALID_TAIL_PTR;
1473 stream->oa_buffer.tail = gtt_offset;
1480 stream->oa_buffer.last_ctx_id = INVALID_CTX_ID;
1482 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
1488 * when re-enabling a stream or in error/reset paths.
1496 memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE);
1499 static void gen12_init_oa_buffer(struct i915_perf_stream *stream)
1501 struct intel_uncore *uncore = stream->uncore;
1502 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);
1505 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1510 stream->oa_buffer.head = gtt_offset;
1526 stream->oa_buffer.aging_tail = INVALID_TAIL_PTR;
1527 stream->oa_buffer.tail = gtt_offset;
1534 stream->oa_buffer.last_ctx_id = INVALID_CTX_ID;
1536 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
1542 * when re-enabling a stream or in error/reset paths.
1550 memset(stream->oa_buffer.vaddr, 0,
1551 stream->oa_buffer.vma->size);
1554 static int alloc_oa_buffer(struct i915_perf_stream *stream)
1556 struct drm_i915_private *i915 = stream->perf->i915;
1561 if (drm_WARN_ON(&i915->drm, stream->oa_buffer.vma))
1567 bo = i915_gem_object_create_shmem(stream->perf->i915, OA_BUFFER_SIZE);
1581 stream->oa_buffer.vma = vma;
1583 stream->oa_buffer.vaddr =
1585 if (IS_ERR(stream->oa_buffer.vaddr)) {
1586 ret = PTR_ERR(stream->oa_buffer.vaddr);
1598 stream->oa_buffer.vaddr = NULL;
1599 stream->oa_buffer.vma = NULL;
1604 static u32 *save_restore_register(struct i915_perf_stream *stream, u32 *cs,
1613 if (INTEL_GEN(stream->perf->i915) >= 8)
1619 *cs++ = intel_gt_scratch_offset(stream->engine->gt,
1627 static int alloc_noa_wait(struct i915_perf_stream *stream)
1629 struct drm_i915_private *i915 = stream->perf->i915;
1633 i915_cs_timestamp_ns_to_ticks(i915, atomic64_read(&stream->perf->noa_programming_delay));
1634 const u32 base = stream->engine->mmio_base;
1657 * needs to be fixed during the lifetime of the i915/perf stream.
1674 stream, cs, true /* save */, CS_GPR(i),
1677 stream, cs, true /* save */, MI_PREDICATE_RESULT_1,
1781 stream, cs, false /* restore */, CS_GPR(i),
1784 stream, cs, false /* restore */, MI_PREDICATE_RESULT_1,
1795 stream->noa_wait = vma;
1839 alloc_oa_config_buffer(struct i915_perf_stream *stream,
1858 obj = i915_gem_object_create_shmem(stream->perf->i915, config_length);
1881 *cs++ = (INTEL_GEN(stream->perf->i915) < 8 ?
1884 *cs++ = i915_ggtt_offset(stream->noa_wait);
1891 &stream->engine->gt->ggtt->vm,
1899 llist_add(&oa_bo->node, &stream->oa_config_bos);
1911 get_oa_vma(struct i915_perf_stream *stream, struct i915_oa_config *oa_config)
1917 * to the stream.
1919 llist_for_each_entry(oa_bo, stream->oa_config_bos.first, node) {
1927 oa_bo = alloc_oa_config_buffer(stream, oa_config);
1936 emit_oa_config(struct i915_perf_stream *stream,
1946 vma = get_oa_vma(stream, oa_config);
2008 static struct intel_context *oa_context(struct i915_perf_stream *stream)
2010 return stream->pinned_ctx ?: stream->engine->kernel_context;
2014 hsw_enable_metric_set(struct i915_perf_stream *stream,
2017 struct intel_uncore *uncore = stream->uncore;
2034 return emit_oa_config(stream,
2035 stream->oa_config, oa_context(stream),
2039 static void hsw_disable_metric_set(struct i915_perf_stream *stream)
2041 struct intel_uncore *uncore = stream->uncore;
2081 const struct i915_perf_stream *stream)
2083 u32 ctx_oactxctrl = stream->perf->ctx_oactxctrl_offset;
2084 u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset;
2099 (stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) |
2100 (stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) |
2105 oa_config_flex_reg(stream->oa_config, flex_regs[i]);
2241 static int gen12_configure_oar_context(struct i915_perf_stream *stream,
2245 struct intel_context *ce = stream->pinned_ctx;
2246 u32 format = stream->oa_buffer.format;
2250 stream->perf->ctx_oactxctrl_offset + 1,
2290 * Manages updating the per-context aspects of the OA stream
2315 oa_configure_all_contexts(struct i915_perf_stream *stream,
2320 struct drm_i915_private *i915 = stream->perf->i915;
2325 lockdep_assert_held(&stream->perf->lock);
2384 gen12_configure_all_contexts(struct i915_perf_stream *stream,
2395 return oa_configure_all_contexts(stream,
2401 lrc_configure_all_contexts(struct i915_perf_stream *stream,
2406 const u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset;
2415 stream->perf->ctx_oactxctrl_offset + 1,
2429 (stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) |
2430 (stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) |
2436 return oa_configure_all_contexts(stream,
2442 gen8_enable_metric_set(struct i915_perf_stream *stream,
2445 struct intel_uncore *uncore = stream->uncore;
2446 struct i915_oa_config *oa_config = stream->oa_config;
2472 if (IS_GEN_RANGE(stream->perf->i915, 9, 11)) {
2483 ret = lrc_configure_all_contexts(stream, oa_config, active);
2487 return emit_oa_config(stream,
2488 stream->oa_config, oa_context(stream),
2492 static u32 oag_report_ctx_switches(const struct i915_perf_stream *stream)
2495 (stream->sample_flags & SAMPLE_OA_REPORT) ?
2500 gen12_enable_metric_set(struct i915_perf_stream *stream,
2503 struct intel_uncore *uncore = stream->uncore;
2504 struct i915_oa_config *oa_config = stream->oa_config;
2505 bool periodic = stream->periodic;
2506 u32 period_exponent = stream->period_exponent;
2517 oag_report_ctx_switches(stream));
2530 ret = gen12_configure_all_contexts(stream, oa_config, active);
2539 if (stream->ctx) {
2540 ret = gen12_configure_oar_context(stream, active);
2545 return emit_oa_config(stream,
2546 stream->oa_config, oa_context(stream),
2550 static void gen8_disable_metric_set(struct i915_perf_stream *stream)
2552 struct intel_uncore *uncore = stream->uncore;
2555 lrc_configure_all_contexts(stream, NULL, NULL);
2560 static void gen10_disable_metric_set(struct i915_perf_stream *stream)
2562 struct intel_uncore *uncore = stream->uncore;
2565 lrc_configure_all_contexts(stream, NULL, NULL);
2571 static void gen12_disable_metric_set(struct i915_perf_stream *stream)
2573 struct intel_uncore *uncore = stream->uncore;
2576 gen12_configure_all_contexts(stream, NULL, NULL);
2579 if (stream->ctx)
2580 gen12_configure_oar_context(stream, NULL);
2586 static void gen7_oa_enable(struct i915_perf_stream *stream)
2588 struct intel_uncore *uncore = stream->uncore;
2589 struct i915_gem_context *ctx = stream->ctx;
2590 u32 ctx_id = stream->specific_ctx_id;
2591 bool periodic = stream->periodic;
2592 u32 period_exponent = stream->period_exponent;
2593 u32 report_format = stream->oa_buffer.format;
2604 gen7_init_oa_buffer(stream);
2616 static void gen8_oa_enable(struct i915_perf_stream *stream)
2618 struct intel_uncore *uncore = stream->uncore;
2619 u32 report_format = stream->oa_buffer.format;
2630 gen8_init_oa_buffer(stream);
2642 static void gen12_oa_enable(struct i915_perf_stream *stream)
2644 struct intel_uncore *uncore = stream->uncore;
2645 u32 report_format = stream->oa_buffer.format;
2651 if (!(stream->sample_flags & SAMPLE_OA_REPORT))
2654 gen12_init_oa_buffer(stream);
2662 * i915_oa_stream_enable - handle `I915_PERF_IOCTL_ENABLE` for OA stream
2663 * @stream: An i915 perf stream opened for OA metrics
2666 * when opening the stream. This also starts a hrtimer that will periodically
2670 static void i915_oa_stream_enable(struct i915_perf_stream *stream)
2672 stream->pollin = false;
2674 stream->perf->ops.oa_enable(stream);
2676 if (stream->sample_flags & SAMPLE_OA_REPORT)
2677 hrtimer_start(&stream->poll_check_timer,
2678 ns_to_ktime(stream->poll_oa_period),
2682 static void gen7_oa_disable(struct i915_perf_stream *stream)
2684 struct intel_uncore *uncore = stream->uncore;
2690 drm_err(&stream->perf->i915->drm,
2694 static void gen8_oa_disable(struct i915_perf_stream *stream)
2696 struct intel_uncore *uncore = stream->uncore;
2702 drm_err(&stream->perf->i915->drm,
2706 static void gen12_oa_disable(struct i915_perf_stream *stream)
2708 struct intel_uncore *uncore = stream->uncore;
2715 drm_err(&stream->perf->i915->drm,
2723 drm_err(&stream->perf->i915->drm,
2728 * i915_oa_stream_disable - handle `I915_PERF_IOCTL_DISABLE` for OA stream
2729 * @stream: An i915 perf stream opened for OA metrics
2735 static void i915_oa_stream_disable(struct i915_perf_stream *stream)
2737 stream->perf->ops.oa_disable(stream);
2739 if (stream->sample_flags & SAMPLE_OA_REPORT)
2740 hrtimer_cancel(&stream->poll_check_timer);
2752 static int i915_perf_stream_enable_sync(struct i915_perf_stream *stream)
2761 err = stream->perf->ops.enable_metric_set(stream, active);
2802 * i915_oa_stream_init - validate combined props for OA stream and init
2803 * @stream: An i915 perf stream
2805 * @props: The property state that configures stream (individually validated)
2810 * At this point it has been determined that userspace wants a stream of
2819 static int i915_oa_stream_init(struct i915_perf_stream *stream,
2823 struct drm_i915_private *i915 = stream->perf->i915;
2824 struct i915_perf *perf = stream->perf;
2844 (INTEL_GEN(perf->i915) < 12 || !stream->ctx)) {
2869 stream->engine = props->engine;
2870 stream->uncore = stream->engine->gt->uncore;
2872 stream->sample_size = sizeof(struct drm_i915_perf_record_header);
2876 stream->sample_flags = props->sample_flags;
2877 stream->sample_size += format_size;
2879 stream->oa_buffer.format_size = format_size;
2880 if (drm_WARN_ON(&i915->drm, stream->oa_buffer.format_size == 0))
2883 stream->hold_preemption = props->hold_preemption;
2885 stream->oa_buffer.format =
2888 stream->periodic = props->oa_periodic;
2889 if (stream->periodic)
2890 stream->period_exponent = props->oa_period_exponent;
2892 if (stream->ctx) {
2893 ret = oa_get_render_ctx_id(stream);
2900 ret = alloc_noa_wait(stream);
2906 stream->oa_config = i915_perf_get_oa_config(perf, props->metrics_set);
2907 if (!stream->oa_config) {
2925 intel_engine_pm_get(stream->engine);
2926 intel_uncore_forcewake_get(stream->uncore, FORCEWAKE_ALL);
2928 ret = alloc_oa_buffer(stream);
2932 stream->ops = &i915_oa_stream_ops;
2935 WRITE_ONCE(perf->exclusive_stream, stream);
2937 ret = i915_perf_stream_enable_sync(stream);
2943 DRM_DEBUG("opening stream oa config uuid=%s\n",
2944 stream->oa_config->uuid);
2946 hrtimer_init(&stream->poll_check_timer,
2948 stream->poll_check_timer.function = oa_poll_check_timer_cb;
2949 init_waitqueue_head(&stream->poll_wq);
2950 spin_lock_init(&stream->oa_buffer.ptr_lock);
2956 perf->ops.disable_metric_set(stream);
2958 free_oa_buffer(stream);
2961 free_oa_configs(stream);
2963 intel_uncore_forcewake_put(stream->uncore, FORCEWAKE_ALL);
2964 intel_engine_pm_put(stream->engine);
2967 free_noa_wait(stream);
2970 if (stream->ctx)
2971 oa_put_render_ctx_id(stream);
2979 struct i915_perf_stream *stream;
2985 stream = READ_ONCE(engine->i915->perf.exclusive_stream);
2986 if (stream && INTEL_GEN(stream->perf->i915) < 12)
2987 gen8_update_reg_state_unlocked(ce, stream);
2991 * i915_perf_read - handles read() FOP for i915 perf stream FDs
2992 * @file: An i915 perf stream file
2997 * The entry point for handling a read() on a stream file descriptor from
2999 * &i915_perf_stream_ops->read but to save having stream implementations (of
3002 * We can also consistently treat trying to read from a disabled stream
3003 * as an IO error so implementations can assume the stream is enabled
3013 struct i915_perf_stream *stream = file->private_data;
3014 struct i915_perf *perf = stream->perf;
3019 * disabled stream as an error. In particular it might otherwise lead
3022 if (!stream->enabled || !(stream->sample_flags & SAMPLE_OA_REPORT))
3027 * stream->ops->wait_unlocked.
3034 ret = stream->ops->wait_unlocked(stream);
3039 ret = stream->ops->read(stream, buf, count, &offset);
3044 ret = stream->ops->read(stream, buf, count, &offset);
3060 stream->pollin = false;
3068 struct i915_perf_stream *stream =
3069 container_of(hrtimer, typeof(*stream), poll_check_timer);
3071 if (oa_buffer_check_unlocked(stream)) {
3072 stream->pollin = true;
3073 wake_up(&stream->poll_wq);
3077 ns_to_ktime(stream->poll_oa_period));
3083 * i915_perf_poll_locked - poll_wait() with a suitable wait queue for stream
3084 * @stream: An i915 perf stream
3085 * @file: An i915 perf stream file
3088 * For handling userspace polling on an i915 perf stream, this calls through to
3090 * will be woken for new stream data.
3097 static __poll_t i915_perf_poll_locked(struct i915_perf_stream *stream,
3103 stream->ops->poll_wait(stream, file, wait);
3111 if (stream->pollin)
3118 * i915_perf_poll - call poll_wait() with a suitable wait queue for stream
3119 * @file: An i915 perf stream file
3122 * For handling userspace polling on an i915 perf stream, this ensures
3123 * poll_wait() gets called with a wait queue that will be woken for new stream
3132 struct i915_perf_stream *stream = file->private_data;
3133 struct i915_perf *perf = stream->perf;
3137 ret = i915_perf_poll_locked(stream, file, wait);
3145 * @stream: A disabled i915 perf stream
3147 * [Re]enables the associated capture of data for this stream.
3149 * If a stream was previously enabled then there's currently no intention
3153 static void i915_perf_enable_locked(struct i915_perf_stream *stream)
3155 if (stream->enabled)
3158 /* Allow stream->ops->enable() to refer to this */
3159 stream->enabled = true;
3161 if (stream->ops->enable)
3162 stream->ops->enable(stream);
3164 if (stream->hold_preemption)
3165 intel_context_set_nopreempt(stream->pinned_ctx);
3170 * @stream: An enabled i915 perf stream
3172 * Disables the associated capture of data for this stream.
3174 * The intention is that disabling an re-enabling a stream will ideally be
3175 * cheaper than destroying and re-opening a stream with the same configuration,
3177 * must be retained between disabling and re-enabling a stream.
3179 * Note: while a stream is disabled it's considered an error for userspace
3180 * to attempt to read from the stream (-EIO).
3182 static void i915_perf_disable_locked(struct i915_perf_stream *stream)
3184 if (!stream->enabled)
3187 /* Allow stream->ops->disable() to refer to this */
3188 stream->enabled = false;
3190 if (stream->hold_preemption)
3191 intel_context_clear_nopreempt(stream->pinned_ctx);
3193 if (stream->ops->disable)
3194 stream->ops->disable(stream);
3197 static long i915_perf_config_locked(struct i915_perf_stream *stream,
3201 long ret = stream->oa_config->id;
3203 config = i915_perf_get_oa_config(stream->perf, metrics_set);
3207 if (config != stream->oa_config) {
3219 err = emit_oa_config(stream, config, oa_context(stream), NULL);
3221 config = xchg(&stream->oa_config, config);
3232 * i915_perf_ioctl - support ioctl() usage with i915 perf stream FDs
3233 * @stream: An i915 perf stream
3243 static long i915_perf_ioctl_locked(struct i915_perf_stream *stream,
3249 i915_perf_enable_locked(stream);
3252 i915_perf_disable_locked(stream);
3255 return i915_perf_config_locked(stream, arg);
3262 * i915_perf_ioctl - support ioctl() usage with i915 perf stream FDs
3263 * @file: An i915 perf stream file
3276 struct i915_perf_stream *stream = file->private_data;
3277 struct i915_perf *perf = stream->perf;
3281 ret = i915_perf_ioctl_locked(stream, cmd, arg);
3288 * i915_perf_destroy_locked - destroy an i915 perf stream
3289 * @stream: An i915 perf stream
3291 * Frees all resources associated with the given i915 perf @stream, disabling
3297 static void i915_perf_destroy_locked(struct i915_perf_stream *stream)
3299 if (stream->enabled)
3300 i915_perf_disable_locked(stream);
3302 if (stream->ops->destroy)
3303 stream->ops->destroy(stream);
3305 if (stream->ctx)
3306 i915_gem_context_put(stream->ctx);
3308 kfree(stream);
3312 * i915_perf_release - handles userspace close() of a stream file
3314 * @file: An i915 perf stream file
3316 * Cleans up any resources associated with an open i915 perf stream file.
3324 struct i915_perf_stream *stream = file->private_data;
3325 struct i915_perf *perf = stream->perf;
3328 i915_perf_destroy_locked(stream);
3331 /* Release the reference the perf stream kept on the driver. */
3353 * i915_perf_open_ioctl_locked - DRM ioctl() for userspace to open a stream FD
3361 * Implements further stream config validation and stream initialization on
3370 * config validation and stream initialization details will be handled by
3372 * will be relevant to all stream types / backends.
3383 struct i915_perf_stream *stream = NULL;
3395 DRM_DEBUG("Failed to look up context with ID %u for opening perf stream\n",
3418 * doesn't request global stream access (i.e. query based sampling
3451 DRM_DEBUG("Insufficient privileges to open i915 perf stream\n");
3456 stream = kzalloc(sizeof(*stream), GFP_KERNEL);
3457 if (!stream) {
3462 stream->perf = perf;
3463 stream->ctx = specific_ctx;
3464 stream->poll_oa_period = props->poll_oa_period;
3466 ret = i915_oa_stream_init(stream, param, props);
3470 /* we avoid simply assigning stream->sample_flags = props->sample_flags
3474 if (WARN_ON(stream->sample_flags != props->sample_flags)) {
3484 stream_fd = anon_inode_getfd("[i915_perf]", &fops, stream, f_flags);
3491 i915_perf_enable_locked(stream);
3501 if (stream->ops->destroy)
3502 stream->ops->destroy(stream);
3504 kfree(stream);
3518 * read_properties_unlocked - validate + copy userspace stream open properties
3522 * @props: The stream configuration built up while validating properties
3526 * properties necessary for a particular kind of stream have been set.
3694 * i915_perf_open_ioctl - DRM ioctl() for userspace to open a stream FD
3699 * Validates the stream open parameters given by userspace including flags
3702 * Very little is assumed up front about the nature of the stream being
3704 * i915-perf stream is expected to be a suitable interface for other forms of
3714 * Return: A newly opened i915 Perf stream file descriptor or negative
3759 * used to open an i915-perf stream.
4158 * and their content will be freed when the stream using the config is closed.
4379 * stream instead of waiting until driver _fini which no one