Lines Matching refs:stream
36 * descriptor representing a stream of GPU metrics which can then be read() as
37 * a stream of sample records.
56 * i915 perf file descriptors represent a "stream" instead of an "event"; where
57 * a perf event primarily corresponds to a single 64bit value, while a stream
61 * of related counters. Samples for an i915 perf stream capturing OA metrics
64 * selected by the user opening the stream. Perf has support for grouping
68 * i915 perf stream configurations are provided as an array of u64 (key,value)
342 * struct perf_open_properties - for validated properties given to open a stream
361 * to open a stream of metrics the configuration is built up in the structure
430 struct i915_perf_regs *__oa_regs(struct i915_perf_stream *stream)
432 return &stream->engine->oa_group->regs;
435 static u32 gen12_oa_hw_tail_read(struct i915_perf_stream *stream)
437 struct intel_uncore *uncore = stream->uncore;
439 return intel_uncore_read(uncore, __oa_regs(stream)->oa_tail_ptr) &
443 static u32 gen8_oa_hw_tail_read(struct i915_perf_stream *stream)
445 struct intel_uncore *uncore = stream->uncore;
450 static u32 gen7_oa_hw_tail_read(struct i915_perf_stream *stream)
452 struct intel_uncore *uncore = stream->uncore;
461 static u64 oa_report_id(struct i915_perf_stream *stream, void *report)
463 return oa_report_header_64bit(stream) ? *(u64 *)report : *(u32 *)report;
466 static u64 oa_report_reason(struct i915_perf_stream *stream, void *report)
468 return (oa_report_id(stream, report) >> OAREPORT_REASON_SHIFT) &
469 (GRAPHICS_VER(stream->perf->i915) == 12 ?
474 static void oa_report_id_clear(struct i915_perf_stream *stream, u32 *report)
476 if (oa_report_header_64bit(stream))
482 static bool oa_report_ctx_invalid(struct i915_perf_stream *stream, void *report)
484 return !(oa_report_id(stream, report) &
485 stream->perf->gen8_valid_ctx_bit);
488 static u64 oa_timestamp(struct i915_perf_stream *stream, void *report)
490 return oa_report_header_64bit(stream) ?
495 static void oa_timestamp_clear(struct i915_perf_stream *stream, u32 *report)
497 if (oa_report_header_64bit(stream))
503 static u32 oa_context_id(struct i915_perf_stream *stream, u32 *report)
505 u32 ctx_id = oa_report_header_64bit(stream) ? report[4] : report[2];
507 return ctx_id & stream->specific_ctx_id_mask;
510 static void oa_context_id_squash(struct i915_perf_stream *stream, u32 *report)
512 if (oa_report_header_64bit(stream))
520 * @stream: i915 stream instance
536 * only called while the stream is enabled, while the global OA configuration
541 static bool oa_buffer_check_unlocked(struct i915_perf_stream *stream)
543 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);
544 int report_size = stream->oa_buffer.format->size;
555 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
557 hw_tail = stream->perf->ops.oa_hw_tail_read(stream);
563 partial_report_size = OA_TAKEN(hw_tail, stream->oa_buffer.tail);
571 * anywhere between this head and stream->oa_buffer.tail.
573 head = stream->oa_buffer.head - gtt_offset;
574 read_tail = stream->oa_buffer.tail - gtt_offset;
578 /* Walk the stream backward until we find a report with report
590 void *report = stream->oa_buffer.vaddr + tail;
592 if (oa_report_id(stream, report) ||
593 oa_timestamp(stream, report))
600 __ratelimit(&stream->perf->tail_pointer_race))
601 drm_notice(&stream->uncore->i915->drm,
605 stream->oa_buffer.tail = gtt_offset + tail;
607 pollin = OA_TAKEN(stream->oa_buffer.tail,
608 stream->oa_buffer.head) >= report_size;
610 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
617 * @stream: An i915-perf stream opened for OA metrics
630 static int append_oa_status(struct i915_perf_stream *stream,
651 * @stream: An i915-perf stream opened for OA metrics
658 * properties when opening a stream, tracked as `stream->sample_flags`. This
666 static int append_oa_sample(struct i915_perf_stream *stream,
672 int report_size = stream->oa_buffer.format->size;
679 header.size = stream->sample_size;
689 oa_buf_end = stream->oa_buffer.vaddr + OA_BUFFER_SIZE;
697 if (copy_to_user(buf, stream->oa_buffer.vaddr,
712 * @stream: An i915-perf stream opened for OA metrics
730 static int gen8_append_oa_reports(struct i915_perf_stream *stream,
735 struct intel_uncore *uncore = stream->uncore;
736 int report_size = stream->oa_buffer.format->size;
737 u8 *oa_buf_base = stream->oa_buffer.vaddr;
738 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);
745 if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled))
748 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
750 head = stream->oa_buffer.head;
751 tail = stream->oa_buffer.tail;
753 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
789 reason = oa_report_reason(stream, report);
790 ctx_id = oa_context_id(stream, report32);
832 if (oa_report_ctx_invalid(stream, report) &&
833 GRAPHICS_VER_FULL(stream->engine->i915) < IP_VER(12, 50)) {
835 oa_context_id_squash(stream, report32);
869 if (!stream->ctx ||
870 stream->specific_ctx_id == ctx_id ||
871 stream->oa_buffer.last_ctx_id == stream->specific_ctx_id ||
878 if (stream->ctx &&
879 stream->specific_ctx_id != ctx_id) {
880 oa_context_id_squash(stream, report32);
883 ret = append_oa_sample(stream, buf, count, offset,
888 stream->oa_buffer.last_ctx_id = ctx_id;
896 oa_report_id_clear(stream, report32);
897 oa_timestamp_clear(stream, report32);
899 u8 *oa_buf_end = stream->oa_buffer.vaddr +
916 oaheadptr = GRAPHICS_VER(stream->perf->i915) == 12 ?
917 __oa_regs(stream)->oa_head_ptr :
920 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
929 stream->oa_buffer.head = head;
931 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
939 * @stream: An i915-perf stream opened for OA metrics
957 static int gen8_oa_read(struct i915_perf_stream *stream,
962 struct intel_uncore *uncore = stream->uncore;
967 if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr))
970 oastatus_reg = GRAPHICS_VER(stream->perf->i915) == 12 ?
971 __oa_regs(stream)->oa_status :
991 ret = append_oa_status(stream, buf, count, offset,
996 drm_dbg(&stream->perf->i915->drm,
998 stream->period_exponent);
1000 stream->perf->ops.oa_disable(stream);
1001 stream->perf->ops.oa_enable(stream);
1011 ret = append_oa_status(stream, buf, count, offset,
1024 return gen8_append_oa_reports(stream, buf, count, offset);
1030 * @stream: An i915-perf stream opened for OA metrics
1048 static int gen7_append_oa_reports(struct i915_perf_stream *stream,
1053 struct intel_uncore *uncore = stream->uncore;
1054 int report_size = stream->oa_buffer.format->size;
1055 u8 *oa_buf_base = stream->oa_buffer.vaddr;
1056 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);
1063 if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled))
1066 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1068 head = stream->oa_buffer.head;
1069 tail = stream->oa_buffer.tail;
1071 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
1121 if (__ratelimit(&stream->perf->spurious_report_rs))
1127 ret = append_oa_sample(stream, buf, count, offset, report);
1139 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1149 stream->oa_buffer.head = head;
1151 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
1159 * @stream: An i915-perf stream opened for OA metrics
1173 static int gen7_oa_read(struct i915_perf_stream *stream,
1178 struct intel_uncore *uncore = stream->uncore;
1182 if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr))
1192 oastatus1 &= ~stream->perf->gen7_latched_oastatus1;
1215 ret = append_oa_status(stream, buf, count, offset,
1220 drm_dbg(&stream->perf->i915->drm,
1222 stream->period_exponent);
1224 stream->perf->ops.oa_disable(stream);
1225 stream->perf->ops.oa_enable(stream);
1231 ret = append_oa_status(stream, buf, count, offset,
1235 stream->perf->gen7_latched_oastatus1 |=
1239 return gen7_append_oa_reports(stream, buf, count, offset);
1244 * @stream: An i915-perf stream opened for OA metrics
1246 * Called when userspace tries to read() from a blocking stream FD opened
1256 static int i915_oa_wait_unlocked(struct i915_perf_stream *stream)
1259 if (!stream->periodic)
1262 return wait_event_interruptible(stream->poll_wq,
1263 oa_buffer_check_unlocked(stream));
1267 * i915_oa_poll_wait - call poll_wait() for an OA stream poll()
1268 * @stream: An i915-perf stream opened for OA metrics
1269 * @file: An i915 perf stream file
1272 * For handling userspace polling on an i915 perf stream opened for OA metrics,
1276 static void i915_oa_poll_wait(struct i915_perf_stream *stream,
1280 poll_wait(file, &stream->poll_wq, wait);
1285 * @stream: An i915-perf stream opened for OA metrics
1295 static int i915_oa_read(struct i915_perf_stream *stream,
1300 return stream->perf->ops.read(stream, buf, count, offset);
1303 static struct intel_context *oa_pin_context(struct i915_perf_stream *stream)
1306 struct i915_gem_context *ctx = stream->ctx;
1312 if (ce->engine != stream->engine) /* first match! */
1340 stream->pinned_ctx = ce;
1341 return stream->pinned_ctx;
1435 static int gen12_get_render_context_id(struct i915_perf_stream *stream)
1440 if (intel_engine_uses_guc(stream->engine)) {
1441 ret = gen12_guc_sw_ctx_id(stream->pinned_ctx, &ctx_id);
1447 } else if (GRAPHICS_VER_FULL(stream->engine->i915) >= IP_VER(12, 50)) {
1460 stream->specific_ctx_id = ctx_id & mask;
1461 stream->specific_ctx_id_mask = mask;
1539 * @stream: An i915-perf stream opened for OA metrics
1542 * lifetime of the stream. This ensures that we don't have to worry about
1547 static int oa_get_render_ctx_id(struct i915_perf_stream *stream)
1552 ce = oa_pin_context(stream);
1556 if (engine_supports_mi_query(stream->engine) &&
1557 HAS_LOGICAL_RING_CONTEXTS(stream->perf->i915)) {
1565 drm_err(&stream->perf->i915->drm,
1567 stream->engine->name);
1578 stream->specific_ctx_id = i915_ggtt_offset(ce->state);
1579 stream->specific_ctx_id_mask = 0;
1596 stream->specific_ctx_id = ce->lrc.lrca >> 12;
1602 stream->specific_ctx_id_mask =
1605 stream->specific_ctx_id_mask =
1607 stream->specific_ctx_id = stream->specific_ctx_id_mask;
1613 ret = gen12_get_render_context_id(stream);
1620 ce->tag = stream->specific_ctx_id;
1622 drm_dbg(&stream->perf->i915->drm,
1624 stream->specific_ctx_id,
1625 stream->specific_ctx_id_mask);
1632 * @stream: An i915-perf stream opened for OA metrics
1635 * for the lifetime of the stream, then that can be undone here.
1637 static void oa_put_render_ctx_id(struct i915_perf_stream *stream)
1641 ce = fetch_and_zero(&stream->pinned_ctx);
1647 stream->specific_ctx_id = INVALID_CTX_ID;
1648 stream->specific_ctx_id_mask = 0;
1652 free_oa_buffer(struct i915_perf_stream *stream)
1654 i915_vma_unpin_and_release(&stream->oa_buffer.vma,
1657 stream->oa_buffer.vaddr = NULL;
1661 free_oa_configs(struct i915_perf_stream *stream)
1665 i915_oa_config_put(stream->oa_config);
1666 llist_for_each_entry_safe(oa_bo, tmp, stream->oa_config_bos.first, node)
1671 free_noa_wait(struct i915_perf_stream *stream)
1673 i915_vma_unpin_and_release(&stream->noa_wait, 0);
1686 static void i915_oa_stream_destroy(struct i915_perf_stream *stream)
1688 struct i915_perf *perf = stream->perf;
1689 struct intel_gt *gt = stream->engine->gt;
1690 struct i915_perf_group *g = stream->engine->oa_group;
1692 if (WARN_ON(stream != g->exclusive_stream))
1702 perf->ops.disable_metric_set(stream);
1704 free_oa_buffer(stream);
1709 if (stream->override_gucrc)
1713 intel_uncore_forcewake_put(stream->uncore, FORCEWAKE_ALL);
1714 intel_engine_pm_put(stream->engine);
1716 if (stream->ctx)
1717 oa_put_render_ctx_id(stream);
1719 free_oa_configs(stream);
1720 free_noa_wait(stream);
1729 static void gen7_init_oa_buffer(struct i915_perf_stream *stream)
1731 struct intel_uncore *uncore = stream->uncore;
1732 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);
1735 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1742 stream->oa_buffer.head = gtt_offset;
1750 stream->oa_buffer.tail = gtt_offset;
1752 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
1758 stream->perf->gen7_latched_oastatus1 = 0;
1763 * when re-enabling a stream or in error/reset paths.
1771 memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE);
1774 static void gen8_init_oa_buffer(struct i915_perf_stream *stream)
1776 struct intel_uncore *uncore = stream->uncore;
1777 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);
1780 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1784 stream->oa_buffer.head = gtt_offset;
1801 stream->oa_buffer.tail = gtt_offset;
1808 stream->oa_buffer.last_ctx_id = INVALID_CTX_ID;
1810 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
1816 * when re-enabling a stream or in error/reset paths.
1824 memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE);
1827 static void gen12_init_oa_buffer(struct i915_perf_stream *stream)
1829 struct intel_uncore *uncore = stream->uncore;
1830 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);
1833 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1835 intel_uncore_write(uncore, __oa_regs(stream)->oa_status, 0);
1836 intel_uncore_write(uncore, __oa_regs(stream)->oa_head_ptr,
1838 stream->oa_buffer.head = gtt_offset;
1848 intel_uncore_write(uncore, __oa_regs(stream)->oa_buffer, gtt_offset |
1850 intel_uncore_write(uncore, __oa_regs(stream)->oa_tail_ptr,
1854 stream->oa_buffer.tail = gtt_offset;
1861 stream->oa_buffer.last_ctx_id = INVALID_CTX_ID;
1863 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
1869 * when re-enabling a stream or in error/reset paths.
1877 memset(stream->oa_buffer.vaddr, 0,
1878 stream->oa_buffer.vma->size);
1881 static int alloc_oa_buffer(struct i915_perf_stream *stream)
1883 struct drm_i915_private *i915 = stream->perf->i915;
1884 struct intel_gt *gt = stream->engine->gt;
1889 if (drm_WARN_ON(&i915->drm, stream->oa_buffer.vma))
1895 bo = i915_gem_object_create_shmem(stream->perf->i915, OA_BUFFER_SIZE);
1920 stream->oa_buffer.vma = vma;
1922 stream->oa_buffer.vaddr =
1924 if (IS_ERR(stream->oa_buffer.vaddr)) {
1925 ret = PTR_ERR(stream->oa_buffer.vaddr);
1937 stream->oa_buffer.vaddr = NULL;
1938 stream->oa_buffer.vma = NULL;
1943 static u32 *save_restore_register(struct i915_perf_stream *stream, u32 *cs,
1952 if (GRAPHICS_VER(stream->perf->i915) >= 8)
1958 *cs++ = i915_ggtt_offset(stream->noa_wait) + offset + 4 * d;
1965 static int alloc_noa_wait(struct i915_perf_stream *stream)
1967 struct drm_i915_private *i915 = stream->perf->i915;
1968 struct intel_gt *gt = stream->engine->gt;
1972 intel_gt_ns_to_clock_interval(to_gt(stream->perf->i915),
1973 atomic64_read(&stream->perf->noa_programming_delay));
1974 const u32 base = stream->engine->mmio_base;
2013 * needs to be fixed during the lifetime of the i915/perf stream.
2031 stream->noa_wait = vma;
2039 stream, cs, true /* save */, CS_GPR(i),
2042 stream, cs, true /* save */, mi_predicate_result,
2158 stream, cs, false /* restore */, CS_GPR(i),
2161 stream, cs, false /* restore */, mi_predicate_result,
2222 alloc_oa_config_buffer(struct i915_perf_stream *stream,
2242 obj = i915_gem_object_create_shmem(stream->perf->i915, config_length);
2271 *cs++ = (GRAPHICS_VER(stream->perf->i915) < 8 ?
2274 *cs++ = i915_ggtt_offset(stream->noa_wait);
2281 &stream->engine->gt->ggtt->vm,
2289 llist_add(&oa_bo->node, &stream->oa_config_bos);
2310 get_oa_vma(struct i915_perf_stream *stream, struct i915_oa_config *oa_config)
2316 * to the stream.
2318 llist_for_each_entry(oa_bo, stream->oa_config_bos.first, node) {
2326 oa_bo = alloc_oa_config_buffer(stream, oa_config);
2335 emit_oa_config(struct i915_perf_stream *stream,
2345 vma = get_oa_vma(stream, oa_config);
2405 static struct intel_context *oa_context(struct i915_perf_stream *stream)
2407 return stream->pinned_ctx ?: stream->engine->kernel_context;
2411 hsw_enable_metric_set(struct i915_perf_stream *stream,
2414 struct intel_uncore *uncore = stream->uncore;
2431 return emit_oa_config(stream,
2432 stream->oa_config, oa_context(stream),
2436 static void hsw_disable_metric_set(struct i915_perf_stream *stream)
2438 struct intel_uncore *uncore = stream->uncore;
2478 const struct i915_perf_stream *stream)
2480 u32 ctx_oactxctrl = stream->perf->ctx_oactxctrl_offset;
2481 u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset;
2496 (stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) |
2497 (stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) |
2502 oa_config_flex_reg(stream->oa_config, flex_regs[i]);
2609 static int gen8_configure_context(struct i915_perf_stream *stream,
2639 static int gen12_configure_oar_context(struct i915_perf_stream *stream,
2643 struct intel_context *ce = stream->pinned_ctx;
2644 u32 format = stream->oa_buffer.format->format;
2645 u32 offset = stream->perf->ctx_oactxctrl_offset;
2690 * Manages updating the per-context aspects of the OA stream
2715 oa_configure_all_contexts(struct i915_perf_stream *stream,
2720 struct drm_i915_private *i915 = stream->perf->i915;
2722 struct intel_gt *gt = stream->engine->gt;
2751 err = gen8_configure_context(stream, ctx, regs, num_regs);
2785 gen12_configure_all_contexts(struct i915_perf_stream *stream,
2796 if (stream->engine->class != RENDER_CLASS)
2799 return oa_configure_all_contexts(stream,
2805 lrc_configure_all_contexts(struct i915_perf_stream *stream,
2809 u32 ctx_oactxctrl = stream->perf->ctx_oactxctrl_offset;
2811 const u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset;
2834 (stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) |
2835 (stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) |
2841 return oa_configure_all_contexts(stream,
2847 gen8_enable_metric_set(struct i915_perf_stream *stream,
2850 struct intel_uncore *uncore = stream->uncore;
2851 struct i915_oa_config *oa_config = stream->oa_config;
2877 if (IS_GRAPHICS_VER(stream->perf->i915, 9, 11)) {
2888 ret = lrc_configure_all_contexts(stream, oa_config, active);
2892 return emit_oa_config(stream,
2893 stream->oa_config, oa_context(stream),
2897 static u32 oag_report_ctx_switches(const struct i915_perf_stream *stream)
2900 (stream->sample_flags & SAMPLE_OA_REPORT) ?
2905 gen12_enable_metric_set(struct i915_perf_stream *stream,
2908 struct drm_i915_private *i915 = stream->perf->i915;
2909 struct intel_uncore *uncore = stream->uncore;
2910 struct i915_oa_config *oa_config = stream->oa_config;
2911 bool periodic = stream->periodic;
2912 u32 period_exponent = stream->period_exponent;
2928 intel_uncore_write(uncore, __oa_regs(stream)->oa_debug,
2936 oag_report_ctx_switches(stream));
2938 intel_uncore_write(uncore, __oa_regs(stream)->oa_ctx_ctrl, periodic ?
2959 ret = gen12_configure_all_contexts(stream, oa_config, active);
2968 if (stream->ctx) {
2969 ret = gen12_configure_oar_context(stream, active);
2974 return emit_oa_config(stream,
2975 stream->oa_config, oa_context(stream),
2979 static void gen8_disable_metric_set(struct i915_perf_stream *stream)
2981 struct intel_uncore *uncore = stream->uncore;
2984 lrc_configure_all_contexts(stream, NULL, NULL);
2989 static void gen11_disable_metric_set(struct i915_perf_stream *stream)
2991 struct intel_uncore *uncore = stream->uncore;
2994 lrc_configure_all_contexts(stream, NULL, NULL);
3000 static void gen12_disable_metric_set(struct i915_perf_stream *stream)
3002 struct intel_uncore *uncore = stream->uncore;
3003 struct drm_i915_private *i915 = stream->perf->i915;
3018 gen12_configure_all_contexts(stream, NULL, NULL);
3021 if (stream->ctx)
3022 gen12_configure_oar_context(stream, NULL);
3034 static void gen7_oa_enable(struct i915_perf_stream *stream)
3036 struct intel_uncore *uncore = stream->uncore;
3037 struct i915_gem_context *ctx = stream->ctx;
3038 u32 ctx_id = stream->specific_ctx_id;
3039 bool periodic = stream->periodic;
3040 u32 period_exponent = stream->period_exponent;
3041 u32 report_format = stream->oa_buffer.format->format;
3052 gen7_init_oa_buffer(stream);
3064 static void gen8_oa_enable(struct i915_perf_stream *stream)
3066 struct intel_uncore *uncore = stream->uncore;
3067 u32 report_format = stream->oa_buffer.format->format;
3078 gen8_init_oa_buffer(stream);
3090 static void gen12_oa_enable(struct i915_perf_stream *stream)
3099 if (!(stream->sample_flags & SAMPLE_OA_REPORT))
3102 gen12_init_oa_buffer(stream);
3104 regs = __oa_regs(stream);
3105 val = (stream->oa_buffer.format->format << regs->oa_ctrl_counter_format_shift) |
3108 intel_uncore_write(stream->uncore, regs->oa_ctrl, val);
3112 * i915_oa_stream_enable - handle `I915_PERF_IOCTL_ENABLE` for OA stream
3113 * @stream: An i915 perf stream opened for OA metrics
3116 * when opening the stream. This also starts a hrtimer that will periodically
3120 static void i915_oa_stream_enable(struct i915_perf_stream *stream)
3122 stream->pollin = false;
3124 stream->perf->ops.oa_enable(stream);
3126 if (stream->sample_flags & SAMPLE_OA_REPORT)
3127 hrtimer_start(&stream->poll_check_timer,
3128 ns_to_ktime(stream->poll_oa_period),
3132 static void gen7_oa_disable(struct i915_perf_stream *stream)
3134 struct intel_uncore *uncore = stream->uncore;
3140 drm_err(&stream->perf->i915->drm,
3144 static void gen8_oa_disable(struct i915_perf_stream *stream)
3146 struct intel_uncore *uncore = stream->uncore;
3152 drm_err(&stream->perf->i915->drm,
3156 static void gen12_oa_disable(struct i915_perf_stream *stream)
3158 struct intel_uncore *uncore = stream->uncore;
3160 intel_uncore_write(uncore, __oa_regs(stream)->oa_ctrl, 0);
3162 __oa_regs(stream)->oa_ctrl,
3165 drm_err(&stream->perf->i915->drm,
3173 drm_err(&stream->perf->i915->drm,
3178 * i915_oa_stream_disable - handle `I915_PERF_IOCTL_DISABLE` for OA stream
3179 * @stream: An i915 perf stream opened for OA metrics
3185 static void i915_oa_stream_disable(struct i915_perf_stream *stream)
3187 stream->perf->ops.oa_disable(stream);
3189 if (stream->sample_flags & SAMPLE_OA_REPORT)
3190 hrtimer_cancel(&stream->poll_check_timer);
3202 static int i915_perf_stream_enable_sync(struct i915_perf_stream *stream)
3211 err = stream->perf->ops.enable_metric_set(stream, active);
3279 * i915_oa_stream_init - validate combined props for OA stream and init
3280 * @stream: An i915 perf stream
3282 * @props: The property state that configures stream (individually validated)
3287 * At this point it has been determined that userspace wants a stream of
3296 static int i915_oa_stream_init(struct i915_perf_stream *stream,
3300 struct drm_i915_private *i915 = stream->perf->i915;
3301 struct i915_perf *perf = stream->perf;
3307 drm_dbg(&stream->perf->i915->drm,
3320 drm_dbg(&stream->perf->i915->drm,
3326 (GRAPHICS_VER(perf->i915) < 12 || !stream->ctx)) {
3327 drm_dbg(&stream->perf->i915->drm,
3333 drm_dbg(&stream->perf->i915->drm,
3344 drm_dbg(&stream->perf->i915->drm,
3350 drm_dbg(&stream->perf->i915->drm,
3355 stream->engine = props->engine;
3356 stream->uncore = stream->engine->gt->uncore;
3358 stream->sample_size = sizeof(struct drm_i915_perf_record_header);
3360 stream->oa_buffer.format = &perf->oa_formats[props->oa_format];
3361 if (drm_WARN_ON(&i915->drm, stream->oa_buffer.format->size == 0))
3364 stream->sample_flags = props->sample_flags;
3365 stream->sample_size += stream->oa_buffer.format->size;
3367 stream->hold_preemption = props->hold_preemption;
3369 stream->periodic = props->oa_periodic;
3370 if (stream->periodic)
3371 stream->period_exponent = props->oa_period_exponent;
3373 if (stream->ctx) {
3374 ret = oa_get_render_ctx_id(stream);
3376 drm_dbg(&stream->perf->i915->drm,
3382 ret = alloc_noa_wait(stream);
3384 drm_dbg(&stream->perf->i915->drm,
3389 stream->oa_config = i915_perf_get_oa_config(perf, props->metrics_set);
3390 if (!stream->oa_config) {
3391 drm_dbg(&stream->perf->i915->drm,
3409 intel_engine_pm_get(stream->engine);
3410 intel_uncore_forcewake_get(stream->uncore, FORCEWAKE_ALL);
3423 drm_dbg(&stream->perf->i915->drm,
3428 stream->override_gucrc = true;
3431 ret = alloc_oa_buffer(stream);
3435 stream->ops = &i915_oa_stream_ops;
3437 stream->engine->gt->perf.sseu = props->sseu;
3438 WRITE_ONCE(g->exclusive_stream, stream);
3440 ret = i915_perf_stream_enable_sync(stream);
3442 drm_dbg(&stream->perf->i915->drm,
3447 drm_dbg(&stream->perf->i915->drm,
3448 "opening stream oa config uuid=%s\n",
3449 stream->oa_config->uuid);
3451 hrtimer_init(&stream->poll_check_timer,
3453 stream->poll_check_timer.function = oa_poll_check_timer_cb;
3454 init_waitqueue_head(&stream->poll_wq);
3455 spin_lock_init(&stream->oa_buffer.ptr_lock);
3456 mutex_init(&stream->lock);
3462 perf->ops.disable_metric_set(stream);
3464 free_oa_buffer(stream);
3467 if (stream->override_gucrc)
3471 intel_uncore_forcewake_put(stream->uncore, FORCEWAKE_ALL);
3472 intel_engine_pm_put(stream->engine);
3474 free_oa_configs(stream);
3477 free_noa_wait(stream);
3480 if (stream->ctx)
3481 oa_put_render_ctx_id(stream);
3489 struct i915_perf_stream *stream;
3495 stream = READ_ONCE(engine->oa_group->exclusive_stream);
3496 if (stream && GRAPHICS_VER(stream->perf->i915) < 12)
3497 gen8_update_reg_state_unlocked(ce, stream);
3501 * i915_perf_read - handles read() FOP for i915 perf stream FDs
3502 * @file: An i915 perf stream file
3507 * The entry point for handling a read() on a stream file descriptor from
3509 * &i915_perf_stream_ops->read but to save having stream implementations (of
3512 * We can also consistently treat trying to read from a disabled stream
3513 * as an IO error so implementations can assume the stream is enabled
3523 struct i915_perf_stream *stream = file->private_data;
3528 * disabled stream as an error. In particular it might otherwise lead
3531 if (!stream->enabled || !(stream->sample_flags & SAMPLE_OA_REPORT))
3536 * stream->ops->wait_unlocked.
3543 ret = stream->ops->wait_unlocked(stream);
3547 mutex_lock(&stream->lock);
3548 ret = stream->ops->read(stream, buf, count, &offset);
3549 mutex_unlock(&stream->lock);
3552 mutex_lock(&stream->lock);
3553 ret = stream->ops->read(stream, buf, count, &offset);
3554 mutex_unlock(&stream->lock);
3569 stream->pollin = false;
3577 struct i915_perf_stream *stream =
3578 container_of(hrtimer, typeof(*stream), poll_check_timer);
3580 if (oa_buffer_check_unlocked(stream)) {
3581 stream->pollin = true;
3582 wake_up(&stream->poll_wq);
3586 ns_to_ktime(stream->poll_oa_period));
3592 * i915_perf_poll_locked - poll_wait() with a suitable wait queue for stream
3593 * @stream: An i915 perf stream
3594 * @file: An i915 perf stream file
3597 * For handling userspace polling on an i915 perf stream, this calls through to
3599 * will be woken for new stream data.
3603 static __poll_t i915_perf_poll_locked(struct i915_perf_stream *stream,
3609 stream->ops->poll_wait(stream, file, wait);
3617 if (stream->pollin)
3624 * i915_perf_poll - call poll_wait() with a suitable wait queue for stream
3625 * @file: An i915 perf stream file
3628 * For handling userspace polling on an i915 perf stream, this ensures
3629 * poll_wait() gets called with a wait queue that will be woken for new stream
3638 struct i915_perf_stream *stream = file->private_data;
3641 mutex_lock(&stream->lock);
3642 ret = i915_perf_poll_locked(stream, file, wait);
3643 mutex_unlock(&stream->lock);
3650 * @stream: A disabled i915 perf stream
3652 * [Re]enables the associated capture of data for this stream.
3654 * If a stream was previously enabled then there's currently no intention
3658 static void i915_perf_enable_locked(struct i915_perf_stream *stream)
3660 if (stream->enabled)
3663 /* Allow stream->ops->enable() to refer to this */
3664 stream->enabled = true;
3666 if (stream->ops->enable)
3667 stream->ops->enable(stream);
3669 if (stream->hold_preemption)
3670 intel_context_set_nopreempt(stream->pinned_ctx);
3675 * @stream: An enabled i915 perf stream
3677 * Disables the associated capture of data for this stream.
3679 * The intention is that disabling an re-enabling a stream will ideally be
3680 * cheaper than destroying and re-opening a stream with the same configuration,
3682 * must be retained between disabling and re-enabling a stream.
3684 * Note: while a stream is disabled it's considered an error for userspace
3685 * to attempt to read from the stream (-EIO).
3687 static void i915_perf_disable_locked(struct i915_perf_stream *stream)
3689 if (!stream->enabled)
3692 /* Allow stream->ops->disable() to refer to this */
3693 stream->enabled = false;
3695 if (stream->hold_preemption)
3696 intel_context_clear_nopreempt(stream->pinned_ctx);
3698 if (stream->ops->disable)
3699 stream->ops->disable(stream);
3702 static long i915_perf_config_locked(struct i915_perf_stream *stream,
3706 long ret = stream->oa_config->id;
3708 config = i915_perf_get_oa_config(stream->perf, metrics_set);
3712 if (config != stream->oa_config) {
3724 err = emit_oa_config(stream, config, oa_context(stream), NULL);
3726 config = xchg(&stream->oa_config, config);
3737 * i915_perf_ioctl_locked - support ioctl() usage with i915 perf stream FDs
3738 * @stream: An i915 perf stream
3745 static long i915_perf_ioctl_locked(struct i915_perf_stream *stream,
3751 i915_perf_enable_locked(stream);
3754 i915_perf_disable_locked(stream);
3757 return i915_perf_config_locked(stream, arg);
3764 * i915_perf_ioctl - support ioctl() usage with i915 perf stream FDs
3765 * @file: An i915 perf stream file
3778 struct i915_perf_stream *stream = file->private_data;
3781 mutex_lock(&stream->lock);
3782 ret = i915_perf_ioctl_locked(stream, cmd, arg);
3783 mutex_unlock(&stream->lock);
3789 * i915_perf_destroy_locked - destroy an i915 perf stream
3790 * @stream: An i915 perf stream
3792 * Frees all resources associated with the given i915 perf @stream, disabling
3798 static void i915_perf_destroy_locked(struct i915_perf_stream *stream)
3800 if (stream->enabled)
3801 i915_perf_disable_locked(stream);
3803 if (stream->ops->destroy)
3804 stream->ops->destroy(stream);
3806 if (stream->ctx)
3807 i915_gem_context_put(stream->ctx);
3809 kfree(stream);
3813 * i915_perf_release - handles userspace close() of a stream file
3815 * @file: An i915 perf stream file
3817 * Cleans up any resources associated with an open i915 perf stream file.
3825 struct i915_perf_stream *stream = file->private_data;
3826 struct i915_perf *perf = stream->perf;
3827 struct intel_gt *gt = stream->engine->gt;
3831 * other user of stream->lock. Use the perf lock to destroy the stream
3835 i915_perf_destroy_locked(stream);
3838 /* Release the reference the perf stream kept on the driver. */
3860 * i915_perf_open_ioctl_locked - DRM ioctl() for userspace to open a stream FD
3868 * Implements further stream config validation and stream initialization on
3877 * config validation and stream initialization details will be handled by
3879 * will be relevant to all stream types / backends.
3890 struct i915_perf_stream *stream = NULL;
3903 "Failed to look up context with ID %u for opening perf stream\n",
3926 * doesn't request global stream access (i.e. query based sampling
3961 "Insufficient privileges to open i915 perf stream\n");
3966 stream = kzalloc(sizeof(*stream), GFP_KERNEL);
3967 if (!stream) {
3972 stream->perf = perf;
3973 stream->ctx = specific_ctx;
3974 stream->poll_oa_period = props->poll_oa_period;
3976 ret = i915_oa_stream_init(stream, param, props);
3980 /* we avoid simply assigning stream->sample_flags = props->sample_flags
3984 if (WARN_ON(stream->sample_flags != props->sample_flags)) {
3994 stream_fd = anon_inode_getfd("[i915_perf]", &fops, stream, f_flags);
4001 i915_perf_enable_locked(stream);
4011 if (stream->ops->destroy)
4012 stream->ops->destroy(stream);
4014 kfree(stream);
4043 * read_properties_unlocked - validate + copy userspace stream open properties
4047 * @props: The stream configuration built up while validating properties
4051 * properties necessary for a particular kind of stream have been set.
4285 * i915_perf_open_ioctl - DRM ioctl() for userspace to open a stream FD
4290 * Validates the stream open parameters given by userspace including flags
4293 * Very little is assumed up front about the nature of the stream being
4295 * i915-perf stream is expected to be a suitable interface for other forms of
4305 * Return: A newly opened i915 Perf stream file descriptor or negative
4352 * used to open an i915-perf stream.
4846 * and their content will be freed when the stream using the config is closed.
5240 * stream instead of waiting until driver _fini which no one