Lines Matching refs:pt
34 #include "intel-pt.h"
42 #include "intel-pt-decoder/intel-pt-log.h"
43 #include "intel-pt-decoder/intel-pt-decoder.h"
44 #include "intel-pt-decoder/intel-pt-insn-decoder.h"
45 #include "intel-pt-decoder/intel-pt-pkt-decoder.h"
146 struct intel_pt *pt;
185 static void intel_pt_dump(struct intel_pt *pt __maybe_unused,
225 static void intel_pt_dump_event(struct intel_pt *pt, unsigned char *buf,
229 intel_pt_dump(pt, buf, len);
245 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
249 intel_pt_dump(pt, sample->aux_sample.data, sample->aux_sample.size);
252 static bool intel_pt_log_events(struct intel_pt *pt, u64 tm)
254 struct perf_time_interval *range = pt->synth_opts.ptime_range;
255 int n = pt->synth_opts.range_num;
257 if (pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_ALL_PERF_EVTS)
260 if (pt->synth_opts.log_minus_flags & AUXTRACE_LOG_FLG_ALL_PERF_EVTS)
270 static int intel_pt_do_fix_overlap(struct intel_pt *pt, struct auxtrace_buffer *a,
277 pt->have_tsc, &consecutive);
295 int fd = perf_data__fd(ptq->pt->session->data);
302 might_overlap = ptq->pt->snapshot_mode || ptq->pt->sampling_mode;
304 intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))
346 queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
396 queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
443 if (!strcmp(var, "intel-pt.cache-divisor")) {
552 static inline u8 intel_pt_cpumode(struct intel_pt *pt, uint64_t ip)
554 return ip >= pt->kernel_start ?
565 struct machine *machine = ptq->pt->machine;
581 cpumode = intel_pt_cpumode(ptq->pt, *ip);
587 thread = ptq->pt->unknown_thread;
691 static bool intel_pt_match_pgd_ip(struct intel_pt *pt, uint64_t ip,
699 list_for_each_entry(filt, &pt->filts.head, list) {
737 if (ip >= ptq->pt->kernel_start)
738 return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL);
751 return intel_pt_match_pgd_ip(ptq->pt, ip, offset,
760 static bool intel_pt_get_config(struct intel_pt *pt,
763 if (attr->type == pt->pmu_type) {
772 static bool intel_pt_exclude_kernel(struct intel_pt *pt)
776 evlist__for_each_entry(pt->session->evlist, evsel) {
777 if (intel_pt_get_config(pt, &evsel->core.attr, NULL) &&
784 static bool intel_pt_return_compression(struct intel_pt *pt)
789 if (!pt->noretcomp_bit)
792 evlist__for_each_entry(pt->session->evlist, evsel) {
793 if (intel_pt_get_config(pt, &evsel->core.attr, &config) &&
794 (config & pt->noretcomp_bit))
800 static bool intel_pt_branch_enable(struct intel_pt *pt)
805 evlist__for_each_entry(pt->session->evlist, evsel) {
806 if (intel_pt_get_config(pt, &evsel->core.attr, &config) &&
813 static unsigned int intel_pt_mtc_period(struct intel_pt *pt)
819 if (!pt->mtc_freq_bits)
822 for (shift = 0, config = pt->mtc_freq_bits; !(config & 1); shift++)
825 evlist__for_each_entry(pt->session->evlist, evsel) {
826 if (intel_pt_get_config(pt, &evsel->core.attr, &config))
827 return (config & pt->mtc_freq_bits) >> shift;
832 static bool intel_pt_timeless_decoding(struct intel_pt *pt)
838 if (!pt->tsc_bit || !pt->cap_user_time_zero)
841 evlist__for_each_entry(pt->session->evlist, evsel) {
844 if (intel_pt_get_config(pt, &evsel->core.attr, &config)) {
845 if (config & pt->tsc_bit)
854 static bool intel_pt_tracing_kernel(struct intel_pt *pt)
858 evlist__for_each_entry(pt->session->evlist, evsel) {
859 if (intel_pt_get_config(pt, &evsel->core.attr, NULL) &&
866 static bool intel_pt_have_tsc(struct intel_pt *pt)
872 if (!pt->tsc_bit)
875 evlist__for_each_entry(pt->session->evlist, evsel) {
876 if (intel_pt_get_config(pt, &evsel->core.attr, &config)) {
877 if (config & pt->tsc_bit)
886 static bool intel_pt_sampling_mode(struct intel_pt *pt)
890 evlist__for_each_entry(pt->session->evlist, evsel) {
898 static u64 intel_pt_ctl(struct intel_pt *pt)
903 evlist__for_each_entry(pt->session->evlist, evsel) {
904 if (intel_pt_get_config(pt, &evsel->core.attr, &config))
910 static u64 intel_pt_ns_to_ticks(const struct intel_pt *pt, u64 ns)
914 quot = ns / pt->tc.time_mult;
915 rem = ns % pt->tc.time_mult;
916 return (quot << pt->tc.time_shift) + (rem << pt->tc.time_shift) /
917 pt->tc.time_mult;
920 static struct ip_callchain *intel_pt_alloc_chain(struct intel_pt *pt)
925 sz += (pt->synth_opts.callchain_sz + 1) * sizeof(u64);
929 static int intel_pt_callchain_init(struct intel_pt *pt)
933 evlist__for_each_entry(pt->session->evlist, evsel) {
938 pt->chain = intel_pt_alloc_chain(pt);
939 if (!pt->chain)
945 static void intel_pt_add_callchain(struct intel_pt *pt,
948 struct thread *thread = machine__findnew_thread(pt->machine,
952 thread_stack__sample_late(thread, sample->cpu, pt->chain,
953 pt->synth_opts.callchain_sz + 1, sample->ip,
954 pt->kernel_start);
956 sample->callchain = pt->chain;
967 static int intel_pt_br_stack_init(struct intel_pt *pt)
971 evlist__for_each_entry(pt->session->evlist, evsel) {
976 pt->br_stack = intel_pt_alloc_br_stack(pt->br_stack_sz);
977 if (!pt->br_stack)
983 static void intel_pt_add_br_stack(struct intel_pt *pt,
986 struct thread *thread = machine__findnew_thread(pt->machine,
990 thread_stack__br_sample_late(thread, sample->cpu, pt->br_stack,
991 pt->br_stack_sz, sample->ip,
992 pt->kernel_start);
994 sample->branch_stack = pt->br_stack;
1000 static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt,
1004 struct perf_env *env = pt->machine->env;
1011 if (pt->synth_opts.callchain) {
1012 ptq->chain = intel_pt_alloc_chain(pt);
1017 if (pt->synth_opts.last_branch || pt->synth_opts.other_events) {
1018 unsigned int entry_cnt = max(LBRS_MAX, pt->br_stack_sz);
1029 ptq->pt = pt;
1031 ptq->exclude_kernel = intel_pt_exclude_kernel(pt);
1041 params.return_compression = intel_pt_return_compression(pt);
1042 params.branch_enable = intel_pt_branch_enable(pt);
1043 params.ctl = intel_pt_ctl(pt);
1044 params.max_non_turbo_ratio = pt->max_non_turbo_ratio;
1045 params.mtc_period = intel_pt_mtc_period(pt);
1046 params.tsc_ctc_ratio_n = pt->tsc_ctc_ratio_n;
1047 params.tsc_ctc_ratio_d = pt->tsc_ctc_ratio_d;
1048 params.quick = pt->synth_opts.quick;
1050 if (pt->filts.cnt > 0)
1053 if (pt->synth_opts.instructions) {
1054 if (pt->synth_opts.period) {
1055 switch (pt->synth_opts.period_type) {
1059 params.period = pt->synth_opts.period;
1063 params.period = pt->synth_opts.period;
1067 params.period = intel_pt_ns_to_ticks(pt,
1068 pt->synth_opts.period);
1112 static void intel_pt_set_pid_tid_cpu(struct intel_pt *pt,
1117 if (queue->tid == -1 || pt->have_sched_switch) {
1118 ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);
1125 ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);
1166 static void intel_pt_setup_time_range(struct intel_pt *pt,
1169 if (!pt->range_cnt)
1172 ptq->sel_timestamp = pt->time_ranges[0].start;
1178 ptq->sel_timestamp = pt->time_ranges[0].end;
1183 static int intel_pt_setup_queue(struct intel_pt *pt,
1193 ptq = intel_pt_alloc_queue(pt, queue_nr);
1204 if (pt->sampling_mode && !pt->snapshot_mode &&
1205 pt->timeless_decoding)
1208 ptq->sync_switch = pt->sync_switch;
1210 intel_pt_setup_time_range(pt, ptq);
1219 if (pt->timeless_decoding)
1256 ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);
1265 static int intel_pt_setup_queues(struct intel_pt *pt)
1270 for (i = 0; i < pt->queues.nr_queues; i++) {
1271 ret = intel_pt_setup_queue(pt, &pt->queues.queue_array[i], i);
1278 static inline bool intel_pt_skip_event(struct intel_pt *pt)
1280 return pt->synth_opts.initial_skip &&
1281 pt->num_events++ < pt->synth_opts.initial_skip;
1289 static inline bool intel_pt_skip_cbr_event(struct intel_pt *pt)
1291 return pt->synth_opts.initial_skip &&
1292 pt->num_events + 4 < pt->synth_opts.initial_skip;
1309 static void intel_pt_prep_b_sample(struct intel_pt *pt,
1316 if (!pt->timeless_decoding)
1317 sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1320 sample->cpumode = intel_pt_cpumode(pt, sample->ip);
1335 static inline int intel_pt_opt_inject(struct intel_pt *pt,
1339 if (!pt->synth_opts.inject)
1345 static int intel_pt_deliver_synth_event(struct intel_pt *pt,
1351 ret = intel_pt_opt_inject(pt, event, sample, type);
1355 ret = perf_session__deliver_synth_event(pt->session, event, sample);
1364 struct intel_pt *pt = ptq->pt;
1373 if (pt->branches_filter && !(pt->branches_filter & ptq->flags))
1376 if (intel_pt_skip_event(pt))
1379 intel_pt_prep_b_sample(pt, ptq, event, &sample);
1381 sample.id = ptq->pt->branches_id;
1382 sample.stream_id = ptq->pt->branches_id;
1388 if (pt->synth_opts.last_branch && sort__mode == SORT_MODE__BRANCH) {
1408 return intel_pt_deliver_synth_event(pt, event, &sample,
1409 pt->branches_sample_type);
1412 static void intel_pt_prep_sample(struct intel_pt *pt,
1417 intel_pt_prep_b_sample(pt, ptq, event, sample);
1419 if (pt->synth_opts.callchain) {
1421 pt->synth_opts.callchain_sz + 1,
1422 sample->ip, pt->kernel_start);
1426 if (pt->synth_opts.last_branch) {
1428 pt->br_stack_sz);
1435 struct intel_pt *pt = ptq->pt;
1439 if (intel_pt_skip_event(pt))
1442 intel_pt_prep_sample(pt, ptq, event, &sample);
1444 sample.id = ptq->pt->instructions_id;
1445 sample.stream_id = ptq->pt->instructions_id;
1446 if (pt->synth_opts.quick)
1461 return intel_pt_deliver_synth_event(pt, event, &sample,
1462 pt->instructions_sample_type);
1467 struct intel_pt *pt = ptq->pt;
1471 if (intel_pt_skip_event(pt))
1474 intel_pt_prep_sample(pt, ptq, event, &sample);
1476 sample.id = ptq->pt->transactions_id;
1477 sample.stream_id = ptq->pt->transactions_id;
1479 return intel_pt_deliver_synth_event(pt, event, &sample,
1480 pt->transactions_sample_type);
1483 static void intel_pt_prep_p_sample(struct intel_pt *pt,
1488 intel_pt_prep_sample(pt, ptq, event, sample);
1500 struct intel_pt *pt = ptq->pt;
1505 if (intel_pt_skip_event(pt))
1508 intel_pt_prep_p_sample(pt, ptq, event, &sample);
1510 sample.id = ptq->pt->ptwrites_id;
1511 sample.stream_id = ptq->pt->ptwrites_id;
1520 return intel_pt_deliver_synth_event(pt, event, &sample,
1521 pt->ptwrites_sample_type);
1526 struct intel_pt *pt = ptq->pt;
1532 if (intel_pt_skip_cbr_event(pt))
1537 intel_pt_prep_p_sample(pt, ptq, event, &sample);
1539 sample.id = ptq->pt->cbr_id;
1540 sample.stream_id = ptq->pt->cbr_id;
1542 flags = (u16)ptq->state->cbr_payload | (pt->max_non_turbo_ratio << 16);
1544 raw.freq = cpu_to_le32(raw.cbr * pt->cbr2khz);
1550 return intel_pt_deliver_synth_event(pt, event, &sample,
1551 pt->pwr_events_sample_type);
1556 struct intel_pt *pt = ptq->pt;
1561 if (intel_pt_skip_event(pt))
1564 intel_pt_prep_p_sample(pt, ptq, event, &sample);
1566 sample.id = ptq->pt->mwait_id;
1567 sample.stream_id = ptq->pt->mwait_id;
1575 return intel_pt_deliver_synth_event(pt, event, &sample,
1576 pt->pwr_events_sample_type);
1581 struct intel_pt *pt = ptq->pt;
1586 if (intel_pt_skip_event(pt))
1589 intel_pt_prep_p_sample(pt, ptq, event, &sample);
1591 sample.id = ptq->pt->pwre_id;
1592 sample.stream_id = ptq->pt->pwre_id;
1600 return intel_pt_deliver_synth_event(pt, event, &sample,
1601 pt->pwr_events_sample_type);
1606 struct intel_pt *pt = ptq->pt;
1611 if (intel_pt_skip_event(pt))
1614 intel_pt_prep_p_sample(pt, ptq, event, &sample);
1616 sample.id = ptq->pt->exstop_id;
1617 sample.stream_id = ptq->pt->exstop_id;
1625 return intel_pt_deliver_synth_event(pt, event, &sample,
1626 pt->pwr_events_sample_type);
1631 struct intel_pt *pt = ptq->pt;
1636 if (intel_pt_skip_event(pt))
1639 intel_pt_prep_p_sample(pt, ptq, event, &sample);
1641 sample.id = ptq->pt->pwrx_id;
1642 sample.stream_id = ptq->pt->pwrx_id;
1650 return intel_pt_deliver_synth_event(pt, event, &sample,
1651 pt->pwr_events_sample_type);
1786 struct intel_pt *pt = ptq->pt;
1787 struct evsel *evsel = pt->pebs_evsel;
1793 if (intel_pt_skip_event(pt))
1813 cpumode = sample.ip < ptq->pt->kernel_start ?
1826 else if (!pt->timeless_decoding)
1829 sample.time = tsc_to_perf_time(timestamp, &pt->tc);
1833 pt->synth_opts.callchain) {
1835 pt->synth_opts.callchain_sz, sample.ip,
1836 pt->kernel_start);
1861 } else if (pt->synth_opts.last_branch) {
1864 pt->br_stack_sz);
1898 return intel_pt_deliver_synth_event(pt, event, &sample, sample_type);
1901 static int intel_pt_synth_error(struct intel_pt *pt, int code, int cpu,
1908 if (pt->synth_opts.error_minus_flags) {
1910 pt->synth_opts.error_minus_flags & AUXTRACE_ERR_FLG_OVERFLOW)
1913 pt->synth_opts.error_minus_flags & AUXTRACE_ERR_FLG_DATA_LOST)
1922 err = perf_session__deliver_synth_event(pt->session, &event, NULL);
1933 struct intel_pt *pt = ptq->pt;
1936 tm = pt->timeless_decoding ? 0 : tsc_to_perf_time(tm, &pt->tc);
1938 return intel_pt_synth_error(pt, state->err, ptq->cpu, ptq->pid,
1942 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq)
1953 err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);
1955 queue = &pt->queues.queue_array[ptq->queue_nr];
1956 intel_pt_set_pid_tid_cpu(pt, queue);
1965 struct intel_pt *pt = ptq->pt;
1967 return ip == pt->switch_ip &&
1979 struct intel_pt *pt = ptq->pt;
1994 if (pt->sample_pebs && state->type & INTEL_PT_BLK_ITEMS) {
2000 if (pt->sample_pwr_events) {
2030 if (pt->sample_instructions && (state->type & INTEL_PT_INSTRUCTION)) {
2036 if (pt->sample_transactions && (state->type & INTEL_PT_TRANSACTION)) {
2042 if (pt->sample_ptwrites && (state->type & INTEL_PT_PTW)) {
2051 if (pt->use_thread_stack) {
2054 state->trace_nr, pt->callstack,
2055 pt->br_stack_sz_plus,
2056 pt->mispred_all);
2061 if (pt->sample_branches) {
2075 err = intel_pt_next_tid(pt, ptq);
2089 state->to_ip == pt->ptss_ip &&
2097 static u64 intel_pt_switch_ip(struct intel_pt *pt, u64 *ptss_ip)
2099 struct machine *machine = pt->machine;
2131 if (pt->have_sched_switch == 1)
2149 static void intel_pt_enable_sync_switch(struct intel_pt *pt)
2153 pt->sync_switch = true;
2155 for (i = 0; i < pt->queues.nr_queues; i++) {
2156 struct auxtrace_queue *queue = &pt->queues.queue_array[i];
2170 struct intel_pt *pt = ptq->pt;
2175 ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].end;
2177 } else if (ptq->sel_idx + 1 < pt->range_cnt) {
2181 ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].start;
2209 intel_pt_next_tid(ptq->pt, ptq);
2239 struct intel_pt *pt = ptq->pt;
2243 if (!pt->kernel_start) {
2244 pt->kernel_start = machine__kernel_start(pt->machine);
2245 if (pt->per_cpu_mmaps &&
2246 (pt->have_sched_switch == 1 || pt->have_sched_switch == 3) &&
2247 !pt->timeless_decoding && intel_pt_tracing_kernel(pt) &&
2248 !pt->sampling_mode) {
2249 pt->switch_ip = intel_pt_switch_ip(pt, &pt->ptss_ip);
2250 if (pt->switch_ip) {
2252 pt->switch_ip, pt->ptss_ip);
2253 intel_pt_enable_sync_switch(pt);
2270 state->from_ip >= pt->kernel_start) {
2272 intel_pt_next_tid(pt, ptq);
2275 if (pt->synth_opts.errors) {
2288 if (pt->est_tsc &&
2289 (state->from_ip >= pt->kernel_start || !state->from_ip) &&
2290 state->to_ip && state->to_ip < pt->kernel_start) {
2312 if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {
2320 static inline int intel_pt_update_queues(struct intel_pt *pt)
2322 if (pt->queues.new_data) {
2323 pt->queues.new_data = false;
2324 return intel_pt_setup_queues(pt);
2329 static int intel_pt_process_queues(struct intel_pt *pt, u64 timestamp)
2339 if (!pt->heap.heap_cnt)
2342 if (pt->heap.heap_array[0].ordinal >= timestamp)
2345 queue_nr = pt->heap.heap_array[0].queue_nr;
2346 queue = &pt->queues.queue_array[queue_nr];
2350 queue_nr, pt->heap.heap_array[0].ordinal,
2353 auxtrace_heap__pop(&pt->heap);
2355 if (pt->heap.heap_cnt) {
2356 ts = pt->heap.heap_array[0].ordinal + 1;
2363 intel_pt_set_pid_tid_cpu(pt, queue);
2368 auxtrace_heap__add(&pt->heap, queue_nr, ts);
2373 ret = auxtrace_heap__add(&pt->heap, queue_nr, ts);
2384 static int intel_pt_process_timeless_queues(struct intel_pt *pt, pid_t tid,
2387 struct auxtrace_queues *queues = &pt->queues;
2392 struct auxtrace_queue *queue = &pt->queues.queue_array[i];
2397 intel_pt_set_pid_tid_cpu(pt, queue);
2408 struct machine *m = ptq->pt->machine;
2432 static int intel_pt_process_timeless_sample(struct intel_pt *pt,
2439 queue = auxtrace_queues__sample_queue(&pt->queues, sample, pt->session);
2454 static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample)
2456 return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu,
2460 static struct intel_pt_queue *intel_pt_cpu_to_ptq(struct intel_pt *pt, int cpu)
2464 if (cpu < 0 || !pt->queues.nr_queues)
2467 if ((unsigned)cpu >= pt->queues.nr_queues)
2468 i = pt->queues.nr_queues - 1;
2472 if (pt->queues.queue_array[i].cpu == cpu)
2473 return pt->queues.queue_array[i].priv;
2476 if (pt->queues.queue_array[--i].cpu == cpu)
2477 return pt->queues.queue_array[i].priv;
2480 for (; j < pt->queues.nr_queues; j++) {
2481 if (pt->queues.queue_array[j].cpu == cpu)
2482 return pt->queues.queue_array[j].priv;
2488 static int intel_pt_sync_switch(struct intel_pt *pt, int cpu, pid_t tid,
2494 if (!pt->sync_switch)
2497 ptq = intel_pt_cpu_to_ptq(pt, cpu);
2512 &pt->tc);
2513 err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,
2533 static int intel_pt_process_switch(struct intel_pt *pt,
2540 evsel = perf_evlist__id2evsel(pt->session->evlist, sample->id);
2541 if (evsel != pt->switch_evsel)
2549 &pt->tc));
2551 ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
2555 return machine__set_current_tid(pt->machine, cpu, -1, tid);
2558 static int intel_pt_context_switch_in(struct intel_pt *pt,
2565 if (pt->sync_switch) {
2568 ptq = intel_pt_cpu_to_ptq(pt, cpu);
2590 if (machine__get_current_tid(pt->machine, cpu) == tid)
2593 return machine__set_current_tid(pt->machine, cpu, pid, tid);
2596 static int intel_pt_context_switch(struct intel_pt *pt, union perf_event *event,
2605 if (pt->have_sched_switch == 3) {
2607 return intel_pt_context_switch_in(pt, sample);
2624 ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
2628 return machine__set_current_tid(pt->machine, cpu, pid, tid);
2631 static int intel_pt_process_itrace_start(struct intel_pt *pt,
2635 if (!pt->per_cpu_mmaps)
2641 perf_time_to_tsc(sample->time, &pt->tc));
2643 return machine__set_current_tid(pt->machine, sample->cpu,
2660 static int intel_pt_text_poke(struct intel_pt *pt, union perf_event *event)
2666 struct thread *thread = pt->unknown_thread;
2668 struct machine *machine = pt->machine;
2714 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2728 timestamp = perf_time_to_tsc(sample->time, &pt->tc);
2732 if (timestamp || pt->timeless_decoding) {
2733 err = intel_pt_update_queues(pt);
2738 if (pt->timeless_decoding) {
2739 if (pt->sampling_mode) {
2741 err = intel_pt_process_timeless_sample(pt,
2744 err = intel_pt_process_timeless_queues(pt,
2749 err = intel_pt_process_queues(pt, timestamp);
2755 if (pt->synth_opts.add_callchain && !sample->callchain)
2756 intel_pt_add_callchain(pt, sample);
2757 if (pt->synth_opts.add_last_branch && !sample->branch_stack)
2758 intel_pt_add_br_stack(pt, sample);
2763 pt->synth_opts.errors) {
2764 err = intel_pt_lost(pt, sample);
2769 if (pt->switch_evsel && event->header.type == PERF_RECORD_SAMPLE)
2770 err = intel_pt_process_switch(pt, sample);
2772 err = intel_pt_process_itrace_start(pt, event, sample);
2775 err = intel_pt_context_switch(pt, event, sample);
2778 err = intel_pt_text_poke(pt, event);
2780 if (intel_pt_enable_logging && intel_pt_log_events(pt, sample->time)) {
2791 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2801 ret = intel_pt_update_queues(pt);
2805 if (pt->timeless_decoding)
2806 return intel_pt_process_timeless_queues(pt, -1,
2809 return intel_pt_process_queues(pt, MAX_TIMESTAMP);
2814 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2816 struct auxtrace_queues *queues = &pt->queues;
2829 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2832 auxtrace_heap__free(&pt->heap);
2835 thread__put(pt->unknown_thread);
2836 addr_filters__exit(&pt->filts);
2837 zfree(&pt->chain);
2838 zfree(&pt->filter);
2839 zfree(&pt->time_ranges);
2840 free(pt);
2846 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2849 return evsel->core.attr.type == pt->pmu_type;
2856 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2859 if (!pt->data_queued) {
2873 err = auxtrace_queues__add_event(&pt->queues, session, event,
2881 intel_pt_dump_event(pt, buffer->data,
2895 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2900 return auxtrace_queues__add_event(&pt->queues, session, event,
2905 timestamp = perf_time_to_tsc(sample->time, &pt->tc);
2909 return auxtrace_queues__add_sample(&pt->queues, session, sample,
2966 static struct evsel *intel_pt_evsel(struct intel_pt *pt,
2972 if (evsel->core.attr.type == pt->pmu_type && evsel->core.ids)
2979 static int intel_pt_synth_events(struct intel_pt *pt,
2983 struct evsel *evsel = intel_pt_evsel(pt, evlist);
2999 if (pt->timeless_decoding)
3003 if (!pt->per_cpu_mmaps)
3017 if (pt->synth_opts.branches) {
3024 pt->sample_branches = true;
3025 pt->branches_sample_type = attr.sample_type;
3026 pt->branches_id = id;
3031 if (pt->synth_opts.callchain)
3033 if (pt->synth_opts.last_branch) {
3043 if (pt->synth_opts.instructions) {
3045 if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS)
3047 intel_pt_ns_to_ticks(pt, pt->synth_opts.period);
3049 attr.sample_period = pt->synth_opts.period;
3053 pt->sample_instructions = true;
3054 pt->instructions_sample_type = attr.sample_type;
3055 pt->instructions_id = id;
3062 if (pt->synth_opts.transactions) {
3067 pt->sample_transactions = true;
3068 pt->transactions_sample_type = attr.sample_type;
3069 pt->transactions_id = id;
3077 if (pt->synth_opts.ptwrites) {
3082 pt->sample_ptwrites = true;
3083 pt->ptwrites_sample_type = attr.sample_type;
3084 pt->ptwrites_id = id;
3089 if (pt->synth_opts.pwr_events) {
3090 pt->sample_pwr_events = true;
3091 pt->pwr_events_sample_type = attr.sample_type;
3097 pt->cbr_id = id;
3102 if (pt->synth_opts.pwr_events && (evsel->core.attr.config & 0x10)) {
3107 pt->mwait_id = id;
3115 pt->pwre_id = id;
3123 pt->exstop_id = id;
3131 pt->pwrx_id = id;
3139 static void intel_pt_setup_pebs_events(struct intel_pt *pt)
3143 if (!pt->synth_opts.other_events)
3146 evlist__for_each_entry(pt->session->evlist, evsel) {
3148 pt->sample_pebs = true;
3149 pt->pebs_evsel = evsel;
3183 struct intel_pt *pt = data;
3185 if (!strcmp(var, "intel-pt.mispred-all"))
3186 pt->mispred_all = perf_config_bool(var, value);
3192 static u64 intel_pt_tsc_start(u64 ns, struct intel_pt *pt)
3196 tsc = perf_time_to_tsc(ns, &pt->tc);
3199 tm = tsc_to_perf_time(tsc, &pt->tc);
3206 tm = tsc_to_perf_time(++tsc, &pt->tc);
3212 static u64 intel_pt_tsc_end(u64 ns, struct intel_pt *pt)
3216 tsc = perf_time_to_tsc(ns, &pt->tc);
3219 tm = tsc_to_perf_time(tsc, &pt->tc);
3226 tm = tsc_to_perf_time(--tsc, &pt->tc);
3231 static int intel_pt_setup_time_ranges(struct intel_pt *pt,
3238 if (!n || !p || pt->timeless_decoding)
3241 pt->time_ranges = calloc(n, sizeof(struct range));
3242 if (!pt->time_ranges)
3245 pt->range_cnt = n;
3250 struct range *r = &pt->time_ranges[i];
3258 r->start = ts ? intel_pt_tsc_start(ts, pt) : 0;
3259 r->end = te ? intel_pt_tsc_end(te, pt) : 0;
3324 struct intel_pt *pt;
3333 pt = zalloc(sizeof(struct intel_pt));
3334 if (!pt)
3337 addr_filters__init(&pt->filts);
3339 err = perf_config(intel_pt_perf_config, pt);
3343 err = auxtrace_queues__init(&pt->queues);
3349 pt->session = session;
3350 pt->machine = &session->machines.host; /* No kvm support */
3351 pt->auxtrace_type = auxtrace_info->type;
3352 pt->pmu_type = auxtrace_info->priv[INTEL_PT_PMU_TYPE];
3353 pt->tc.time_shift = auxtrace_info->priv[INTEL_PT_TIME_SHIFT];
3354 pt->tc.time_mult = auxtrace_info->priv[INTEL_PT_TIME_MULT];
3355 pt->tc.time_zero = auxtrace_info->priv[INTEL_PT_TIME_ZERO];
3356 pt->cap_user_time_zero = auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO];
3357 pt->tsc_bit = auxtrace_info->priv[INTEL_PT_TSC_BIT];
3358 pt->noretcomp_bit = auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT];
3359 pt->have_sched_switch = auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH];
3360 pt->snapshot_mode = auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE];
3361 pt->per_cpu_mmaps = auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS];
3366 pt->mtc_bit = auxtrace_info->priv[INTEL_PT_MTC_BIT];
3367 pt->mtc_freq_bits = auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS];
3368 pt->tsc_ctc_ratio_n = auxtrace_info->priv[INTEL_PT_TSC_CTC_N];
3369 pt->tsc_ctc_ratio_d = auxtrace_info->priv[INTEL_PT_TSC_CTC_D];
3370 pt->cyc_bit = auxtrace_info->priv[INTEL_PT_CYC_BIT];
3376 pt->max_non_turbo_ratio =
3403 pt->filter = memdup(filter, len);
3404 if (!pt->filter) {
3409 mem_bswap_64(pt->filter, len);
3410 if (pt->filter[len - 1]) {
3415 err = addr_filters__parse_bare_filter(&pt->filts,
3420 intel_pt_print_info_str("Filter string", pt->filter);
3423 pt->timeless_decoding = intel_pt_timeless_decoding(pt);
3424 if (pt->timeless_decoding && !pt->tc.time_mult)
3425 pt->tc.time_mult = 1;
3426 pt->have_tsc = intel_pt_have_tsc(pt);
3427 pt->sampling_mode = intel_pt_sampling_mode(pt);
3428 pt->est_tsc = !pt->timeless_decoding;
3430 pt->unknown_thread = thread__new(999999999, 999999999);
3431 if (!pt->unknown_thread) {
3442 INIT_LIST_HEAD(&pt->unknown_thread->node);
3444 err = thread__set_comm(pt->unknown_thread, "unknown", 0);
3447 if (thread__init_maps(pt->unknown_thread, pt->machine)) {
3452 pt->auxtrace.process_event = intel_pt_process_event;
3453 pt->auxtrace.process_auxtrace_event = intel_pt_process_auxtrace_event;
3454 pt->auxtrace.queue_data = intel_pt_queue_data;
3455 pt->auxtrace.dump_auxtrace_sample = intel_pt_dump_sample;
3456 pt->auxtrace.flush_events = intel_pt_flush;
3457 pt->auxtrace.free_events = intel_pt_free_events;
3458 pt->auxtrace.free = intel_pt_free;
3459 pt->auxtrace.evsel_is_auxtrace = intel_pt_evsel_is_auxtrace;
3460 session->auxtrace = &pt->auxtrace;
3465 if (pt->have_sched_switch == 1) {
3466 pt->switch_evsel = intel_pt_find_sched_switch(session->evlist);
3467 if (!pt->switch_evsel) {
3472 } else if (pt->have_sched_switch == 2 &&
3480 pt->synth_opts = *session->itrace_synth_opts;
3482 itrace_synth_opts__set_default(&pt->synth_opts,
3486 pt->synth_opts.branches = false;
3487 pt->synth_opts.callchain = true;
3488 pt->synth_opts.add_callchain = true;
3490 pt->synth_opts.thread_stack =
3494 if (pt->synth_opts.log)
3498 if (pt->tc.time_mult) {
3499 u64 tsc_freq = intel_pt_ns_to_ticks(pt, 1000000000);
3501 if (!pt->max_non_turbo_ratio)
3502 pt->max_non_turbo_ratio =
3506 pt->max_non_turbo_ratio);
3507 pt->cbr2khz = tsc_freq / pt->max_non_turbo_ratio / 1000;
3510 err = intel_pt_setup_time_ranges(pt, session->itrace_synth_opts);
3514 if (pt->synth_opts.calls)
3515 pt->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC |
3517 if (pt->synth_opts.returns)
3518 pt->branches_filter |= PERF_IP_FLAG_RETURN |
3521 if ((pt->synth_opts.callchain || pt->synth_opts.add_callchain) &&
3526 pt->synth_opts.callchain = false;
3527 pt->synth_opts.add_callchain = false;
3531 if (pt->synth_opts.add_callchain) {
3532 err = intel_pt_callchain_init(pt);
3537 if (pt->synth_opts.last_branch || pt->synth_opts.add_last_branch) {
3538 pt->br_stack_sz = pt->synth_opts.last_branch_sz;
3539 pt->br_stack_sz_plus = pt->br_stack_sz;
3542 if (pt->synth_opts.add_last_branch) {
3543 err = intel_pt_br_stack_init(pt);
3553 if (intel_pt_tracing_kernel(pt))
3554 pt->br_stack_sz_plus += 1024;
3556 pt->br_stack_sz_plus += 1;
3559 pt->use_thread_stack = pt->synth_opts.callchain ||
3560 pt->synth_opts.add_callchain ||
3561 pt->synth_opts.thread_stack ||
3562 pt->synth_opts.last_branch ||
3563 pt->synth_opts.add_last_branch;
3565 pt->callstack = pt->synth_opts.callchain ||
3566 pt->synth_opts.add_callchain ||
3567 pt->synth_opts.thread_stack;
3569 err = intel_pt_synth_events(pt, session);
3573 intel_pt_setup_pebs_events(pt);
3575 if (pt->sampling_mode || list_empty(&session->auxtrace_index))
3578 err = auxtrace_queues__process_index(&pt->queues, session);
3582 if (pt->queues.populated)
3583 pt->data_queued = true;
3585 if (pt->timeless_decoding)
3591 zfree(&pt->chain);
3592 thread__zput(pt->unknown_thread);
3595 auxtrace_queues__free(&pt->queues);
3598 addr_filters__exit(&pt->filts);
3599 zfree(&pt->filter);
3600 zfree(&pt->time_ranges);
3601 free(pt);