Lines Matching defs:ptq
287 static int intel_pt_get_buffer(struct intel_pt_queue *ptq,
295 int fd = perf_data__fd(ptq->pt->session->data);
302 might_overlap = ptq->pt->snapshot_mode || ptq->pt->sampling_mode;
304 intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))
327 static void intel_pt_lookahead_drop_buffer(struct intel_pt_queue *ptq,
330 if (!buffer || buffer == ptq->buffer || buffer == ptq->old_buffer)
340 struct intel_pt_queue *ptq = data;
341 struct auxtrace_buffer *buffer = ptq->buffer;
342 struct auxtrace_buffer *old_buffer = ptq->old_buffer;
346 queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
355 err = intel_pt_get_buffer(ptq, buffer, old_buffer, &b);
360 intel_pt_lookahead_drop_buffer(ptq, old_buffer);
363 intel_pt_lookahead_drop_buffer(ptq, buffer);
373 intel_pt_lookahead_drop_buffer(ptq, buffer);
374 intel_pt_lookahead_drop_buffer(ptq, old_buffer);
385 struct intel_pt_queue *ptq = data;
386 struct auxtrace_buffer *buffer = ptq->buffer;
387 struct auxtrace_buffer *old_buffer = ptq->old_buffer;
391 if (ptq->stop) {
396 queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
406 ptq->buffer = buffer;
408 err = intel_pt_get_buffer(ptq, buffer, old_buffer, b);
412 if (ptq->step_through_buffers)
413 ptq->stop = true;
418 ptq->old_buffer = buffer;
564 struct intel_pt_queue *ptq = data;
565 struct machine *machine = ptq->pt->machine;
581 cpumode = intel_pt_cpumode(ptq->pt, *ip);
583 thread = ptq->thread;
587 thread = ptq->pt->unknown_thread;
731 struct intel_pt_queue *ptq = data;
737 if (ip >= ptq->pt->kernel_start)
738 return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL);
742 thread = ptq->thread;
751 return intel_pt_match_pgd_ip(ptq->pt, ip, offset,
1005 struct intel_pt_queue *ptq;
1007 ptq = zalloc(sizeof(struct intel_pt_queue));
1008 if (!ptq)
1012 ptq->chain = intel_pt_alloc_chain(pt);
1013 if (!ptq->chain)
1020 ptq->last_branch = intel_pt_alloc_br_stack(entry_cnt);
1021 if (!ptq->last_branch)
1025 ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
1026 if (!ptq->event_buf)
1029 ptq->pt = pt;
1030 ptq->queue_nr = queue_nr;
1031 ptq->exclude_kernel = intel_pt_exclude_kernel(pt);
1032 ptq->pid = -1;
1033 ptq->tid = -1;
1034 ptq->cpu = -1;
1035 ptq->next_tid = -1;
1040 params.data = ptq;
1084 ptq->decoder = intel_pt_decoder_new(¶ms);
1085 if (!ptq->decoder)
1088 return ptq;
1091 zfree(&ptq->event_buf);
1092 zfree(&ptq->last_branch);
1093 zfree(&ptq->chain);
1094 free(ptq);
1100 struct intel_pt_queue *ptq = priv;
1102 if (!ptq)
1104 thread__zput(ptq->thread);
1105 intel_pt_decoder_free(ptq->decoder);
1106 zfree(&ptq->event_buf);
1107 zfree(&ptq->last_branch);
1108 zfree(&ptq->chain);
1109 free(ptq);
1115 struct intel_pt_queue *ptq = queue->priv;
1118 ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);
1119 if (ptq->tid == -1)
1120 ptq->pid = -1;
1121 thread__zput(ptq->thread);
1124 if (!ptq->thread && ptq->tid != -1)
1125 ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);
1127 if (ptq->thread) {
1128 ptq->pid = ptq->thread->pid_;
1130 ptq->cpu = ptq->thread->cpu;
1134 static void intel_pt_sample_flags(struct intel_pt_queue *ptq)
1136 ptq->insn_len = 0;
1137 if (ptq->state->flags & INTEL_PT_ABORT_TX) {
1138 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT;
1139 } else if (ptq->state->flags & INTEL_PT_ASYNC) {
1140 if (ptq->state->to_ip)
1141 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL |
1145 ptq->flags = PERF_IP_FLAG_BRANCH |
1147 ptq->insn_len = 0;
1149 if (ptq->state->from_ip)
1150 ptq->flags = intel_pt_insn_type(ptq->state->insn_op);
1152 ptq->flags = PERF_IP_FLAG_BRANCH |
1154 if (ptq->state->flags & INTEL_PT_IN_TX)
1155 ptq->flags |= PERF_IP_FLAG_IN_TX;
1156 ptq->insn_len = ptq->state->insn_len;
1157 memcpy(ptq->insn, ptq->state->insn, INTEL_PT_INSN_BUF_SZ);
1160 if (ptq->state->type & INTEL_PT_TRACE_BEGIN)
1161 ptq->flags |= PERF_IP_FLAG_TRACE_BEGIN;
1162 if (ptq->state->type & INTEL_PT_TRACE_END)
1163 ptq->flags |= PERF_IP_FLAG_TRACE_END;
1167 struct intel_pt_queue *ptq)
1172 ptq->sel_timestamp = pt->time_ranges[0].start;
1173 ptq->sel_idx = 0;
1175 if (ptq->sel_timestamp) {
1176 ptq->sel_start = true;
1178 ptq->sel_timestamp = pt->time_ranges[0].end;
1179 ptq->sel_start = false;
1187 struct intel_pt_queue *ptq = queue->priv;
1192 if (!ptq) {
1193 ptq = intel_pt_alloc_queue(pt, queue_nr);
1194 if (!ptq)
1196 queue->priv = ptq;
1199 ptq->cpu = queue->cpu;
1200 ptq->tid = queue->tid;
1202 ptq->cbr_seen = UINT_MAX;
1206 ptq->step_through_buffers = true;
1208 ptq->sync_switch = pt->sync_switch;
1210 intel_pt_setup_time_range(pt, ptq);
1213 if (!ptq->on_heap &&
1214 (!ptq->sync_switch ||
1215 ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) {
1224 queue_nr, ptq->cpu, ptq->pid, ptq->tid);
1226 if (ptq->sel_start && ptq->sel_timestamp) {
1227 ret = intel_pt_fast_forward(ptq->decoder,
1228 ptq->sel_timestamp);
1234 state = intel_pt_decode(ptq->decoder);
1247 ptq->timestamp = state->timestamp;
1249 queue_nr, ptq->timestamp);
1250 ptq->state = state;
1251 ptq->have_sample = true;
1252 if (ptq->sel_start && ptq->sel_timestamp &&
1253 ptq->timestamp < ptq->sel_timestamp)
1254 ptq->have_sample = false;
1255 intel_pt_sample_flags(ptq);
1256 ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);
1259 ptq->on_heap = true;
1295 static void intel_pt_prep_a_sample(struct intel_pt_queue *ptq,
1302 sample->pid = ptq->pid;
1303 sample->tid = ptq->tid;
1304 sample->cpu = ptq->cpu;
1305 sample->insn_len = ptq->insn_len;
1306 memcpy(sample->insn, ptq->insn, INTEL_PT_INSN_BUF_SZ);
1310 struct intel_pt_queue *ptq,
1314 intel_pt_prep_a_sample(ptq, event, sample);
1317 sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1319 sample->ip = ptq->state->from_ip;
1321 sample->addr = ptq->state->to_ip;
1323 sample->flags = ptq->flags;
1362 static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq)
1364 struct intel_pt *pt = ptq->pt;
1365 union perf_event *event = ptq->event_buf;
1373 if (pt->branches_filter && !(pt->branches_filter & ptq->flags))
1379 intel_pt_prep_b_sample(pt, ptq, event, &sample);
1381 sample.id = ptq->pt->branches_id;
1382 sample.stream_id = ptq->pt->branches_id;
1400 if (ptq->state->flags & INTEL_PT_SAMPLE_IPC)
1401 sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_br_cyc_cnt;
1403 sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_br_insn_cnt;
1404 ptq->last_br_insn_cnt = ptq->ipc_insn_cnt;
1405 ptq->last_br_cyc_cnt = ptq->ipc_cyc_cnt;
1413 struct intel_pt_queue *ptq,
1417 intel_pt_prep_b_sample(pt, ptq, event, sample);
1420 thread_stack__sample(ptq->thread, ptq->cpu, ptq->chain,
1423 sample->callchain = ptq->chain;
1427 thread_stack__br_sample(ptq->thread, ptq->cpu, ptq->last_branch,
1429 sample->branch_stack = ptq->last_branch;
1433 static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq)
1435 struct intel_pt *pt = ptq->pt;
1436 union perf_event *event = ptq->event_buf;
1442 intel_pt_prep_sample(pt, ptq, event, &sample);
1444 sample.id = ptq->pt->instructions_id;
1445 sample.stream_id = ptq->pt->instructions_id;
1449 sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;
1451 if (ptq->state->flags & INTEL_PT_SAMPLE_IPC)
1452 sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_in_cyc_cnt;
1454 sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_in_insn_cnt;
1455 ptq->last_in_insn_cnt = ptq->ipc_insn_cnt;
1456 ptq->last_in_cyc_cnt = ptq->ipc_cyc_cnt;
1459 ptq->last_insn_cnt = ptq->state->tot_insn_cnt;
1465 static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq)
1467 struct intel_pt *pt = ptq->pt;
1468 union perf_event *event = ptq->event_buf;
1474 intel_pt_prep_sample(pt, ptq, event, &sample);
1476 sample.id = ptq->pt->transactions_id;
1477 sample.stream_id = ptq->pt->transactions_id;
1484 struct intel_pt_queue *ptq,
1488 intel_pt_prep_sample(pt, ptq, event, sample);
1498 static int intel_pt_synth_ptwrite_sample(struct intel_pt_queue *ptq)
1500 struct intel_pt *pt = ptq->pt;
1501 union perf_event *event = ptq->event_buf;
1508 intel_pt_prep_p_sample(pt, ptq, event, &sample);
1510 sample.id = ptq->pt->ptwrites_id;
1511 sample.stream_id = ptq->pt->ptwrites_id;
1514 raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);
1515 raw.payload = cpu_to_le64(ptq->state->ptw_payload);
1524 static int intel_pt_synth_cbr_sample(struct intel_pt_queue *ptq)
1526 struct intel_pt *pt = ptq->pt;
1527 union perf_event *event = ptq->event_buf;
1535 ptq->cbr_seen = ptq->state->cbr;
1537 intel_pt_prep_p_sample(pt, ptq, event, &sample);
1539 sample.id = ptq->pt->cbr_id;
1540 sample.stream_id = ptq->pt->cbr_id;
1542 flags = (u16)ptq->state->cbr_payload | (pt->max_non_turbo_ratio << 16);
1554 static int intel_pt_synth_mwait_sample(struct intel_pt_queue *ptq)
1556 struct intel_pt *pt = ptq->pt;
1557 union perf_event *event = ptq->event_buf;
1564 intel_pt_prep_p_sample(pt, ptq, event, &sample);
1566 sample.id = ptq->pt->mwait_id;
1567 sample.stream_id = ptq->pt->mwait_id;
1570 raw.payload = cpu_to_le64(ptq->state->mwait_payload);
1579 static int intel_pt_synth_pwre_sample(struct intel_pt_queue *ptq)
1581 struct intel_pt *pt = ptq->pt;
1582 union perf_event *event = ptq->event_buf;
1589 intel_pt_prep_p_sample(pt, ptq, event, &sample);
1591 sample.id = ptq->pt->pwre_id;
1592 sample.stream_id = ptq->pt->pwre_id;
1595 raw.payload = cpu_to_le64(ptq->state->pwre_payload);
1604 static int intel_pt_synth_exstop_sample(struct intel_pt_queue *ptq)
1606 struct intel_pt *pt = ptq->pt;
1607 union perf_event *event = ptq->event_buf;
1614 intel_pt_prep_p_sample(pt, ptq, event, &sample);
1616 sample.id = ptq->pt->exstop_id;
1617 sample.stream_id = ptq->pt->exstop_id;
1620 raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);
1629 static int intel_pt_synth_pwrx_sample(struct intel_pt_queue *ptq)
1631 struct intel_pt *pt = ptq->pt;
1632 union perf_event *event = ptq->event_buf;
1639 intel_pt_prep_p_sample(pt, ptq, event, &sample);
1641 sample.id = ptq->pt->pwrx_id;
1642 sample.stream_id = ptq->pt->pwrx_id;
1645 raw.payload = cpu_to_le64(ptq->state->pwrx_payload);
1781 static int intel_pt_synth_pebs_sample(struct intel_pt_queue *ptq)
1783 const struct intel_pt_blk_items *items = &ptq->state->items;
1785 union perf_event *event = ptq->event_buf;
1786 struct intel_pt *pt = ptq->pt;
1796 intel_pt_prep_a_sample(ptq, event, &sample);
1810 sample.ip = ptq->state->from_ip;
1813 cpumode = sample.ip < ptq->pt->kernel_start ?
1827 timestamp = ptq->timestamp;
1834 thread_stack__sample(ptq->thread, ptq->cpu, ptq->chain,
1837 sample.callchain = ptq->chain;
1860 intel_pt_add_lbrs(ptq->last_branch, items);
1862 thread_stack__br_sample(ptq->thread, ptq->cpu,
1863 ptq->last_branch,
1866 ptq->last_branch->nr = 0;
1868 sample.branch_stack = ptq->last_branch;
1930 static int intel_ptq_synth_error(struct intel_pt_queue *ptq,
1933 struct intel_pt *pt = ptq->pt;
1934 u64 tm = ptq->timestamp;
1938 return intel_pt_synth_error(pt, state->err, ptq->cpu, ptq->pid,
1939 ptq->tid, state->from_ip, tm);
1942 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq)
1945 pid_t tid = ptq->next_tid;
1951 intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid);
1953 err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);
1955 queue = &pt->queues.queue_array[ptq->queue_nr];
1958 ptq->next_tid = -1;
1963 static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip)
1965 struct intel_pt *pt = ptq->pt;
1968 (ptq->flags & PERF_IP_FLAG_BRANCH) &&
1969 !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC |
1976 static int intel_pt_sample(struct intel_pt_queue *ptq)
1978 const struct intel_pt_state *state = ptq->state;
1979 struct intel_pt *pt = ptq->pt;
1982 if (!ptq->have_sample)
1985 ptq->have_sample = false;
1987 ptq->ipc_insn_cnt = ptq->state->tot_insn_cnt;
1988 ptq->ipc_cyc_cnt = ptq->state->tot_cyc_cnt;
1995 err = intel_pt_synth_pebs_sample(ptq);
2001 if (ptq->state->cbr != ptq->cbr_seen) {
2002 err = intel_pt_synth_cbr_sample(ptq);
2008 err = intel_pt_synth_mwait_sample(ptq);
2013 err = intel_pt_synth_pwre_sample(ptq);
2018 err = intel_pt_synth_exstop_sample(ptq);
2023 err = intel_pt_synth_pwrx_sample(ptq);
2031 err = intel_pt_synth_instruction_sample(ptq);
2037 err = intel_pt_synth_transaction_sample(ptq);
2043 err = intel_pt_synth_ptwrite_sample(ptq);
2052 thread_stack__event(ptq->thread, ptq->cpu, ptq->flags,
2053 state->from_ip, state->to_ip, ptq->insn_len,
2058 thread_stack__set_trace_nr(ptq->thread, ptq->cpu, state->trace_nr);
2062 err = intel_pt_synth_branch_sample(ptq);
2067 if (!ptq->sync_switch)
2070 if (intel_pt_is_switch_ip(ptq, state->to_ip)) {
2071 switch (ptq->switch_state) {
2075 err = intel_pt_next_tid(pt, ptq);
2078 ptq->switch_state = INTEL_PT_SS_TRACING;
2081 ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT;
2085 ptq->switch_state = INTEL_PT_SS_NOT_TRACING;
2086 } else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) {
2087 ptq->switch_state = INTEL_PT_SS_UNKNOWN;
2088 } else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
2090 (ptq->flags & PERF_IP_FLAG_CALL)) {
2091 ptq->switch_state = INTEL_PT_SS_TRACING;
2157 struct intel_pt_queue *ptq = queue->priv;
2159 if (ptq)
2160 ptq->sync_switch = true;
2168 static bool intel_pt_next_time(struct intel_pt_queue *ptq)
2170 struct intel_pt *pt = ptq->pt;
2172 if (ptq->sel_start) {
2174 ptq->sel_start = false;
2175 ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].end;
2177 } else if (ptq->sel_idx + 1 < pt->range_cnt) {
2179 ptq->sel_start = true;
2180 ptq->sel_idx += 1;
2181 ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].start;
2189 static int intel_pt_time_filter(struct intel_pt_queue *ptq, u64 *ff_timestamp)
2194 if (ptq->sel_start) {
2195 if (ptq->timestamp >= ptq->sel_timestamp) {
2197 intel_pt_next_time(ptq);
2198 if (!ptq->sel_timestamp) {
2206 ptq->have_sample = false;
2207 if (ptq->sel_timestamp > *ff_timestamp) {
2208 if (ptq->sync_switch) {
2209 intel_pt_next_tid(ptq->pt, ptq);
2210 ptq->switch_state = INTEL_PT_SS_UNKNOWN;
2212 *ff_timestamp = ptq->sel_timestamp;
2213 err = intel_pt_fast_forward(ptq->decoder,
2214 ptq->sel_timestamp);
2219 } else if (ptq->timestamp > ptq->sel_timestamp) {
2221 if (!intel_pt_next_time(ptq)) {
2223 ptq->have_sample = false;
2224 ptq->switch_state = INTEL_PT_SS_NOT_TRACING;
2236 static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp)
2238 const struct intel_pt_state *state = ptq->state;
2239 struct intel_pt *pt = ptq->pt;
2259 ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
2261 err = intel_pt_sample(ptq);
2265 state = intel_pt_decode(ptq->decoder);
2269 if (ptq->sync_switch &&
2271 ptq->sync_switch = false;
2272 intel_pt_next_tid(pt, ptq);
2274 ptq->timestamp = state->est_timestamp;
2276 err = intel_ptq_synth_error(ptq, state);
2283 ptq->state = state;
2284 ptq->have_sample = true;
2285 intel_pt_sample_flags(ptq);
2293 ptq->timestamp = state->est_timestamp;
2295 } else if (ptq->sync_switch &&
2296 ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
2297 intel_pt_is_switch_ip(ptq, state->to_ip) &&
2298 ptq->next_tid == -1) {
2301 ptq->timestamp = state->est_timestamp;
2302 } else if (state->timestamp > ptq->timestamp) {
2303 ptq->timestamp = state->timestamp;
2306 if (ptq->sel_timestamp) {
2307 err = intel_pt_time_filter(ptq, &ff_timestamp);
2312 if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {
2313 *timestamp = ptq->timestamp;
2337 struct intel_pt_queue *ptq;
2347 ptq = queue->priv;
2365 ret = intel_pt_run_decoder(ptq, &ts);
2377 ptq->on_heap = false;
2393 struct intel_pt_queue *ptq = queue->priv;
2395 if (ptq && (tid == -1 || ptq->tid == tid)) {
2396 ptq->time = time_;
2398 intel_pt_run_decoder(ptq, &ts);
2404 static void intel_pt_sample_set_pid_tid_cpu(struct intel_pt_queue *ptq,
2408 struct machine *m = ptq->pt->machine;
2410 ptq->pid = sample->pid;
2411 ptq->tid = sample->tid;
2412 ptq->cpu = queue->cpu;
2415 ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
2417 thread__zput(ptq->thread);
2419 if (ptq->tid == -1)
2422 if (ptq->pid == -1) {
2423 ptq->thread = machine__find_thread(m, -1, ptq->tid);
2424 if (ptq->thread)
2425 ptq->pid = ptq->thread->pid_;
2429 ptq->thread = machine__findnew_thread(m, ptq->pid, ptq->tid);
2436 struct intel_pt_queue *ptq;
2443 ptq = queue->priv;
2444 if (!ptq)
2447 ptq->stop = false;
2448 ptq->time = sample->time;
2449 intel_pt_sample_set_pid_tid_cpu(ptq, queue, sample);
2450 intel_pt_run_decoder(ptq, &ts);
2491 struct intel_pt_queue *ptq;
2497 ptq = intel_pt_cpu_to_ptq(pt, cpu);
2498 if (!ptq || !ptq->sync_switch)
2501 switch (ptq->switch_state) {
2506 ptq->next_tid = tid;
2507 ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP;
2510 if (!ptq->on_heap) {
2511 ptq->timestamp = perf_time_to_tsc(timestamp,
2513 err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,
2514 ptq->timestamp);
2517 ptq->on_heap = true;
2519 ptq->switch_state = INTEL_PT_SS_TRACING;
2528 ptq->next_tid = -1;
2566 struct intel_pt_queue *ptq;
2568 ptq = intel_pt_cpu_to_ptq(pt, cpu);
2569 if (ptq && ptq->sync_switch) {
2570 ptq->next_tid = -1;
2571 switch (ptq->switch_state) {
2578 ptq->switch_state = INTEL_PT_SS_TRACING;