1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * intel_pt_decoder.c: Intel Processor Trace support 4 * Copyright (c) 2013-2014, Intel Corporation. 5 */ 6 7#ifndef _GNU_SOURCE 8#define _GNU_SOURCE 9#endif 10#include <stdlib.h> 11#include <stdbool.h> 12#include <string.h> 13#include <errno.h> 14#include <stdint.h> 15#include <inttypes.h> 16#include <linux/compiler.h> 17#include <linux/string.h> 18#include <linux/zalloc.h> 19 20#include "../auxtrace.h" 21 22#include "intel-pt-insn-decoder.h" 23#include "intel-pt-pkt-decoder.h" 24#include "intel-pt-decoder.h" 25#include "intel-pt-log.h" 26 27#define BITULL(x) (1ULL << (x)) 28 29/* IA32_RTIT_CTL MSR bits */ 30#define INTEL_PT_CYC_ENABLE BITULL(1) 31#define INTEL_PT_CYC_THRESHOLD (BITULL(22) | BITULL(21) | BITULL(20) | BITULL(19)) 32#define INTEL_PT_CYC_THRESHOLD_SHIFT 19 33 34#define INTEL_PT_BLK_SIZE 1024 35 36#define BIT63 (((uint64_t)1 << 63)) 37 38#define INTEL_PT_RETURN 1 39 40/* Maximum number of loops with no packets consumed i.e. stuck in a loop */ 41#define INTEL_PT_MAX_LOOPS 10000 42 43struct intel_pt_blk { 44 struct intel_pt_blk *prev; 45 uint64_t ip[INTEL_PT_BLK_SIZE]; 46}; 47 48struct intel_pt_stack { 49 struct intel_pt_blk *blk; 50 struct intel_pt_blk *spare; 51 int pos; 52}; 53 54enum intel_pt_pkt_state { 55 INTEL_PT_STATE_NO_PSB, 56 INTEL_PT_STATE_NO_IP, 57 INTEL_PT_STATE_ERR_RESYNC, 58 INTEL_PT_STATE_IN_SYNC, 59 INTEL_PT_STATE_TNT_CONT, 60 INTEL_PT_STATE_TNT, 61 INTEL_PT_STATE_TIP, 62 INTEL_PT_STATE_TIP_PGD, 63 INTEL_PT_STATE_FUP, 64 INTEL_PT_STATE_FUP_NO_TIP, 65 INTEL_PT_STATE_RESAMPLE, 66}; 67 68static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state) 69{ 70 switch (pkt_state) { 71 case INTEL_PT_STATE_NO_PSB: 72 case INTEL_PT_STATE_NO_IP: 73 case INTEL_PT_STATE_ERR_RESYNC: 74 case INTEL_PT_STATE_IN_SYNC: 75 case INTEL_PT_STATE_TNT_CONT: 76 case INTEL_PT_STATE_RESAMPLE: 77 return true; 78 case INTEL_PT_STATE_TNT: 79 case INTEL_PT_STATE_TIP: 80 case INTEL_PT_STATE_TIP_PGD: 81 case INTEL_PT_STATE_FUP: 82 case INTEL_PT_STATE_FUP_NO_TIP: 83 return false; 84 default: 85 return true; 86 }; 87} 88 89#ifdef INTEL_PT_STRICT 90#define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB 91#define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB 92#define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB 93#define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB 94#else 95#define INTEL_PT_STATE_ERR1 (decoder->pkt_state) 96#define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP 97#define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC 98#define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC 99#endif 100 101struct intel_pt_decoder { 102 int (*get_trace)(struct intel_pt_buffer *buffer, void *data); 103 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn, 104 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip, 105 uint64_t max_insn_cnt, void *data); 106 bool (*pgd_ip)(uint64_t ip, void *data); 107 int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data); 108 void *data; 109 struct intel_pt_state state; 110 const unsigned char *buf; 111 size_t len; 112 bool return_compression; 113 bool branch_enable; 114 bool mtc_insn; 115 bool pge; 116 bool have_tma; 117 bool have_cyc; 118 bool fixup_last_mtc; 119 bool have_last_ip; 120 bool in_psb; 121 bool hop; 122 bool hop_psb_fup; 123 bool leap; 124 enum intel_pt_param_flags flags; 125 uint64_t pos; 126 uint64_t last_ip; 127 uint64_t ip; 128 uint64_t cr3; 129 uint64_t timestamp; 130 uint64_t tsc_timestamp; 131 uint64_t ref_timestamp; 132 uint64_t buf_timestamp; 133 uint64_t sample_timestamp; 134 uint64_t ret_addr; 135 uint64_t ctc_timestamp; 136 uint64_t ctc_delta; 137 uint64_t cycle_cnt; 138 uint64_t cyc_ref_timestamp; 139 uint32_t last_mtc; 140 uint32_t tsc_ctc_ratio_n; 141 uint32_t tsc_ctc_ratio_d; 142 uint32_t tsc_ctc_mult; 143 uint32_t tsc_slip; 144 uint32_t ctc_rem_mask; 145 int mtc_shift; 146 struct intel_pt_stack stack; 147 enum intel_pt_pkt_state pkt_state; 148 enum intel_pt_pkt_ctx pkt_ctx; 149 enum intel_pt_pkt_ctx prev_pkt_ctx; 150 enum intel_pt_blk_type blk_type; 151 int blk_type_pos; 152 struct intel_pt_pkt packet; 153 struct intel_pt_pkt tnt; 154 int pkt_step; 155 int pkt_len; 156 int last_packet_type; 157 unsigned int cbr; 158 unsigned int cbr_seen; 159 unsigned int max_non_turbo_ratio; 160 double max_non_turbo_ratio_fp; 161 double cbr_cyc_to_tsc; 162 double calc_cyc_to_tsc; 163 bool have_calc_cyc_to_tsc; 164 int exec_mode; 165 unsigned int insn_bytes; 166 uint64_t period; 167 enum intel_pt_period_type period_type; 168 uint64_t tot_insn_cnt; 169 uint64_t period_insn_cnt; 170 uint64_t period_mask; 171 uint64_t period_ticks; 172 uint64_t last_masked_timestamp; 173 uint64_t tot_cyc_cnt; 174 uint64_t sample_tot_cyc_cnt; 175 uint64_t base_cyc_cnt; 176 uint64_t cyc_cnt_timestamp; 177 uint64_t ctl; 178 uint64_t cyc_threshold; 179 double tsc_to_cyc; 180 bool continuous_period; 181 bool overflow; 182 bool set_fup_tx_flags; 183 bool set_fup_ptw; 184 bool set_fup_mwait; 185 bool set_fup_pwre; 186 bool set_fup_exstop; 187 bool set_fup_bep; 188 bool sample_cyc; 189 unsigned int fup_tx_flags; 190 unsigned int tx_flags; 191 uint64_t fup_ptw_payload; 192 uint64_t fup_mwait_payload; 193 uint64_t fup_pwre_payload; 194 uint64_t cbr_payload; 195 uint64_t timestamp_insn_cnt; 196 uint64_t sample_insn_cnt; 197 uint64_t stuck_ip; 198 int no_progress; 199 int stuck_ip_prd; 200 int stuck_ip_cnt; 201 const unsigned char *next_buf; 202 size_t next_len; 203 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ]; 204}; 205 206static uint64_t intel_pt_lower_power_of_2(uint64_t x) 207{ 208 int i; 209 210 for (i = 0; x != 1; i++) 211 x >>= 1; 212 213 return x << i; 214} 215 216static uint64_t intel_pt_cyc_threshold(uint64_t ctl) 217{ 218 if (!(ctl & INTEL_PT_CYC_ENABLE)) 219 return 0; 220 221 return (ctl & INTEL_PT_CYC_THRESHOLD) >> INTEL_PT_CYC_THRESHOLD_SHIFT; 222} 223 224static void intel_pt_setup_period(struct intel_pt_decoder *decoder) 225{ 226 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) { 227 uint64_t period; 228 229 period = intel_pt_lower_power_of_2(decoder->period); 230 decoder->period_mask = ~(period - 1); 231 decoder->period_ticks = period; 232 } 233} 234 235static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d) 236{ 237 if (!d) 238 return 0; 239 return (t / d) * n + ((t % d) * n) / d; 240} 241 242struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params) 243{ 244 struct intel_pt_decoder *decoder; 245 246 if (!params->get_trace || !params->walk_insn) 247 return NULL; 248 249 decoder = zalloc(sizeof(struct intel_pt_decoder)); 250 if (!decoder) 251 return NULL; 252 253 decoder->get_trace = params->get_trace; 254 decoder->walk_insn = params->walk_insn; 255 decoder->pgd_ip = params->pgd_ip; 256 decoder->lookahead = params->lookahead; 257 decoder->data = params->data; 258 decoder->return_compression = params->return_compression; 259 decoder->branch_enable = params->branch_enable; 260 decoder->hop = params->quick >= 1; 261 decoder->leap = params->quick >= 2; 262 263 decoder->flags = params->flags; 264 265 decoder->ctl = params->ctl; 266 decoder->period = params->period; 267 decoder->period_type = params->period_type; 268 269 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio; 270 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio; 271 272 decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl); 273 274 intel_pt_setup_period(decoder); 275 276 decoder->mtc_shift = params->mtc_period; 277 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1; 278 279 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n; 280 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d; 281 282 if (!decoder->tsc_ctc_ratio_n) 283 decoder->tsc_ctc_ratio_d = 0; 284 285 if (decoder->tsc_ctc_ratio_d) { 286 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d)) 287 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n / 288 decoder->tsc_ctc_ratio_d; 289 } 290 291 /* 292 * A TSC packet can slip past MTC packets so that the timestamp appears 293 * to go backwards. One estimate is that can be up to about 40 CPU 294 * cycles, which is certainly less than 0x1000 TSC ticks, but accept 295 * slippage an order of magnitude more to be on the safe side. 296 */ 297 decoder->tsc_slip = 0x10000; 298 299 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift); 300 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n); 301 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d); 302 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult); 303 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip); 304 305 if (decoder->hop) 306 intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n"); 307 308 return decoder; 309} 310 311static void intel_pt_pop_blk(struct intel_pt_stack *stack) 312{ 313 struct intel_pt_blk *blk = stack->blk; 314 315 stack->blk = blk->prev; 316 if (!stack->spare) 317 stack->spare = blk; 318 else 319 free(blk); 320} 321 322static uint64_t intel_pt_pop(struct intel_pt_stack *stack) 323{ 324 if (!stack->pos) { 325 if (!stack->blk) 326 return 0; 327 intel_pt_pop_blk(stack); 328 if (!stack->blk) 329 return 0; 330 stack->pos = INTEL_PT_BLK_SIZE; 331 } 332 return stack->blk->ip[--stack->pos]; 333} 334 335static int intel_pt_alloc_blk(struct intel_pt_stack *stack) 336{ 337 struct intel_pt_blk *blk; 338 339 if (stack->spare) { 340 blk = stack->spare; 341 stack->spare = NULL; 342 } else { 343 blk = malloc(sizeof(struct intel_pt_blk)); 344 if (!blk) 345 return -ENOMEM; 346 } 347 348 blk->prev = stack->blk; 349 stack->blk = blk; 350 stack->pos = 0; 351 return 0; 352} 353 354static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip) 355{ 356 int err; 357 358 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) { 359 err = intel_pt_alloc_blk(stack); 360 if (err) 361 return err; 362 } 363 364 stack->blk->ip[stack->pos++] = ip; 365 return 0; 366} 367 368static void intel_pt_clear_stack(struct intel_pt_stack *stack) 369{ 370 while (stack->blk) 371 intel_pt_pop_blk(stack); 372 stack->pos = 0; 373} 374 375static void intel_pt_free_stack(struct intel_pt_stack *stack) 376{ 377 intel_pt_clear_stack(stack); 378 zfree(&stack->blk); 379 zfree(&stack->spare); 380} 381 382void intel_pt_decoder_free(struct intel_pt_decoder *decoder) 383{ 384 intel_pt_free_stack(&decoder->stack); 385 free(decoder); 386} 387 388static int intel_pt_ext_err(int code) 389{ 390 switch (code) { 391 case -ENOMEM: 392 return INTEL_PT_ERR_NOMEM; 393 case -ENOSYS: 394 return INTEL_PT_ERR_INTERN; 395 case -EBADMSG: 396 return INTEL_PT_ERR_BADPKT; 397 case -ENODATA: 398 return INTEL_PT_ERR_NODATA; 399 case -EILSEQ: 400 return INTEL_PT_ERR_NOINSN; 401 case -ENOENT: 402 return INTEL_PT_ERR_MISMAT; 403 case -EOVERFLOW: 404 return INTEL_PT_ERR_OVR; 405 case -ENOSPC: 406 return INTEL_PT_ERR_LOST; 407 case -ELOOP: 408 return INTEL_PT_ERR_NELOOP; 409 default: 410 return INTEL_PT_ERR_UNK; 411 } 412} 413 414static const char *intel_pt_err_msgs[] = { 415 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed", 416 [INTEL_PT_ERR_INTERN] = "Internal error", 417 [INTEL_PT_ERR_BADPKT] = "Bad packet", 418 [INTEL_PT_ERR_NODATA] = "No more data", 419 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction", 420 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction", 421 [INTEL_PT_ERR_OVR] = "Overflow packet", 422 [INTEL_PT_ERR_LOST] = "Lost trace data", 423 [INTEL_PT_ERR_UNK] = "Unknown error!", 424 [INTEL_PT_ERR_NELOOP] = "Never-ending loop", 425}; 426 427int intel_pt__strerror(int code, char *buf, size_t buflen) 428{ 429 if (code < 1 || code >= INTEL_PT_ERR_MAX) 430 code = INTEL_PT_ERR_UNK; 431 strlcpy(buf, intel_pt_err_msgs[code], buflen); 432 return 0; 433} 434 435static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet, 436 uint64_t last_ip) 437{ 438 uint64_t ip; 439 440 switch (packet->count) { 441 case 1: 442 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) | 443 packet->payload; 444 break; 445 case 2: 446 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) | 447 packet->payload; 448 break; 449 case 3: 450 ip = packet->payload; 451 /* Sign-extend 6-byte ip */ 452 if (ip & (uint64_t)0x800000000000ULL) 453 ip |= (uint64_t)0xffff000000000000ULL; 454 break; 455 case 4: 456 ip = (last_ip & (uint64_t)0xffff000000000000ULL) | 457 packet->payload; 458 break; 459 case 6: 460 ip = packet->payload; 461 break; 462 default: 463 return 0; 464 } 465 466 return ip; 467} 468 469static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder) 470{ 471 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip); 472 decoder->have_last_ip = true; 473} 474 475static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder) 476{ 477 intel_pt_set_last_ip(decoder); 478 decoder->ip = decoder->last_ip; 479} 480 481static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder) 482{ 483 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos, 484 decoder->buf); 485} 486 487static int intel_pt_bug(struct intel_pt_decoder *decoder) 488{ 489 intel_pt_log("ERROR: Internal error\n"); 490 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 491 return -ENOSYS; 492} 493 494static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder) 495{ 496 decoder->tx_flags = 0; 497} 498 499static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder) 500{ 501 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX; 502} 503 504static int intel_pt_bad_packet(struct intel_pt_decoder *decoder) 505{ 506 intel_pt_clear_tx_flags(decoder); 507 decoder->have_tma = false; 508 decoder->pkt_len = 1; 509 decoder->pkt_step = 1; 510 intel_pt_decoder_log_packet(decoder); 511 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) { 512 intel_pt_log("ERROR: Bad packet\n"); 513 decoder->pkt_state = INTEL_PT_STATE_ERR1; 514 } 515 return -EBADMSG; 516} 517 518static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder) 519{ 520 decoder->sample_timestamp = decoder->timestamp; 521 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt; 522} 523 524static void intel_pt_reposition(struct intel_pt_decoder *decoder) 525{ 526 decoder->ip = 0; 527 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 528 decoder->timestamp = 0; 529 decoder->have_tma = false; 530} 531 532static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition) 533{ 534 struct intel_pt_buffer buffer = { .buf = 0, }; 535 int ret; 536 537 decoder->pkt_step = 0; 538 539 intel_pt_log("Getting more data\n"); 540 ret = decoder->get_trace(&buffer, decoder->data); 541 if (ret) 542 return ret; 543 decoder->buf = buffer.buf; 544 decoder->len = buffer.len; 545 if (!decoder->len) { 546 intel_pt_log("No more data\n"); 547 return -ENODATA; 548 } 549 decoder->buf_timestamp = buffer.ref_timestamp; 550 if (!buffer.consecutive || reposition) { 551 intel_pt_reposition(decoder); 552 decoder->ref_timestamp = buffer.ref_timestamp; 553 decoder->state.trace_nr = buffer.trace_nr; 554 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n", 555 decoder->ref_timestamp); 556 return -ENOLINK; 557 } 558 559 return 0; 560} 561 562static int intel_pt_get_next_data(struct intel_pt_decoder *decoder, 563 bool reposition) 564{ 565 if (!decoder->next_buf) 566 return intel_pt_get_data(decoder, reposition); 567 568 decoder->buf = decoder->next_buf; 569 decoder->len = decoder->next_len; 570 decoder->next_buf = 0; 571 decoder->next_len = 0; 572 return 0; 573} 574 575static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder) 576{ 577 unsigned char *buf = decoder->temp_buf; 578 size_t old_len, len, n; 579 int ret; 580 581 old_len = decoder->len; 582 len = decoder->len; 583 memcpy(buf, decoder->buf, len); 584 585 ret = intel_pt_get_data(decoder, false); 586 if (ret) { 587 decoder->pos += old_len; 588 return ret < 0 ? ret : -EINVAL; 589 } 590 591 n = INTEL_PT_PKT_MAX_SZ - len; 592 if (n > decoder->len) 593 n = decoder->len; 594 memcpy(buf + len, decoder->buf, n); 595 len += n; 596 597 decoder->prev_pkt_ctx = decoder->pkt_ctx; 598 ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx); 599 if (ret < (int)old_len) { 600 decoder->next_buf = decoder->buf; 601 decoder->next_len = decoder->len; 602 decoder->buf = buf; 603 decoder->len = old_len; 604 return intel_pt_bad_packet(decoder); 605 } 606 607 decoder->next_buf = decoder->buf + (ret - old_len); 608 decoder->next_len = decoder->len - (ret - old_len); 609 610 decoder->buf = buf; 611 decoder->len = ret; 612 613 return ret; 614} 615 616struct intel_pt_pkt_info { 617 struct intel_pt_decoder *decoder; 618 struct intel_pt_pkt packet; 619 uint64_t pos; 620 int pkt_len; 621 int last_packet_type; 622 void *data; 623}; 624 625typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info); 626 627/* Lookahead packets in current buffer */ 628static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder, 629 intel_pt_pkt_cb_t cb, void *data) 630{ 631 struct intel_pt_pkt_info pkt_info; 632 const unsigned char *buf = decoder->buf; 633 enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx; 634 size_t len = decoder->len; 635 int ret; 636 637 pkt_info.decoder = decoder; 638 pkt_info.pos = decoder->pos; 639 pkt_info.pkt_len = decoder->pkt_step; 640 pkt_info.last_packet_type = decoder->last_packet_type; 641 pkt_info.data = data; 642 643 while (1) { 644 do { 645 pkt_info.pos += pkt_info.pkt_len; 646 buf += pkt_info.pkt_len; 647 len -= pkt_info.pkt_len; 648 649 if (!len) 650 return INTEL_PT_NEED_MORE_BYTES; 651 652 ret = intel_pt_get_packet(buf, len, &pkt_info.packet, 653 &pkt_ctx); 654 if (!ret) 655 return INTEL_PT_NEED_MORE_BYTES; 656 if (ret < 0) 657 return ret; 658 659 pkt_info.pkt_len = ret; 660 } while (pkt_info.packet.type == INTEL_PT_PAD); 661 662 ret = cb(&pkt_info); 663 if (ret) 664 return 0; 665 666 pkt_info.last_packet_type = pkt_info.packet.type; 667 } 668} 669 670struct intel_pt_calc_cyc_to_tsc_info { 671 uint64_t cycle_cnt; 672 unsigned int cbr; 673 uint32_t last_mtc; 674 uint64_t ctc_timestamp; 675 uint64_t ctc_delta; 676 uint64_t tsc_timestamp; 677 uint64_t timestamp; 678 bool have_tma; 679 bool fixup_last_mtc; 680 bool from_mtc; 681 double cbr_cyc_to_tsc; 682}; 683 684/* 685 * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower 686 * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC 687 * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA 688 * packet by copying the missing bits from the current MTC assuming the least 689 * difference between the two, and that the current MTC comes after last_mtc. 690 */ 691static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift, 692 uint32_t *last_mtc) 693{ 694 uint32_t first_missing_bit = 1U << (16 - mtc_shift); 695 uint32_t mask = ~(first_missing_bit - 1); 696 697 *last_mtc |= mtc & mask; 698 if (*last_mtc >= mtc) { 699 *last_mtc -= first_missing_bit; 700 *last_mtc &= 0xff; 701 } 702} 703 704static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info) 705{ 706 struct intel_pt_decoder *decoder = pkt_info->decoder; 707 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data; 708 uint64_t timestamp; 709 double cyc_to_tsc; 710 unsigned int cbr; 711 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem; 712 713 switch (pkt_info->packet.type) { 714 case INTEL_PT_TNT: 715 case INTEL_PT_TIP_PGE: 716 case INTEL_PT_TIP: 717 case INTEL_PT_FUP: 718 case INTEL_PT_PSB: 719 case INTEL_PT_PIP: 720 case INTEL_PT_MODE_EXEC: 721 case INTEL_PT_MODE_TSX: 722 case INTEL_PT_PSBEND: 723 case INTEL_PT_PAD: 724 case INTEL_PT_VMCS: 725 case INTEL_PT_MNT: 726 case INTEL_PT_PTWRITE: 727 case INTEL_PT_PTWRITE_IP: 728 case INTEL_PT_BBP: 729 case INTEL_PT_BIP: 730 case INTEL_PT_BEP: 731 case INTEL_PT_BEP_IP: 732 return 0; 733 734 case INTEL_PT_MTC: 735 if (!data->have_tma) 736 return 0; 737 738 mtc = pkt_info->packet.payload; 739 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) { 740 data->fixup_last_mtc = false; 741 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 742 &data->last_mtc); 743 } 744 if (mtc > data->last_mtc) 745 mtc_delta = mtc - data->last_mtc; 746 else 747 mtc_delta = mtc + 256 - data->last_mtc; 748 data->ctc_delta += mtc_delta << decoder->mtc_shift; 749 data->last_mtc = mtc; 750 751 if (decoder->tsc_ctc_mult) { 752 timestamp = data->ctc_timestamp + 753 data->ctc_delta * decoder->tsc_ctc_mult; 754 } else { 755 timestamp = data->ctc_timestamp + 756 multdiv(data->ctc_delta, 757 decoder->tsc_ctc_ratio_n, 758 decoder->tsc_ctc_ratio_d); 759 } 760 761 if (timestamp < data->timestamp) 762 return 1; 763 764 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 765 data->timestamp = timestamp; 766 return 0; 767 } 768 769 break; 770 771 case INTEL_PT_TSC: 772 /* 773 * For now, do not support using TSC packets - refer 774 * intel_pt_calc_cyc_to_tsc(). 775 */ 776 if (data->from_mtc) 777 return 1; 778 timestamp = pkt_info->packet.payload | 779 (data->timestamp & (0xffULL << 56)); 780 if (data->from_mtc && timestamp < data->timestamp && 781 data->timestamp - timestamp < decoder->tsc_slip) 782 return 1; 783 if (timestamp < data->timestamp) 784 timestamp += (1ULL << 56); 785 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 786 if (data->from_mtc) 787 return 1; 788 data->tsc_timestamp = timestamp; 789 data->timestamp = timestamp; 790 return 0; 791 } 792 break; 793 794 case INTEL_PT_TMA: 795 if (data->from_mtc) 796 return 1; 797 798 if (!decoder->tsc_ctc_ratio_d) 799 return 0; 800 801 ctc = pkt_info->packet.payload; 802 fc = pkt_info->packet.count; 803 ctc_rem = ctc & decoder->ctc_rem_mask; 804 805 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 806 807 data->ctc_timestamp = data->tsc_timestamp - fc; 808 if (decoder->tsc_ctc_mult) { 809 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 810 } else { 811 data->ctc_timestamp -= 812 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n, 813 decoder->tsc_ctc_ratio_d); 814 } 815 816 data->ctc_delta = 0; 817 data->have_tma = true; 818 data->fixup_last_mtc = true; 819 820 return 0; 821 822 case INTEL_PT_CYC: 823 data->cycle_cnt += pkt_info->packet.payload; 824 return 0; 825 826 case INTEL_PT_CBR: 827 cbr = pkt_info->packet.payload; 828 if (data->cbr && data->cbr != cbr) 829 return 1; 830 data->cbr = cbr; 831 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 832 return 0; 833 834 case INTEL_PT_TIP_PGD: 835 case INTEL_PT_TRACESTOP: 836 case INTEL_PT_EXSTOP: 837 case INTEL_PT_EXSTOP_IP: 838 case INTEL_PT_MWAIT: 839 case INTEL_PT_PWRE: 840 case INTEL_PT_PWRX: 841 case INTEL_PT_OVF: 842 case INTEL_PT_BAD: /* Does not happen */ 843 default: 844 return 1; 845 } 846 847 if (!data->cbr && decoder->cbr) { 848 data->cbr = decoder->cbr; 849 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc; 850 } 851 852 if (!data->cycle_cnt) 853 return 1; 854 855 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt; 856 857 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc && 858 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) { 859 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n", 860 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 861 return 1; 862 } 863 864 decoder->calc_cyc_to_tsc = cyc_to_tsc; 865 decoder->have_calc_cyc_to_tsc = true; 866 867 if (data->cbr) { 868 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n", 869 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 870 } else { 871 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n", 872 cyc_to_tsc, pkt_info->pos); 873 } 874 875 return 1; 876} 877 878static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder, 879 bool from_mtc) 880{ 881 struct intel_pt_calc_cyc_to_tsc_info data = { 882 .cycle_cnt = 0, 883 .cbr = 0, 884 .last_mtc = decoder->last_mtc, 885 .ctc_timestamp = decoder->ctc_timestamp, 886 .ctc_delta = decoder->ctc_delta, 887 .tsc_timestamp = decoder->tsc_timestamp, 888 .timestamp = decoder->timestamp, 889 .have_tma = decoder->have_tma, 890 .fixup_last_mtc = decoder->fixup_last_mtc, 891 .from_mtc = from_mtc, 892 .cbr_cyc_to_tsc = 0, 893 }; 894 895 /* 896 * For now, do not support using TSC packets for at least the reasons: 897 * 1) timing might have stopped 898 * 2) TSC packets within PSB+ can slip against CYC packets 899 */ 900 if (!from_mtc) 901 return; 902 903 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data); 904} 905 906static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder) 907{ 908 int ret; 909 910 decoder->last_packet_type = decoder->packet.type; 911 912 do { 913 decoder->pos += decoder->pkt_step; 914 decoder->buf += decoder->pkt_step; 915 decoder->len -= decoder->pkt_step; 916 917 if (!decoder->len) { 918 ret = intel_pt_get_next_data(decoder, false); 919 if (ret) 920 return ret; 921 } 922 923 decoder->prev_pkt_ctx = decoder->pkt_ctx; 924 ret = intel_pt_get_packet(decoder->buf, decoder->len, 925 &decoder->packet, &decoder->pkt_ctx); 926 if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 && 927 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) { 928 ret = intel_pt_get_split_packet(decoder); 929 if (ret < 0) 930 return ret; 931 } 932 if (ret <= 0) 933 return intel_pt_bad_packet(decoder); 934 935 decoder->pkt_len = ret; 936 decoder->pkt_step = ret; 937 intel_pt_decoder_log_packet(decoder); 938 } while (decoder->packet.type == INTEL_PT_PAD); 939 940 return 0; 941} 942 943static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder) 944{ 945 uint64_t timestamp, masked_timestamp; 946 947 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 948 masked_timestamp = timestamp & decoder->period_mask; 949 if (decoder->continuous_period) { 950 if (masked_timestamp > decoder->last_masked_timestamp) 951 return 1; 952 } else { 953 timestamp += 1; 954 masked_timestamp = timestamp & decoder->period_mask; 955 if (masked_timestamp > decoder->last_masked_timestamp) { 956 decoder->last_masked_timestamp = masked_timestamp; 957 decoder->continuous_period = true; 958 } 959 } 960 961 if (masked_timestamp < decoder->last_masked_timestamp) 962 return decoder->period_ticks; 963 964 return decoder->period_ticks - (timestamp - masked_timestamp); 965} 966 967static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder) 968{ 969 switch (decoder->period_type) { 970 case INTEL_PT_PERIOD_INSTRUCTIONS: 971 return decoder->period - decoder->period_insn_cnt; 972 case INTEL_PT_PERIOD_TICKS: 973 return intel_pt_next_period(decoder); 974 case INTEL_PT_PERIOD_NONE: 975 case INTEL_PT_PERIOD_MTC: 976 default: 977 return 0; 978 } 979} 980 981static void intel_pt_sample_insn(struct intel_pt_decoder *decoder) 982{ 983 uint64_t timestamp, masked_timestamp; 984 985 switch (decoder->period_type) { 986 case INTEL_PT_PERIOD_INSTRUCTIONS: 987 decoder->period_insn_cnt = 0; 988 break; 989 case INTEL_PT_PERIOD_TICKS: 990 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 991 masked_timestamp = timestamp & decoder->period_mask; 992 if (masked_timestamp > decoder->last_masked_timestamp) 993 decoder->last_masked_timestamp = masked_timestamp; 994 else 995 decoder->last_masked_timestamp += decoder->period_ticks; 996 break; 997 case INTEL_PT_PERIOD_NONE: 998 case INTEL_PT_PERIOD_MTC: 999 default: 1000 break; 1001 } 1002 1003 decoder->state.type |= INTEL_PT_INSTRUCTION; 1004} 1005 1006static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, 1007 struct intel_pt_insn *intel_pt_insn, uint64_t ip) 1008{ 1009 uint64_t max_insn_cnt, insn_cnt = 0; 1010 int err; 1011 1012 if (!decoder->mtc_insn) 1013 decoder->mtc_insn = true; 1014 1015 max_insn_cnt = intel_pt_next_sample(decoder); 1016 1017 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip, 1018 max_insn_cnt, decoder->data); 1019 1020 decoder->tot_insn_cnt += insn_cnt; 1021 decoder->timestamp_insn_cnt += insn_cnt; 1022 decoder->sample_insn_cnt += insn_cnt; 1023 decoder->period_insn_cnt += insn_cnt; 1024 1025 if (err) { 1026 decoder->no_progress = 0; 1027 decoder->pkt_state = INTEL_PT_STATE_ERR2; 1028 intel_pt_log_at("ERROR: Failed to get instruction", 1029 decoder->ip); 1030 if (err == -ENOENT) 1031 return -ENOLINK; 1032 return -EILSEQ; 1033 } 1034 1035 if (ip && decoder->ip == ip) { 1036 err = -EAGAIN; 1037 goto out; 1038 } 1039 1040 if (max_insn_cnt && insn_cnt >= max_insn_cnt) 1041 intel_pt_sample_insn(decoder); 1042 1043 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) { 1044 decoder->state.type = INTEL_PT_INSTRUCTION; 1045 decoder->state.from_ip = decoder->ip; 1046 decoder->state.to_ip = 0; 1047 decoder->ip += intel_pt_insn->length; 1048 err = INTEL_PT_RETURN; 1049 goto out; 1050 } 1051 1052 if (intel_pt_insn->op == INTEL_PT_OP_CALL) { 1053 /* Zero-length calls are excluded */ 1054 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL || 1055 intel_pt_insn->rel) { 1056 err = intel_pt_push(&decoder->stack, decoder->ip + 1057 intel_pt_insn->length); 1058 if (err) 1059 goto out; 1060 } 1061 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) { 1062 decoder->ret_addr = intel_pt_pop(&decoder->stack); 1063 } 1064 1065 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) { 1066 int cnt = decoder->no_progress++; 1067 1068 decoder->state.from_ip = decoder->ip; 1069 decoder->ip += intel_pt_insn->length + 1070 intel_pt_insn->rel; 1071 decoder->state.to_ip = decoder->ip; 1072 err = INTEL_PT_RETURN; 1073 1074 /* 1075 * Check for being stuck in a loop. This can happen if a 1076 * decoder error results in the decoder erroneously setting the 1077 * ip to an address that is itself in an infinite loop that 1078 * consumes no packets. When that happens, there must be an 1079 * unconditional branch. 1080 */ 1081 if (cnt) { 1082 if (cnt == 1) { 1083 decoder->stuck_ip = decoder->state.to_ip; 1084 decoder->stuck_ip_prd = 1; 1085 decoder->stuck_ip_cnt = 1; 1086 } else if (cnt > INTEL_PT_MAX_LOOPS || 1087 decoder->state.to_ip == decoder->stuck_ip) { 1088 intel_pt_log_at("ERROR: Never-ending loop", 1089 decoder->state.to_ip); 1090 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1091 err = -ELOOP; 1092 goto out; 1093 } else if (!--decoder->stuck_ip_cnt) { 1094 decoder->stuck_ip_prd += 1; 1095 decoder->stuck_ip_cnt = decoder->stuck_ip_prd; 1096 decoder->stuck_ip = decoder->state.to_ip; 1097 } 1098 } 1099 goto out_no_progress; 1100 } 1101out: 1102 decoder->no_progress = 0; 1103out_no_progress: 1104 decoder->state.insn_op = intel_pt_insn->op; 1105 decoder->state.insn_len = intel_pt_insn->length; 1106 memcpy(decoder->state.insn, intel_pt_insn->buf, 1107 INTEL_PT_INSN_BUF_SZ); 1108 1109 if (decoder->tx_flags & INTEL_PT_IN_TX) 1110 decoder->state.flags |= INTEL_PT_IN_TX; 1111 1112 return err; 1113} 1114 1115static bool intel_pt_fup_event(struct intel_pt_decoder *decoder) 1116{ 1117 enum intel_pt_sample_type type = decoder->state.type; 1118 bool ret = false; 1119 1120 decoder->state.type &= ~INTEL_PT_BRANCH; 1121 1122 if (decoder->set_fup_tx_flags) { 1123 decoder->set_fup_tx_flags = false; 1124 decoder->tx_flags = decoder->fup_tx_flags; 1125 decoder->state.type |= INTEL_PT_TRANSACTION; 1126 if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX) 1127 decoder->state.type |= INTEL_PT_BRANCH; 1128 decoder->state.flags = decoder->fup_tx_flags; 1129 ret = true; 1130 } 1131 if (decoder->set_fup_ptw) { 1132 decoder->set_fup_ptw = false; 1133 decoder->state.type |= INTEL_PT_PTW; 1134 decoder->state.flags |= INTEL_PT_FUP_IP; 1135 decoder->state.ptw_payload = decoder->fup_ptw_payload; 1136 ret = true; 1137 } 1138 if (decoder->set_fup_mwait) { 1139 decoder->set_fup_mwait = false; 1140 decoder->state.type |= INTEL_PT_MWAIT_OP; 1141 decoder->state.mwait_payload = decoder->fup_mwait_payload; 1142 ret = true; 1143 } 1144 if (decoder->set_fup_pwre) { 1145 decoder->set_fup_pwre = false; 1146 decoder->state.type |= INTEL_PT_PWR_ENTRY; 1147 decoder->state.pwre_payload = decoder->fup_pwre_payload; 1148 ret = true; 1149 } 1150 if (decoder->set_fup_exstop) { 1151 decoder->set_fup_exstop = false; 1152 decoder->state.type |= INTEL_PT_EX_STOP; 1153 decoder->state.flags |= INTEL_PT_FUP_IP; 1154 ret = true; 1155 } 1156 if (decoder->set_fup_bep) { 1157 decoder->set_fup_bep = false; 1158 decoder->state.type |= INTEL_PT_BLK_ITEMS; 1159 ret = true; 1160 } 1161 if (decoder->overflow) { 1162 decoder->overflow = false; 1163 if (!ret && !decoder->pge) { 1164 if (decoder->hop) { 1165 decoder->state.type = 0; 1166 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 1167 } 1168 decoder->pge = true; 1169 decoder->state.type |= INTEL_PT_BRANCH | INTEL_PT_TRACE_BEGIN; 1170 decoder->state.from_ip = 0; 1171 decoder->state.to_ip = decoder->ip; 1172 return true; 1173 } 1174 } 1175 if (ret) { 1176 decoder->state.from_ip = decoder->ip; 1177 decoder->state.to_ip = 0; 1178 } else { 1179 decoder->state.type = type; 1180 } 1181 return ret; 1182} 1183 1184static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder, 1185 struct intel_pt_insn *intel_pt_insn, 1186 uint64_t ip, int err) 1187{ 1188 return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err && 1189 intel_pt_insn->branch == INTEL_PT_BR_INDIRECT && 1190 ip == decoder->ip + intel_pt_insn->length; 1191} 1192 1193static int intel_pt_walk_fup(struct intel_pt_decoder *decoder) 1194{ 1195 struct intel_pt_insn intel_pt_insn; 1196 uint64_t ip; 1197 int err; 1198 1199 ip = decoder->last_ip; 1200 1201 while (1) { 1202 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip); 1203 if (err == INTEL_PT_RETURN) 1204 return 0; 1205 if (err == -EAGAIN || 1206 intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) { 1207 bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP; 1208 1209 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1210 if (intel_pt_fup_event(decoder) && no_tip) 1211 return 0; 1212 return -EAGAIN; 1213 } 1214 decoder->set_fup_tx_flags = false; 1215 if (err) 1216 return err; 1217 1218 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1219 intel_pt_log_at("ERROR: Unexpected indirect branch", 1220 decoder->ip); 1221 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1222 return -ENOENT; 1223 } 1224 1225 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1226 intel_pt_log_at("ERROR: Unexpected conditional branch", 1227 decoder->ip); 1228 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1229 return -ENOENT; 1230 } 1231 1232 intel_pt_bug(decoder); 1233 } 1234} 1235 1236static int intel_pt_walk_tip(struct intel_pt_decoder *decoder) 1237{ 1238 struct intel_pt_insn intel_pt_insn; 1239 int err; 1240 1241 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1242 if (err == INTEL_PT_RETURN && 1243 decoder->pgd_ip && 1244 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1245 (decoder->state.type & INTEL_PT_BRANCH) && 1246 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) { 1247 /* Unconditional branch leaving filter region */ 1248 decoder->no_progress = 0; 1249 decoder->pge = false; 1250 decoder->continuous_period = false; 1251 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1252 decoder->state.type |= INTEL_PT_TRACE_END; 1253 return 0; 1254 } 1255 if (err == INTEL_PT_RETURN) 1256 return 0; 1257 if (err) 1258 return err; 1259 1260 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1261 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) { 1262 decoder->pge = false; 1263 decoder->continuous_period = false; 1264 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1265 decoder->state.from_ip = decoder->ip; 1266 if (decoder->packet.count == 0) { 1267 decoder->state.to_ip = 0; 1268 } else { 1269 decoder->state.to_ip = decoder->last_ip; 1270 decoder->ip = decoder->last_ip; 1271 } 1272 decoder->state.type |= INTEL_PT_TRACE_END; 1273 } else { 1274 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1275 decoder->state.from_ip = decoder->ip; 1276 if (decoder->packet.count == 0) { 1277 decoder->state.to_ip = 0; 1278 } else { 1279 decoder->state.to_ip = decoder->last_ip; 1280 decoder->ip = decoder->last_ip; 1281 } 1282 } 1283 return 0; 1284 } 1285 1286 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1287 uint64_t to_ip = decoder->ip + intel_pt_insn.length + 1288 intel_pt_insn.rel; 1289 1290 if (decoder->pgd_ip && 1291 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1292 decoder->pgd_ip(to_ip, decoder->data)) { 1293 /* Conditional branch leaving filter region */ 1294 decoder->pge = false; 1295 decoder->continuous_period = false; 1296 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1297 decoder->ip = to_ip; 1298 decoder->state.from_ip = decoder->ip; 1299 decoder->state.to_ip = to_ip; 1300 decoder->state.type |= INTEL_PT_TRACE_END; 1301 return 0; 1302 } 1303 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch", 1304 decoder->ip); 1305 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1306 return -ENOENT; 1307 } 1308 1309 return intel_pt_bug(decoder); 1310} 1311 1312static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder) 1313{ 1314 struct intel_pt_insn intel_pt_insn; 1315 int err; 1316 1317 while (1) { 1318 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1319 if (err == INTEL_PT_RETURN) 1320 return 0; 1321 if (err) 1322 return err; 1323 1324 if (intel_pt_insn.op == INTEL_PT_OP_RET) { 1325 if (!decoder->return_compression) { 1326 intel_pt_log_at("ERROR: RET when expecting conditional branch", 1327 decoder->ip); 1328 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1329 return -ENOENT; 1330 } 1331 if (!decoder->ret_addr) { 1332 intel_pt_log_at("ERROR: Bad RET compression (stack empty)", 1333 decoder->ip); 1334 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1335 return -ENOENT; 1336 } 1337 if (!(decoder->tnt.payload & BIT63)) { 1338 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)", 1339 decoder->ip); 1340 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1341 return -ENOENT; 1342 } 1343 decoder->tnt.count -= 1; 1344 if (decoder->tnt.count) 1345 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; 1346 else 1347 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1348 decoder->tnt.payload <<= 1; 1349 decoder->state.from_ip = decoder->ip; 1350 decoder->ip = decoder->ret_addr; 1351 decoder->state.to_ip = decoder->ip; 1352 return 0; 1353 } 1354 1355 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1356 /* Handle deferred TIPs */ 1357 err = intel_pt_get_next_packet(decoder); 1358 if (err) 1359 return err; 1360 if (decoder->packet.type != INTEL_PT_TIP || 1361 decoder->packet.count == 0) { 1362 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch", 1363 decoder->ip); 1364 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1365 decoder->pkt_step = 0; 1366 return -ENOENT; 1367 } 1368 intel_pt_set_last_ip(decoder); 1369 decoder->state.from_ip = decoder->ip; 1370 decoder->state.to_ip = decoder->last_ip; 1371 decoder->ip = decoder->last_ip; 1372 return 0; 1373 } 1374 1375 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1376 decoder->tnt.count -= 1; 1377 if (decoder->tnt.count) 1378 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; 1379 else 1380 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1381 if (decoder->tnt.payload & BIT63) { 1382 decoder->tnt.payload <<= 1; 1383 decoder->state.from_ip = decoder->ip; 1384 decoder->ip += intel_pt_insn.length + 1385 intel_pt_insn.rel; 1386 decoder->state.to_ip = decoder->ip; 1387 return 0; 1388 } 1389 /* Instruction sample for a non-taken branch */ 1390 if (decoder->state.type & INTEL_PT_INSTRUCTION) { 1391 decoder->tnt.payload <<= 1; 1392 decoder->state.type = INTEL_PT_INSTRUCTION; 1393 decoder->state.from_ip = decoder->ip; 1394 decoder->state.to_ip = 0; 1395 decoder->ip += intel_pt_insn.length; 1396 return 0; 1397 } 1398 decoder->sample_cyc = false; 1399 decoder->ip += intel_pt_insn.length; 1400 if (!decoder->tnt.count) { 1401 intel_pt_update_sample_time(decoder); 1402 return -EAGAIN; 1403 } 1404 decoder->tnt.payload <<= 1; 1405 continue; 1406 } 1407 1408 return intel_pt_bug(decoder); 1409 } 1410} 1411 1412static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip) 1413{ 1414 unsigned int fup_tx_flags; 1415 int err; 1416 1417 fup_tx_flags = decoder->packet.payload & 1418 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX); 1419 err = intel_pt_get_next_packet(decoder); 1420 if (err) 1421 return err; 1422 if (decoder->packet.type == INTEL_PT_FUP) { 1423 decoder->fup_tx_flags = fup_tx_flags; 1424 decoder->set_fup_tx_flags = true; 1425 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX)) 1426 *no_tip = true; 1427 } else { 1428 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX", 1429 decoder->pos); 1430 intel_pt_update_in_tx(decoder); 1431 } 1432 return 0; 1433} 1434 1435static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp) 1436{ 1437 timestamp |= (ref_timestamp & (0xffULL << 56)); 1438 1439 if (timestamp < ref_timestamp) { 1440 if (ref_timestamp - timestamp > (1ULL << 55)) 1441 timestamp += (1ULL << 56); 1442 } else { 1443 if (timestamp - ref_timestamp > (1ULL << 55)) 1444 timestamp -= (1ULL << 56); 1445 } 1446 1447 return timestamp; 1448} 1449 1450static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder) 1451{ 1452 uint64_t timestamp; 1453 1454 decoder->have_tma = false; 1455 1456 if (decoder->ref_timestamp) { 1457 timestamp = intel_pt_8b_tsc(decoder->packet.payload, 1458 decoder->ref_timestamp); 1459 decoder->tsc_timestamp = timestamp; 1460 decoder->timestamp = timestamp; 1461 decoder->ref_timestamp = 0; 1462 decoder->timestamp_insn_cnt = 0; 1463 } else if (decoder->timestamp) { 1464 timestamp = decoder->packet.payload | 1465 (decoder->timestamp & (0xffULL << 56)); 1466 decoder->tsc_timestamp = timestamp; 1467 if (timestamp < decoder->timestamp && 1468 decoder->timestamp - timestamp < decoder->tsc_slip) { 1469 intel_pt_log_to("Suppressing backwards timestamp", 1470 timestamp); 1471 timestamp = decoder->timestamp; 1472 } 1473 if (timestamp < decoder->timestamp) { 1474 intel_pt_log_to("Wraparound timestamp", timestamp); 1475 timestamp += (1ULL << 56); 1476 decoder->tsc_timestamp = timestamp; 1477 } 1478 decoder->timestamp = timestamp; 1479 decoder->timestamp_insn_cnt = 0; 1480 } 1481 1482 if (decoder->last_packet_type == INTEL_PT_CYC) { 1483 decoder->cyc_ref_timestamp = decoder->timestamp; 1484 decoder->cycle_cnt = 0; 1485 decoder->have_calc_cyc_to_tsc = false; 1486 intel_pt_calc_cyc_to_tsc(decoder, false); 1487 } 1488 1489 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1490} 1491 1492static int intel_pt_overflow(struct intel_pt_decoder *decoder) 1493{ 1494 intel_pt_log("ERROR: Buffer overflow\n"); 1495 intel_pt_clear_tx_flags(decoder); 1496 decoder->timestamp_insn_cnt = 0; 1497 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1498 decoder->state.from_ip = decoder->ip; 1499 decoder->ip = 0; 1500 decoder->pge = false; 1501 decoder->set_fup_tx_flags = false; 1502 decoder->set_fup_ptw = false; 1503 decoder->set_fup_mwait = false; 1504 decoder->set_fup_pwre = false; 1505 decoder->set_fup_exstop = false; 1506 decoder->set_fup_bep = false; 1507 decoder->overflow = true; 1508 return -EOVERFLOW; 1509} 1510 1511static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder) 1512{ 1513 if (decoder->have_cyc) 1514 return; 1515 1516 decoder->cyc_cnt_timestamp = decoder->timestamp; 1517 decoder->base_cyc_cnt = decoder->tot_cyc_cnt; 1518} 1519 1520static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder) 1521{ 1522 decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp; 1523 1524 if (decoder->pge) 1525 intel_pt_mtc_cyc_cnt_pge(decoder); 1526} 1527 1528static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder) 1529{ 1530 uint64_t tot_cyc_cnt, tsc_delta; 1531 1532 if (decoder->have_cyc) 1533 return; 1534 1535 decoder->sample_cyc = true; 1536 1537 if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp) 1538 return; 1539 1540 tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp; 1541 tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt; 1542 1543 if (tot_cyc_cnt > decoder->tot_cyc_cnt) 1544 decoder->tot_cyc_cnt = tot_cyc_cnt; 1545} 1546 1547static void intel_pt_calc_tma(struct intel_pt_decoder *decoder) 1548{ 1549 uint32_t ctc = decoder->packet.payload; 1550 uint32_t fc = decoder->packet.count; 1551 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask; 1552 1553 if (!decoder->tsc_ctc_ratio_d) 1554 return; 1555 1556 if (decoder->pge && !decoder->in_psb) 1557 intel_pt_mtc_cyc_cnt_pge(decoder); 1558 else 1559 intel_pt_mtc_cyc_cnt_upd(decoder); 1560 1561 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 1562 decoder->ctc_timestamp = decoder->tsc_timestamp - fc; 1563 if (decoder->tsc_ctc_mult) { 1564 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 1565 } else { 1566 decoder->ctc_timestamp -= multdiv(ctc_rem, 1567 decoder->tsc_ctc_ratio_n, 1568 decoder->tsc_ctc_ratio_d); 1569 } 1570 decoder->ctc_delta = 0; 1571 decoder->have_tma = true; 1572 decoder->fixup_last_mtc = true; 1573 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n", 1574 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem); 1575} 1576 1577static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder) 1578{ 1579 uint64_t timestamp; 1580 uint32_t mtc, mtc_delta; 1581 1582 if (!decoder->have_tma) 1583 return; 1584 1585 mtc = decoder->packet.payload; 1586 1587 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) { 1588 decoder->fixup_last_mtc = false; 1589 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 1590 &decoder->last_mtc); 1591 } 1592 1593 if (mtc > decoder->last_mtc) 1594 mtc_delta = mtc - decoder->last_mtc; 1595 else 1596 mtc_delta = mtc + 256 - decoder->last_mtc; 1597 1598 decoder->ctc_delta += mtc_delta << decoder->mtc_shift; 1599 1600 if (decoder->tsc_ctc_mult) { 1601 timestamp = decoder->ctc_timestamp + 1602 decoder->ctc_delta * decoder->tsc_ctc_mult; 1603 } else { 1604 timestamp = decoder->ctc_timestamp + 1605 multdiv(decoder->ctc_delta, 1606 decoder->tsc_ctc_ratio_n, 1607 decoder->tsc_ctc_ratio_d); 1608 } 1609 1610 if (timestamp < decoder->timestamp) 1611 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1612 timestamp, decoder->timestamp); 1613 else 1614 decoder->timestamp = timestamp; 1615 1616 intel_pt_mtc_cyc_cnt_upd(decoder); 1617 1618 decoder->timestamp_insn_cnt = 0; 1619 decoder->last_mtc = mtc; 1620 1621 if (decoder->last_packet_type == INTEL_PT_CYC) { 1622 decoder->cyc_ref_timestamp = decoder->timestamp; 1623 decoder->cycle_cnt = 0; 1624 decoder->have_calc_cyc_to_tsc = false; 1625 intel_pt_calc_cyc_to_tsc(decoder, true); 1626 } 1627 1628 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1629} 1630 1631static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder) 1632{ 1633 unsigned int cbr = decoder->packet.payload & 0xff; 1634 1635 decoder->cbr_payload = decoder->packet.payload; 1636 1637 if (decoder->cbr == cbr) 1638 return; 1639 1640 decoder->cbr = cbr; 1641 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 1642 decoder->cyc_ref_timestamp = decoder->timestamp; 1643 decoder->cycle_cnt = 0; 1644 1645 intel_pt_mtc_cyc_cnt_cbr(decoder); 1646} 1647 1648static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder) 1649{ 1650 uint64_t timestamp = decoder->cyc_ref_timestamp; 1651 1652 decoder->have_cyc = true; 1653 1654 decoder->cycle_cnt += decoder->packet.payload; 1655 if (decoder->pge) 1656 decoder->tot_cyc_cnt += decoder->packet.payload; 1657 decoder->sample_cyc = true; 1658 1659 if (!decoder->cyc_ref_timestamp) 1660 return; 1661 1662 if (decoder->have_calc_cyc_to_tsc) 1663 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc; 1664 else if (decoder->cbr) 1665 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc; 1666 else 1667 return; 1668 1669 if (timestamp < decoder->timestamp) 1670 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1671 timestamp, decoder->timestamp); 1672 else 1673 decoder->timestamp = timestamp; 1674 1675 decoder->timestamp_insn_cnt = 0; 1676 1677 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1678} 1679 1680static void intel_pt_bbp(struct intel_pt_decoder *decoder) 1681{ 1682 if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) { 1683 memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask)); 1684 decoder->state.items.is_32_bit = false; 1685 } 1686 decoder->blk_type = decoder->packet.payload; 1687 decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type); 1688 if (decoder->blk_type == INTEL_PT_GP_REGS) 1689 decoder->state.items.is_32_bit = decoder->packet.count; 1690 if (decoder->blk_type_pos < 0) { 1691 intel_pt_log("WARNING: Unknown block type %u\n", 1692 decoder->blk_type); 1693 } else if (decoder->state.items.mask[decoder->blk_type_pos]) { 1694 intel_pt_log("WARNING: Duplicate block type %u\n", 1695 decoder->blk_type); 1696 } 1697} 1698 1699static void intel_pt_bip(struct intel_pt_decoder *decoder) 1700{ 1701 uint32_t id = decoder->packet.count; 1702 uint32_t bit = 1 << id; 1703 int pos = decoder->blk_type_pos; 1704 1705 if (pos < 0 || id >= INTEL_PT_BLK_ITEM_ID_CNT) { 1706 intel_pt_log("WARNING: Unknown block item %u type %d\n", 1707 id, decoder->blk_type); 1708 return; 1709 } 1710 1711 if (decoder->state.items.mask[pos] & bit) { 1712 intel_pt_log("WARNING: Duplicate block item %u type %d\n", 1713 id, decoder->blk_type); 1714 } 1715 1716 decoder->state.items.mask[pos] |= bit; 1717 decoder->state.items.val[pos][id] = decoder->packet.payload; 1718} 1719 1720/* Walk PSB+ packets when already in sync. */ 1721static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder) 1722{ 1723 int err; 1724 1725 decoder->in_psb = true; 1726 1727 while (1) { 1728 err = intel_pt_get_next_packet(decoder); 1729 if (err) 1730 goto out; 1731 1732 switch (decoder->packet.type) { 1733 case INTEL_PT_PSBEND: 1734 err = 0; 1735 goto out; 1736 1737 case INTEL_PT_TIP_PGD: 1738 case INTEL_PT_TIP_PGE: 1739 case INTEL_PT_TIP: 1740 case INTEL_PT_TNT: 1741 case INTEL_PT_TRACESTOP: 1742 case INTEL_PT_BAD: 1743 case INTEL_PT_PSB: 1744 case INTEL_PT_PTWRITE: 1745 case INTEL_PT_PTWRITE_IP: 1746 case INTEL_PT_EXSTOP: 1747 case INTEL_PT_EXSTOP_IP: 1748 case INTEL_PT_MWAIT: 1749 case INTEL_PT_PWRE: 1750 case INTEL_PT_PWRX: 1751 case INTEL_PT_BBP: 1752 case INTEL_PT_BIP: 1753 case INTEL_PT_BEP: 1754 case INTEL_PT_BEP_IP: 1755 decoder->have_tma = false; 1756 intel_pt_log("ERROR: Unexpected packet\n"); 1757 err = -EAGAIN; 1758 goto out; 1759 1760 case INTEL_PT_OVF: 1761 err = intel_pt_overflow(decoder); 1762 goto out; 1763 1764 case INTEL_PT_TSC: 1765 intel_pt_calc_tsc_timestamp(decoder); 1766 break; 1767 1768 case INTEL_PT_TMA: 1769 intel_pt_calc_tma(decoder); 1770 break; 1771 1772 case INTEL_PT_CBR: 1773 intel_pt_calc_cbr(decoder); 1774 break; 1775 1776 case INTEL_PT_MODE_EXEC: 1777 decoder->exec_mode = decoder->packet.payload; 1778 break; 1779 1780 case INTEL_PT_PIP: 1781 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1782 break; 1783 1784 case INTEL_PT_FUP: 1785 decoder->pge = true; 1786 if (decoder->packet.count) { 1787 intel_pt_set_last_ip(decoder); 1788 if (decoder->hop) { 1789 /* Act on FUP at PSBEND */ 1790 decoder->ip = decoder->last_ip; 1791 decoder->hop_psb_fup = true; 1792 } 1793 } 1794 break; 1795 1796 case INTEL_PT_MODE_TSX: 1797 intel_pt_update_in_tx(decoder); 1798 break; 1799 1800 case INTEL_PT_MTC: 1801 intel_pt_calc_mtc_timestamp(decoder); 1802 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1803 decoder->state.type |= INTEL_PT_INSTRUCTION; 1804 break; 1805 1806 case INTEL_PT_CYC: 1807 intel_pt_calc_cyc_timestamp(decoder); 1808 break; 1809 1810 case INTEL_PT_VMCS: 1811 case INTEL_PT_MNT: 1812 case INTEL_PT_PAD: 1813 default: 1814 break; 1815 } 1816 } 1817out: 1818 decoder->in_psb = false; 1819 1820 return err; 1821} 1822 1823static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder) 1824{ 1825 int err; 1826 1827 if (decoder->tx_flags & INTEL_PT_ABORT_TX) { 1828 decoder->tx_flags = 0; 1829 decoder->state.flags &= ~INTEL_PT_IN_TX; 1830 decoder->state.flags |= INTEL_PT_ABORT_TX; 1831 } else { 1832 decoder->state.flags |= INTEL_PT_ASYNC; 1833 } 1834 1835 while (1) { 1836 err = intel_pt_get_next_packet(decoder); 1837 if (err) 1838 return err; 1839 1840 switch (decoder->packet.type) { 1841 case INTEL_PT_TNT: 1842 case INTEL_PT_FUP: 1843 case INTEL_PT_TRACESTOP: 1844 case INTEL_PT_PSB: 1845 case INTEL_PT_TSC: 1846 case INTEL_PT_TMA: 1847 case INTEL_PT_MODE_TSX: 1848 case INTEL_PT_BAD: 1849 case INTEL_PT_PSBEND: 1850 case INTEL_PT_PTWRITE: 1851 case INTEL_PT_PTWRITE_IP: 1852 case INTEL_PT_EXSTOP: 1853 case INTEL_PT_EXSTOP_IP: 1854 case INTEL_PT_MWAIT: 1855 case INTEL_PT_PWRE: 1856 case INTEL_PT_PWRX: 1857 case INTEL_PT_BBP: 1858 case INTEL_PT_BIP: 1859 case INTEL_PT_BEP: 1860 case INTEL_PT_BEP_IP: 1861 intel_pt_log("ERROR: Missing TIP after FUP\n"); 1862 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1863 decoder->pkt_step = 0; 1864 return -ENOENT; 1865 1866 case INTEL_PT_CBR: 1867 intel_pt_calc_cbr(decoder); 1868 break; 1869 1870 case INTEL_PT_OVF: 1871 return intel_pt_overflow(decoder); 1872 1873 case INTEL_PT_TIP_PGD: 1874 decoder->state.from_ip = decoder->ip; 1875 if (decoder->packet.count == 0) { 1876 decoder->state.to_ip = 0; 1877 } else { 1878 intel_pt_set_ip(decoder); 1879 decoder->state.to_ip = decoder->ip; 1880 } 1881 decoder->pge = false; 1882 decoder->continuous_period = false; 1883 decoder->state.type |= INTEL_PT_TRACE_END; 1884 return 0; 1885 1886 case INTEL_PT_TIP_PGE: 1887 decoder->pge = true; 1888 intel_pt_log("Omitting PGE ip " x64_fmt "\n", 1889 decoder->ip); 1890 decoder->state.from_ip = 0; 1891 if (decoder->packet.count == 0) { 1892 decoder->state.to_ip = 0; 1893 } else { 1894 intel_pt_set_ip(decoder); 1895 decoder->state.to_ip = decoder->ip; 1896 } 1897 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 1898 intel_pt_mtc_cyc_cnt_pge(decoder); 1899 return 0; 1900 1901 case INTEL_PT_TIP: 1902 decoder->state.from_ip = decoder->ip; 1903 if (decoder->packet.count == 0) { 1904 decoder->state.to_ip = 0; 1905 } else { 1906 intel_pt_set_ip(decoder); 1907 decoder->state.to_ip = decoder->ip; 1908 } 1909 return 0; 1910 1911 case INTEL_PT_PIP: 1912 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1913 break; 1914 1915 case INTEL_PT_MTC: 1916 intel_pt_calc_mtc_timestamp(decoder); 1917 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1918 decoder->state.type |= INTEL_PT_INSTRUCTION; 1919 break; 1920 1921 case INTEL_PT_CYC: 1922 intel_pt_calc_cyc_timestamp(decoder); 1923 break; 1924 1925 case INTEL_PT_MODE_EXEC: 1926 decoder->exec_mode = decoder->packet.payload; 1927 break; 1928 1929 case INTEL_PT_VMCS: 1930 case INTEL_PT_MNT: 1931 case INTEL_PT_PAD: 1932 break; 1933 1934 default: 1935 return intel_pt_bug(decoder); 1936 } 1937 } 1938} 1939 1940static int intel_pt_resample(struct intel_pt_decoder *decoder) 1941{ 1942 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1943 decoder->state.type = INTEL_PT_INSTRUCTION; 1944 decoder->state.from_ip = decoder->ip; 1945 decoder->state.to_ip = 0; 1946 return 0; 1947} 1948 1949#define HOP_PROCESS 0 1950#define HOP_IGNORE 1 1951#define HOP_RETURN 2 1952#define HOP_AGAIN 3 1953 1954static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder); 1955 1956/* Hop mode: Ignore TNT, do not walk code, but get ip from FUPs and TIPs */ 1957static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err) 1958{ 1959 *err = 0; 1960 1961 /* Leap from PSB to PSB, getting ip from FUP within PSB+ */ 1962 if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) { 1963 *err = intel_pt_scan_for_psb(decoder); 1964 if (*err) 1965 return HOP_RETURN; 1966 } 1967 1968 switch (decoder->packet.type) { 1969 case INTEL_PT_TNT: 1970 return HOP_IGNORE; 1971 1972 case INTEL_PT_TIP_PGD: 1973 decoder->pge = false; 1974 if (!decoder->packet.count) 1975 return HOP_IGNORE; 1976 intel_pt_set_ip(decoder); 1977 decoder->state.type |= INTEL_PT_TRACE_END; 1978 decoder->state.from_ip = 0; 1979 decoder->state.to_ip = decoder->ip; 1980 return HOP_RETURN; 1981 1982 case INTEL_PT_TIP: 1983 if (!decoder->packet.count) 1984 return HOP_IGNORE; 1985 intel_pt_set_ip(decoder); 1986 decoder->state.type = INTEL_PT_INSTRUCTION; 1987 decoder->state.from_ip = decoder->ip; 1988 decoder->state.to_ip = 0; 1989 return HOP_RETURN; 1990 1991 case INTEL_PT_FUP: 1992 if (!decoder->packet.count) 1993 return HOP_IGNORE; 1994 intel_pt_set_ip(decoder); 1995 if (decoder->set_fup_mwait || decoder->set_fup_pwre) 1996 *no_tip = true; 1997 if (!decoder->branch_enable || !decoder->pge) 1998 *no_tip = true; 1999 if (*no_tip) { 2000 decoder->state.type = INTEL_PT_INSTRUCTION; 2001 decoder->state.from_ip = decoder->ip; 2002 decoder->state.to_ip = 0; 2003 intel_pt_fup_event(decoder); 2004 return HOP_RETURN; 2005 } 2006 intel_pt_fup_event(decoder); 2007 decoder->state.type |= INTEL_PT_INSTRUCTION | INTEL_PT_BRANCH; 2008 *err = intel_pt_walk_fup_tip(decoder); 2009 if (!*err && decoder->state.to_ip) 2010 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 2011 return HOP_RETURN; 2012 2013 case INTEL_PT_PSB: 2014 decoder->last_ip = 0; 2015 decoder->have_last_ip = true; 2016 decoder->hop_psb_fup = false; 2017 *err = intel_pt_walk_psbend(decoder); 2018 if (*err == -EAGAIN) 2019 return HOP_AGAIN; 2020 if (*err) 2021 return HOP_RETURN; 2022 if (decoder->hop_psb_fup) { 2023 decoder->hop_psb_fup = false; 2024 decoder->state.type = INTEL_PT_INSTRUCTION; 2025 decoder->state.from_ip = decoder->ip; 2026 decoder->state.to_ip = 0; 2027 return HOP_RETURN; 2028 } 2029 if (decoder->cbr != decoder->cbr_seen) { 2030 decoder->state.type = 0; 2031 return HOP_RETURN; 2032 } 2033 return HOP_IGNORE; 2034 2035 case INTEL_PT_BAD: 2036 case INTEL_PT_PAD: 2037 case INTEL_PT_TIP_PGE: 2038 case INTEL_PT_TSC: 2039 case INTEL_PT_TMA: 2040 case INTEL_PT_MODE_EXEC: 2041 case INTEL_PT_MODE_TSX: 2042 case INTEL_PT_MTC: 2043 case INTEL_PT_CYC: 2044 case INTEL_PT_VMCS: 2045 case INTEL_PT_PSBEND: 2046 case INTEL_PT_CBR: 2047 case INTEL_PT_TRACESTOP: 2048 case INTEL_PT_PIP: 2049 case INTEL_PT_OVF: 2050 case INTEL_PT_MNT: 2051 case INTEL_PT_PTWRITE: 2052 case INTEL_PT_PTWRITE_IP: 2053 case INTEL_PT_EXSTOP: 2054 case INTEL_PT_EXSTOP_IP: 2055 case INTEL_PT_MWAIT: 2056 case INTEL_PT_PWRE: 2057 case INTEL_PT_PWRX: 2058 case INTEL_PT_BBP: 2059 case INTEL_PT_BIP: 2060 case INTEL_PT_BEP: 2061 case INTEL_PT_BEP_IP: 2062 default: 2063 return HOP_PROCESS; 2064 } 2065} 2066 2067static int intel_pt_walk_trace(struct intel_pt_decoder *decoder) 2068{ 2069 int last_packet_type = INTEL_PT_PAD; 2070 bool no_tip = false; 2071 int err; 2072 2073 while (1) { 2074 err = intel_pt_get_next_packet(decoder); 2075 if (err) 2076 return err; 2077next: 2078 err = 0; 2079 if (decoder->cyc_threshold) { 2080 if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC) 2081 decoder->sample_cyc = false; 2082 last_packet_type = decoder->packet.type; 2083 } 2084 2085 if (decoder->hop) { 2086 switch (intel_pt_hop_trace(decoder, &no_tip, &err)) { 2087 case HOP_IGNORE: 2088 continue; 2089 case HOP_RETURN: 2090 return err; 2091 case HOP_AGAIN: 2092 goto next; 2093 default: 2094 break; 2095 } 2096 } 2097 2098 switch (decoder->packet.type) { 2099 case INTEL_PT_TNT: 2100 if (!decoder->packet.count) 2101 break; 2102 decoder->tnt = decoder->packet; 2103 decoder->pkt_state = INTEL_PT_STATE_TNT; 2104 err = intel_pt_walk_tnt(decoder); 2105 if (err == -EAGAIN) 2106 break; 2107 return err; 2108 2109 case INTEL_PT_TIP_PGD: 2110 if (decoder->packet.count != 0) 2111 intel_pt_set_last_ip(decoder); 2112 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD; 2113 return intel_pt_walk_tip(decoder); 2114 2115 case INTEL_PT_TIP_PGE: { 2116 decoder->pge = true; 2117 decoder->overflow = false; 2118 intel_pt_mtc_cyc_cnt_pge(decoder); 2119 if (decoder->packet.count == 0) { 2120 intel_pt_log_at("Skipping zero TIP.PGE", 2121 decoder->pos); 2122 break; 2123 } 2124 intel_pt_set_ip(decoder); 2125 decoder->state.from_ip = 0; 2126 decoder->state.to_ip = decoder->ip; 2127 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 2128 /* 2129 * In hop mode, resample to get the to_ip as an 2130 * "instruction" sample. 2131 */ 2132 if (decoder->hop) 2133 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 2134 return 0; 2135 } 2136 2137 case INTEL_PT_OVF: 2138 return intel_pt_overflow(decoder); 2139 2140 case INTEL_PT_TIP: 2141 if (decoder->packet.count != 0) 2142 intel_pt_set_last_ip(decoder); 2143 decoder->pkt_state = INTEL_PT_STATE_TIP; 2144 return intel_pt_walk_tip(decoder); 2145 2146 case INTEL_PT_FUP: 2147 if (decoder->packet.count == 0) { 2148 intel_pt_log_at("Skipping zero FUP", 2149 decoder->pos); 2150 no_tip = false; 2151 break; 2152 } 2153 intel_pt_set_last_ip(decoder); 2154 if (!decoder->branch_enable || !decoder->pge) { 2155 decoder->ip = decoder->last_ip; 2156 if (intel_pt_fup_event(decoder)) 2157 return 0; 2158 no_tip = false; 2159 break; 2160 } 2161 if (decoder->set_fup_mwait) 2162 no_tip = true; 2163 if (no_tip) 2164 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP; 2165 else 2166 decoder->pkt_state = INTEL_PT_STATE_FUP; 2167 err = intel_pt_walk_fup(decoder); 2168 if (err != -EAGAIN) 2169 return err; 2170 if (no_tip) { 2171 no_tip = false; 2172 break; 2173 } 2174 return intel_pt_walk_fup_tip(decoder); 2175 2176 case INTEL_PT_TRACESTOP: 2177 decoder->pge = false; 2178 decoder->continuous_period = false; 2179 intel_pt_clear_tx_flags(decoder); 2180 decoder->have_tma = false; 2181 break; 2182 2183 case INTEL_PT_PSB: 2184 decoder->last_ip = 0; 2185 decoder->have_last_ip = true; 2186 intel_pt_clear_stack(&decoder->stack); 2187 err = intel_pt_walk_psbend(decoder); 2188 if (err == -EAGAIN) 2189 goto next; 2190 if (err) 2191 return err; 2192 /* 2193 * PSB+ CBR will not have changed but cater for the 2194 * possibility of another CBR change that gets caught up 2195 * in the PSB+. 2196 */ 2197 if (decoder->cbr != decoder->cbr_seen) { 2198 decoder->state.type = 0; 2199 return 0; 2200 } 2201 break; 2202 2203 case INTEL_PT_PIP: 2204 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 2205 break; 2206 2207 case INTEL_PT_MTC: 2208 intel_pt_calc_mtc_timestamp(decoder); 2209 if (decoder->period_type != INTEL_PT_PERIOD_MTC) 2210 break; 2211 /* 2212 * Ensure that there has been an instruction since the 2213 * last MTC. 2214 */ 2215 if (!decoder->mtc_insn) 2216 break; 2217 decoder->mtc_insn = false; 2218 /* Ensure that there is a timestamp */ 2219 if (!decoder->timestamp) 2220 break; 2221 decoder->state.type = INTEL_PT_INSTRUCTION; 2222 decoder->state.from_ip = decoder->ip; 2223 decoder->state.to_ip = 0; 2224 decoder->mtc_insn = false; 2225 return 0; 2226 2227 case INTEL_PT_TSC: 2228 intel_pt_calc_tsc_timestamp(decoder); 2229 break; 2230 2231 case INTEL_PT_TMA: 2232 intel_pt_calc_tma(decoder); 2233 break; 2234 2235 case INTEL_PT_CYC: 2236 intel_pt_calc_cyc_timestamp(decoder); 2237 break; 2238 2239 case INTEL_PT_CBR: 2240 intel_pt_calc_cbr(decoder); 2241 if (decoder->cbr != decoder->cbr_seen) { 2242 decoder->state.type = 0; 2243 return 0; 2244 } 2245 break; 2246 2247 case INTEL_PT_MODE_EXEC: 2248 decoder->exec_mode = decoder->packet.payload; 2249 break; 2250 2251 case INTEL_PT_MODE_TSX: 2252 /* MODE_TSX need not be followed by FUP */ 2253 if (!decoder->pge || decoder->in_psb) { 2254 intel_pt_update_in_tx(decoder); 2255 break; 2256 } 2257 err = intel_pt_mode_tsx(decoder, &no_tip); 2258 if (err) 2259 return err; 2260 goto next; 2261 2262 case INTEL_PT_BAD: /* Does not happen */ 2263 return intel_pt_bug(decoder); 2264 2265 case INTEL_PT_PSBEND: 2266 case INTEL_PT_VMCS: 2267 case INTEL_PT_MNT: 2268 case INTEL_PT_PAD: 2269 break; 2270 2271 case INTEL_PT_PTWRITE_IP: 2272 decoder->fup_ptw_payload = decoder->packet.payload; 2273 err = intel_pt_get_next_packet(decoder); 2274 if (err) 2275 return err; 2276 if (decoder->packet.type == INTEL_PT_FUP) { 2277 decoder->set_fup_ptw = true; 2278 no_tip = true; 2279 } else { 2280 intel_pt_log_at("ERROR: Missing FUP after PTWRITE", 2281 decoder->pos); 2282 } 2283 goto next; 2284 2285 case INTEL_PT_PTWRITE: 2286 decoder->state.type = INTEL_PT_PTW; 2287 decoder->state.from_ip = decoder->ip; 2288 decoder->state.to_ip = 0; 2289 decoder->state.ptw_payload = decoder->packet.payload; 2290 return 0; 2291 2292 case INTEL_PT_MWAIT: 2293 decoder->fup_mwait_payload = decoder->packet.payload; 2294 decoder->set_fup_mwait = true; 2295 break; 2296 2297 case INTEL_PT_PWRE: 2298 if (decoder->set_fup_mwait) { 2299 decoder->fup_pwre_payload = 2300 decoder->packet.payload; 2301 decoder->set_fup_pwre = true; 2302 break; 2303 } 2304 decoder->state.type = INTEL_PT_PWR_ENTRY; 2305 decoder->state.from_ip = decoder->ip; 2306 decoder->state.to_ip = 0; 2307 decoder->state.pwrx_payload = decoder->packet.payload; 2308 return 0; 2309 2310 case INTEL_PT_EXSTOP_IP: 2311 err = intel_pt_get_next_packet(decoder); 2312 if (err) 2313 return err; 2314 if (decoder->packet.type == INTEL_PT_FUP) { 2315 decoder->set_fup_exstop = true; 2316 no_tip = true; 2317 } else { 2318 intel_pt_log_at("ERROR: Missing FUP after EXSTOP", 2319 decoder->pos); 2320 } 2321 goto next; 2322 2323 case INTEL_PT_EXSTOP: 2324 decoder->state.type = INTEL_PT_EX_STOP; 2325 decoder->state.from_ip = decoder->ip; 2326 decoder->state.to_ip = 0; 2327 return 0; 2328 2329 case INTEL_PT_PWRX: 2330 decoder->state.type = INTEL_PT_PWR_EXIT; 2331 decoder->state.from_ip = decoder->ip; 2332 decoder->state.to_ip = 0; 2333 decoder->state.pwrx_payload = decoder->packet.payload; 2334 return 0; 2335 2336 case INTEL_PT_BBP: 2337 intel_pt_bbp(decoder); 2338 break; 2339 2340 case INTEL_PT_BIP: 2341 intel_pt_bip(decoder); 2342 break; 2343 2344 case INTEL_PT_BEP: 2345 decoder->state.type = INTEL_PT_BLK_ITEMS; 2346 decoder->state.from_ip = decoder->ip; 2347 decoder->state.to_ip = 0; 2348 return 0; 2349 2350 case INTEL_PT_BEP_IP: 2351 err = intel_pt_get_next_packet(decoder); 2352 if (err) 2353 return err; 2354 if (decoder->packet.type == INTEL_PT_FUP) { 2355 decoder->set_fup_bep = true; 2356 no_tip = true; 2357 } else { 2358 intel_pt_log_at("ERROR: Missing FUP after BEP", 2359 decoder->pos); 2360 } 2361 goto next; 2362 2363 default: 2364 return intel_pt_bug(decoder); 2365 } 2366 } 2367} 2368 2369static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder) 2370{ 2371 return decoder->packet.count && 2372 (decoder->have_last_ip || decoder->packet.count == 3 || 2373 decoder->packet.count == 6); 2374} 2375 2376/* Walk PSB+ packets to get in sync. */ 2377static int intel_pt_walk_psb(struct intel_pt_decoder *decoder) 2378{ 2379 int err; 2380 2381 decoder->in_psb = true; 2382 2383 while (1) { 2384 err = intel_pt_get_next_packet(decoder); 2385 if (err) 2386 goto out; 2387 2388 switch (decoder->packet.type) { 2389 case INTEL_PT_TIP_PGD: 2390 decoder->continuous_period = false; 2391 __fallthrough; 2392 case INTEL_PT_TIP_PGE: 2393 case INTEL_PT_TIP: 2394 case INTEL_PT_PTWRITE: 2395 case INTEL_PT_PTWRITE_IP: 2396 case INTEL_PT_EXSTOP: 2397 case INTEL_PT_EXSTOP_IP: 2398 case INTEL_PT_MWAIT: 2399 case INTEL_PT_PWRE: 2400 case INTEL_PT_PWRX: 2401 case INTEL_PT_BBP: 2402 case INTEL_PT_BIP: 2403 case INTEL_PT_BEP: 2404 case INTEL_PT_BEP_IP: 2405 intel_pt_log("ERROR: Unexpected packet\n"); 2406 err = -ENOENT; 2407 goto out; 2408 2409 case INTEL_PT_FUP: 2410 decoder->pge = true; 2411 if (intel_pt_have_ip(decoder)) { 2412 uint64_t current_ip = decoder->ip; 2413 2414 intel_pt_set_ip(decoder); 2415 if (current_ip) 2416 intel_pt_log_to("Setting IP", 2417 decoder->ip); 2418 } 2419 break; 2420 2421 case INTEL_PT_MTC: 2422 intel_pt_calc_mtc_timestamp(decoder); 2423 break; 2424 2425 case INTEL_PT_TSC: 2426 intel_pt_calc_tsc_timestamp(decoder); 2427 break; 2428 2429 case INTEL_PT_TMA: 2430 intel_pt_calc_tma(decoder); 2431 break; 2432 2433 case INTEL_PT_CYC: 2434 intel_pt_calc_cyc_timestamp(decoder); 2435 break; 2436 2437 case INTEL_PT_CBR: 2438 intel_pt_calc_cbr(decoder); 2439 break; 2440 2441 case INTEL_PT_PIP: 2442 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 2443 break; 2444 2445 case INTEL_PT_MODE_EXEC: 2446 decoder->exec_mode = decoder->packet.payload; 2447 break; 2448 2449 case INTEL_PT_MODE_TSX: 2450 intel_pt_update_in_tx(decoder); 2451 break; 2452 2453 case INTEL_PT_TRACESTOP: 2454 decoder->pge = false; 2455 decoder->continuous_period = false; 2456 intel_pt_clear_tx_flags(decoder); 2457 __fallthrough; 2458 2459 case INTEL_PT_TNT: 2460 decoder->have_tma = false; 2461 intel_pt_log("ERROR: Unexpected packet\n"); 2462 if (decoder->ip) 2463 decoder->pkt_state = INTEL_PT_STATE_ERR4; 2464 else 2465 decoder->pkt_state = INTEL_PT_STATE_ERR3; 2466 err = -ENOENT; 2467 goto out; 2468 2469 case INTEL_PT_BAD: /* Does not happen */ 2470 err = intel_pt_bug(decoder); 2471 goto out; 2472 2473 case INTEL_PT_OVF: 2474 err = intel_pt_overflow(decoder); 2475 goto out; 2476 2477 case INTEL_PT_PSBEND: 2478 err = 0; 2479 goto out; 2480 2481 case INTEL_PT_PSB: 2482 case INTEL_PT_VMCS: 2483 case INTEL_PT_MNT: 2484 case INTEL_PT_PAD: 2485 default: 2486 break; 2487 } 2488 } 2489out: 2490 decoder->in_psb = false; 2491 2492 return err; 2493} 2494 2495static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder) 2496{ 2497 int err; 2498 2499 while (1) { 2500 err = intel_pt_get_next_packet(decoder); 2501 if (err) 2502 return err; 2503 2504 switch (decoder->packet.type) { 2505 case INTEL_PT_TIP_PGD: 2506 decoder->continuous_period = false; 2507 decoder->pge = false; 2508 if (intel_pt_have_ip(decoder)) 2509 intel_pt_set_ip(decoder); 2510 if (!decoder->ip) 2511 break; 2512 decoder->state.type |= INTEL_PT_TRACE_END; 2513 return 0; 2514 2515 case INTEL_PT_TIP_PGE: 2516 decoder->pge = true; 2517 intel_pt_mtc_cyc_cnt_pge(decoder); 2518 if (intel_pt_have_ip(decoder)) 2519 intel_pt_set_ip(decoder); 2520 if (!decoder->ip) 2521 break; 2522 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 2523 return 0; 2524 2525 case INTEL_PT_TIP: 2526 decoder->pge = true; 2527 if (intel_pt_have_ip(decoder)) 2528 intel_pt_set_ip(decoder); 2529 if (!decoder->ip) 2530 break; 2531 return 0; 2532 2533 case INTEL_PT_FUP: 2534 if (intel_pt_have_ip(decoder)) 2535 intel_pt_set_ip(decoder); 2536 if (decoder->ip) 2537 return 0; 2538 break; 2539 2540 case INTEL_PT_MTC: 2541 intel_pt_calc_mtc_timestamp(decoder); 2542 break; 2543 2544 case INTEL_PT_TSC: 2545 intel_pt_calc_tsc_timestamp(decoder); 2546 break; 2547 2548 case INTEL_PT_TMA: 2549 intel_pt_calc_tma(decoder); 2550 break; 2551 2552 case INTEL_PT_CYC: 2553 intel_pt_calc_cyc_timestamp(decoder); 2554 break; 2555 2556 case INTEL_PT_CBR: 2557 intel_pt_calc_cbr(decoder); 2558 break; 2559 2560 case INTEL_PT_PIP: 2561 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 2562 break; 2563 2564 case INTEL_PT_MODE_EXEC: 2565 decoder->exec_mode = decoder->packet.payload; 2566 break; 2567 2568 case INTEL_PT_MODE_TSX: 2569 intel_pt_update_in_tx(decoder); 2570 break; 2571 2572 case INTEL_PT_OVF: 2573 return intel_pt_overflow(decoder); 2574 2575 case INTEL_PT_BAD: /* Does not happen */ 2576 return intel_pt_bug(decoder); 2577 2578 case INTEL_PT_TRACESTOP: 2579 decoder->pge = false; 2580 decoder->continuous_period = false; 2581 intel_pt_clear_tx_flags(decoder); 2582 decoder->have_tma = false; 2583 break; 2584 2585 case INTEL_PT_PSB: 2586 decoder->last_ip = 0; 2587 decoder->have_last_ip = true; 2588 intel_pt_clear_stack(&decoder->stack); 2589 err = intel_pt_walk_psb(decoder); 2590 if (err) 2591 return err; 2592 if (decoder->ip) { 2593 /* Do not have a sample */ 2594 decoder->state.type = 0; 2595 return 0; 2596 } 2597 break; 2598 2599 case INTEL_PT_TNT: 2600 case INTEL_PT_PSBEND: 2601 case INTEL_PT_VMCS: 2602 case INTEL_PT_MNT: 2603 case INTEL_PT_PAD: 2604 case INTEL_PT_PTWRITE: 2605 case INTEL_PT_PTWRITE_IP: 2606 case INTEL_PT_EXSTOP: 2607 case INTEL_PT_EXSTOP_IP: 2608 case INTEL_PT_MWAIT: 2609 case INTEL_PT_PWRE: 2610 case INTEL_PT_PWRX: 2611 case INTEL_PT_BBP: 2612 case INTEL_PT_BIP: 2613 case INTEL_PT_BEP: 2614 case INTEL_PT_BEP_IP: 2615 default: 2616 break; 2617 } 2618 } 2619} 2620 2621static int intel_pt_sync_ip(struct intel_pt_decoder *decoder) 2622{ 2623 int err; 2624 2625 decoder->set_fup_tx_flags = false; 2626 decoder->set_fup_ptw = false; 2627 decoder->set_fup_mwait = false; 2628 decoder->set_fup_pwre = false; 2629 decoder->set_fup_exstop = false; 2630 decoder->set_fup_bep = false; 2631 decoder->overflow = false; 2632 2633 if (!decoder->branch_enable) { 2634 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2635 decoder->state.type = 0; /* Do not have a sample */ 2636 return 0; 2637 } 2638 2639 intel_pt_log("Scanning for full IP\n"); 2640 err = intel_pt_walk_to_ip(decoder); 2641 if (err) 2642 return err; 2643 2644 /* In hop mode, resample to get the to_ip as an "instruction" sample */ 2645 if (decoder->hop) 2646 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 2647 else 2648 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2649 2650 decoder->state.from_ip = 0; 2651 decoder->state.to_ip = decoder->ip; 2652 intel_pt_log_to("Setting IP", decoder->ip); 2653 2654 return 0; 2655} 2656 2657static int intel_pt_part_psb(struct intel_pt_decoder *decoder) 2658{ 2659 const unsigned char *end = decoder->buf + decoder->len; 2660 size_t i; 2661 2662 for (i = INTEL_PT_PSB_LEN - 1; i; i--) { 2663 if (i > decoder->len) 2664 continue; 2665 if (!memcmp(end - i, INTEL_PT_PSB_STR, i)) 2666 return i; 2667 } 2668 return 0; 2669} 2670 2671static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb) 2672{ 2673 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb; 2674 const char *psb = INTEL_PT_PSB_STR; 2675 2676 if (rest_psb > decoder->len || 2677 memcmp(decoder->buf, psb + part_psb, rest_psb)) 2678 return 0; 2679 2680 return rest_psb; 2681} 2682 2683static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder, 2684 int part_psb) 2685{ 2686 int rest_psb, ret; 2687 2688 decoder->pos += decoder->len; 2689 decoder->len = 0; 2690 2691 ret = intel_pt_get_next_data(decoder, false); 2692 if (ret) 2693 return ret; 2694 2695 rest_psb = intel_pt_rest_psb(decoder, part_psb); 2696 if (!rest_psb) 2697 return 0; 2698 2699 decoder->pos -= part_psb; 2700 decoder->next_buf = decoder->buf + rest_psb; 2701 decoder->next_len = decoder->len - rest_psb; 2702 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 2703 decoder->buf = decoder->temp_buf; 2704 decoder->len = INTEL_PT_PSB_LEN; 2705 2706 return 0; 2707} 2708 2709static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder) 2710{ 2711 unsigned char *next; 2712 int ret; 2713 2714 intel_pt_log("Scanning for PSB\n"); 2715 while (1) { 2716 if (!decoder->len) { 2717 ret = intel_pt_get_next_data(decoder, false); 2718 if (ret) 2719 return ret; 2720 } 2721 2722 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR, 2723 INTEL_PT_PSB_LEN); 2724 if (!next) { 2725 int part_psb; 2726 2727 part_psb = intel_pt_part_psb(decoder); 2728 if (part_psb) { 2729 ret = intel_pt_get_split_psb(decoder, part_psb); 2730 if (ret) 2731 return ret; 2732 } else { 2733 decoder->pos += decoder->len; 2734 decoder->len = 0; 2735 } 2736 continue; 2737 } 2738 2739 decoder->pkt_step = next - decoder->buf; 2740 return intel_pt_get_next_packet(decoder); 2741 } 2742} 2743 2744static int intel_pt_sync(struct intel_pt_decoder *decoder) 2745{ 2746 int err; 2747 2748 decoder->pge = false; 2749 decoder->continuous_period = false; 2750 decoder->have_last_ip = false; 2751 decoder->last_ip = 0; 2752 decoder->ip = 0; 2753 intel_pt_clear_stack(&decoder->stack); 2754 2755leap: 2756 err = intel_pt_scan_for_psb(decoder); 2757 if (err) 2758 return err; 2759 2760 decoder->have_last_ip = true; 2761 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2762 2763 err = intel_pt_walk_psb(decoder); 2764 if (err) 2765 return err; 2766 2767 if (decoder->ip) { 2768 decoder->state.type = 0; /* Do not have a sample */ 2769 /* 2770 * In hop mode, resample to get the PSB FUP ip as an 2771 * "instruction" sample. 2772 */ 2773 if (decoder->hop) 2774 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 2775 else 2776 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2777 } else if (decoder->leap) { 2778 /* 2779 * In leap mode, only PSB+ is decoded, so keeping leaping to the 2780 * next PSB until there is an ip. 2781 */ 2782 goto leap; 2783 } else { 2784 return intel_pt_sync_ip(decoder); 2785 } 2786 2787 return 0; 2788} 2789 2790static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder) 2791{ 2792 uint64_t est = decoder->sample_insn_cnt << 1; 2793 2794 if (!decoder->cbr || !decoder->max_non_turbo_ratio) 2795 goto out; 2796 2797 est *= decoder->max_non_turbo_ratio; 2798 est /= decoder->cbr; 2799out: 2800 return decoder->sample_timestamp + est; 2801} 2802 2803const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder) 2804{ 2805 int err; 2806 2807 do { 2808 decoder->state.type = INTEL_PT_BRANCH; 2809 decoder->state.flags = 0; 2810 2811 switch (decoder->pkt_state) { 2812 case INTEL_PT_STATE_NO_PSB: 2813 err = intel_pt_sync(decoder); 2814 break; 2815 case INTEL_PT_STATE_NO_IP: 2816 decoder->have_last_ip = false; 2817 decoder->last_ip = 0; 2818 decoder->ip = 0; 2819 __fallthrough; 2820 case INTEL_PT_STATE_ERR_RESYNC: 2821 err = intel_pt_sync_ip(decoder); 2822 break; 2823 case INTEL_PT_STATE_IN_SYNC: 2824 err = intel_pt_walk_trace(decoder); 2825 break; 2826 case INTEL_PT_STATE_TNT: 2827 case INTEL_PT_STATE_TNT_CONT: 2828 err = intel_pt_walk_tnt(decoder); 2829 if (err == -EAGAIN) 2830 err = intel_pt_walk_trace(decoder); 2831 break; 2832 case INTEL_PT_STATE_TIP: 2833 case INTEL_PT_STATE_TIP_PGD: 2834 err = intel_pt_walk_tip(decoder); 2835 break; 2836 case INTEL_PT_STATE_FUP: 2837 err = intel_pt_walk_fup(decoder); 2838 if (err == -EAGAIN) 2839 err = intel_pt_walk_fup_tip(decoder); 2840 break; 2841 case INTEL_PT_STATE_FUP_NO_TIP: 2842 err = intel_pt_walk_fup(decoder); 2843 if (err == -EAGAIN) 2844 err = intel_pt_walk_trace(decoder); 2845 break; 2846 case INTEL_PT_STATE_RESAMPLE: 2847 err = intel_pt_resample(decoder); 2848 break; 2849 default: 2850 err = intel_pt_bug(decoder); 2851 break; 2852 } 2853 } while (err == -ENOLINK); 2854 2855 if (err) { 2856 decoder->state.err = intel_pt_ext_err(err); 2857 if (err != -EOVERFLOW) 2858 decoder->state.from_ip = decoder->ip; 2859 intel_pt_update_sample_time(decoder); 2860 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; 2861 } else { 2862 decoder->state.err = 0; 2863 if (decoder->cbr != decoder->cbr_seen) { 2864 decoder->cbr_seen = decoder->cbr; 2865 if (!decoder->state.type) { 2866 decoder->state.from_ip = decoder->ip; 2867 decoder->state.to_ip = 0; 2868 } 2869 decoder->state.type |= INTEL_PT_CBR_CHG; 2870 decoder->state.cbr_payload = decoder->cbr_payload; 2871 decoder->state.cbr = decoder->cbr; 2872 } 2873 if (intel_pt_sample_time(decoder->pkt_state)) { 2874 intel_pt_update_sample_time(decoder); 2875 if (decoder->sample_cyc) { 2876 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; 2877 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; 2878 decoder->sample_cyc = false; 2879 } 2880 } 2881 /* 2882 * When using only TSC/MTC to compute cycles, IPC can be 2883 * sampled as soon as the cycle count changes. 2884 */ 2885 if (!decoder->have_cyc) 2886 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; 2887 } 2888 2889 decoder->state.timestamp = decoder->sample_timestamp; 2890 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder); 2891 decoder->state.cr3 = decoder->cr3; 2892 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt; 2893 decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt; 2894 2895 return &decoder->state; 2896} 2897 2898/** 2899 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet. 2900 * @buf: pointer to buffer pointer 2901 * @len: size of buffer 2902 * 2903 * Updates the buffer pointer to point to the start of the next PSB packet if 2904 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated, 2905 * @len is adjusted accordingly. 2906 * 2907 * Return: %true if a PSB packet is found, %false otherwise. 2908 */ 2909static bool intel_pt_next_psb(unsigned char **buf, size_t *len) 2910{ 2911 unsigned char *next; 2912 2913 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 2914 if (next) { 2915 *len -= next - *buf; 2916 *buf = next; 2917 return true; 2918 } 2919 return false; 2920} 2921 2922/** 2923 * intel_pt_step_psb - move buffer pointer to the start of the following PSB 2924 * packet. 2925 * @buf: pointer to buffer pointer 2926 * @len: size of buffer 2927 * 2928 * Updates the buffer pointer to point to the start of the following PSB packet 2929 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer 2930 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly. 2931 * 2932 * Return: %true if a PSB packet is found, %false otherwise. 2933 */ 2934static bool intel_pt_step_psb(unsigned char **buf, size_t *len) 2935{ 2936 unsigned char *next; 2937 2938 if (!*len) 2939 return false; 2940 2941 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 2942 if (next) { 2943 *len -= next - *buf; 2944 *buf = next; 2945 return true; 2946 } 2947 return false; 2948} 2949 2950/** 2951 * intel_pt_last_psb - find the last PSB packet in a buffer. 2952 * @buf: buffer 2953 * @len: size of buffer 2954 * 2955 * This function finds the last PSB in a buffer. 2956 * 2957 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise. 2958 */ 2959static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len) 2960{ 2961 const char *n = INTEL_PT_PSB_STR; 2962 unsigned char *p; 2963 size_t k; 2964 2965 if (len < INTEL_PT_PSB_LEN) 2966 return NULL; 2967 2968 k = len - INTEL_PT_PSB_LEN + 1; 2969 while (1) { 2970 p = memrchr(buf, n[0], k); 2971 if (!p) 2972 return NULL; 2973 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1)) 2974 return p; 2975 k = p - buf; 2976 if (!k) 2977 return NULL; 2978 } 2979} 2980 2981/** 2982 * intel_pt_next_tsc - find and return next TSC. 2983 * @buf: buffer 2984 * @len: size of buffer 2985 * @tsc: TSC value returned 2986 * @rem: returns remaining size when TSC is found 2987 * 2988 * Find a TSC packet in @buf and return the TSC value. This function assumes 2989 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a 2990 * PSBEND packet is found. 2991 * 2992 * Return: %true if TSC is found, false otherwise. 2993 */ 2994static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc, 2995 size_t *rem) 2996{ 2997 enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX; 2998 struct intel_pt_pkt packet; 2999 int ret; 3000 3001 while (len) { 3002 ret = intel_pt_get_packet(buf, len, &packet, &ctx); 3003 if (ret <= 0) 3004 return false; 3005 if (packet.type == INTEL_PT_TSC) { 3006 *tsc = packet.payload; 3007 *rem = len; 3008 return true; 3009 } 3010 if (packet.type == INTEL_PT_PSBEND) 3011 return false; 3012 buf += ret; 3013 len -= ret; 3014 } 3015 return false; 3016} 3017 3018/** 3019 * intel_pt_tsc_cmp - compare 7-byte TSCs. 3020 * @tsc1: first TSC to compare 3021 * @tsc2: second TSC to compare 3022 * 3023 * This function compares 7-byte TSC values allowing for the possibility that 3024 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped 3025 * around so for that purpose this function assumes the absolute difference is 3026 * less than half the maximum difference. 3027 * 3028 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is 3029 * after @tsc2. 3030 */ 3031static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2) 3032{ 3033 const uint64_t halfway = (1ULL << 55); 3034 3035 if (tsc1 == tsc2) 3036 return 0; 3037 3038 if (tsc1 < tsc2) { 3039 if (tsc2 - tsc1 < halfway) 3040 return -1; 3041 else 3042 return 1; 3043 } else { 3044 if (tsc1 - tsc2 < halfway) 3045 return 1; 3046 else 3047 return -1; 3048 } 3049} 3050 3051#define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1) 3052 3053/** 3054 * adj_for_padding - adjust overlap to account for padding. 3055 * @buf_b: second buffer 3056 * @buf_a: first buffer 3057 * @len_a: size of first buffer 3058 * 3059 * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap 3060 * accordingly. 3061 * 3062 * Return: A pointer into @buf_b from where non-overlapped data starts 3063 */ 3064static unsigned char *adj_for_padding(unsigned char *buf_b, 3065 unsigned char *buf_a, size_t len_a) 3066{ 3067 unsigned char *p = buf_b - MAX_PADDING; 3068 unsigned char *q = buf_a + len_a - MAX_PADDING; 3069 int i; 3070 3071 for (i = MAX_PADDING; i; i--, p++, q++) { 3072 if (*p != *q) 3073 break; 3074 } 3075 3076 return p; 3077} 3078 3079/** 3080 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data 3081 * using TSC. 3082 * @buf_a: first buffer 3083 * @len_a: size of first buffer 3084 * @buf_b: second buffer 3085 * @len_b: size of second buffer 3086 * @consecutive: returns true if there is data in buf_b that is consecutive 3087 * to buf_a 3088 * 3089 * If the trace contains TSC we can look at the last TSC of @buf_a and the 3090 * first TSC of @buf_b in order to determine if the buffers overlap, and then 3091 * walk forward in @buf_b until a later TSC is found. A precondition is that 3092 * @buf_a and @buf_b are positioned at a PSB. 3093 * 3094 * Return: A pointer into @buf_b from where non-overlapped data starts, or 3095 * @buf_b + @len_b if there is no non-overlapped data. 3096 */ 3097static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a, 3098 size_t len_a, 3099 unsigned char *buf_b, 3100 size_t len_b, bool *consecutive) 3101{ 3102 uint64_t tsc_a, tsc_b; 3103 unsigned char *p; 3104 size_t len, rem_a, rem_b; 3105 3106 p = intel_pt_last_psb(buf_a, len_a); 3107 if (!p) 3108 return buf_b; /* No PSB in buf_a => no overlap */ 3109 3110 len = len_a - (p - buf_a); 3111 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) { 3112 /* The last PSB+ in buf_a is incomplete, so go back one more */ 3113 len_a -= len; 3114 p = intel_pt_last_psb(buf_a, len_a); 3115 if (!p) 3116 return buf_b; /* No full PSB+ => assume no overlap */ 3117 len = len_a - (p - buf_a); 3118 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) 3119 return buf_b; /* No TSC in buf_a => assume no overlap */ 3120 } 3121 3122 while (1) { 3123 /* Ignore PSB+ with no TSC */ 3124 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) { 3125 int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b); 3126 3127 /* Same TSC, so buffers are consecutive */ 3128 if (!cmp && rem_b >= rem_a) { 3129 unsigned char *start; 3130 3131 *consecutive = true; 3132 start = buf_b + len_b - (rem_b - rem_a); 3133 return adj_for_padding(start, buf_a, len_a); 3134 } 3135 if (cmp < 0) 3136 return buf_b; /* tsc_a < tsc_b => no overlap */ 3137 } 3138 3139 if (!intel_pt_step_psb(&buf_b, &len_b)) 3140 return buf_b + len_b; /* No PSB in buf_b => no data */ 3141 } 3142} 3143 3144/** 3145 * intel_pt_find_overlap - determine start of non-overlapped trace data. 3146 * @buf_a: first buffer 3147 * @len_a: size of first buffer 3148 * @buf_b: second buffer 3149 * @len_b: size of second buffer 3150 * @have_tsc: can use TSC packets to detect overlap 3151 * @consecutive: returns true if there is data in buf_b that is consecutive 3152 * to buf_a 3153 * 3154 * When trace samples or snapshots are recorded there is the possibility that 3155 * the data overlaps. Note that, for the purposes of decoding, data is only 3156 * useful if it begins with a PSB packet. 3157 * 3158 * Return: A pointer into @buf_b from where non-overlapped data starts, or 3159 * @buf_b + @len_b if there is no non-overlapped data. 3160 */ 3161unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a, 3162 unsigned char *buf_b, size_t len_b, 3163 bool have_tsc, bool *consecutive) 3164{ 3165 unsigned char *found; 3166 3167 /* Buffer 'b' must start at PSB so throw away everything before that */ 3168 if (!intel_pt_next_psb(&buf_b, &len_b)) 3169 return buf_b + len_b; /* No PSB */ 3170 3171 if (!intel_pt_next_psb(&buf_a, &len_a)) 3172 return buf_b; /* No overlap */ 3173 3174 if (have_tsc) { 3175 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b, 3176 consecutive); 3177 if (found) 3178 return found; 3179 } 3180 3181 /* 3182 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes, 3183 * we can ignore the first part of buffer 'a'. 3184 */ 3185 while (len_b < len_a) { 3186 if (!intel_pt_step_psb(&buf_a, &len_a)) 3187 return buf_b; /* No overlap */ 3188 } 3189 3190 /* Now len_b >= len_a */ 3191 while (1) { 3192 /* Potential overlap so check the bytes */ 3193 found = memmem(buf_a, len_a, buf_b, len_a); 3194 if (found) { 3195 *consecutive = true; 3196 return adj_for_padding(buf_b + len_a, buf_a, len_a); 3197 } 3198 3199 /* Try again at next PSB in buffer 'a' */ 3200 if (!intel_pt_step_psb(&buf_a, &len_a)) 3201 return buf_b; /* No overlap */ 3202 } 3203} 3204 3205/** 3206 * struct fast_forward_data - data used by intel_pt_ff_cb(). 3207 * @timestamp: timestamp to fast forward towards 3208 * @buf_timestamp: buffer timestamp of last buffer with trace data earlier than 3209 * the fast forward timestamp. 3210 */ 3211struct fast_forward_data { 3212 uint64_t timestamp; 3213 uint64_t buf_timestamp; 3214}; 3215 3216/** 3217 * intel_pt_ff_cb - fast forward lookahead callback. 3218 * @buffer: Intel PT trace buffer 3219 * @data: opaque pointer to fast forward data (struct fast_forward_data) 3220 * 3221 * Determine if @buffer trace is past the fast forward timestamp. 3222 * 3223 * Return: 1 (stop lookahead) if @buffer trace is past the fast forward 3224 * timestamp, and 0 otherwise. 3225 */ 3226static int intel_pt_ff_cb(struct intel_pt_buffer *buffer, void *data) 3227{ 3228 struct fast_forward_data *d = data; 3229 unsigned char *buf; 3230 uint64_t tsc; 3231 size_t rem; 3232 size_t len; 3233 3234 buf = (unsigned char *)buffer->buf; 3235 len = buffer->len; 3236 3237 if (!intel_pt_next_psb(&buf, &len) || 3238 !intel_pt_next_tsc(buf, len, &tsc, &rem)) 3239 return 0; 3240 3241 tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp); 3242 3243 intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n", 3244 tsc, buffer->ref_timestamp); 3245 3246 /* 3247 * If the buffer contains a timestamp earlier that the fast forward 3248 * timestamp, then record it, else stop. 3249 */ 3250 if (tsc < d->timestamp) 3251 d->buf_timestamp = buffer->ref_timestamp; 3252 else 3253 return 1; 3254 3255 return 0; 3256} 3257 3258/** 3259 * intel_pt_fast_forward - reposition decoder forwards. 3260 * @decoder: Intel PT decoder 3261 * @timestamp: timestamp to fast forward towards 3262 * 3263 * Reposition decoder at the last PSB with a timestamp earlier than @timestamp. 3264 * 3265 * Return: 0 on success or negative error code on failure. 3266 */ 3267int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp) 3268{ 3269 struct fast_forward_data d = { .timestamp = timestamp }; 3270 unsigned char *buf; 3271 size_t len; 3272 int err; 3273 3274 intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp); 3275 3276 /* Find buffer timestamp of buffer to fast forward to */ 3277 err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d); 3278 if (err < 0) 3279 return err; 3280 3281 /* Walk to buffer with same buffer timestamp */ 3282 if (d.buf_timestamp) { 3283 do { 3284 decoder->pos += decoder->len; 3285 decoder->len = 0; 3286 err = intel_pt_get_next_data(decoder, true); 3287 /* -ENOLINK means non-consecutive trace */ 3288 if (err && err != -ENOLINK) 3289 return err; 3290 } while (decoder->buf_timestamp != d.buf_timestamp); 3291 } 3292 3293 if (!decoder->buf) 3294 return 0; 3295 3296 buf = (unsigned char *)decoder->buf; 3297 len = decoder->len; 3298 3299 if (!intel_pt_next_psb(&buf, &len)) 3300 return 0; 3301 3302 /* 3303 * Walk PSBs while the PSB timestamp is less than the fast forward 3304 * timestamp. 3305 */ 3306 do { 3307 uint64_t tsc; 3308 size_t rem; 3309 3310 if (!intel_pt_next_tsc(buf, len, &tsc, &rem)) 3311 break; 3312 tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp); 3313 /* 3314 * A TSC packet can slip past MTC packets but, after fast 3315 * forward, decoding starts at the TSC timestamp. That means 3316 * the timestamps may not be exactly the same as the timestamps 3317 * that would have been decoded without fast forward. 3318 */ 3319 if (tsc < timestamp) { 3320 intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc); 3321 decoder->pos += decoder->len - len; 3322 decoder->buf = buf; 3323 decoder->len = len; 3324 intel_pt_reposition(decoder); 3325 } else { 3326 break; 3327 } 3328 } while (intel_pt_step_psb(&buf, &len)); 3329 3330 return 0; 3331} 3332