Lines Matching refs:ts
118 static int thread_stack__grow(struct thread_stack *ts)
123 new_sz = ts->sz + STACK_GROWTH;
126 new_stack = realloc(ts->stack, sz);
130 ts->stack = new_stack;
131 ts->sz = new_sz;
136 static int thread_stack__init(struct thread_stack *ts, struct thread *thread,
143 err = thread_stack__grow(ts);
152 ts->br_stack_rb = zalloc(sz);
153 if (!ts->br_stack_rb)
155 ts->br_stack_sz = br_stack_sz;
162 ts->kernel_start = machine__kernel_start(machine);
164 ts->rstate = X86_RETPOLINE_POSSIBLE;
166 ts->kernel_start = 1ULL << 63;
168 ts->crp = crp;
178 struct thread_stack *ts = thread->ts, *new_ts;
179 unsigned int old_sz = ts ? ts->arr_sz : 0;
185 if (!ts || new_sz > old_sz) {
186 new_ts = calloc(new_sz, sizeof(*ts));
189 if (ts)
190 memcpy(new_ts, ts, old_sz * sizeof(*ts));
192 zfree(&thread->ts);
193 thread->ts = new_ts;
194 ts = new_ts;
198 (unsigned int)cpu < ts->arr_sz)
199 ts += cpu;
201 if (!ts->stack &&
202 thread_stack__init(ts, thread, crp, callstack, br_stack_sz))
205 return ts;
210 struct thread_stack *ts = thread->ts;
215 if (!ts || (unsigned int)cpu >= ts->arr_sz)
218 ts += cpu;
220 if (!ts->stack)
223 return ts;
235 return thread->ts;
238 static int thread_stack__push(struct thread_stack *ts, u64 ret_addr,
243 if (ts->cnt == ts->sz) {
244 err = thread_stack__grow(ts);
247 ts->cnt = 0;
251 ts->stack[ts->cnt].trace_end = trace_end;
252 ts->stack[ts->cnt++].ret_addr = ret_addr;
257 static void thread_stack__pop(struct thread_stack *ts, u64 ret_addr)
270 for (i = ts->cnt; i; ) {
271 if (ts->stack[--i].ret_addr == ret_addr) {
272 ts->cnt = i;
278 static void thread_stack__pop_trace_end(struct thread_stack *ts)
282 for (i = ts->cnt; i; ) {
283 if (ts->stack[--i].trace_end)
284 ts->cnt = i;
290 static bool thread_stack__in_kernel(struct thread_stack *ts)
292 if (!ts->cnt)
295 return ts->stack[ts->cnt - 1].cp->in_kernel;
299 struct thread_stack *ts, size_t idx,
302 struct call_return_processor *crp = ts->crp;
306 .comm = ts->comm,
311 tse = &ts->stack[idx];
315 cr.branch_count = ts->branch_count - tse->branch_count;
316 cr.insn_count = ts->insn_count - tse->insn_count;
317 cr.cyc_count = ts->cyc_count - tse->cyc_count;
338 static int __thread_stack__flush(struct thread *thread, struct thread_stack *ts)
340 struct call_return_processor *crp = ts->crp;
344 ts->cnt = 0;
345 ts->br_stack_pos = 0;
346 if (ts->br_stack_rb)
347 ts->br_stack_rb->nr = 0;
351 while (ts->cnt) {
352 err = thread_stack__call_return(thread, ts, --ts->cnt,
353 ts->last_time, 0, true);
356 ts->cnt = 0;
366 struct thread_stack *ts = thread->ts;
370 if (ts) {
371 for (pos = 0; pos < ts->arr_sz; pos++) {
372 int ret = __thread_stack__flush(thread, ts + pos);
382 static void thread_stack__update_br_stack(struct thread_stack *ts, u32 flags,
385 struct branch_stack *bs = ts->br_stack_rb;
388 if (!ts->br_stack_pos)
389 ts->br_stack_pos = ts->br_stack_sz;
391 ts->br_stack_pos -= 1;
393 be = &bs->entries[ts->br_stack_pos];
400 be->flags.mispred = ts->mispred_all;
402 if (bs->nr < ts->br_stack_sz)
410 struct thread_stack *ts = thread__stack(thread, cpu);
415 if (!ts) {
416 ts = thread_stack__new(thread, cpu, NULL, callstack, br_stack_sz);
417 if (!ts) {
421 ts->trace_nr = trace_nr;
422 ts->mispred_all = mispred_all;
430 if (trace_nr != ts->trace_nr) {
431 if (ts->trace_nr)
432 __thread_stack__flush(thread, ts);
433 ts->trace_nr = trace_nr;
437 thread_stack__update_br_stack(ts, flags, from_ip, to_ip);
443 if (ts->crp || !callstack)
454 return thread_stack__push(ts, ret_addr,
464 thread_stack__pop(ts, to_ip);
465 thread_stack__pop_trace_end(ts);
467 thread_stack__pop(ts, to_ip);
475 struct thread_stack *ts = thread__stack(thread, cpu);
477 if (!ts)
480 if (trace_nr != ts->trace_nr) {
481 if (ts->trace_nr)
482 __thread_stack__flush(thread, ts);
483 ts->trace_nr = trace_nr;
487 static void __thread_stack__free(struct thread *thread, struct thread_stack *ts)
489 __thread_stack__flush(thread, ts);
490 zfree(&ts->stack);
491 zfree(&ts->br_stack_rb);
494 static void thread_stack__reset(struct thread *thread, struct thread_stack *ts)
496 unsigned int arr_sz = ts->arr_sz;
498 __thread_stack__free(thread, ts);
499 memset(ts, 0, sizeof(*ts));
500 ts->arr_sz = arr_sz;
505 struct thread_stack *ts = thread->ts;
508 if (ts) {
509 for (pos = 0; pos < ts->arr_sz; pos++)
510 __thread_stack__free(thread, ts + pos);
511 zfree(&thread->ts);
524 struct thread_stack *ts = thread__stack(thread, cpu);
537 if (!ts) {
544 for (i = 2, j = 1; i < sz && j <= ts->cnt; i++, j++) {
545 ip = ts->stack[ts->cnt - j].ret_addr;
567 struct thread_stack *ts = thread__stack(thread, cpu);
577 if (!ts)
585 for (j = 1; j <= ts->cnt; j++) {
586 ip = ts->stack[ts->cnt - j].ret_addr;
595 for (; nr < sz && j <= ts->cnt; nr++, j++) {
596 ip = ts->stack[ts->cnt - j].ret_addr;
619 struct thread_stack *ts = thread__stack(thread, cpu);
627 if (!ts)
630 src = ts->br_stack_rb;
637 nr = min(ts->br_stack_sz - ts->br_stack_pos, (unsigned int)dst->nr);
638 memcpy(be, &src->entries[ts->br_stack_pos], bsz * nr);
640 if (src->nr >= ts->br_stack_sz) {
643 nr = min(ts->br_stack_pos, sz);
644 memcpy(be, &src->entries[0], bsz * ts->br_stack_pos);
681 struct thread_stack *ts = thread__stack(thread, cpu);
689 if (!ts)
692 src = ts->br_stack_rb;
696 spos = &src->entries[ts->br_stack_pos];
697 ssz = &src->entries[ts->br_stack_sz];
714 if (src->nr >= ts->br_stack_sz) {
738 if (src->nr >= ts->br_stack_sz) {
781 static int thread_stack__push_cp(struct thread_stack *ts, u64 ret_addr,
791 if (ts->cnt == ts->sz) {
792 err = thread_stack__grow(ts);
797 tse = &ts->stack[ts->cnt++];
801 tse->branch_count = ts->branch_count;
802 tse->insn_count = ts->insn_count;
803 tse->cyc_count = ts->cyc_count;
813 static int thread_stack__pop_cp(struct thread *thread, struct thread_stack *ts,
819 if (!ts->cnt)
822 if (ts->cnt == 1) {
823 struct thread_stack_entry *tse = &ts->stack[0];
826 return thread_stack__call_return(thread, ts, --ts->cnt,
830 if (ts->stack[ts->cnt - 1].ret_addr == ret_addr &&
831 !ts->stack[ts->cnt - 1].non_call) {
832 return thread_stack__call_return(thread, ts, --ts->cnt,
835 size_t i = ts->cnt - 1;
838 if (ts->stack[i].ret_addr != ret_addr ||
839 ts->stack[i].non_call)
842 while (ts->cnt > i) {
843 err = thread_stack__call_return(thread, ts,
844 --ts->cnt,
850 return thread_stack__call_return(thread, ts, --ts->cnt,
858 static int thread_stack__bottom(struct thread_stack *ts,
863 struct call_path_root *cpr = ts->crp->cpr;
879 ts->kernel_start);
881 return thread_stack__push_cp(ts, ip, sample->time, ref, cp,
885 static int thread_stack__pop_ks(struct thread *thread, struct thread_stack *ts,
892 while (thread_stack__in_kernel(ts)) {
893 err = thread_stack__call_return(thread, ts, --ts->cnt,
903 struct thread_stack *ts,
908 struct call_path_root *cpr = ts->crp->cpr;
913 u64 ks = ts->kernel_start;
921 err = thread_stack__pop_ks(thread, ts, sample, ref);
926 if (!ts->cnt) {
928 return thread_stack__push_cp(ts, 0, tm, ref, cp, true,
931 } else if (thread_stack__in_kernel(ts) && ip < ks) {
933 err = thread_stack__pop_ks(thread, ts, sample, ref);
938 if (ts->cnt)
939 parent = ts->stack[ts->cnt - 1].cp;
949 if (ts->cnt == 1) {
950 err = thread_stack__call_return(thread, ts, --ts->cnt,
956 if (!ts->cnt) {
959 return thread_stack__push_cp(ts, addr, tm, ref, cp,
969 err = thread_stack__push_cp(ts, 0, tm, ref, cp, true, false);
971 ts->stack[ts->cnt - 1].non_call = true;
983 err = thread_stack__push_cp(ts, addr, tm, ref, cp, true, false);
989 err = thread_stack__push_cp(ts, ip, tm, ref, cp, true, false);
993 return thread_stack__call_return(thread, ts, --ts->cnt, tm, ref, false);
997 struct thread_stack *ts, u64 timestamp,
1003 if (!ts->cnt)
1007 tse = &ts->stack[ts->cnt - 1];
1009 err = thread_stack__call_return(thread, ts, --ts->cnt,
1018 static int thread_stack__trace_end(struct thread_stack *ts,
1021 struct call_path_root *cpr = ts->crp->cpr;
1026 if (!ts->cnt || (ts->cnt == 1 && ts->stack[0].ref == ref))
1029 cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp, NULL, 0,
1030 ts->kernel_start);
1034 return thread_stack__push_cp(ts, ret_addr, sample->time, ref, cp,
1050 static int thread_stack__x86_retpoline(struct thread_stack *ts,
1054 struct thread_stack_entry *tse = &ts->stack[ts->cnt - 1];
1055 struct call_path_root *cpr = ts->crp->cpr;
1071 ts->cnt -= 1;
1072 sym = ts->stack[ts->cnt - 2].cp->sym;
1079 ts->cnt -= 1;
1087 ts->cnt -= 1;
1091 cp = call_path__findnew(cpr, ts->stack[ts->cnt - 2].cp, tsym,
1092 sample->addr, ts->kernel_start);
1097 ts->stack[ts->cnt - 1].cp = cp;
1108 struct thread_stack *ts = thread__stack(thread, sample->cpu);
1112 if (ts && !ts->crp) {
1114 thread_stack__reset(thread, ts);
1115 ts = NULL;
1118 if (!ts) {
1119 ts = thread_stack__new(thread, sample->cpu, crp, true, 0);
1120 if (!ts)
1122 ts->comm = comm;
1125 rstate = ts->rstate;
1127 ts->rstate = X86_RETPOLINE_POSSIBLE;
1130 if (ts->comm != comm && thread->pid_ == thread->tid) {
1131 err = __thread_stack__flush(thread, ts);
1134 ts->comm = comm;
1138 if (!ts->cnt) {
1139 err = thread_stack__bottom(ts, sample, from_al, to_al, ref);
1144 ts->branch_count += 1;
1145 ts->insn_count += sample->insn_cnt;
1146 ts->cyc_count += sample->cyc_cnt;
1147 ts->last_time = sample->time;
1151 struct call_path_root *cpr = ts->crp->cpr;
1162 cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp,
1164 ts->kernel_start);
1165 err = thread_stack__push_cp(ts, ret_addr, sample->time, ref,
1175 ts->rstate = X86_RETPOLINE_DETECTED;
1186 return thread_stack__pop_ks(thread, ts, sample, ref);
1193 if (rstate == X86_RETPOLINE_DETECTED && ts->cnt > 2 &&
1194 ts->stack[ts->cnt - 1].ret_addr != sample->addr)
1195 return thread_stack__x86_retpoline(ts, sample, to_al);
1197 err = thread_stack__pop_cp(thread, ts, sample->addr,
1202 err = thread_stack__no_call_return(thread, ts, sample,
1206 err = thread_stack__trace_begin(thread, ts, sample->time, ref);
1208 err = thread_stack__trace_end(ts, sample, ref);
1212 struct call_path_root *cpr = ts->crp->cpr;
1221 cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp,
1223 ts->kernel_start);
1224 err = thread_stack__push_cp(ts, 0, sample->time, ref, cp, false,
1227 ts->stack[ts->cnt - 1].non_call = true;
1235 struct thread_stack *ts = thread__stack(thread, cpu);
1237 if (!ts)
1239 return ts->cnt;