1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright 2014, Michael Ellerman, IBM Corp.
4 */
5
6#define _GNU_SOURCE	/* For CPU_ZERO etc. */
7
8#include <sched.h>
9#include <sys/wait.h>
10#include <setjmp.h>
11#include <signal.h>
12#include <stdio.h>
13#include <stdlib.h>
14#include <string.h>
15#include <sys/ioctl.h>
16
17#include "trace.h"
18#include "ebb.h"
19
20
21void (*ebb_user_func)(void);
22
23void ebb_hook(void)
24{
25	if (ebb_user_func)
26		ebb_user_func();
27}
28
29struct ebb_state ebb_state;
30
31u64 sample_period = 0x40000000ull;
32
33void reset_ebb_with_clear_mask(unsigned long mmcr0_clear_mask)
34{
35	u64 val;
36
37	/* 2) clear MMCR0[PMAO] - docs say BESCR[PMEO] should do this */
38	/* 3) set MMCR0[PMAE]	- docs say BESCR[PME] should do this */
39	val = mfspr(SPRN_MMCR0);
40	mtspr(SPRN_MMCR0, (val & ~mmcr0_clear_mask) | MMCR0_PMAE);
41
42	/* 4) clear BESCR[PMEO] */
43	mtspr(SPRN_BESCRR, BESCR_PMEO);
44
45	/* 5) set BESCR[PME] */
46	mtspr(SPRN_BESCRS, BESCR_PME);
47
48	/* 6) rfebb 1 - done in our caller */
49}
50
51void reset_ebb(void)
52{
53	reset_ebb_with_clear_mask(MMCR0_PMAO | MMCR0_FC);
54}
55
56/* Called outside of the EBB handler to check MMCR0 is sane */
57int ebb_check_mmcr0(void)
58{
59	u64 val;
60
61	val = mfspr(SPRN_MMCR0);
62	if ((val & (MMCR0_FC | MMCR0_PMAO)) == MMCR0_FC) {
63		/* It's OK if we see FC & PMAO, but not FC by itself */
64		printf("Outside of loop, only FC set 0x%llx\n", val);
65		return 1;
66	}
67
68	return 0;
69}
70
71bool ebb_check_count(int pmc, u64 sample_period, int fudge)
72{
73	u64 count, upper, lower;
74
75	count = ebb_state.stats.pmc_count[PMC_INDEX(pmc)];
76
77	lower = ebb_state.stats.ebb_count * (sample_period - fudge);
78
79	if (count < lower) {
80		printf("PMC%d count (0x%llx) below lower limit 0x%llx (-0x%llx)\n",
81			pmc, count, lower, lower - count);
82		return false;
83	}
84
85	upper = ebb_state.stats.ebb_count * (sample_period + fudge);
86
87	if (count > upper) {
88		printf("PMC%d count (0x%llx) above upper limit 0x%llx (+0x%llx)\n",
89			pmc, count, upper, count - upper);
90		return false;
91	}
92
93	printf("PMC%d count (0x%llx) is between 0x%llx and 0x%llx delta +0x%llx/-0x%llx\n",
94		pmc, count, lower, upper, count - lower, upper - count);
95
96	return true;
97}
98
99void standard_ebb_callee(void)
100{
101	int found, i;
102	u64 val;
103
104	val = mfspr(SPRN_BESCR);
105	if (!(val & BESCR_PMEO)) {
106		ebb_state.stats.spurious++;
107		goto out;
108	}
109
110	ebb_state.stats.ebb_count++;
111	trace_log_counter(ebb_state.trace, ebb_state.stats.ebb_count);
112
113	val = mfspr(SPRN_MMCR0);
114	trace_log_reg(ebb_state.trace, SPRN_MMCR0, val);
115
116	found = 0;
117	for (i = 1; i <= 6; i++) {
118		if (ebb_state.pmc_enable[PMC_INDEX(i)])
119			found += count_pmc(i, sample_period);
120	}
121
122	if (!found)
123		ebb_state.stats.no_overflow++;
124
125out:
126	reset_ebb();
127}
128
129extern void ebb_handler(void);
130
131void setup_ebb_handler(void (*callee)(void))
132{
133	u64 entry;
134
135#if defined(_CALL_ELF) && _CALL_ELF == 2
136	entry = (u64)ebb_handler;
137#else
138	struct opd
139	{
140	    u64 entry;
141	    u64 toc;
142	} *opd;
143
144	opd = (struct opd *)ebb_handler;
145	entry = opd->entry;
146#endif
147	printf("EBB Handler is at %#llx\n", entry);
148
149	ebb_user_func = callee;
150
151	/* Ensure ebb_user_func is set before we set the handler */
152	mb();
153	mtspr(SPRN_EBBHR, entry);
154
155	/* Make sure the handler is set before we return */
156	mb();
157}
158
159void clear_ebb_stats(void)
160{
161	memset(&ebb_state.stats, 0, sizeof(ebb_state.stats));
162}
163
164void dump_summary_ebb_state(void)
165{
166	printf("ebb_state:\n"			\
167	       "  ebb_count    = %d\n"		\
168	       "  spurious     = %d\n"		\
169	       "  negative     = %d\n"		\
170	       "  no_overflow  = %d\n"		\
171	       "  pmc[1] count = 0x%llx\n"	\
172	       "  pmc[2] count = 0x%llx\n"	\
173	       "  pmc[3] count = 0x%llx\n"	\
174	       "  pmc[4] count = 0x%llx\n"	\
175	       "  pmc[5] count = 0x%llx\n"	\
176	       "  pmc[6] count = 0x%llx\n",
177		ebb_state.stats.ebb_count, ebb_state.stats.spurious,
178		ebb_state.stats.negative, ebb_state.stats.no_overflow,
179		ebb_state.stats.pmc_count[0], ebb_state.stats.pmc_count[1],
180		ebb_state.stats.pmc_count[2], ebb_state.stats.pmc_count[3],
181		ebb_state.stats.pmc_count[4], ebb_state.stats.pmc_count[5]);
182}
183
184static char *decode_mmcr0(u32 value)
185{
186	static char buf[16];
187
188	buf[0] = '\0';
189
190	if (value & (1 << 31))
191		strcat(buf, "FC ");
192	if (value & (1 << 26))
193		strcat(buf, "PMAE ");
194	if (value & (1 << 7))
195		strcat(buf, "PMAO ");
196
197	return buf;
198}
199
200static char *decode_bescr(u64 value)
201{
202	static char buf[16];
203
204	buf[0] = '\0';
205
206	if (value & (1ull << 63))
207		strcat(buf, "GE ");
208	if (value & (1ull << 32))
209		strcat(buf, "PMAE ");
210	if (value & 1)
211		strcat(buf, "PMAO ");
212
213	return buf;
214}
215
216void dump_ebb_hw_state(void)
217{
218	u64 bescr;
219	u32 mmcr0;
220
221	mmcr0 = mfspr(SPRN_MMCR0);
222	bescr = mfspr(SPRN_BESCR);
223
224	printf("HW state:\n"		\
225	       "MMCR0 0x%016x %s\n"	\
226	       "MMCR2 0x%016lx\n"	\
227	       "EBBHR 0x%016lx\n"	\
228	       "BESCR 0x%016llx %s\n"	\
229	       "PMC1  0x%016lx\n"	\
230	       "PMC2  0x%016lx\n"	\
231	       "PMC3  0x%016lx\n"	\
232	       "PMC4  0x%016lx\n"	\
233	       "PMC5  0x%016lx\n"	\
234	       "PMC6  0x%016lx\n"	\
235	       "SIAR  0x%016lx\n",
236	       mmcr0, decode_mmcr0(mmcr0), mfspr(SPRN_MMCR2),
237	       mfspr(SPRN_EBBHR), bescr, decode_bescr(bescr),
238	       mfspr(SPRN_PMC1), mfspr(SPRN_PMC2), mfspr(SPRN_PMC3),
239	       mfspr(SPRN_PMC4), mfspr(SPRN_PMC5), mfspr(SPRN_PMC6),
240	       mfspr(SPRN_SIAR));
241}
242
243void dump_ebb_state(void)
244{
245	dump_summary_ebb_state();
246
247	dump_ebb_hw_state();
248
249	trace_buffer_print(ebb_state.trace);
250}
251
252int count_pmc(int pmc, uint32_t sample_period)
253{
254	uint32_t start_value;
255	u64 val;
256
257	/* 0) Read PMC */
258	start_value = pmc_sample_period(sample_period);
259
260	val = read_pmc(pmc);
261	if (val < start_value)
262		ebb_state.stats.negative++;
263	else
264		ebb_state.stats.pmc_count[PMC_INDEX(pmc)] += val - start_value;
265
266	trace_log_reg(ebb_state.trace, SPRN_PMC1 + pmc - 1, val);
267
268	/* 1) Reset PMC */
269	write_pmc(pmc, start_value);
270
271	/* Report if we overflowed */
272	return val >= COUNTER_OVERFLOW;
273}
274
275int ebb_event_enable(struct event *e)
276{
277	int rc;
278
279	/* Ensure any SPR writes are ordered vs us */
280	mb();
281
282	rc = ioctl(e->fd, PERF_EVENT_IOC_ENABLE);
283	if (rc)
284		return rc;
285
286	rc = event_read(e);
287
288	/* Ditto */
289	mb();
290
291	return rc;
292}
293
294void ebb_freeze_pmcs(void)
295{
296	mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) | MMCR0_FC);
297	mb();
298}
299
300void ebb_unfreeze_pmcs(void)
301{
302	/* Unfreeze counters */
303	mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) & ~MMCR0_FC);
304	mb();
305}
306
307void ebb_global_enable(void)
308{
309	/* Enable EBBs globally and PMU EBBs */
310	mtspr(SPRN_BESCR, 0x8000000100000000ull);
311	mb();
312}
313
314void ebb_global_disable(void)
315{
316	/* Disable EBBs & freeze counters, events are still scheduled */
317	mtspr(SPRN_BESCRR, BESCR_PME);
318	mb();
319}
320
321bool ebb_is_supported(void)
322{
323#ifdef PPC_FEATURE2_EBB
324	/* EBB requires at least POWER8 */
325	return have_hwcap2(PPC_FEATURE2_EBB);
326#else
327	return false;
328#endif
329}
330
331void event_ebb_init(struct event *e)
332{
333	e->attr.config |= (1ull << 63);
334}
335
336void event_bhrb_init(struct event *e, unsigned ifm)
337{
338	e->attr.config |= (1ull << 62) | ((u64)ifm << 60);
339}
340
341void event_leader_ebb_init(struct event *e)
342{
343	event_ebb_init(e);
344
345	e->attr.exclusive = 1;
346	e->attr.pinned = 1;
347}
348
349int ebb_child(union pipe read_pipe, union pipe write_pipe)
350{
351	struct event event;
352	uint64_t val;
353
354	FAIL_IF(wait_for_parent(read_pipe));
355
356	event_init_named(&event, 0x1001e, "cycles");
357	event_leader_ebb_init(&event);
358
359	event.attr.exclude_kernel = 1;
360	event.attr.exclude_hv = 1;
361	event.attr.exclude_idle = 1;
362
363	FAIL_IF(event_open(&event));
364
365	ebb_enable_pmc_counting(1);
366	setup_ebb_handler(standard_ebb_callee);
367	ebb_global_enable();
368
369	FAIL_IF(event_enable(&event));
370
371	if (event_read(&event)) {
372		/*
373		 * Some tests expect to fail here, so don't report an error on
374		 * this line, and return a distinguisable error code. Tell the
375		 * parent an error happened.
376		 */
377		notify_parent_of_error(write_pipe);
378		return 2;
379	}
380
381	mtspr(SPRN_PMC1, pmc_sample_period(sample_period));
382
383	FAIL_IF(notify_parent(write_pipe));
384	FAIL_IF(wait_for_parent(read_pipe));
385	FAIL_IF(notify_parent(write_pipe));
386
387	while (ebb_state.stats.ebb_count < 20) {
388		FAIL_IF(core_busy_loop());
389
390		/* To try and hit SIGILL case */
391		val  = mfspr(SPRN_MMCRA);
392		val |= mfspr(SPRN_MMCR2);
393		val |= mfspr(SPRN_MMCR0);
394	}
395
396	ebb_global_disable();
397	ebb_freeze_pmcs();
398
399	dump_ebb_state();
400
401	event_close(&event);
402
403	FAIL_IF(ebb_state.stats.ebb_count == 0);
404
405	return 0;
406}
407
408static jmp_buf setjmp_env;
409
410static void sigill_handler(int signal)
411{
412	printf("Took sigill\n");
413	longjmp(setjmp_env, 1);
414}
415
416static struct sigaction sigill_action = {
417	.sa_handler = sigill_handler,
418};
419
420int catch_sigill(void (*func)(void))
421{
422	if (sigaction(SIGILL, &sigill_action, NULL)) {
423		perror("sigaction");
424		return 1;
425	}
426
427	if (setjmp(setjmp_env) == 0) {
428		func();
429		return 1;
430	}
431
432	return 0;
433}
434
435void write_pmc1(void)
436{
437	mtspr(SPRN_PMC1, 0);
438}
439
440void write_pmc(int pmc, u64 value)
441{
442	switch (pmc) {
443		case 1: mtspr(SPRN_PMC1, value); break;
444		case 2: mtspr(SPRN_PMC2, value); break;
445		case 3: mtspr(SPRN_PMC3, value); break;
446		case 4: mtspr(SPRN_PMC4, value); break;
447		case 5: mtspr(SPRN_PMC5, value); break;
448		case 6: mtspr(SPRN_PMC6, value); break;
449	}
450}
451
452u64 read_pmc(int pmc)
453{
454	switch (pmc) {
455		case 1: return mfspr(SPRN_PMC1);
456		case 2: return mfspr(SPRN_PMC2);
457		case 3: return mfspr(SPRN_PMC3);
458		case 4: return mfspr(SPRN_PMC4);
459		case 5: return mfspr(SPRN_PMC5);
460		case 6: return mfspr(SPRN_PMC6);
461	}
462
463	return 0;
464}
465
466static void term_handler(int signal)
467{
468	dump_summary_ebb_state();
469	dump_ebb_hw_state();
470	abort();
471}
472
473struct sigaction term_action = {
474	.sa_handler = term_handler,
475};
476
477static void __attribute__((constructor)) ebb_init(void)
478{
479	clear_ebb_stats();
480
481	if (sigaction(SIGTERM, &term_action, NULL))
482		perror("sigaction");
483
484	ebb_state.trace = trace_buffer_allocate(1 * 1024 * 1024);
485}
486