1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Cell Broadband Engine Performance Monitor
4 *
5 * (C) Copyright IBM Corporation 2001,2006
6 *
7 * Author:
8 *    David Erb (djerb@us.ibm.com)
9 *    Kevin Corry (kevcorry@us.ibm.com)
10 */
11
12#include <linux/interrupt.h>
13#include <linux/types.h>
14#include <linux/export.h>
15#include <asm/io.h>
16#include <asm/irq_regs.h>
17#include <asm/machdep.h>
18#include <asm/pmc.h>
19#include <asm/reg.h>
20#include <asm/spu.h>
21#include <asm/cell-regs.h>
22
23#include "interrupt.h"
24
25/*
26 * When writing to write-only mmio addresses, save a shadow copy. All of the
27 * registers are 32-bit, but stored in the upper-half of a 64-bit field in
28 * pmd_regs.
29 */
30
31#define WRITE_WO_MMIO(reg, x)					\
32	do {							\
33		u32 _x = (x);					\
34		struct cbe_pmd_regs __iomem *pmd_regs;		\
35		struct cbe_pmd_shadow_regs *shadow_regs;	\
36		pmd_regs = cbe_get_cpu_pmd_regs(cpu);		\
37		shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu);	\
38		out_be64(&(pmd_regs->reg), (((u64)_x) << 32));	\
39		shadow_regs->reg = _x;				\
40	} while (0)
41
42#define READ_SHADOW_REG(val, reg)				\
43	do {							\
44		struct cbe_pmd_shadow_regs *shadow_regs;	\
45		shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu);	\
46		(val) = shadow_regs->reg;			\
47	} while (0)
48
49#define READ_MMIO_UPPER32(val, reg)				\
50	do {							\
51		struct cbe_pmd_regs __iomem *pmd_regs;		\
52		pmd_regs = cbe_get_cpu_pmd_regs(cpu);		\
53		(val) = (u32)(in_be64(&pmd_regs->reg) >> 32);	\
54	} while (0)
55
56/*
57 * Physical counter registers.
58 * Each physical counter can act as one 32-bit counter or two 16-bit counters.
59 */
60
61u32 cbe_read_phys_ctr(u32 cpu, u32 phys_ctr)
62{
63	u32 val_in_latch, val = 0;
64
65	if (phys_ctr < NR_PHYS_CTRS) {
66		READ_SHADOW_REG(val_in_latch, counter_value_in_latch);
67
68		/* Read the latch or the actual counter, whichever is newer. */
69		if (val_in_latch & (1 << phys_ctr)) {
70			READ_SHADOW_REG(val, pm_ctr[phys_ctr]);
71		} else {
72			READ_MMIO_UPPER32(val, pm_ctr[phys_ctr]);
73		}
74	}
75
76	return val;
77}
78EXPORT_SYMBOL_GPL(cbe_read_phys_ctr);
79
80void cbe_write_phys_ctr(u32 cpu, u32 phys_ctr, u32 val)
81{
82	struct cbe_pmd_shadow_regs *shadow_regs;
83	u32 pm_ctrl;
84
85	if (phys_ctr < NR_PHYS_CTRS) {
86		/* Writing to a counter only writes to a hardware latch.
87		 * The new value is not propagated to the actual counter
88		 * until the performance monitor is enabled.
89		 */
90		WRITE_WO_MMIO(pm_ctr[phys_ctr], val);
91
92		pm_ctrl = cbe_read_pm(cpu, pm_control);
93		if (pm_ctrl & CBE_PM_ENABLE_PERF_MON) {
94			/* The counters are already active, so we need to
95			 * rewrite the pm_control register to "re-enable"
96			 * the PMU.
97			 */
98			cbe_write_pm(cpu, pm_control, pm_ctrl);
99		} else {
100			shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu);
101			shadow_regs->counter_value_in_latch |= (1 << phys_ctr);
102		}
103	}
104}
105EXPORT_SYMBOL_GPL(cbe_write_phys_ctr);
106
107/*
108 * "Logical" counter registers.
109 * These will read/write 16-bits or 32-bits depending on the
110 * current size of the counter. Counters 4 - 7 are always 16-bit.
111 */
112
113u32 cbe_read_ctr(u32 cpu, u32 ctr)
114{
115	u32 val;
116	u32 phys_ctr = ctr & (NR_PHYS_CTRS - 1);
117
118	val = cbe_read_phys_ctr(cpu, phys_ctr);
119
120	if (cbe_get_ctr_size(cpu, phys_ctr) == 16)
121		val = (ctr < NR_PHYS_CTRS) ? (val >> 16) : (val & 0xffff);
122
123	return val;
124}
125EXPORT_SYMBOL_GPL(cbe_read_ctr);
126
127void cbe_write_ctr(u32 cpu, u32 ctr, u32 val)
128{
129	u32 phys_ctr;
130	u32 phys_val;
131
132	phys_ctr = ctr & (NR_PHYS_CTRS - 1);
133
134	if (cbe_get_ctr_size(cpu, phys_ctr) == 16) {
135		phys_val = cbe_read_phys_ctr(cpu, phys_ctr);
136
137		if (ctr < NR_PHYS_CTRS)
138			val = (val << 16) | (phys_val & 0xffff);
139		else
140			val = (val & 0xffff) | (phys_val & 0xffff0000);
141	}
142
143	cbe_write_phys_ctr(cpu, phys_ctr, val);
144}
145EXPORT_SYMBOL_GPL(cbe_write_ctr);
146
147/*
148 * Counter-control registers.
149 * Each "logical" counter has a corresponding control register.
150 */
151
152u32 cbe_read_pm07_control(u32 cpu, u32 ctr)
153{
154	u32 pm07_control = 0;
155
156	if (ctr < NR_CTRS)
157		READ_SHADOW_REG(pm07_control, pm07_control[ctr]);
158
159	return pm07_control;
160}
161EXPORT_SYMBOL_GPL(cbe_read_pm07_control);
162
163void cbe_write_pm07_control(u32 cpu, u32 ctr, u32 val)
164{
165	if (ctr < NR_CTRS)
166		WRITE_WO_MMIO(pm07_control[ctr], val);
167}
168EXPORT_SYMBOL_GPL(cbe_write_pm07_control);
169
170/*
171 * Other PMU control registers. Most of these are write-only.
172 */
173
174u32 cbe_read_pm(u32 cpu, enum pm_reg_name reg)
175{
176	u32 val = 0;
177
178	switch (reg) {
179	case group_control:
180		READ_SHADOW_REG(val, group_control);
181		break;
182
183	case debug_bus_control:
184		READ_SHADOW_REG(val, debug_bus_control);
185		break;
186
187	case trace_address:
188		READ_MMIO_UPPER32(val, trace_address);
189		break;
190
191	case ext_tr_timer:
192		READ_SHADOW_REG(val, ext_tr_timer);
193		break;
194
195	case pm_status:
196		READ_MMIO_UPPER32(val, pm_status);
197		break;
198
199	case pm_control:
200		READ_SHADOW_REG(val, pm_control);
201		break;
202
203	case pm_interval:
204		READ_MMIO_UPPER32(val, pm_interval);
205		break;
206
207	case pm_start_stop:
208		READ_SHADOW_REG(val, pm_start_stop);
209		break;
210	}
211
212	return val;
213}
214EXPORT_SYMBOL_GPL(cbe_read_pm);
215
216void cbe_write_pm(u32 cpu, enum pm_reg_name reg, u32 val)
217{
218	switch (reg) {
219	case group_control:
220		WRITE_WO_MMIO(group_control, val);
221		break;
222
223	case debug_bus_control:
224		WRITE_WO_MMIO(debug_bus_control, val);
225		break;
226
227	case trace_address:
228		WRITE_WO_MMIO(trace_address, val);
229		break;
230
231	case ext_tr_timer:
232		WRITE_WO_MMIO(ext_tr_timer, val);
233		break;
234
235	case pm_status:
236		WRITE_WO_MMIO(pm_status, val);
237		break;
238
239	case pm_control:
240		WRITE_WO_MMIO(pm_control, val);
241		break;
242
243	case pm_interval:
244		WRITE_WO_MMIO(pm_interval, val);
245		break;
246
247	case pm_start_stop:
248		WRITE_WO_MMIO(pm_start_stop, val);
249		break;
250	}
251}
252EXPORT_SYMBOL_GPL(cbe_write_pm);
253
254/*
255 * Get/set the size of a physical counter to either 16 or 32 bits.
256 */
257
258u32 cbe_get_ctr_size(u32 cpu, u32 phys_ctr)
259{
260	u32 pm_ctrl, size = 0;
261
262	if (phys_ctr < NR_PHYS_CTRS) {
263		pm_ctrl = cbe_read_pm(cpu, pm_control);
264		size = (pm_ctrl & CBE_PM_16BIT_CTR(phys_ctr)) ? 16 : 32;
265	}
266
267	return size;
268}
269EXPORT_SYMBOL_GPL(cbe_get_ctr_size);
270
271void cbe_set_ctr_size(u32 cpu, u32 phys_ctr, u32 ctr_size)
272{
273	u32 pm_ctrl;
274
275	if (phys_ctr < NR_PHYS_CTRS) {
276		pm_ctrl = cbe_read_pm(cpu, pm_control);
277		switch (ctr_size) {
278		case 16:
279			pm_ctrl |= CBE_PM_16BIT_CTR(phys_ctr);
280			break;
281
282		case 32:
283			pm_ctrl &= ~CBE_PM_16BIT_CTR(phys_ctr);
284			break;
285		}
286		cbe_write_pm(cpu, pm_control, pm_ctrl);
287	}
288}
289EXPORT_SYMBOL_GPL(cbe_set_ctr_size);
290
291/*
292 * Enable/disable the entire performance monitoring unit.
293 * When we enable the PMU, all pending writes to counters get committed.
294 */
295
296void cbe_enable_pm(u32 cpu)
297{
298	struct cbe_pmd_shadow_regs *shadow_regs;
299	u32 pm_ctrl;
300
301	shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu);
302	shadow_regs->counter_value_in_latch = 0;
303
304	pm_ctrl = cbe_read_pm(cpu, pm_control) | CBE_PM_ENABLE_PERF_MON;
305	cbe_write_pm(cpu, pm_control, pm_ctrl);
306}
307EXPORT_SYMBOL_GPL(cbe_enable_pm);
308
309void cbe_disable_pm(u32 cpu)
310{
311	u32 pm_ctrl;
312	pm_ctrl = cbe_read_pm(cpu, pm_control) & ~CBE_PM_ENABLE_PERF_MON;
313	cbe_write_pm(cpu, pm_control, pm_ctrl);
314}
315EXPORT_SYMBOL_GPL(cbe_disable_pm);
316
317/*
318 * Reading from the trace_buffer.
319 * The trace buffer is two 64-bit registers. Reading from
320 * the second half automatically increments the trace_address.
321 */
322
323void cbe_read_trace_buffer(u32 cpu, u64 *buf)
324{
325	struct cbe_pmd_regs __iomem *pmd_regs = cbe_get_cpu_pmd_regs(cpu);
326
327	*buf++ = in_be64(&pmd_regs->trace_buffer_0_63);
328	*buf++ = in_be64(&pmd_regs->trace_buffer_64_127);
329}
330EXPORT_SYMBOL_GPL(cbe_read_trace_buffer);
331
332/*
333 * Enabling/disabling interrupts for the entire performance monitoring unit.
334 */
335
336u32 cbe_get_and_clear_pm_interrupts(u32 cpu)
337{
338	/* Reading pm_status clears the interrupt bits. */
339	return cbe_read_pm(cpu, pm_status);
340}
341EXPORT_SYMBOL_GPL(cbe_get_and_clear_pm_interrupts);
342
343void cbe_enable_pm_interrupts(u32 cpu, u32 thread, u32 mask)
344{
345	/* Set which node and thread will handle the next interrupt. */
346	iic_set_interrupt_routing(cpu, thread, 0);
347
348	/* Enable the interrupt bits in the pm_status register. */
349	if (mask)
350		cbe_write_pm(cpu, pm_status, mask);
351}
352EXPORT_SYMBOL_GPL(cbe_enable_pm_interrupts);
353
354void cbe_disable_pm_interrupts(u32 cpu)
355{
356	cbe_get_and_clear_pm_interrupts(cpu);
357	cbe_write_pm(cpu, pm_status, 0);
358}
359EXPORT_SYMBOL_GPL(cbe_disable_pm_interrupts);
360
361static irqreturn_t cbe_pm_irq(int irq, void *dev_id)
362{
363	perf_irq(get_irq_regs());
364	return IRQ_HANDLED;
365}
366
367static int __init cbe_init_pm_irq(void)
368{
369	unsigned int irq;
370	int rc, node;
371
372	for_each_online_node(node) {
373		irq = irq_create_mapping(NULL, IIC_IRQ_IOEX_PMI |
374					       (node << IIC_IRQ_NODE_SHIFT));
375		if (!irq) {
376			printk("ERROR: Unable to allocate irq for node %d\n",
377			       node);
378			return -EINVAL;
379		}
380
381		rc = request_irq(irq, cbe_pm_irq,
382				 0, "cbe-pmu-0", NULL);
383		if (rc) {
384			printk("ERROR: Request for irq on node %d failed\n",
385			       node);
386			return rc;
387		}
388	}
389
390	return 0;
391}
392machine_arch_initcall(cell, cbe_init_pm_irq);
393
394void cbe_sync_irq(int node)
395{
396	unsigned int irq;
397
398	irq = irq_find_mapping(NULL,
399			       IIC_IRQ_IOEX_PMI
400			       | (node << IIC_IRQ_NODE_SHIFT));
401
402	if (!irq) {
403		printk(KERN_WARNING "ERROR, unable to get existing irq %d " \
404		"for node %d\n", irq, node);
405		return;
406	}
407
408	synchronize_irq(irq);
409}
410EXPORT_SYMBOL_GPL(cbe_sync_irq);
411
412