1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * ARM DynamIQ Shared Unit (DSU) PMU driver 4 * 5 * Copyright (C) ARM Limited, 2017. 6 * 7 * Based on ARM CCI-PMU, ARMv8 PMU-v3 drivers. 8 */ 9 10#define PMUNAME "arm_dsu" 11#define DRVNAME PMUNAME "_pmu" 12#define pr_fmt(fmt) DRVNAME ": " fmt 13 14#include <linux/acpi.h> 15#include <linux/bitmap.h> 16#include <linux/bitops.h> 17#include <linux/bug.h> 18#include <linux/cpumask.h> 19#include <linux/device.h> 20#include <linux/interrupt.h> 21#include <linux/kernel.h> 22#include <linux/module.h> 23#include <linux/of_device.h> 24#include <linux/perf_event.h> 25#include <linux/platform_device.h> 26#include <linux/spinlock.h> 27#include <linux/smp.h> 28#include <linux/sysfs.h> 29#include <linux/types.h> 30 31#include <asm/arm_dsu_pmu.h> 32#include <asm/local64.h> 33 34/* PMU event codes */ 35#define DSU_PMU_EVT_CYCLES 0x11 36#define DSU_PMU_EVT_CHAIN 0x1e 37 38#define DSU_PMU_MAX_COMMON_EVENTS 0x40 39 40#define DSU_PMU_MAX_HW_CNTRS 32 41#define DSU_PMU_HW_COUNTER_MASK (DSU_PMU_MAX_HW_CNTRS - 1) 42 43#define CLUSTERPMCR_E BIT(0) 44#define CLUSTERPMCR_P BIT(1) 45#define CLUSTERPMCR_C BIT(2) 46#define CLUSTERPMCR_N_SHIFT 11 47#define CLUSTERPMCR_N_MASK 0x1f 48#define CLUSTERPMCR_IDCODE_SHIFT 16 49#define CLUSTERPMCR_IDCODE_MASK 0xff 50#define CLUSTERPMCR_IMP_SHIFT 24 51#define CLUSTERPMCR_IMP_MASK 0xff 52#define CLUSTERPMCR_RES_MASK 0x7e8 53#define CLUSTERPMCR_RES_VAL 0x40 54 55#define DSU_ACTIVE_CPU_MASK 0x0 56#define DSU_ASSOCIATED_CPU_MASK 0x1 57 58/* 59 * We use the index of the counters as they appear in the counter 60 * bit maps in the PMU registers (e.g CLUSTERPMSELR). 61 * i.e, 62 * counter 0 - Bit 0 63 * counter 1 - Bit 1 64 * ... 65 * Cycle counter - Bit 31 66 */ 67#define DSU_PMU_IDX_CYCLE_COUNTER 31 68 69/* All event counters are 32bit, with a 64bit Cycle counter */ 70#define DSU_PMU_COUNTER_WIDTH(idx) \ 71 (((idx) == DSU_PMU_IDX_CYCLE_COUNTER) ? 64 : 32) 72 73#define DSU_PMU_COUNTER_MASK(idx) \ 74 GENMASK_ULL((DSU_PMU_COUNTER_WIDTH((idx)) - 1), 0) 75 76#define DSU_EXT_ATTR(_name, _func, _config) \ 77 (&((struct dev_ext_attribute[]) { \ 78 { \ 79 .attr = __ATTR(_name, 0444, _func, NULL), \ 80 .var = (void *)_config \ 81 } \ 82 })[0].attr.attr) 83 84#define DSU_EVENT_ATTR(_name, _config) \ 85 DSU_EXT_ATTR(_name, dsu_pmu_sysfs_event_show, (unsigned long)_config) 86 87#define DSU_FORMAT_ATTR(_name, _config) \ 88 DSU_EXT_ATTR(_name, dsu_pmu_sysfs_format_show, (char *)_config) 89 90#define DSU_CPUMASK_ATTR(_name, _config) \ 91 DSU_EXT_ATTR(_name, dsu_pmu_cpumask_show, (unsigned long)_config) 92 93struct dsu_hw_events { 94 DECLARE_BITMAP(used_mask, DSU_PMU_MAX_HW_CNTRS); 95 struct perf_event *events[DSU_PMU_MAX_HW_CNTRS]; 96}; 97 98/* 99 * struct dsu_pmu - DSU PMU descriptor 100 * 101 * @pmu_lock : Protects accesses to DSU PMU register from normal vs 102 * interrupt handler contexts. 103 * @hw_events : Holds the event counter state. 104 * @associated_cpus : CPUs attached to the DSU. 105 * @active_cpu : CPU to which the PMU is bound for accesses. 106 * @cpuhp_node : Node for CPU hotplug notifier link. 107 * @num_counters : Number of event counters implemented by the PMU, 108 * excluding the cycle counter. 109 * @irq : Interrupt line for counter overflow. 110 * @cpmceid_bitmap : Bitmap for the availability of architected common 111 * events (event_code < 0x40). 112 */ 113struct dsu_pmu { 114 struct pmu pmu; 115 struct device *dev; 116 raw_spinlock_t pmu_lock; 117 struct dsu_hw_events hw_events; 118 cpumask_t associated_cpus; 119 cpumask_t active_cpu; 120 struct hlist_node cpuhp_node; 121 s8 num_counters; 122 int irq; 123 DECLARE_BITMAP(cpmceid_bitmap, DSU_PMU_MAX_COMMON_EVENTS); 124}; 125 126static unsigned long dsu_pmu_cpuhp_state; 127 128static inline struct dsu_pmu *to_dsu_pmu(struct pmu *pmu) 129{ 130 return container_of(pmu, struct dsu_pmu, pmu); 131} 132 133static ssize_t dsu_pmu_sysfs_event_show(struct device *dev, 134 struct device_attribute *attr, 135 char *buf) 136{ 137 struct dev_ext_attribute *eattr = container_of(attr, 138 struct dev_ext_attribute, attr); 139 return snprintf(buf, PAGE_SIZE, "event=0x%lx\n", 140 (unsigned long)eattr->var); 141} 142 143static ssize_t dsu_pmu_sysfs_format_show(struct device *dev, 144 struct device_attribute *attr, 145 char *buf) 146{ 147 struct dev_ext_attribute *eattr = container_of(attr, 148 struct dev_ext_attribute, attr); 149 return snprintf(buf, PAGE_SIZE, "%s\n", (char *)eattr->var); 150} 151 152static ssize_t dsu_pmu_cpumask_show(struct device *dev, 153 struct device_attribute *attr, 154 char *buf) 155{ 156 struct pmu *pmu = dev_get_drvdata(dev); 157 struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu); 158 struct dev_ext_attribute *eattr = container_of(attr, 159 struct dev_ext_attribute, attr); 160 unsigned long mask_id = (unsigned long)eattr->var; 161 const cpumask_t *cpumask; 162 163 switch (mask_id) { 164 case DSU_ACTIVE_CPU_MASK: 165 cpumask = &dsu_pmu->active_cpu; 166 break; 167 case DSU_ASSOCIATED_CPU_MASK: 168 cpumask = &dsu_pmu->associated_cpus; 169 break; 170 default: 171 return 0; 172 } 173 return cpumap_print_to_pagebuf(true, buf, cpumask); 174} 175 176static struct attribute *dsu_pmu_format_attrs[] = { 177 DSU_FORMAT_ATTR(event, "config:0-31"), 178 NULL, 179}; 180 181static const struct attribute_group dsu_pmu_format_attr_group = { 182 .name = "format", 183 .attrs = dsu_pmu_format_attrs, 184}; 185 186static struct attribute *dsu_pmu_event_attrs[] = { 187 DSU_EVENT_ATTR(cycles, 0x11), 188 DSU_EVENT_ATTR(bus_access, 0x19), 189 DSU_EVENT_ATTR(memory_error, 0x1a), 190 DSU_EVENT_ATTR(bus_cycles, 0x1d), 191 DSU_EVENT_ATTR(l3d_cache_allocate, 0x29), 192 DSU_EVENT_ATTR(l3d_cache_refill, 0x2a), 193 DSU_EVENT_ATTR(l3d_cache, 0x2b), 194 DSU_EVENT_ATTR(l3d_cache_wb, 0x2c), 195 NULL, 196}; 197 198static umode_t 199dsu_pmu_event_attr_is_visible(struct kobject *kobj, struct attribute *attr, 200 int unused) 201{ 202 struct pmu *pmu = dev_get_drvdata(kobj_to_dev(kobj)); 203 struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu); 204 struct dev_ext_attribute *eattr = container_of(attr, 205 struct dev_ext_attribute, attr.attr); 206 unsigned long evt = (unsigned long)eattr->var; 207 208 return test_bit(evt, dsu_pmu->cpmceid_bitmap) ? attr->mode : 0; 209} 210 211static const struct attribute_group dsu_pmu_events_attr_group = { 212 .name = "events", 213 .attrs = dsu_pmu_event_attrs, 214 .is_visible = dsu_pmu_event_attr_is_visible, 215}; 216 217static struct attribute *dsu_pmu_cpumask_attrs[] = { 218 DSU_CPUMASK_ATTR(cpumask, DSU_ACTIVE_CPU_MASK), 219 DSU_CPUMASK_ATTR(associated_cpus, DSU_ASSOCIATED_CPU_MASK), 220 NULL, 221}; 222 223static const struct attribute_group dsu_pmu_cpumask_attr_group = { 224 .attrs = dsu_pmu_cpumask_attrs, 225}; 226 227static const struct attribute_group *dsu_pmu_attr_groups[] = { 228 &dsu_pmu_cpumask_attr_group, 229 &dsu_pmu_events_attr_group, 230 &dsu_pmu_format_attr_group, 231 NULL, 232}; 233 234static int dsu_pmu_get_online_cpu_any_but(struct dsu_pmu *dsu_pmu, int cpu) 235{ 236 struct cpumask online_supported; 237 238 cpumask_and(&online_supported, 239 &dsu_pmu->associated_cpus, cpu_online_mask); 240 return cpumask_any_but(&online_supported, cpu); 241} 242 243static inline bool dsu_pmu_counter_valid(struct dsu_pmu *dsu_pmu, u32 idx) 244{ 245 return (idx < dsu_pmu->num_counters) || 246 (idx == DSU_PMU_IDX_CYCLE_COUNTER); 247} 248 249static inline u64 dsu_pmu_read_counter(struct perf_event *event) 250{ 251 u64 val; 252 unsigned long flags; 253 struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu); 254 int idx = event->hw.idx; 255 256 if (WARN_ON(!cpumask_test_cpu(smp_processor_id(), 257 &dsu_pmu->associated_cpus))) 258 return 0; 259 260 if (!dsu_pmu_counter_valid(dsu_pmu, idx)) { 261 dev_err(event->pmu->dev, 262 "Trying reading invalid counter %d\n", idx); 263 return 0; 264 } 265 266 raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags); 267 if (idx == DSU_PMU_IDX_CYCLE_COUNTER) 268 val = __dsu_pmu_read_pmccntr(); 269 else 270 val = __dsu_pmu_read_counter(idx); 271 raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags); 272 273 return val; 274} 275 276static void dsu_pmu_write_counter(struct perf_event *event, u64 val) 277{ 278 unsigned long flags; 279 struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu); 280 int idx = event->hw.idx; 281 282 if (WARN_ON(!cpumask_test_cpu(smp_processor_id(), 283 &dsu_pmu->associated_cpus))) 284 return; 285 286 if (!dsu_pmu_counter_valid(dsu_pmu, idx)) { 287 dev_err(event->pmu->dev, 288 "writing to invalid counter %d\n", idx); 289 return; 290 } 291 292 raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags); 293 if (idx == DSU_PMU_IDX_CYCLE_COUNTER) 294 __dsu_pmu_write_pmccntr(val); 295 else 296 __dsu_pmu_write_counter(idx, val); 297 raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags); 298} 299 300static int dsu_pmu_get_event_idx(struct dsu_hw_events *hw_events, 301 struct perf_event *event) 302{ 303 int idx; 304 unsigned long evtype = event->attr.config; 305 struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu); 306 unsigned long *used_mask = hw_events->used_mask; 307 308 if (evtype == DSU_PMU_EVT_CYCLES) { 309 if (test_and_set_bit(DSU_PMU_IDX_CYCLE_COUNTER, used_mask)) 310 return -EAGAIN; 311 return DSU_PMU_IDX_CYCLE_COUNTER; 312 } 313 314 idx = find_first_zero_bit(used_mask, dsu_pmu->num_counters); 315 if (idx >= dsu_pmu->num_counters) 316 return -EAGAIN; 317 set_bit(idx, hw_events->used_mask); 318 return idx; 319} 320 321static void dsu_pmu_enable_counter(struct dsu_pmu *dsu_pmu, int idx) 322{ 323 __dsu_pmu_counter_interrupt_enable(idx); 324 __dsu_pmu_enable_counter(idx); 325} 326 327static void dsu_pmu_disable_counter(struct dsu_pmu *dsu_pmu, int idx) 328{ 329 __dsu_pmu_disable_counter(idx); 330 __dsu_pmu_counter_interrupt_disable(idx); 331} 332 333static inline void dsu_pmu_set_event(struct dsu_pmu *dsu_pmu, 334 struct perf_event *event) 335{ 336 int idx = event->hw.idx; 337 unsigned long flags; 338 339 if (!dsu_pmu_counter_valid(dsu_pmu, idx)) { 340 dev_err(event->pmu->dev, 341 "Trying to set invalid counter %d\n", idx); 342 return; 343 } 344 345 raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags); 346 __dsu_pmu_set_event(idx, event->hw.config_base); 347 raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags); 348} 349 350static void dsu_pmu_event_update(struct perf_event *event) 351{ 352 struct hw_perf_event *hwc = &event->hw; 353 u64 delta, prev_count, new_count; 354 355 do { 356 /* We may also be called from the irq handler */ 357 prev_count = local64_read(&hwc->prev_count); 358 new_count = dsu_pmu_read_counter(event); 359 } while (local64_cmpxchg(&hwc->prev_count, prev_count, new_count) != 360 prev_count); 361 delta = (new_count - prev_count) & DSU_PMU_COUNTER_MASK(hwc->idx); 362 local64_add(delta, &event->count); 363} 364 365static void dsu_pmu_read(struct perf_event *event) 366{ 367 dsu_pmu_event_update(event); 368} 369 370static inline u32 dsu_pmu_get_reset_overflow(void) 371{ 372 return __dsu_pmu_get_reset_overflow(); 373} 374 375/** 376 * dsu_pmu_set_event_period: Set the period for the counter. 377 * 378 * All DSU PMU event counters, except the cycle counter are 32bit 379 * counters. To handle cases of extreme interrupt latency, we program 380 * the counter with half of the max count for the counters. 381 */ 382static void dsu_pmu_set_event_period(struct perf_event *event) 383{ 384 int idx = event->hw.idx; 385 u64 val = DSU_PMU_COUNTER_MASK(idx) >> 1; 386 387 local64_set(&event->hw.prev_count, val); 388 dsu_pmu_write_counter(event, val); 389} 390 391static irqreturn_t dsu_pmu_handle_irq(int irq_num, void *dev) 392{ 393 int i; 394 bool handled = false; 395 struct dsu_pmu *dsu_pmu = dev; 396 struct dsu_hw_events *hw_events = &dsu_pmu->hw_events; 397 unsigned long overflow; 398 399 overflow = dsu_pmu_get_reset_overflow(); 400 if (!overflow) 401 return IRQ_NONE; 402 403 for_each_set_bit(i, &overflow, DSU_PMU_MAX_HW_CNTRS) { 404 struct perf_event *event = hw_events->events[i]; 405 406 if (!event) 407 continue; 408 dsu_pmu_event_update(event); 409 dsu_pmu_set_event_period(event); 410 handled = true; 411 } 412 413 return IRQ_RETVAL(handled); 414} 415 416static void dsu_pmu_start(struct perf_event *event, int pmu_flags) 417{ 418 struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu); 419 420 /* We always reprogram the counter */ 421 if (pmu_flags & PERF_EF_RELOAD) 422 WARN_ON(!(event->hw.state & PERF_HES_UPTODATE)); 423 dsu_pmu_set_event_period(event); 424 if (event->hw.idx != DSU_PMU_IDX_CYCLE_COUNTER) 425 dsu_pmu_set_event(dsu_pmu, event); 426 event->hw.state = 0; 427 dsu_pmu_enable_counter(dsu_pmu, event->hw.idx); 428} 429 430static void dsu_pmu_stop(struct perf_event *event, int pmu_flags) 431{ 432 struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu); 433 434 if (event->hw.state & PERF_HES_STOPPED) 435 return; 436 dsu_pmu_disable_counter(dsu_pmu, event->hw.idx); 437 dsu_pmu_event_update(event); 438 event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 439} 440 441static int dsu_pmu_add(struct perf_event *event, int flags) 442{ 443 struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu); 444 struct dsu_hw_events *hw_events = &dsu_pmu->hw_events; 445 struct hw_perf_event *hwc = &event->hw; 446 int idx; 447 448 if (WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), 449 &dsu_pmu->associated_cpus))) 450 return -ENOENT; 451 452 idx = dsu_pmu_get_event_idx(hw_events, event); 453 if (idx < 0) 454 return idx; 455 456 hwc->idx = idx; 457 hw_events->events[idx] = event; 458 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 459 460 if (flags & PERF_EF_START) 461 dsu_pmu_start(event, PERF_EF_RELOAD); 462 463 perf_event_update_userpage(event); 464 return 0; 465} 466 467static void dsu_pmu_del(struct perf_event *event, int flags) 468{ 469 struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu); 470 struct dsu_hw_events *hw_events = &dsu_pmu->hw_events; 471 struct hw_perf_event *hwc = &event->hw; 472 int idx = hwc->idx; 473 474 dsu_pmu_stop(event, PERF_EF_UPDATE); 475 hw_events->events[idx] = NULL; 476 clear_bit(idx, hw_events->used_mask); 477 perf_event_update_userpage(event); 478} 479 480static void dsu_pmu_enable(struct pmu *pmu) 481{ 482 u32 pmcr; 483 unsigned long flags; 484 struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu); 485 486 /* If no counters are added, skip enabling the PMU */ 487 if (bitmap_empty(dsu_pmu->hw_events.used_mask, DSU_PMU_MAX_HW_CNTRS)) 488 return; 489 490 raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags); 491 pmcr = __dsu_pmu_read_pmcr(); 492 pmcr |= CLUSTERPMCR_E; 493 __dsu_pmu_write_pmcr(pmcr); 494 raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags); 495} 496 497static void dsu_pmu_disable(struct pmu *pmu) 498{ 499 u32 pmcr; 500 unsigned long flags; 501 struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu); 502 503 raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags); 504 pmcr = __dsu_pmu_read_pmcr(); 505 pmcr &= ~CLUSTERPMCR_E; 506 __dsu_pmu_write_pmcr(pmcr); 507 raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags); 508} 509 510static bool dsu_pmu_validate_event(struct pmu *pmu, 511 struct dsu_hw_events *hw_events, 512 struct perf_event *event) 513{ 514 if (is_software_event(event)) 515 return true; 516 /* Reject groups spanning multiple HW PMUs. */ 517 if (event->pmu != pmu) 518 return false; 519 return dsu_pmu_get_event_idx(hw_events, event) >= 0; 520} 521 522/* 523 * Make sure the group of events can be scheduled at once 524 * on the PMU. 525 */ 526static bool dsu_pmu_validate_group(struct perf_event *event) 527{ 528 struct perf_event *sibling, *leader = event->group_leader; 529 struct dsu_hw_events fake_hw; 530 531 if (event->group_leader == event) 532 return true; 533 534 memset(fake_hw.used_mask, 0, sizeof(fake_hw.used_mask)); 535 if (!dsu_pmu_validate_event(event->pmu, &fake_hw, leader)) 536 return false; 537 for_each_sibling_event(sibling, leader) { 538 if (!dsu_pmu_validate_event(event->pmu, &fake_hw, sibling)) 539 return false; 540 } 541 return dsu_pmu_validate_event(event->pmu, &fake_hw, event); 542} 543 544static int dsu_pmu_event_init(struct perf_event *event) 545{ 546 struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu); 547 548 if (event->attr.type != event->pmu->type) 549 return -ENOENT; 550 551 /* We don't support sampling */ 552 if (is_sampling_event(event)) { 553 dev_dbg(dsu_pmu->pmu.dev, "Can't support sampling events\n"); 554 return -EOPNOTSUPP; 555 } 556 557 /* We cannot support task bound events */ 558 if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK) { 559 dev_dbg(dsu_pmu->pmu.dev, "Can't support per-task counters\n"); 560 return -EINVAL; 561 } 562 563 if (has_branch_stack(event)) { 564 dev_dbg(dsu_pmu->pmu.dev, "Can't support filtering\n"); 565 return -EINVAL; 566 } 567 568 if (!cpumask_test_cpu(event->cpu, &dsu_pmu->associated_cpus)) { 569 dev_dbg(dsu_pmu->pmu.dev, 570 "Requested cpu is not associated with the DSU\n"); 571 return -EINVAL; 572 } 573 /* 574 * Choose the current active CPU to read the events. We don't want 575 * to migrate the event contexts, irq handling etc to the requested 576 * CPU. As long as the requested CPU is within the same DSU, we 577 * are fine. 578 */ 579 event->cpu = cpumask_first(&dsu_pmu->active_cpu); 580 if (event->cpu >= nr_cpu_ids) 581 return -EINVAL; 582 if (!dsu_pmu_validate_group(event)) 583 return -EINVAL; 584 585 event->hw.config_base = event->attr.config; 586 return 0; 587} 588 589static struct dsu_pmu *dsu_pmu_alloc(struct platform_device *pdev) 590{ 591 struct dsu_pmu *dsu_pmu; 592 593 dsu_pmu = devm_kzalloc(&pdev->dev, sizeof(*dsu_pmu), GFP_KERNEL); 594 if (!dsu_pmu) 595 return ERR_PTR(-ENOMEM); 596 597 raw_spin_lock_init(&dsu_pmu->pmu_lock); 598 /* 599 * Initialise the number of counters to -1, until we probe 600 * the real number on a connected CPU. 601 */ 602 dsu_pmu->num_counters = -1; 603 return dsu_pmu; 604} 605 606/** 607 * dsu_pmu_dt_get_cpus: Get the list of CPUs in the cluster 608 * from device tree. 609 */ 610static int dsu_pmu_dt_get_cpus(struct device *dev, cpumask_t *mask) 611{ 612 int i = 0, n, cpu; 613 struct device_node *cpu_node; 614 615 n = of_count_phandle_with_args(dev->of_node, "cpus", NULL); 616 if (n <= 0) 617 return -ENODEV; 618 for (; i < n; i++) { 619 cpu_node = of_parse_phandle(dev->of_node, "cpus", i); 620 if (!cpu_node) 621 break; 622 cpu = of_cpu_node_to_id(cpu_node); 623 of_node_put(cpu_node); 624 /* 625 * We have to ignore the failures here and continue scanning 626 * the list to handle cases where the nr_cpus could be capped 627 * in the running kernel. 628 */ 629 if (cpu < 0) 630 continue; 631 cpumask_set_cpu(cpu, mask); 632 } 633 return 0; 634} 635 636/** 637 * dsu_pmu_acpi_get_cpus: Get the list of CPUs in the cluster 638 * from ACPI. 639 */ 640static int dsu_pmu_acpi_get_cpus(struct device *dev, cpumask_t *mask) 641{ 642#ifdef CONFIG_ACPI 643 int cpu; 644 645 /* 646 * A dsu pmu node is inside a cluster parent node along with cpu nodes. 647 * We need to find out all cpus that have the same parent with this pmu. 648 */ 649 for_each_possible_cpu(cpu) { 650 struct acpi_device *acpi_dev; 651 struct device *cpu_dev = get_cpu_device(cpu); 652 653 if (!cpu_dev) 654 continue; 655 656 acpi_dev = ACPI_COMPANION(cpu_dev); 657 if (acpi_dev && 658 acpi_dev->parent == ACPI_COMPANION(dev)->parent) 659 cpumask_set_cpu(cpu, mask); 660 } 661#endif 662 663 return 0; 664} 665 666/* 667 * dsu_pmu_probe_pmu: Probe the PMU details on a CPU in the cluster. 668 */ 669static void dsu_pmu_probe_pmu(struct dsu_pmu *dsu_pmu) 670{ 671 u64 num_counters; 672 u32 cpmceid[2]; 673 674 num_counters = (__dsu_pmu_read_pmcr() >> CLUSTERPMCR_N_SHIFT) & 675 CLUSTERPMCR_N_MASK; 676 /* We can only support up to 31 independent counters */ 677 if (WARN_ON(num_counters > 31)) 678 num_counters = 31; 679 dsu_pmu->num_counters = num_counters; 680 if (!dsu_pmu->num_counters) 681 return; 682 cpmceid[0] = __dsu_pmu_read_pmceid(0); 683 cpmceid[1] = __dsu_pmu_read_pmceid(1); 684 bitmap_from_arr32(dsu_pmu->cpmceid_bitmap, cpmceid, 685 DSU_PMU_MAX_COMMON_EVENTS); 686} 687 688static void dsu_pmu_set_active_cpu(int cpu, struct dsu_pmu *dsu_pmu) 689{ 690 cpumask_set_cpu(cpu, &dsu_pmu->active_cpu); 691 if (irq_set_affinity_hint(dsu_pmu->irq, &dsu_pmu->active_cpu)) 692 pr_warn("Failed to set irq affinity to %d\n", cpu); 693} 694 695/* 696 * dsu_pmu_init_pmu: Initialise the DSU PMU configurations if 697 * we haven't done it already. 698 */ 699static void dsu_pmu_init_pmu(struct dsu_pmu *dsu_pmu) 700{ 701 if (dsu_pmu->num_counters == -1) 702 dsu_pmu_probe_pmu(dsu_pmu); 703 /* Reset the interrupt overflow mask */ 704 dsu_pmu_get_reset_overflow(); 705} 706 707static int dsu_pmu_device_probe(struct platform_device *pdev) 708{ 709 int irq, rc; 710 struct dsu_pmu *dsu_pmu; 711 struct fwnode_handle *fwnode = dev_fwnode(&pdev->dev); 712 char *name; 713 static atomic_t pmu_idx = ATOMIC_INIT(-1); 714 715 dsu_pmu = dsu_pmu_alloc(pdev); 716 if (IS_ERR(dsu_pmu)) 717 return PTR_ERR(dsu_pmu); 718 719 if (IS_ERR_OR_NULL(fwnode)) 720 return -ENOENT; 721 722 if (is_of_node(fwnode)) 723 rc = dsu_pmu_dt_get_cpus(&pdev->dev, &dsu_pmu->associated_cpus); 724 else if (is_acpi_device_node(fwnode)) 725 rc = dsu_pmu_acpi_get_cpus(&pdev->dev, &dsu_pmu->associated_cpus); 726 else 727 return -ENOENT; 728 729 if (rc) { 730 dev_warn(&pdev->dev, "Failed to parse the CPUs\n"); 731 return rc; 732 } 733 734 irq = platform_get_irq(pdev, 0); 735 if (irq < 0) 736 return -EINVAL; 737 738 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_%d", 739 PMUNAME, atomic_inc_return(&pmu_idx)); 740 if (!name) 741 return -ENOMEM; 742 rc = devm_request_irq(&pdev->dev, irq, dsu_pmu_handle_irq, 743 IRQF_NOBALANCING, name, dsu_pmu); 744 if (rc) { 745 dev_warn(&pdev->dev, "Failed to request IRQ %d\n", irq); 746 return rc; 747 } 748 749 dsu_pmu->irq = irq; 750 platform_set_drvdata(pdev, dsu_pmu); 751 rc = cpuhp_state_add_instance(dsu_pmu_cpuhp_state, 752 &dsu_pmu->cpuhp_node); 753 if (rc) 754 return rc; 755 756 dsu_pmu->pmu = (struct pmu) { 757 .task_ctx_nr = perf_invalid_context, 758 .module = THIS_MODULE, 759 .pmu_enable = dsu_pmu_enable, 760 .pmu_disable = dsu_pmu_disable, 761 .event_init = dsu_pmu_event_init, 762 .add = dsu_pmu_add, 763 .del = dsu_pmu_del, 764 .start = dsu_pmu_start, 765 .stop = dsu_pmu_stop, 766 .read = dsu_pmu_read, 767 768 .attr_groups = dsu_pmu_attr_groups, 769 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 770 }; 771 772 rc = perf_pmu_register(&dsu_pmu->pmu, name, -1); 773 if (rc) { 774 cpuhp_state_remove_instance(dsu_pmu_cpuhp_state, 775 &dsu_pmu->cpuhp_node); 776 irq_set_affinity_hint(dsu_pmu->irq, NULL); 777 } 778 779 return rc; 780} 781 782static int dsu_pmu_device_remove(struct platform_device *pdev) 783{ 784 struct dsu_pmu *dsu_pmu = platform_get_drvdata(pdev); 785 786 perf_pmu_unregister(&dsu_pmu->pmu); 787 cpuhp_state_remove_instance(dsu_pmu_cpuhp_state, &dsu_pmu->cpuhp_node); 788 irq_set_affinity_hint(dsu_pmu->irq, NULL); 789 790 return 0; 791} 792 793static const struct of_device_id dsu_pmu_of_match[] = { 794 { .compatible = "arm,dsu-pmu", }, 795 {}, 796}; 797MODULE_DEVICE_TABLE(of, dsu_pmu_of_match); 798 799#ifdef CONFIG_ACPI 800static const struct acpi_device_id dsu_pmu_acpi_match[] = { 801 { "ARMHD500", 0}, 802 {}, 803}; 804MODULE_DEVICE_TABLE(acpi, dsu_pmu_acpi_match); 805#endif 806 807static struct platform_driver dsu_pmu_driver = { 808 .driver = { 809 .name = DRVNAME, 810 .of_match_table = of_match_ptr(dsu_pmu_of_match), 811 .acpi_match_table = ACPI_PTR(dsu_pmu_acpi_match), 812 .suppress_bind_attrs = true, 813 }, 814 .probe = dsu_pmu_device_probe, 815 .remove = dsu_pmu_device_remove, 816}; 817 818static int dsu_pmu_cpu_online(unsigned int cpu, struct hlist_node *node) 819{ 820 struct dsu_pmu *dsu_pmu = hlist_entry_safe(node, struct dsu_pmu, 821 cpuhp_node); 822 823 if (!cpumask_test_cpu(cpu, &dsu_pmu->associated_cpus)) 824 return 0; 825 826 /* If the PMU is already managed, there is nothing to do */ 827 if (!cpumask_empty(&dsu_pmu->active_cpu)) 828 return 0; 829 830 dsu_pmu_init_pmu(dsu_pmu); 831 dsu_pmu_set_active_cpu(cpu, dsu_pmu); 832 833 return 0; 834} 835 836static int dsu_pmu_cpu_teardown(unsigned int cpu, struct hlist_node *node) 837{ 838 int dst; 839 struct dsu_pmu *dsu_pmu = hlist_entry_safe(node, struct dsu_pmu, 840 cpuhp_node); 841 842 if (!cpumask_test_and_clear_cpu(cpu, &dsu_pmu->active_cpu)) 843 return 0; 844 845 dst = dsu_pmu_get_online_cpu_any_but(dsu_pmu, cpu); 846 /* If there are no active CPUs in the DSU, leave IRQ disabled */ 847 if (dst >= nr_cpu_ids) { 848 irq_set_affinity_hint(dsu_pmu->irq, NULL); 849 return 0; 850 } 851 852 perf_pmu_migrate_context(&dsu_pmu->pmu, cpu, dst); 853 dsu_pmu_set_active_cpu(dst, dsu_pmu); 854 855 return 0; 856} 857 858static int __init dsu_pmu_init(void) 859{ 860 int ret; 861 862 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, 863 DRVNAME, 864 dsu_pmu_cpu_online, 865 dsu_pmu_cpu_teardown); 866 if (ret < 0) 867 return ret; 868 dsu_pmu_cpuhp_state = ret; 869 ret = platform_driver_register(&dsu_pmu_driver); 870 if (ret) 871 cpuhp_remove_multi_state(dsu_pmu_cpuhp_state); 872 873 return ret; 874} 875 876static void __exit dsu_pmu_exit(void) 877{ 878 platform_driver_unregister(&dsu_pmu_driver); 879 cpuhp_remove_multi_state(dsu_pmu_cpuhp_state); 880} 881 882module_init(dsu_pmu_init); 883module_exit(dsu_pmu_exit); 884 885MODULE_DESCRIPTION("Perf driver for ARM DynamIQ Shared Unit"); 886MODULE_AUTHOR("Suzuki K Poulose <suzuki.poulose@arm.com>"); 887MODULE_LICENSE("GPL v2"); 888