1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * linux/drivers/cpufreq/cpufreq.c 4 * 5 * Copyright (C) 2001 Russell King 6 * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de> 7 * (C) 2013 Viresh Kumar <viresh.kumar@linaro.org> 8 * 9 * Oct 2005 - Ashok Raj <ashok.raj@intel.com> 10 * Added handling for CPU hotplug 11 * Feb 2006 - Jacob Shin <jacob.shin@amd.com> 12 * Fix handling for CPU hotplug -- affected CPUs 13 */ 14 15#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 16 17#include <linux/cpu.h> 18#include <linux/cpufreq.h> 19#include <linux/cpu_cooling.h> 20#include <linux/delay.h> 21#include <linux/device.h> 22#include <linux/init.h> 23#include <linux/kernel_stat.h> 24#include <linux/module.h> 25#include <linux/mutex.h> 26#include <linux/pm_qos.h> 27#include <linux/slab.h> 28#include <linux/suspend.h> 29#include <linux/syscore_ops.h> 30#include <linux/tick.h> 31#include <trace/events/power.h> 32 33static LIST_HEAD(cpufreq_policy_list); 34 35/* Macros to iterate over CPU policies */ 36#define for_each_suitable_policy(__policy, __active) \ 37 list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) \ 38 if ((__active) == !policy_is_inactive(__policy)) 39 40#define for_each_active_policy(__policy) \ 41 for_each_suitable_policy(__policy, true) 42#define for_each_inactive_policy(__policy) \ 43 for_each_suitable_policy(__policy, false) 44 45#define for_each_policy(__policy) \ 46 list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) 47 48/* Iterate over governors */ 49static LIST_HEAD(cpufreq_governor_list); 50#define for_each_governor(__governor) \ 51 list_for_each_entry(__governor, &cpufreq_governor_list, governor_list) 52 53static char default_governor[CPUFREQ_NAME_LEN]; 54 55/* 56 * The "cpufreq driver" - the arch- or hardware-dependent low 57 * level driver of CPUFreq support, and its spinlock. This lock 58 * also protects the cpufreq_cpu_data array. 59 */ 60static struct cpufreq_driver *cpufreq_driver; 61static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data); 62static DEFINE_RWLOCK(cpufreq_driver_lock); 63 64static DEFINE_STATIC_KEY_FALSE(cpufreq_freq_invariance); 65bool cpufreq_supports_freq_invariance(void) 66{ 67 return static_branch_likely(&cpufreq_freq_invariance); 68} 69 70/* Flag to suspend/resume CPUFreq governors */ 71static bool cpufreq_suspended; 72 73static inline bool has_target(void) 74{ 75 return cpufreq_driver->target_index || cpufreq_driver->target; 76} 77 78/* internal prototypes */ 79static unsigned int __cpufreq_get(struct cpufreq_policy *policy); 80static int cpufreq_init_governor(struct cpufreq_policy *policy); 81static void cpufreq_exit_governor(struct cpufreq_policy *policy); 82static void cpufreq_governor_limits(struct cpufreq_policy *policy); 83static int cpufreq_set_policy(struct cpufreq_policy *policy, 84 struct cpufreq_governor *new_gov, 85 unsigned int new_pol); 86 87/* 88 * Two notifier lists: the "policy" list is involved in the 89 * validation process for a new CPU frequency policy; the 90 * "transition" list for kernel code that needs to handle 91 * changes to devices when the CPU clock speed changes. 92 * The mutex locks both lists. 93 */ 94static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list); 95SRCU_NOTIFIER_HEAD_STATIC(cpufreq_transition_notifier_list); 96 97static int off __read_mostly; 98static int cpufreq_disabled(void) 99{ 100 return off; 101} 102void disable_cpufreq(void) 103{ 104 off = 1; 105} 106static DEFINE_MUTEX(cpufreq_governor_mutex); 107 108bool have_governor_per_policy(void) 109{ 110 return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY); 111} 112EXPORT_SYMBOL_GPL(have_governor_per_policy); 113 114static struct kobject *cpufreq_global_kobject; 115 116struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy) 117{ 118 if (have_governor_per_policy()) 119 return &policy->kobj; 120 else 121 return cpufreq_global_kobject; 122} 123EXPORT_SYMBOL_GPL(get_governor_parent_kobj); 124 125static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall) 126{ 127 struct kernel_cpustat kcpustat; 128 u64 cur_wall_time; 129 u64 idle_time; 130 u64 busy_time; 131 132 cur_wall_time = jiffies64_to_nsecs(get_jiffies_64()); 133 134 kcpustat_cpu_fetch(&kcpustat, cpu); 135 136 busy_time = kcpustat.cpustat[CPUTIME_USER]; 137 busy_time += kcpustat.cpustat[CPUTIME_SYSTEM]; 138 busy_time += kcpustat.cpustat[CPUTIME_IRQ]; 139 busy_time += kcpustat.cpustat[CPUTIME_SOFTIRQ]; 140 busy_time += kcpustat.cpustat[CPUTIME_STEAL]; 141 busy_time += kcpustat.cpustat[CPUTIME_NICE]; 142 143 idle_time = cur_wall_time - busy_time; 144 if (wall) 145 *wall = div_u64(cur_wall_time, NSEC_PER_USEC); 146 147 return div_u64(idle_time, NSEC_PER_USEC); 148} 149 150u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy) 151{ 152 u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL); 153 154 if (idle_time == -1ULL) 155 return get_cpu_idle_time_jiffy(cpu, wall); 156 else if (!io_busy) 157 idle_time += get_cpu_iowait_time_us(cpu, wall); 158 159 return idle_time; 160} 161EXPORT_SYMBOL_GPL(get_cpu_idle_time); 162 163/* 164 * This is a generic cpufreq init() routine which can be used by cpufreq 165 * drivers of SMP systems. It will do following: 166 * - validate & show freq table passed 167 * - set policies transition latency 168 * - policy->cpus with all possible CPUs 169 */ 170void cpufreq_generic_init(struct cpufreq_policy *policy, 171 struct cpufreq_frequency_table *table, 172 unsigned int transition_latency) 173{ 174 policy->freq_table = table; 175 policy->cpuinfo.transition_latency = transition_latency; 176 177 /* 178 * The driver only supports the SMP configuration where all processors 179 * share the clock and voltage and clock. 180 */ 181 cpumask_setall(policy->cpus); 182} 183EXPORT_SYMBOL_GPL(cpufreq_generic_init); 184 185struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu) 186{ 187 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); 188 189 return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL; 190} 191EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw); 192 193unsigned int cpufreq_generic_get(unsigned int cpu) 194{ 195 struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); 196 197 if (!policy || IS_ERR(policy->clk)) { 198 pr_err("%s: No %s associated to cpu: %d\n", 199 __func__, policy ? "clk" : "policy", cpu); 200 return 0; 201 } 202 203 return clk_get_rate(policy->clk) / 1000; 204} 205EXPORT_SYMBOL_GPL(cpufreq_generic_get); 206 207/** 208 * cpufreq_cpu_get - Return policy for a CPU and mark it as busy. 209 * @cpu: CPU to find the policy for. 210 * 211 * Call cpufreq_cpu_get_raw() to obtain a cpufreq policy for @cpu and increment 212 * the kobject reference counter of that policy. Return a valid policy on 213 * success or NULL on failure. 214 * 215 * The policy returned by this function has to be released with the help of 216 * cpufreq_cpu_put() to balance its kobject reference counter properly. 217 */ 218struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu) 219{ 220 struct cpufreq_policy *policy = NULL; 221 unsigned long flags; 222 223 if (WARN_ON(cpu >= nr_cpu_ids)) 224 return NULL; 225 226 /* get the cpufreq driver */ 227 read_lock_irqsave(&cpufreq_driver_lock, flags); 228 229 if (cpufreq_driver) { 230 /* get the CPU */ 231 policy = cpufreq_cpu_get_raw(cpu); 232 if (policy) 233 kobject_get(&policy->kobj); 234 } 235 236 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 237 238 return policy; 239} 240EXPORT_SYMBOL_GPL(cpufreq_cpu_get); 241 242/** 243 * cpufreq_cpu_put - Decrement kobject usage counter for cpufreq policy. 244 * @policy: cpufreq policy returned by cpufreq_cpu_get(). 245 */ 246void cpufreq_cpu_put(struct cpufreq_policy *policy) 247{ 248 kobject_put(&policy->kobj); 249} 250EXPORT_SYMBOL_GPL(cpufreq_cpu_put); 251 252/** 253 * cpufreq_cpu_release - Unlock a policy and decrement its usage counter. 254 * @policy: cpufreq policy returned by cpufreq_cpu_acquire(). 255 */ 256void cpufreq_cpu_release(struct cpufreq_policy *policy) 257{ 258 if (WARN_ON(!policy)) 259 return; 260 261 lockdep_assert_held(&policy->rwsem); 262 263 up_write(&policy->rwsem); 264 265 cpufreq_cpu_put(policy); 266} 267 268/** 269 * cpufreq_cpu_acquire - Find policy for a CPU, mark it as busy and lock it. 270 * @cpu: CPU to find the policy for. 271 * 272 * Call cpufreq_cpu_get() to get a reference on the cpufreq policy for @cpu and 273 * if the policy returned by it is not NULL, acquire its rwsem for writing. 274 * Return the policy if it is active or release it and return NULL otherwise. 275 * 276 * The policy returned by this function has to be released with the help of 277 * cpufreq_cpu_release() in order to release its rwsem and balance its usage 278 * counter properly. 279 */ 280struct cpufreq_policy *cpufreq_cpu_acquire(unsigned int cpu) 281{ 282 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 283 284 if (!policy) 285 return NULL; 286 287 down_write(&policy->rwsem); 288 289 if (policy_is_inactive(policy)) { 290 cpufreq_cpu_release(policy); 291 return NULL; 292 } 293 294 return policy; 295} 296 297/********************************************************************* 298 * EXTERNALLY AFFECTING FREQUENCY CHANGES * 299 *********************************************************************/ 300 301/* 302 * adjust_jiffies - adjust the system "loops_per_jiffy" 303 * 304 * This function alters the system "loops_per_jiffy" for the clock 305 * speed change. Note that loops_per_jiffy cannot be updated on SMP 306 * systems as each CPU might be scaled differently. So, use the arch 307 * per-CPU loops_per_jiffy value wherever possible. 308 */ 309static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci) 310{ 311#ifndef CONFIG_SMP 312 static unsigned long l_p_j_ref; 313 static unsigned int l_p_j_ref_freq; 314 315 if (ci->flags & CPUFREQ_CONST_LOOPS) 316 return; 317 318 if (!l_p_j_ref_freq) { 319 l_p_j_ref = loops_per_jiffy; 320 l_p_j_ref_freq = ci->old; 321 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n", 322 l_p_j_ref, l_p_j_ref_freq); 323 } 324 if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) { 325 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq, 326 ci->new); 327 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n", 328 loops_per_jiffy, ci->new); 329 } 330#endif 331} 332 333/** 334 * cpufreq_notify_transition - Notify frequency transition and adjust_jiffies. 335 * @policy: cpufreq policy to enable fast frequency switching for. 336 * @freqs: contain details of the frequency update. 337 * @state: set to CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE. 338 * 339 * This function calls the transition notifiers and the "adjust_jiffies" 340 * function. It is called twice on all CPU frequency changes that have 341 * external effects. 342 */ 343static void cpufreq_notify_transition(struct cpufreq_policy *policy, 344 struct cpufreq_freqs *freqs, 345 unsigned int state) 346{ 347 int cpu; 348 349 BUG_ON(irqs_disabled()); 350 351 if (cpufreq_disabled()) 352 return; 353 354 freqs->policy = policy; 355 freqs->flags = cpufreq_driver->flags; 356 pr_debug("notification %u of frequency transition to %u kHz\n", 357 state, freqs->new); 358 359 switch (state) { 360 case CPUFREQ_PRECHANGE: 361 /* 362 * Detect if the driver reported a value as "old frequency" 363 * which is not equal to what the cpufreq core thinks is 364 * "old frequency". 365 */ 366 if (policy->cur && policy->cur != freqs->old) { 367 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n", 368 freqs->old, policy->cur); 369 freqs->old = policy->cur; 370 } 371 372 srcu_notifier_call_chain(&cpufreq_transition_notifier_list, 373 CPUFREQ_PRECHANGE, freqs); 374 375 adjust_jiffies(CPUFREQ_PRECHANGE, freqs); 376 break; 377 378 case CPUFREQ_POSTCHANGE: 379 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs); 380 pr_debug("FREQ: %u - CPUs: %*pbl\n", freqs->new, 381 cpumask_pr_args(policy->cpus)); 382 383 for_each_cpu(cpu, policy->cpus) 384 trace_cpu_frequency(freqs->new, cpu); 385 386 srcu_notifier_call_chain(&cpufreq_transition_notifier_list, 387 CPUFREQ_POSTCHANGE, freqs); 388 389 cpufreq_stats_record_transition(policy, freqs->new); 390 policy->cur = freqs->new; 391 } 392} 393 394/* Do post notifications when there are chances that transition has failed */ 395static void cpufreq_notify_post_transition(struct cpufreq_policy *policy, 396 struct cpufreq_freqs *freqs, int transition_failed) 397{ 398 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE); 399 if (!transition_failed) 400 return; 401 402 swap(freqs->old, freqs->new); 403 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE); 404 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE); 405} 406 407void cpufreq_freq_transition_begin(struct cpufreq_policy *policy, 408 struct cpufreq_freqs *freqs) 409{ 410 411 /* 412 * Catch double invocations of _begin() which lead to self-deadlock. 413 * ASYNC_NOTIFICATION drivers are left out because the cpufreq core 414 * doesn't invoke _begin() on their behalf, and hence the chances of 415 * double invocations are very low. Moreover, there are scenarios 416 * where these checks can emit false-positive warnings in these 417 * drivers; so we avoid that by skipping them altogether. 418 */ 419 WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION) 420 && current == policy->transition_task); 421 422wait: 423 wait_event(policy->transition_wait, !policy->transition_ongoing); 424 425 spin_lock(&policy->transition_lock); 426 427 if (unlikely(policy->transition_ongoing)) { 428 spin_unlock(&policy->transition_lock); 429 goto wait; 430 } 431 432 policy->transition_ongoing = true; 433 policy->transition_task = current; 434 435 spin_unlock(&policy->transition_lock); 436 437 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE); 438} 439EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin); 440 441void cpufreq_freq_transition_end(struct cpufreq_policy *policy, 442 struct cpufreq_freqs *freqs, int transition_failed) 443{ 444 if (WARN_ON(!policy->transition_ongoing)) 445 return; 446 447 cpufreq_notify_post_transition(policy, freqs, transition_failed); 448 449 arch_set_freq_scale(policy->related_cpus, 450 policy->cur, 451 policy->cpuinfo.max_freq); 452 453 spin_lock(&policy->transition_lock); 454 policy->transition_ongoing = false; 455 policy->transition_task = NULL; 456 spin_unlock(&policy->transition_lock); 457 458 wake_up(&policy->transition_wait); 459} 460EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end); 461 462/* 463 * Fast frequency switching status count. Positive means "enabled", negative 464 * means "disabled" and 0 means "not decided yet". 465 */ 466static int cpufreq_fast_switch_count; 467static DEFINE_MUTEX(cpufreq_fast_switch_lock); 468 469static void cpufreq_list_transition_notifiers(void) 470{ 471 struct notifier_block *nb; 472 473 pr_info("Registered transition notifiers:\n"); 474 475 mutex_lock(&cpufreq_transition_notifier_list.mutex); 476 477 for (nb = cpufreq_transition_notifier_list.head; nb; nb = nb->next) 478 pr_info("%pS\n", nb->notifier_call); 479 480 mutex_unlock(&cpufreq_transition_notifier_list.mutex); 481} 482 483/** 484 * cpufreq_enable_fast_switch - Enable fast frequency switching for policy. 485 * @policy: cpufreq policy to enable fast frequency switching for. 486 * 487 * Try to enable fast frequency switching for @policy. 488 * 489 * The attempt will fail if there is at least one transition notifier registered 490 * at this point, as fast frequency switching is quite fundamentally at odds 491 * with transition notifiers. Thus if successful, it will make registration of 492 * transition notifiers fail going forward. 493 */ 494void cpufreq_enable_fast_switch(struct cpufreq_policy *policy) 495{ 496 lockdep_assert_held(&policy->rwsem); 497 498 if (!policy->fast_switch_possible) 499 return; 500 501 mutex_lock(&cpufreq_fast_switch_lock); 502 if (cpufreq_fast_switch_count >= 0) { 503 cpufreq_fast_switch_count++; 504 policy->fast_switch_enabled = true; 505 } else { 506 pr_warn("CPU%u: Fast frequency switching not enabled\n", 507 policy->cpu); 508 cpufreq_list_transition_notifiers(); 509 } 510 mutex_unlock(&cpufreq_fast_switch_lock); 511} 512EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch); 513 514/** 515 * cpufreq_disable_fast_switch - Disable fast frequency switching for policy. 516 * @policy: cpufreq policy to disable fast frequency switching for. 517 */ 518void cpufreq_disable_fast_switch(struct cpufreq_policy *policy) 519{ 520 mutex_lock(&cpufreq_fast_switch_lock); 521 if (policy->fast_switch_enabled) { 522 policy->fast_switch_enabled = false; 523 if (!WARN_ON(cpufreq_fast_switch_count <= 0)) 524 cpufreq_fast_switch_count--; 525 } 526 mutex_unlock(&cpufreq_fast_switch_lock); 527} 528EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch); 529 530/** 531 * cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported 532 * one. 533 * @policy: associated policy to interrogate 534 * @target_freq: target frequency to resolve. 535 * 536 * The target to driver frequency mapping is cached in the policy. 537 * 538 * Return: Lowest driver-supported frequency greater than or equal to the 539 * given target_freq, subject to policy (min/max) and driver limitations. 540 */ 541unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy, 542 unsigned int target_freq) 543{ 544 target_freq = clamp_val(target_freq, policy->min, policy->max); 545 policy->cached_target_freq = target_freq; 546 547 if (cpufreq_driver->target_index) { 548 unsigned int idx; 549 550 idx = cpufreq_frequency_table_target(policy, target_freq, 551 CPUFREQ_RELATION_L); 552 policy->cached_resolved_idx = idx; 553 return policy->freq_table[idx].frequency; 554 } 555 556 if (cpufreq_driver->resolve_freq) 557 return cpufreq_driver->resolve_freq(policy, target_freq); 558 559 return target_freq; 560} 561EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq); 562 563unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy) 564{ 565 unsigned int latency; 566 567 if (policy->transition_delay_us) 568 return policy->transition_delay_us; 569 570 latency = policy->cpuinfo.transition_latency / NSEC_PER_USEC; 571 if (latency) { 572 /* 573 * For platforms that can change the frequency very fast (< 10 574 * us), the above formula gives a decent transition delay. But 575 * for platforms where transition_latency is in milliseconds, it 576 * ends up giving unrealistic values. 577 * 578 * Cap the default transition delay to 10 ms, which seems to be 579 * a reasonable amount of time after which we should reevaluate 580 * the frequency. 581 */ 582 return min(latency * LATENCY_MULTIPLIER, (unsigned int)10000); 583 } 584 585 return LATENCY_MULTIPLIER; 586} 587EXPORT_SYMBOL_GPL(cpufreq_policy_transition_delay_us); 588 589/********************************************************************* 590 * SYSFS INTERFACE * 591 *********************************************************************/ 592static ssize_t show_boost(struct kobject *kobj, 593 struct kobj_attribute *attr, char *buf) 594{ 595 return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled); 596} 597 598static ssize_t store_boost(struct kobject *kobj, struct kobj_attribute *attr, 599 const char *buf, size_t count) 600{ 601 int ret, enable; 602 603 ret = sscanf(buf, "%d", &enable); 604 if (ret != 1 || enable < 0 || enable > 1) 605 return -EINVAL; 606 607 if (cpufreq_boost_trigger_state(enable)) { 608 pr_err("%s: Cannot %s BOOST!\n", 609 __func__, enable ? "enable" : "disable"); 610 return -EINVAL; 611 } 612 613 pr_debug("%s: cpufreq BOOST %s\n", 614 __func__, enable ? "enabled" : "disabled"); 615 616 return count; 617} 618define_one_global_rw(boost); 619 620static struct cpufreq_governor *find_governor(const char *str_governor) 621{ 622 struct cpufreq_governor *t; 623 624 for_each_governor(t) 625 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN)) 626 return t; 627 628 return NULL; 629} 630 631static struct cpufreq_governor *get_governor(const char *str_governor) 632{ 633 struct cpufreq_governor *t; 634 635 mutex_lock(&cpufreq_governor_mutex); 636 t = find_governor(str_governor); 637 if (!t) 638 goto unlock; 639 640 if (!try_module_get(t->owner)) 641 t = NULL; 642 643unlock: 644 mutex_unlock(&cpufreq_governor_mutex); 645 646 return t; 647} 648 649static unsigned int cpufreq_parse_policy(char *str_governor) 650{ 651 if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) 652 return CPUFREQ_POLICY_PERFORMANCE; 653 654 if (!strncasecmp(str_governor, "powersave", CPUFREQ_NAME_LEN)) 655 return CPUFREQ_POLICY_POWERSAVE; 656 657 return CPUFREQ_POLICY_UNKNOWN; 658} 659 660/** 661 * cpufreq_parse_governor - parse a governor string only for has_target() 662 * @str_governor: Governor name. 663 */ 664static struct cpufreq_governor *cpufreq_parse_governor(char *str_governor) 665{ 666 struct cpufreq_governor *t; 667 668 t = get_governor(str_governor); 669 if (t) 670 return t; 671 672 if (request_module("cpufreq_%s", str_governor)) 673 return NULL; 674 675 return get_governor(str_governor); 676} 677 678/* 679 * cpufreq_per_cpu_attr_read() / show_##file_name() - 680 * print out cpufreq information 681 * 682 * Write out information from cpufreq_driver->policy[cpu]; object must be 683 * "unsigned int". 684 */ 685 686#define show_one(file_name, object) \ 687static ssize_t show_##file_name \ 688(struct cpufreq_policy *policy, char *buf) \ 689{ \ 690 return sprintf(buf, "%u\n", policy->object); \ 691} 692 693show_one(cpuinfo_min_freq, cpuinfo.min_freq); 694show_one(cpuinfo_max_freq, cpuinfo.max_freq); 695show_one(cpuinfo_transition_latency, cpuinfo.transition_latency); 696show_one(scaling_min_freq, min); 697show_one(scaling_max_freq, max); 698 699__weak unsigned int arch_freq_get_on_cpu(int cpu) 700{ 701 return 0; 702} 703 704static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf) 705{ 706 ssize_t ret; 707 unsigned int freq; 708 709 freq = arch_freq_get_on_cpu(policy->cpu); 710 if (freq) 711 ret = sprintf(buf, "%u\n", freq); 712 else if (cpufreq_driver->setpolicy && cpufreq_driver->get) 713 ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu)); 714 else 715 ret = sprintf(buf, "%u\n", policy->cur); 716 return ret; 717} 718 719/* 720 * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access 721 */ 722#define store_one(file_name, object) \ 723static ssize_t store_##file_name \ 724(struct cpufreq_policy *policy, const char *buf, size_t count) \ 725{ \ 726 unsigned long val; \ 727 int ret; \ 728 \ 729 ret = sscanf(buf, "%lu", &val); \ 730 if (ret != 1) \ 731 return -EINVAL; \ 732 \ 733 ret = freq_qos_update_request(policy->object##_freq_req, val);\ 734 return ret >= 0 ? count : ret; \ 735} 736 737store_one(scaling_min_freq, min); 738store_one(scaling_max_freq, max); 739 740/* 741 * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware 742 */ 743static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy, 744 char *buf) 745{ 746 unsigned int cur_freq = __cpufreq_get(policy); 747 748 if (cur_freq) 749 return sprintf(buf, "%u\n", cur_freq); 750 751 return sprintf(buf, "<unknown>\n"); 752} 753 754/* 755 * show_scaling_governor - show the current policy for the specified CPU 756 */ 757static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf) 758{ 759 if (policy->policy == CPUFREQ_POLICY_POWERSAVE) 760 return sprintf(buf, "powersave\n"); 761 else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE) 762 return sprintf(buf, "performance\n"); 763 else if (policy->governor) 764 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", 765 policy->governor->name); 766 return -EINVAL; 767} 768 769/* 770 * store_scaling_governor - store policy for the specified CPU 771 */ 772static ssize_t store_scaling_governor(struct cpufreq_policy *policy, 773 const char *buf, size_t count) 774{ 775 char str_governor[16]; 776 int ret; 777 778 ret = sscanf(buf, "%15s", str_governor); 779 if (ret != 1) 780 return -EINVAL; 781 782 if (cpufreq_driver->setpolicy) { 783 unsigned int new_pol; 784 785 new_pol = cpufreq_parse_policy(str_governor); 786 if (!new_pol) 787 return -EINVAL; 788 789 ret = cpufreq_set_policy(policy, NULL, new_pol); 790 } else { 791 struct cpufreq_governor *new_gov; 792 793 new_gov = cpufreq_parse_governor(str_governor); 794 if (!new_gov) 795 return -EINVAL; 796 797 ret = cpufreq_set_policy(policy, new_gov, 798 CPUFREQ_POLICY_UNKNOWN); 799 800 module_put(new_gov->owner); 801 } 802 803 return ret ? ret : count; 804} 805 806/* 807 * show_scaling_driver - show the cpufreq driver currently loaded 808 */ 809static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf) 810{ 811 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name); 812} 813 814/* 815 * show_scaling_available_governors - show the available CPUfreq governors 816 */ 817static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy, 818 char *buf) 819{ 820 ssize_t i = 0; 821 struct cpufreq_governor *t; 822 823 if (!has_target()) { 824 i += sprintf(buf, "performance powersave"); 825 goto out; 826 } 827 828 mutex_lock(&cpufreq_governor_mutex); 829 for_each_governor(t) { 830 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char)) 831 - (CPUFREQ_NAME_LEN + 2))) 832 break; 833 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name); 834 } 835 mutex_unlock(&cpufreq_governor_mutex); 836out: 837 i += sprintf(&buf[i], "\n"); 838 return i; 839} 840 841ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf) 842{ 843 ssize_t i = 0; 844 unsigned int cpu; 845 846 for_each_cpu(cpu, mask) { 847 if (i) 848 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " "); 849 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu); 850 if (i >= (PAGE_SIZE - 5)) 851 break; 852 } 853 i += sprintf(&buf[i], "\n"); 854 return i; 855} 856EXPORT_SYMBOL_GPL(cpufreq_show_cpus); 857 858/* 859 * show_related_cpus - show the CPUs affected by each transition even if 860 * hw coordination is in use 861 */ 862static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf) 863{ 864 return cpufreq_show_cpus(policy->related_cpus, buf); 865} 866 867/* 868 * show_affected_cpus - show the CPUs affected by each transition 869 */ 870static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf) 871{ 872 return cpufreq_show_cpus(policy->cpus, buf); 873} 874 875static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy, 876 const char *buf, size_t count) 877{ 878 unsigned int freq = 0; 879 unsigned int ret; 880 881 if (!policy->governor || !policy->governor->store_setspeed) 882 return -EINVAL; 883 884 ret = sscanf(buf, "%u", &freq); 885 if (ret != 1) 886 return -EINVAL; 887 888 policy->governor->store_setspeed(policy, freq); 889 890 return count; 891} 892 893static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf) 894{ 895 if (!policy->governor || !policy->governor->show_setspeed) 896 return sprintf(buf, "<unsupported>\n"); 897 898 return policy->governor->show_setspeed(policy, buf); 899} 900 901/* 902 * show_bios_limit - show the current cpufreq HW/BIOS limitation 903 */ 904static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf) 905{ 906 unsigned int limit; 907 int ret; 908 ret = cpufreq_driver->bios_limit(policy->cpu, &limit); 909 if (!ret) 910 return sprintf(buf, "%u\n", limit); 911 return sprintf(buf, "%u\n", policy->cpuinfo.max_freq); 912} 913 914cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400); 915cpufreq_freq_attr_ro(cpuinfo_min_freq); 916cpufreq_freq_attr_ro(cpuinfo_max_freq); 917cpufreq_freq_attr_ro(cpuinfo_transition_latency); 918cpufreq_freq_attr_ro(scaling_available_governors); 919cpufreq_freq_attr_ro(scaling_driver); 920cpufreq_freq_attr_ro(scaling_cur_freq); 921cpufreq_freq_attr_ro(bios_limit); 922cpufreq_freq_attr_ro(related_cpus); 923cpufreq_freq_attr_ro(affected_cpus); 924cpufreq_freq_attr_rw(scaling_min_freq); 925cpufreq_freq_attr_rw(scaling_max_freq); 926cpufreq_freq_attr_rw(scaling_governor); 927cpufreq_freq_attr_rw(scaling_setspeed); 928 929static struct attribute *default_attrs[] = { 930 &cpuinfo_min_freq.attr, 931 &cpuinfo_max_freq.attr, 932 &cpuinfo_transition_latency.attr, 933 &scaling_min_freq.attr, 934 &scaling_max_freq.attr, 935 &affected_cpus.attr, 936 &related_cpus.attr, 937 &scaling_governor.attr, 938 &scaling_driver.attr, 939 &scaling_available_governors.attr, 940 &scaling_setspeed.attr, 941 NULL 942}; 943 944#define to_policy(k) container_of(k, struct cpufreq_policy, kobj) 945#define to_attr(a) container_of(a, struct freq_attr, attr) 946 947static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf) 948{ 949 struct cpufreq_policy *policy = to_policy(kobj); 950 struct freq_attr *fattr = to_attr(attr); 951 ssize_t ret; 952 953 if (!fattr->show) 954 return -EIO; 955 956 down_read(&policy->rwsem); 957 ret = fattr->show(policy, buf); 958 up_read(&policy->rwsem); 959 960 return ret; 961} 962 963static ssize_t store(struct kobject *kobj, struct attribute *attr, 964 const char *buf, size_t count) 965{ 966 struct cpufreq_policy *policy = to_policy(kobj); 967 struct freq_attr *fattr = to_attr(attr); 968 ssize_t ret = -EINVAL; 969 970 if (!fattr->store) 971 return -EIO; 972 973 /* 974 * cpus_read_trylock() is used here to work around a circular lock 975 * dependency problem with respect to the cpufreq_register_driver(). 976 */ 977 if (!cpus_read_trylock()) 978 return -EBUSY; 979 980 if (cpu_online(policy->cpu)) { 981 down_write(&policy->rwsem); 982 ret = fattr->store(policy, buf, count); 983 up_write(&policy->rwsem); 984 } 985 986 cpus_read_unlock(); 987 988 return ret; 989} 990 991static void cpufreq_sysfs_release(struct kobject *kobj) 992{ 993 struct cpufreq_policy *policy = to_policy(kobj); 994 pr_debug("last reference is dropped\n"); 995 complete(&policy->kobj_unregister); 996} 997 998static const struct sysfs_ops sysfs_ops = { 999 .show = show, 1000 .store = store, 1001}; 1002 1003static struct kobj_type ktype_cpufreq = { 1004 .sysfs_ops = &sysfs_ops, 1005 .default_attrs = default_attrs, 1006 .release = cpufreq_sysfs_release, 1007}; 1008 1009static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu, 1010 struct device *dev) 1011{ 1012 if (unlikely(!dev)) 1013 return; 1014 1015 if (cpumask_test_and_set_cpu(cpu, policy->real_cpus)) 1016 return; 1017 1018 dev_dbg(dev, "%s: Adding symlink\n", __func__); 1019 if (sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq")) 1020 dev_err(dev, "cpufreq symlink creation failed\n"); 1021} 1022 1023static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, 1024 struct device *dev) 1025{ 1026 dev_dbg(dev, "%s: Removing symlink\n", __func__); 1027 sysfs_remove_link(&dev->kobj, "cpufreq"); 1028} 1029 1030static int cpufreq_add_dev_interface(struct cpufreq_policy *policy) 1031{ 1032 struct freq_attr **drv_attr; 1033 int ret = 0; 1034 1035 /* set up files for this cpu device */ 1036 drv_attr = cpufreq_driver->attr; 1037 while (drv_attr && *drv_attr) { 1038 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr)); 1039 if (ret) 1040 return ret; 1041 drv_attr++; 1042 } 1043 if (cpufreq_driver->get) { 1044 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr); 1045 if (ret) 1046 return ret; 1047 } 1048 1049 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr); 1050 if (ret) 1051 return ret; 1052 1053 if (cpufreq_driver->bios_limit) { 1054 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr); 1055 if (ret) 1056 return ret; 1057 } 1058 1059 return 0; 1060} 1061 1062static int cpufreq_init_policy(struct cpufreq_policy *policy) 1063{ 1064 struct cpufreq_governor *gov = NULL; 1065 unsigned int pol = CPUFREQ_POLICY_UNKNOWN; 1066 int ret; 1067 1068 if (has_target()) { 1069 /* Update policy governor to the one used before hotplug. */ 1070 gov = get_governor(policy->last_governor); 1071 if (gov) { 1072 pr_debug("Restoring governor %s for cpu %d\n", 1073 gov->name, policy->cpu); 1074 } else { 1075 gov = get_governor(default_governor); 1076 } 1077 1078 if (!gov) { 1079 gov = cpufreq_default_governor(); 1080 __module_get(gov->owner); 1081 } 1082 1083 } else { 1084 1085 /* Use the default policy if there is no last_policy. */ 1086 if (policy->last_policy) { 1087 pol = policy->last_policy; 1088 } else { 1089 pol = cpufreq_parse_policy(default_governor); 1090 /* 1091 * In case the default governor is neither "performance" 1092 * nor "powersave", fall back to the initial policy 1093 * value set by the driver. 1094 */ 1095 if (pol == CPUFREQ_POLICY_UNKNOWN) 1096 pol = policy->policy; 1097 } 1098 if (pol != CPUFREQ_POLICY_PERFORMANCE && 1099 pol != CPUFREQ_POLICY_POWERSAVE) 1100 return -ENODATA; 1101 } 1102 1103 ret = cpufreq_set_policy(policy, gov, pol); 1104 if (gov) 1105 module_put(gov->owner); 1106 1107 return ret; 1108} 1109 1110static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu) 1111{ 1112 int ret = 0; 1113 1114 /* Has this CPU been taken care of already? */ 1115 if (cpumask_test_cpu(cpu, policy->cpus)) 1116 return 0; 1117 1118 down_write(&policy->rwsem); 1119 if (has_target()) 1120 cpufreq_stop_governor(policy); 1121 1122 cpumask_set_cpu(cpu, policy->cpus); 1123 1124 if (has_target()) { 1125 ret = cpufreq_start_governor(policy); 1126 if (ret) 1127 pr_err("%s: Failed to start governor\n", __func__); 1128 } 1129 up_write(&policy->rwsem); 1130 return ret; 1131} 1132 1133void refresh_frequency_limits(struct cpufreq_policy *policy) 1134{ 1135 if (!policy_is_inactive(policy)) { 1136 pr_debug("updating policy for CPU %u\n", policy->cpu); 1137 1138 cpufreq_set_policy(policy, policy->governor, policy->policy); 1139 } 1140} 1141EXPORT_SYMBOL(refresh_frequency_limits); 1142 1143static void handle_update(struct work_struct *work) 1144{ 1145 struct cpufreq_policy *policy = 1146 container_of(work, struct cpufreq_policy, update); 1147 1148 pr_debug("handle_update for cpu %u called\n", policy->cpu); 1149 down_write(&policy->rwsem); 1150 refresh_frequency_limits(policy); 1151 up_write(&policy->rwsem); 1152} 1153 1154static int cpufreq_notifier_min(struct notifier_block *nb, unsigned long freq, 1155 void *data) 1156{ 1157 struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_min); 1158 1159 schedule_work(&policy->update); 1160 return 0; 1161} 1162 1163static int cpufreq_notifier_max(struct notifier_block *nb, unsigned long freq, 1164 void *data) 1165{ 1166 struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_max); 1167 1168 schedule_work(&policy->update); 1169 return 0; 1170} 1171 1172static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy) 1173{ 1174 struct kobject *kobj; 1175 struct completion *cmp; 1176 1177 down_write(&policy->rwsem); 1178 cpufreq_stats_free_table(policy); 1179 kobj = &policy->kobj; 1180 cmp = &policy->kobj_unregister; 1181 up_write(&policy->rwsem); 1182 kobject_put(kobj); 1183 1184 /* 1185 * We need to make sure that the underlying kobj is 1186 * actually not referenced anymore by anybody before we 1187 * proceed with unloading. 1188 */ 1189 pr_debug("waiting for dropping of refcount\n"); 1190 wait_for_completion(cmp); 1191 pr_debug("wait complete\n"); 1192} 1193 1194static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu) 1195{ 1196 struct cpufreq_policy *policy; 1197 struct device *dev = get_cpu_device(cpu); 1198 int ret; 1199 1200 if (!dev) 1201 return NULL; 1202 1203 policy = kzalloc(sizeof(*policy), GFP_KERNEL); 1204 if (!policy) 1205 return NULL; 1206 1207 if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL)) 1208 goto err_free_policy; 1209 1210 if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL)) 1211 goto err_free_cpumask; 1212 1213 if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL)) 1214 goto err_free_rcpumask; 1215 1216 init_completion(&policy->kobj_unregister); 1217 ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq, 1218 cpufreq_global_kobject, "policy%u", cpu); 1219 if (ret) { 1220 dev_err(dev, "%s: failed to init policy->kobj: %d\n", __func__, ret); 1221 /* 1222 * The entire policy object will be freed below, but the extra 1223 * memory allocated for the kobject name needs to be freed by 1224 * releasing the kobject. 1225 */ 1226 kobject_put(&policy->kobj); 1227 goto err_free_real_cpus; 1228 } 1229 1230 freq_constraints_init(&policy->constraints); 1231 1232 policy->nb_min.notifier_call = cpufreq_notifier_min; 1233 policy->nb_max.notifier_call = cpufreq_notifier_max; 1234 1235 ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MIN, 1236 &policy->nb_min); 1237 if (ret) { 1238 dev_err(dev, "Failed to register MIN QoS notifier: %d (%*pbl)\n", 1239 ret, cpumask_pr_args(policy->cpus)); 1240 goto err_kobj_remove; 1241 } 1242 1243 ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MAX, 1244 &policy->nb_max); 1245 if (ret) { 1246 dev_err(dev, "Failed to register MAX QoS notifier: %d (%*pbl)\n", 1247 ret, cpumask_pr_args(policy->cpus)); 1248 goto err_min_qos_notifier; 1249 } 1250 1251 INIT_LIST_HEAD(&policy->policy_list); 1252 init_rwsem(&policy->rwsem); 1253 spin_lock_init(&policy->transition_lock); 1254 init_waitqueue_head(&policy->transition_wait); 1255 INIT_WORK(&policy->update, handle_update); 1256 1257 policy->cpu = cpu; 1258 return policy; 1259 1260err_min_qos_notifier: 1261 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN, 1262 &policy->nb_min); 1263err_kobj_remove: 1264 cpufreq_policy_put_kobj(policy); 1265err_free_real_cpus: 1266 free_cpumask_var(policy->real_cpus); 1267err_free_rcpumask: 1268 free_cpumask_var(policy->related_cpus); 1269err_free_cpumask: 1270 free_cpumask_var(policy->cpus); 1271err_free_policy: 1272 kfree(policy); 1273 1274 return NULL; 1275} 1276 1277static void cpufreq_policy_free(struct cpufreq_policy *policy) 1278{ 1279 unsigned long flags; 1280 int cpu; 1281 1282 /* Remove policy from list */ 1283 write_lock_irqsave(&cpufreq_driver_lock, flags); 1284 list_del(&policy->policy_list); 1285 1286 for_each_cpu(cpu, policy->related_cpus) 1287 per_cpu(cpufreq_cpu_data, cpu) = NULL; 1288 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 1289 1290 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MAX, 1291 &policy->nb_max); 1292 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN, 1293 &policy->nb_min); 1294 1295 /* Cancel any pending policy->update work before freeing the policy. */ 1296 cancel_work_sync(&policy->update); 1297 1298 if (policy->max_freq_req) { 1299 /* 1300 * CPUFREQ_CREATE_POLICY notification is sent only after 1301 * successfully adding max_freq_req request. 1302 */ 1303 blocking_notifier_call_chain(&cpufreq_policy_notifier_list, 1304 CPUFREQ_REMOVE_POLICY, policy); 1305 freq_qos_remove_request(policy->max_freq_req); 1306 } 1307 1308 freq_qos_remove_request(policy->min_freq_req); 1309 kfree(policy->min_freq_req); 1310 1311 cpufreq_policy_put_kobj(policy); 1312 free_cpumask_var(policy->real_cpus); 1313 free_cpumask_var(policy->related_cpus); 1314 free_cpumask_var(policy->cpus); 1315 kfree(policy); 1316} 1317 1318static int cpufreq_online(unsigned int cpu) 1319{ 1320 struct cpufreq_policy *policy; 1321 bool new_policy; 1322 unsigned long flags; 1323 unsigned int j; 1324 int ret; 1325 1326 pr_debug("%s: bringing CPU%u online\n", __func__, cpu); 1327 1328 /* Check if this CPU already has a policy to manage it */ 1329 policy = per_cpu(cpufreq_cpu_data, cpu); 1330 if (policy) { 1331 WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus)); 1332 if (!policy_is_inactive(policy)) 1333 return cpufreq_add_policy_cpu(policy, cpu); 1334 1335 /* This is the only online CPU for the policy. Start over. */ 1336 new_policy = false; 1337 down_write(&policy->rwsem); 1338 policy->cpu = cpu; 1339 policy->governor = NULL; 1340 up_write(&policy->rwsem); 1341 } else { 1342 new_policy = true; 1343 policy = cpufreq_policy_alloc(cpu); 1344 if (!policy) 1345 return -ENOMEM; 1346 } 1347 1348 if (!new_policy && cpufreq_driver->online) { 1349 ret = cpufreq_driver->online(policy); 1350 if (ret) { 1351 pr_debug("%s: %d: initialization failed\n", __func__, 1352 __LINE__); 1353 goto out_exit_policy; 1354 } 1355 1356 /* Recover policy->cpus using related_cpus */ 1357 cpumask_copy(policy->cpus, policy->related_cpus); 1358 } else { 1359 cpumask_copy(policy->cpus, cpumask_of(cpu)); 1360 1361 /* 1362 * Call driver. From then on the cpufreq must be able 1363 * to accept all calls to ->verify and ->setpolicy for this CPU. 1364 */ 1365 ret = cpufreq_driver->init(policy); 1366 if (ret) { 1367 pr_debug("%s: %d: initialization failed\n", __func__, 1368 __LINE__); 1369 goto out_free_policy; 1370 } 1371 1372 /* 1373 * The initialization has succeeded and the policy is online. 1374 * If there is a problem with its frequency table, take it 1375 * offline and drop it. 1376 */ 1377 ret = cpufreq_table_validate_and_sort(policy); 1378 if (ret) 1379 goto out_offline_policy; 1380 1381 /* related_cpus should at least include policy->cpus. */ 1382 cpumask_copy(policy->related_cpus, policy->cpus); 1383 } 1384 1385 down_write(&policy->rwsem); 1386 /* 1387 * affected cpus must always be the one, which are online. We aren't 1388 * managing offline cpus here. 1389 */ 1390 cpumask_and(policy->cpus, policy->cpus, cpu_online_mask); 1391 1392 if (new_policy) { 1393 for_each_cpu(j, policy->related_cpus) { 1394 per_cpu(cpufreq_cpu_data, j) = policy; 1395 add_cpu_dev_symlink(policy, j, get_cpu_device(j)); 1396 } 1397 1398 policy->min_freq_req = kzalloc(2 * sizeof(*policy->min_freq_req), 1399 GFP_KERNEL); 1400 if (!policy->min_freq_req) 1401 goto out_destroy_policy; 1402 1403 ret = freq_qos_add_request(&policy->constraints, 1404 policy->min_freq_req, FREQ_QOS_MIN, 1405 FREQ_QOS_MIN_DEFAULT_VALUE); 1406 if (ret < 0) { 1407 /* 1408 * So we don't call freq_qos_remove_request() for an 1409 * uninitialized request. 1410 */ 1411 kfree(policy->min_freq_req); 1412 policy->min_freq_req = NULL; 1413 goto out_destroy_policy; 1414 } 1415 1416 /* 1417 * This must be initialized right here to avoid calling 1418 * freq_qos_remove_request() on uninitialized request in case 1419 * of errors. 1420 */ 1421 policy->max_freq_req = policy->min_freq_req + 1; 1422 1423 ret = freq_qos_add_request(&policy->constraints, 1424 policy->max_freq_req, FREQ_QOS_MAX, 1425 FREQ_QOS_MAX_DEFAULT_VALUE); 1426 if (ret < 0) { 1427 policy->max_freq_req = NULL; 1428 goto out_destroy_policy; 1429 } 1430 1431 blocking_notifier_call_chain(&cpufreq_policy_notifier_list, 1432 CPUFREQ_CREATE_POLICY, policy); 1433 } 1434 1435 if (cpufreq_driver->get && has_target()) { 1436 policy->cur = cpufreq_driver->get(policy->cpu); 1437 if (!policy->cur) { 1438 pr_err("%s: ->get() failed\n", __func__); 1439 goto out_destroy_policy; 1440 } 1441 } 1442 1443 /* 1444 * Sometimes boot loaders set CPU frequency to a value outside of 1445 * frequency table present with cpufreq core. In such cases CPU might be 1446 * unstable if it has to run on that frequency for long duration of time 1447 * and so its better to set it to a frequency which is specified in 1448 * freq-table. This also makes cpufreq stats inconsistent as 1449 * cpufreq-stats would fail to register because current frequency of CPU 1450 * isn't found in freq-table. 1451 * 1452 * Because we don't want this change to effect boot process badly, we go 1453 * for the next freq which is >= policy->cur ('cur' must be set by now, 1454 * otherwise we will end up setting freq to lowest of the table as 'cur' 1455 * is initialized to zero). 1456 * 1457 * We are passing target-freq as "policy->cur - 1" otherwise 1458 * __cpufreq_driver_target() would simply fail, as policy->cur will be 1459 * equal to target-freq. 1460 */ 1461 if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK) 1462 && has_target()) { 1463 unsigned int old_freq = policy->cur; 1464 1465 /* Are we running at unknown frequency ? */ 1466 ret = cpufreq_frequency_table_get_index(policy, old_freq); 1467 if (ret == -EINVAL) { 1468 ret = __cpufreq_driver_target(policy, old_freq - 1, 1469 CPUFREQ_RELATION_L); 1470 1471 /* 1472 * Reaching here after boot in a few seconds may not 1473 * mean that system will remain stable at "unknown" 1474 * frequency for longer duration. Hence, a BUG_ON(). 1475 */ 1476 BUG_ON(ret); 1477 pr_info("%s: CPU%d: Running at unlisted initial frequency: %u KHz, changing to: %u KHz\n", 1478 __func__, policy->cpu, old_freq, policy->cur); 1479 } 1480 } 1481 1482 if (new_policy) { 1483 ret = cpufreq_add_dev_interface(policy); 1484 if (ret) 1485 goto out_destroy_policy; 1486 1487 cpufreq_stats_create_table(policy); 1488 1489 write_lock_irqsave(&cpufreq_driver_lock, flags); 1490 list_add(&policy->policy_list, &cpufreq_policy_list); 1491 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 1492 } 1493 1494 ret = cpufreq_init_policy(policy); 1495 if (ret) { 1496 pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n", 1497 __func__, cpu, ret); 1498 goto out_destroy_policy; 1499 } 1500 1501 up_write(&policy->rwsem); 1502 1503 kobject_uevent(&policy->kobj, KOBJ_ADD); 1504 1505 /* Callback for handling stuff after policy is ready */ 1506 if (cpufreq_driver->ready) 1507 cpufreq_driver->ready(policy); 1508 1509 if (cpufreq_thermal_control_enabled(cpufreq_driver)) 1510 policy->cdev = of_cpufreq_cooling_register(policy); 1511 1512 pr_debug("initialization complete\n"); 1513 1514 return 0; 1515 1516out_destroy_policy: 1517 for_each_cpu(j, policy->real_cpus) 1518 remove_cpu_dev_symlink(policy, get_cpu_device(j)); 1519 1520 up_write(&policy->rwsem); 1521 1522out_offline_policy: 1523 if (cpufreq_driver->offline) 1524 cpufreq_driver->offline(policy); 1525 1526out_exit_policy: 1527 if (cpufreq_driver->exit) 1528 cpufreq_driver->exit(policy); 1529 1530out_free_policy: 1531 cpufreq_policy_free(policy); 1532 return ret; 1533} 1534 1535/** 1536 * cpufreq_add_dev - the cpufreq interface for a CPU device. 1537 * @dev: CPU device. 1538 * @sif: Subsystem interface structure pointer (not used) 1539 */ 1540static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif) 1541{ 1542 struct cpufreq_policy *policy; 1543 unsigned cpu = dev->id; 1544 int ret; 1545 1546 dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu); 1547 1548 if (cpu_online(cpu)) { 1549 ret = cpufreq_online(cpu); 1550 if (ret) 1551 return ret; 1552 } 1553 1554 /* Create sysfs link on CPU registration */ 1555 policy = per_cpu(cpufreq_cpu_data, cpu); 1556 if (policy) 1557 add_cpu_dev_symlink(policy, cpu, dev); 1558 1559 return 0; 1560} 1561 1562static void __cpufreq_offline(unsigned int cpu, struct cpufreq_policy *policy) 1563{ 1564 int ret; 1565 1566 if (has_target()) 1567 cpufreq_stop_governor(policy); 1568 1569 cpumask_clear_cpu(cpu, policy->cpus); 1570 1571 if (!policy_is_inactive(policy)) { 1572 /* Nominate a new CPU if necessary. */ 1573 if (cpu == policy->cpu) 1574 policy->cpu = cpumask_any(policy->cpus); 1575 1576 /* Start the governor again for the active policy. */ 1577 if (has_target()) { 1578 ret = cpufreq_start_governor(policy); 1579 if (ret) 1580 pr_err("%s: Failed to start governor\n", __func__); 1581 } 1582 1583 return; 1584 } 1585 1586 if (has_target()) 1587 strncpy(policy->last_governor, policy->governor->name, 1588 CPUFREQ_NAME_LEN); 1589 else 1590 policy->last_policy = policy->policy; 1591 1592 if (cpufreq_thermal_control_enabled(cpufreq_driver)) { 1593 cpufreq_cooling_unregister(policy->cdev); 1594 policy->cdev = NULL; 1595 } 1596 1597 if (cpufreq_driver->stop_cpu) 1598 cpufreq_driver->stop_cpu(policy); 1599 1600 if (has_target()) 1601 cpufreq_exit_governor(policy); 1602 1603 /* 1604 * Perform the ->offline() during light-weight tear-down, as 1605 * that allows fast recovery when the CPU comes back. 1606 */ 1607 if (cpufreq_driver->offline) { 1608 cpufreq_driver->offline(policy); 1609 return; 1610 } 1611 1612 if (cpufreq_driver->exit) 1613 cpufreq_driver->exit(policy); 1614 1615 policy->freq_table = NULL; 1616} 1617 1618static int cpufreq_offline(unsigned int cpu) 1619{ 1620 struct cpufreq_policy *policy; 1621 1622 pr_debug("%s: unregistering CPU %u\n", __func__, cpu); 1623 1624 policy = cpufreq_cpu_get_raw(cpu); 1625 if (!policy) { 1626 pr_debug("%s: No cpu_data found\n", __func__); 1627 return 0; 1628 } 1629 1630 down_write(&policy->rwsem); 1631 1632 __cpufreq_offline(cpu, policy); 1633 1634 up_write(&policy->rwsem); 1635 return 0; 1636} 1637 1638/* 1639 * cpufreq_remove_dev - remove a CPU device 1640 * 1641 * Removes the cpufreq interface for a CPU device. 1642 */ 1643static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif) 1644{ 1645 unsigned int cpu = dev->id; 1646 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); 1647 1648 if (!policy) 1649 return; 1650 1651 down_write(&policy->rwsem); 1652 1653 if (cpu_online(cpu)) 1654 __cpufreq_offline(cpu, policy); 1655 1656 cpumask_clear_cpu(cpu, policy->real_cpus); 1657 remove_cpu_dev_symlink(policy, dev); 1658 1659 if (!cpumask_empty(policy->real_cpus)) { 1660 up_write(&policy->rwsem); 1661 return; 1662 } 1663 1664 /* We did light-weight exit earlier, do full tear down now */ 1665 if (cpufreq_driver->offline && cpufreq_driver->exit) 1666 cpufreq_driver->exit(policy); 1667 1668 up_write(&policy->rwsem); 1669 1670 cpufreq_policy_free(policy); 1671} 1672 1673/** 1674 * cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're 1675 * in deep trouble. 1676 * @policy: policy managing CPUs 1677 * @new_freq: CPU frequency the CPU actually runs at 1678 * 1679 * We adjust to current frequency first, and need to clean up later. 1680 * So either call to cpufreq_update_policy() or schedule handle_update()). 1681 */ 1682static void cpufreq_out_of_sync(struct cpufreq_policy *policy, 1683 unsigned int new_freq) 1684{ 1685 struct cpufreq_freqs freqs; 1686 1687 pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n", 1688 policy->cur, new_freq); 1689 1690 freqs.old = policy->cur; 1691 freqs.new = new_freq; 1692 1693 cpufreq_freq_transition_begin(policy, &freqs); 1694 cpufreq_freq_transition_end(policy, &freqs, 0); 1695} 1696 1697static unsigned int cpufreq_verify_current_freq(struct cpufreq_policy *policy, bool update) 1698{ 1699 unsigned int new_freq; 1700 1701 new_freq = cpufreq_driver->get(policy->cpu); 1702 if (!new_freq) 1703 return 0; 1704 1705 /* 1706 * If fast frequency switching is used with the given policy, the check 1707 * against policy->cur is pointless, so skip it in that case. 1708 */ 1709 if (policy->fast_switch_enabled || !has_target()) 1710 return new_freq; 1711 1712 if (policy->cur != new_freq) { 1713 cpufreq_out_of_sync(policy, new_freq); 1714 if (update) 1715 schedule_work(&policy->update); 1716 } 1717 1718 return new_freq; 1719} 1720 1721/** 1722 * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur 1723 * @cpu: CPU number 1724 * 1725 * This is the last known freq, without actually getting it from the driver. 1726 * Return value will be same as what is shown in scaling_cur_freq in sysfs. 1727 */ 1728unsigned int cpufreq_quick_get(unsigned int cpu) 1729{ 1730 struct cpufreq_policy *policy; 1731 unsigned int ret_freq = 0; 1732 unsigned long flags; 1733 1734 read_lock_irqsave(&cpufreq_driver_lock, flags); 1735 1736 if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) { 1737 ret_freq = cpufreq_driver->get(cpu); 1738 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 1739 return ret_freq; 1740 } 1741 1742 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 1743 1744 policy = cpufreq_cpu_get(cpu); 1745 if (policy) { 1746 ret_freq = policy->cur; 1747 cpufreq_cpu_put(policy); 1748 } 1749 1750 return ret_freq; 1751} 1752EXPORT_SYMBOL(cpufreq_quick_get); 1753 1754/** 1755 * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU 1756 * @cpu: CPU number 1757 * 1758 * Just return the max possible frequency for a given CPU. 1759 */ 1760unsigned int cpufreq_quick_get_max(unsigned int cpu) 1761{ 1762 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 1763 unsigned int ret_freq = 0; 1764 1765 if (policy) { 1766 ret_freq = policy->max; 1767 cpufreq_cpu_put(policy); 1768 } 1769 1770 return ret_freq; 1771} 1772EXPORT_SYMBOL(cpufreq_quick_get_max); 1773 1774/** 1775 * cpufreq_get_hw_max_freq - get the max hardware frequency of the CPU 1776 * @cpu: CPU number 1777 * 1778 * The default return value is the max_freq field of cpuinfo. 1779 */ 1780__weak unsigned int cpufreq_get_hw_max_freq(unsigned int cpu) 1781{ 1782 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 1783 unsigned int ret_freq = 0; 1784 1785 if (policy) { 1786 ret_freq = policy->cpuinfo.max_freq; 1787 cpufreq_cpu_put(policy); 1788 } 1789 1790 return ret_freq; 1791} 1792EXPORT_SYMBOL(cpufreq_get_hw_max_freq); 1793 1794static unsigned int __cpufreq_get(struct cpufreq_policy *policy) 1795{ 1796 if (unlikely(policy_is_inactive(policy))) 1797 return 0; 1798 1799 return cpufreq_verify_current_freq(policy, true); 1800} 1801 1802/** 1803 * cpufreq_get - get the current CPU frequency (in kHz) 1804 * @cpu: CPU number 1805 * 1806 * Get the CPU current (static) CPU frequency 1807 */ 1808unsigned int cpufreq_get(unsigned int cpu) 1809{ 1810 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 1811 unsigned int ret_freq = 0; 1812 1813 if (policy) { 1814 down_read(&policy->rwsem); 1815 if (cpufreq_driver->get) 1816 ret_freq = __cpufreq_get(policy); 1817 up_read(&policy->rwsem); 1818 1819 cpufreq_cpu_put(policy); 1820 } 1821 1822 return ret_freq; 1823} 1824EXPORT_SYMBOL(cpufreq_get); 1825 1826static struct subsys_interface cpufreq_interface = { 1827 .name = "cpufreq", 1828 .subsys = &cpu_subsys, 1829 .add_dev = cpufreq_add_dev, 1830 .remove_dev = cpufreq_remove_dev, 1831}; 1832 1833/* 1834 * In case platform wants some specific frequency to be configured 1835 * during suspend.. 1836 */ 1837int cpufreq_generic_suspend(struct cpufreq_policy *policy) 1838{ 1839 int ret; 1840 1841 if (!policy->suspend_freq) { 1842 pr_debug("%s: suspend_freq not defined\n", __func__); 1843 return 0; 1844 } 1845 1846 pr_debug("%s: Setting suspend-freq: %u\n", __func__, 1847 policy->suspend_freq); 1848 1849 ret = __cpufreq_driver_target(policy, policy->suspend_freq, 1850 CPUFREQ_RELATION_H); 1851 if (ret) 1852 pr_err("%s: unable to set suspend-freq: %u. err: %d\n", 1853 __func__, policy->suspend_freq, ret); 1854 1855 return ret; 1856} 1857EXPORT_SYMBOL(cpufreq_generic_suspend); 1858 1859/** 1860 * cpufreq_suspend() - Suspend CPUFreq governors 1861 * 1862 * Called during system wide Suspend/Hibernate cycles for suspending governors 1863 * as some platforms can't change frequency after this point in suspend cycle. 1864 * Because some of the devices (like: i2c, regulators, etc) they use for 1865 * changing frequency are suspended quickly after this point. 1866 */ 1867void cpufreq_suspend(void) 1868{ 1869 struct cpufreq_policy *policy; 1870 1871 if (!cpufreq_driver) 1872 return; 1873 1874 if (!has_target() && !cpufreq_driver->suspend) 1875 goto suspend; 1876 1877 pr_debug("%s: Suspending Governors\n", __func__); 1878 1879 for_each_active_policy(policy) { 1880 if (has_target()) { 1881 down_write(&policy->rwsem); 1882 cpufreq_stop_governor(policy); 1883 up_write(&policy->rwsem); 1884 } 1885 1886 if (cpufreq_driver->suspend && cpufreq_driver->suspend(policy)) 1887 pr_err("%s: Failed to suspend driver: %s\n", __func__, 1888 cpufreq_driver->name); 1889 } 1890 1891suspend: 1892 cpufreq_suspended = true; 1893} 1894 1895/** 1896 * cpufreq_resume() - Resume CPUFreq governors 1897 * 1898 * Called during system wide Suspend/Hibernate cycle for resuming governors that 1899 * are suspended with cpufreq_suspend(). 1900 */ 1901void cpufreq_resume(void) 1902{ 1903 struct cpufreq_policy *policy; 1904 int ret; 1905 1906 if (!cpufreq_driver) 1907 return; 1908 1909 if (unlikely(!cpufreq_suspended)) 1910 return; 1911 1912 cpufreq_suspended = false; 1913 1914 if (!has_target() && !cpufreq_driver->resume) 1915 return; 1916 1917 pr_debug("%s: Resuming Governors\n", __func__); 1918 1919 for_each_active_policy(policy) { 1920 if (cpufreq_driver->resume && cpufreq_driver->resume(policy)) { 1921 pr_err("%s: Failed to resume driver: %p\n", __func__, 1922 policy); 1923 } else if (has_target()) { 1924 down_write(&policy->rwsem); 1925 ret = cpufreq_start_governor(policy); 1926 up_write(&policy->rwsem); 1927 1928 if (ret) 1929 pr_err("%s: Failed to start governor for policy: %p\n", 1930 __func__, policy); 1931 } 1932 } 1933} 1934 1935/** 1936 * cpufreq_driver_test_flags - Test cpufreq driver's flags against given ones. 1937 * @flags: Flags to test against the current cpufreq driver's flags. 1938 * 1939 * Assumes that the driver is there, so callers must ensure that this is the 1940 * case. 1941 */ 1942bool cpufreq_driver_test_flags(u16 flags) 1943{ 1944 return !!(cpufreq_driver->flags & flags); 1945} 1946 1947/** 1948 * cpufreq_get_current_driver - return current driver's name 1949 * 1950 * Return the name string of the currently loaded cpufreq driver 1951 * or NULL, if none. 1952 */ 1953const char *cpufreq_get_current_driver(void) 1954{ 1955 if (cpufreq_driver) 1956 return cpufreq_driver->name; 1957 1958 return NULL; 1959} 1960EXPORT_SYMBOL_GPL(cpufreq_get_current_driver); 1961 1962/** 1963 * cpufreq_get_driver_data - return current driver data 1964 * 1965 * Return the private data of the currently loaded cpufreq 1966 * driver, or NULL if no cpufreq driver is loaded. 1967 */ 1968void *cpufreq_get_driver_data(void) 1969{ 1970 if (cpufreq_driver) 1971 return cpufreq_driver->driver_data; 1972 1973 return NULL; 1974} 1975EXPORT_SYMBOL_GPL(cpufreq_get_driver_data); 1976 1977/********************************************************************* 1978 * NOTIFIER LISTS INTERFACE * 1979 *********************************************************************/ 1980 1981/** 1982 * cpufreq_register_notifier - register a driver with cpufreq 1983 * @nb: notifier function to register 1984 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER 1985 * 1986 * Add a driver to one of two lists: either a list of drivers that 1987 * are notified about clock rate changes (once before and once after 1988 * the transition), or a list of drivers that are notified about 1989 * changes in cpufreq policy. 1990 * 1991 * This function may sleep, and has the same return conditions as 1992 * blocking_notifier_chain_register. 1993 */ 1994int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list) 1995{ 1996 int ret; 1997 1998 if (cpufreq_disabled()) 1999 return -EINVAL; 2000 2001 switch (list) { 2002 case CPUFREQ_TRANSITION_NOTIFIER: 2003 mutex_lock(&cpufreq_fast_switch_lock); 2004 2005 if (cpufreq_fast_switch_count > 0) { 2006 mutex_unlock(&cpufreq_fast_switch_lock); 2007 return -EBUSY; 2008 } 2009 ret = srcu_notifier_chain_register( 2010 &cpufreq_transition_notifier_list, nb); 2011 if (!ret) 2012 cpufreq_fast_switch_count--; 2013 2014 mutex_unlock(&cpufreq_fast_switch_lock); 2015 break; 2016 case CPUFREQ_POLICY_NOTIFIER: 2017 ret = blocking_notifier_chain_register( 2018 &cpufreq_policy_notifier_list, nb); 2019 break; 2020 default: 2021 ret = -EINVAL; 2022 } 2023 2024 return ret; 2025} 2026EXPORT_SYMBOL(cpufreq_register_notifier); 2027 2028/** 2029 * cpufreq_unregister_notifier - unregister a driver with cpufreq 2030 * @nb: notifier block to be unregistered 2031 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER 2032 * 2033 * Remove a driver from the CPU frequency notifier list. 2034 * 2035 * This function may sleep, and has the same return conditions as 2036 * blocking_notifier_chain_unregister. 2037 */ 2038int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list) 2039{ 2040 int ret; 2041 2042 if (cpufreq_disabled()) 2043 return -EINVAL; 2044 2045 switch (list) { 2046 case CPUFREQ_TRANSITION_NOTIFIER: 2047 mutex_lock(&cpufreq_fast_switch_lock); 2048 2049 ret = srcu_notifier_chain_unregister( 2050 &cpufreq_transition_notifier_list, nb); 2051 if (!ret && !WARN_ON(cpufreq_fast_switch_count >= 0)) 2052 cpufreq_fast_switch_count++; 2053 2054 mutex_unlock(&cpufreq_fast_switch_lock); 2055 break; 2056 case CPUFREQ_POLICY_NOTIFIER: 2057 ret = blocking_notifier_chain_unregister( 2058 &cpufreq_policy_notifier_list, nb); 2059 break; 2060 default: 2061 ret = -EINVAL; 2062 } 2063 2064 return ret; 2065} 2066EXPORT_SYMBOL(cpufreq_unregister_notifier); 2067 2068 2069/********************************************************************* 2070 * GOVERNORS * 2071 *********************************************************************/ 2072 2073/** 2074 * cpufreq_driver_fast_switch - Carry out a fast CPU frequency switch. 2075 * @policy: cpufreq policy to switch the frequency for. 2076 * @target_freq: New frequency to set (may be approximate). 2077 * 2078 * Carry out a fast frequency switch without sleeping. 2079 * 2080 * The driver's ->fast_switch() callback invoked by this function must be 2081 * suitable for being called from within RCU-sched read-side critical sections 2082 * and it is expected to select the minimum available frequency greater than or 2083 * equal to @target_freq (CPUFREQ_RELATION_L). 2084 * 2085 * This function must not be called if policy->fast_switch_enabled is unset. 2086 * 2087 * Governors calling this function must guarantee that it will never be invoked 2088 * twice in parallel for the same policy and that it will never be called in 2089 * parallel with either ->target() or ->target_index() for the same policy. 2090 * 2091 * Returns the actual frequency set for the CPU. 2092 * 2093 * If 0 is returned by the driver's ->fast_switch() callback to indicate an 2094 * error condition, the hardware configuration must be preserved. 2095 */ 2096unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy, 2097 unsigned int target_freq) 2098{ 2099 unsigned int freq; 2100 int cpu; 2101 2102 target_freq = clamp_val(target_freq, policy->min, policy->max); 2103 freq = cpufreq_driver->fast_switch(policy, target_freq); 2104 2105 if (!freq) 2106 return 0; 2107 2108 policy->cur = freq; 2109 arch_set_freq_scale(policy->related_cpus, freq, 2110 policy->cpuinfo.max_freq); 2111 cpufreq_stats_record_transition(policy, freq); 2112 2113 if (trace_cpu_frequency_enabled()) { 2114 for_each_cpu(cpu, policy->cpus) 2115 trace_cpu_frequency(freq, cpu); 2116 } 2117 2118 return freq; 2119} 2120EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch); 2121 2122/* Must set freqs->new to intermediate frequency */ 2123static int __target_intermediate(struct cpufreq_policy *policy, 2124 struct cpufreq_freqs *freqs, int index) 2125{ 2126 int ret; 2127 2128 freqs->new = cpufreq_driver->get_intermediate(policy, index); 2129 2130 /* We don't need to switch to intermediate freq */ 2131 if (!freqs->new) 2132 return 0; 2133 2134 pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n", 2135 __func__, policy->cpu, freqs->old, freqs->new); 2136 2137 cpufreq_freq_transition_begin(policy, freqs); 2138 ret = cpufreq_driver->target_intermediate(policy, index); 2139 cpufreq_freq_transition_end(policy, freqs, ret); 2140 2141 if (ret) 2142 pr_err("%s: Failed to change to intermediate frequency: %d\n", 2143 __func__, ret); 2144 2145 return ret; 2146} 2147 2148static int __target_index(struct cpufreq_policy *policy, int index) 2149{ 2150 struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0}; 2151 unsigned int intermediate_freq = 0; 2152 unsigned int newfreq = policy->freq_table[index].frequency; 2153 int retval = -EINVAL; 2154 bool notify; 2155 2156 if (newfreq == policy->cur) 2157 return 0; 2158 2159 notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION); 2160 if (notify) { 2161 /* Handle switching to intermediate frequency */ 2162 if (cpufreq_driver->get_intermediate) { 2163 retval = __target_intermediate(policy, &freqs, index); 2164 if (retval) 2165 return retval; 2166 2167 intermediate_freq = freqs.new; 2168 /* Set old freq to intermediate */ 2169 if (intermediate_freq) 2170 freqs.old = freqs.new; 2171 } 2172 2173 freqs.new = newfreq; 2174 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n", 2175 __func__, policy->cpu, freqs.old, freqs.new); 2176 2177 cpufreq_freq_transition_begin(policy, &freqs); 2178 } 2179 2180 retval = cpufreq_driver->target_index(policy, index); 2181 if (retval) 2182 pr_err("%s: Failed to change cpu frequency: %d\n", __func__, 2183 retval); 2184 2185 if (notify) { 2186 cpufreq_freq_transition_end(policy, &freqs, retval); 2187 2188 /* 2189 * Failed after setting to intermediate freq? Driver should have 2190 * reverted back to initial frequency and so should we. Check 2191 * here for intermediate_freq instead of get_intermediate, in 2192 * case we haven't switched to intermediate freq at all. 2193 */ 2194 if (unlikely(retval && intermediate_freq)) { 2195 freqs.old = intermediate_freq; 2196 freqs.new = policy->restore_freq; 2197 cpufreq_freq_transition_begin(policy, &freqs); 2198 cpufreq_freq_transition_end(policy, &freqs, 0); 2199 } 2200 } 2201 2202 return retval; 2203} 2204 2205int __cpufreq_driver_target(struct cpufreq_policy *policy, 2206 unsigned int target_freq, 2207 unsigned int relation) 2208{ 2209 unsigned int old_target_freq = target_freq; 2210 int index; 2211 2212 if (cpufreq_disabled()) 2213 return -ENODEV; 2214 2215 /* Make sure that target_freq is within supported range */ 2216 target_freq = clamp_val(target_freq, policy->min, policy->max); 2217 2218 pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n", 2219 policy->cpu, target_freq, relation, old_target_freq); 2220 2221 /* 2222 * This might look like a redundant call as we are checking it again 2223 * after finding index. But it is left intentionally for cases where 2224 * exactly same freq is called again and so we can save on few function 2225 * calls. 2226 */ 2227 if (target_freq == policy->cur && 2228 !(cpufreq_driver->flags & CPUFREQ_NEED_UPDATE_LIMITS)) 2229 return 0; 2230 2231 /* Save last value to restore later on errors */ 2232 policy->restore_freq = policy->cur; 2233 2234 if (cpufreq_driver->target) 2235 return cpufreq_driver->target(policy, target_freq, relation); 2236 2237 if (!cpufreq_driver->target_index) 2238 return -EINVAL; 2239 2240 index = cpufreq_frequency_table_target(policy, target_freq, relation); 2241 2242 return __target_index(policy, index); 2243} 2244EXPORT_SYMBOL_GPL(__cpufreq_driver_target); 2245 2246int cpufreq_driver_target(struct cpufreq_policy *policy, 2247 unsigned int target_freq, 2248 unsigned int relation) 2249{ 2250 int ret; 2251 2252 down_write(&policy->rwsem); 2253 2254 ret = __cpufreq_driver_target(policy, target_freq, relation); 2255 2256 up_write(&policy->rwsem); 2257 2258 return ret; 2259} 2260EXPORT_SYMBOL_GPL(cpufreq_driver_target); 2261 2262__weak struct cpufreq_governor *cpufreq_fallback_governor(void) 2263{ 2264 return NULL; 2265} 2266 2267static int cpufreq_init_governor(struct cpufreq_policy *policy) 2268{ 2269 int ret; 2270 2271 /* Don't start any governor operations if we are entering suspend */ 2272 if (cpufreq_suspended) 2273 return 0; 2274 /* 2275 * Governor might not be initiated here if ACPI _PPC changed 2276 * notification happened, so check it. 2277 */ 2278 if (!policy->governor) 2279 return -EINVAL; 2280 2281 /* Platform doesn't want dynamic frequency switching ? */ 2282 if (policy->governor->flags & CPUFREQ_GOV_DYNAMIC_SWITCHING && 2283 cpufreq_driver->flags & CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING) { 2284 struct cpufreq_governor *gov = cpufreq_fallback_governor(); 2285 2286 if (gov) { 2287 pr_warn("Can't use %s governor as dynamic switching is disallowed. Fallback to %s governor\n", 2288 policy->governor->name, gov->name); 2289 policy->governor = gov; 2290 } else { 2291 return -EINVAL; 2292 } 2293 } 2294 2295 if (!try_module_get(policy->governor->owner)) 2296 return -EINVAL; 2297 2298 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2299 2300 if (policy->governor->init) { 2301 ret = policy->governor->init(policy); 2302 if (ret) { 2303 module_put(policy->governor->owner); 2304 return ret; 2305 } 2306 } 2307 2308 policy->strict_target = !!(policy->governor->flags & CPUFREQ_GOV_STRICT_TARGET); 2309 2310 return 0; 2311} 2312 2313static void cpufreq_exit_governor(struct cpufreq_policy *policy) 2314{ 2315 if (cpufreq_suspended || !policy->governor) 2316 return; 2317 2318 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2319 2320 if (policy->governor->exit) 2321 policy->governor->exit(policy); 2322 2323 module_put(policy->governor->owner); 2324} 2325 2326int cpufreq_start_governor(struct cpufreq_policy *policy) 2327{ 2328 int ret; 2329 2330 if (cpufreq_suspended) 2331 return 0; 2332 2333 if (!policy->governor) 2334 return -EINVAL; 2335 2336 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2337 2338 if (cpufreq_driver->get) 2339 cpufreq_verify_current_freq(policy, false); 2340 2341 if (policy->governor->start) { 2342 ret = policy->governor->start(policy); 2343 if (ret) 2344 return ret; 2345 } 2346 2347 if (policy->governor->limits) 2348 policy->governor->limits(policy); 2349 2350 return 0; 2351} 2352 2353void cpufreq_stop_governor(struct cpufreq_policy *policy) 2354{ 2355 if (cpufreq_suspended || !policy->governor) 2356 return; 2357 2358 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2359 2360 if (policy->governor->stop) 2361 policy->governor->stop(policy); 2362} 2363 2364static void cpufreq_governor_limits(struct cpufreq_policy *policy) 2365{ 2366 if (cpufreq_suspended || !policy->governor) 2367 return; 2368 2369 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2370 2371 if (policy->governor->limits) 2372 policy->governor->limits(policy); 2373} 2374 2375int cpufreq_register_governor(struct cpufreq_governor *governor) 2376{ 2377 int err; 2378 2379 if (!governor) 2380 return -EINVAL; 2381 2382 if (cpufreq_disabled()) 2383 return -ENODEV; 2384 2385 mutex_lock(&cpufreq_governor_mutex); 2386 2387 err = -EBUSY; 2388 if (!find_governor(governor->name)) { 2389 err = 0; 2390 list_add(&governor->governor_list, &cpufreq_governor_list); 2391 } 2392 2393 mutex_unlock(&cpufreq_governor_mutex); 2394 return err; 2395} 2396EXPORT_SYMBOL_GPL(cpufreq_register_governor); 2397 2398void cpufreq_unregister_governor(struct cpufreq_governor *governor) 2399{ 2400 struct cpufreq_policy *policy; 2401 unsigned long flags; 2402 2403 if (!governor) 2404 return; 2405 2406 if (cpufreq_disabled()) 2407 return; 2408 2409 /* clear last_governor for all inactive policies */ 2410 read_lock_irqsave(&cpufreq_driver_lock, flags); 2411 for_each_inactive_policy(policy) { 2412 if (!strcmp(policy->last_governor, governor->name)) { 2413 policy->governor = NULL; 2414 strcpy(policy->last_governor, "\0"); 2415 } 2416 } 2417 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 2418 2419 mutex_lock(&cpufreq_governor_mutex); 2420 list_del(&governor->governor_list); 2421 mutex_unlock(&cpufreq_governor_mutex); 2422} 2423EXPORT_SYMBOL_GPL(cpufreq_unregister_governor); 2424 2425 2426/********************************************************************* 2427 * POLICY INTERFACE * 2428 *********************************************************************/ 2429 2430/** 2431 * cpufreq_get_policy - get the current cpufreq_policy 2432 * @policy: struct cpufreq_policy into which the current cpufreq_policy 2433 * is written 2434 * @cpu: CPU to find the policy for 2435 * 2436 * Reads the current cpufreq policy. 2437 */ 2438int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu) 2439{ 2440 struct cpufreq_policy *cpu_policy; 2441 if (!policy) 2442 return -EINVAL; 2443 2444 cpu_policy = cpufreq_cpu_get(cpu); 2445 if (!cpu_policy) 2446 return -EINVAL; 2447 2448 memcpy(policy, cpu_policy, sizeof(*policy)); 2449 2450 cpufreq_cpu_put(cpu_policy); 2451 return 0; 2452} 2453EXPORT_SYMBOL(cpufreq_get_policy); 2454 2455/** 2456 * cpufreq_set_policy - Modify cpufreq policy parameters. 2457 * @policy: Policy object to modify. 2458 * @new_gov: Policy governor pointer. 2459 * @new_pol: Policy value (for drivers with built-in governors). 2460 * 2461 * Invoke the cpufreq driver's ->verify() callback to sanity-check the frequency 2462 * limits to be set for the policy, update @policy with the verified limits 2463 * values and either invoke the driver's ->setpolicy() callback (if present) or 2464 * carry out a governor update for @policy. That is, run the current governor's 2465 * ->limits() callback (if @new_gov points to the same object as the one in 2466 * @policy) or replace the governor for @policy with @new_gov. 2467 * 2468 * The cpuinfo part of @policy is not updated by this function. 2469 */ 2470static int cpufreq_set_policy(struct cpufreq_policy *policy, 2471 struct cpufreq_governor *new_gov, 2472 unsigned int new_pol) 2473{ 2474 struct cpufreq_policy_data new_data; 2475 struct cpufreq_governor *old_gov; 2476 int ret; 2477 2478 memcpy(&new_data.cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo)); 2479 new_data.freq_table = policy->freq_table; 2480 new_data.cpu = policy->cpu; 2481 /* 2482 * PM QoS framework collects all the requests from users and provide us 2483 * the final aggregated value here. 2484 */ 2485 new_data.min = freq_qos_read_value(&policy->constraints, FREQ_QOS_MIN); 2486 new_data.max = freq_qos_read_value(&policy->constraints, FREQ_QOS_MAX); 2487 2488 pr_debug("setting new policy for CPU %u: %u - %u kHz\n", 2489 new_data.cpu, new_data.min, new_data.max); 2490 2491 /* 2492 * Verify that the CPU speed can be set within these limits and make sure 2493 * that min <= max. 2494 */ 2495 ret = cpufreq_driver->verify(&new_data); 2496 if (ret) 2497 return ret; 2498 2499 policy->min = new_data.min; 2500 policy->max = new_data.max; 2501 trace_cpu_frequency_limits(policy); 2502 2503 policy->cached_target_freq = UINT_MAX; 2504 2505 pr_debug("new min and max freqs are %u - %u kHz\n", 2506 policy->min, policy->max); 2507 2508 if (cpufreq_driver->setpolicy) { 2509 policy->policy = new_pol; 2510 pr_debug("setting range\n"); 2511 return cpufreq_driver->setpolicy(policy); 2512 } 2513 2514 if (new_gov == policy->governor) { 2515 pr_debug("governor limits update\n"); 2516 cpufreq_governor_limits(policy); 2517 return 0; 2518 } 2519 2520 pr_debug("governor switch\n"); 2521 2522 /* save old, working values */ 2523 old_gov = policy->governor; 2524 /* end old governor */ 2525 if (old_gov) { 2526 cpufreq_stop_governor(policy); 2527 cpufreq_exit_governor(policy); 2528 } 2529 2530 /* start new governor */ 2531 policy->governor = new_gov; 2532 ret = cpufreq_init_governor(policy); 2533 if (!ret) { 2534 ret = cpufreq_start_governor(policy); 2535 if (!ret) { 2536 pr_debug("governor change\n"); 2537 sched_cpufreq_governor_change(policy, old_gov); 2538 return 0; 2539 } 2540 cpufreq_exit_governor(policy); 2541 } 2542 2543 /* new governor failed, so re-start old one */ 2544 pr_debug("starting governor %s failed\n", policy->governor->name); 2545 if (old_gov) { 2546 policy->governor = old_gov; 2547 if (cpufreq_init_governor(policy)) 2548 policy->governor = NULL; 2549 else 2550 cpufreq_start_governor(policy); 2551 } 2552 2553 return ret; 2554} 2555 2556/** 2557 * cpufreq_update_policy - Re-evaluate an existing cpufreq policy. 2558 * @cpu: CPU to re-evaluate the policy for. 2559 * 2560 * Update the current frequency for the cpufreq policy of @cpu and use 2561 * cpufreq_set_policy() to re-apply the min and max limits, which triggers the 2562 * evaluation of policy notifiers and the cpufreq driver's ->verify() callback 2563 * for the policy in question, among other things. 2564 */ 2565void cpufreq_update_policy(unsigned int cpu) 2566{ 2567 struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu); 2568 2569 if (!policy) 2570 return; 2571 2572 /* 2573 * BIOS might change freq behind our back 2574 * -> ask driver for current freq and notify governors about a change 2575 */ 2576 if (cpufreq_driver->get && has_target() && 2577 (cpufreq_suspended || WARN_ON(!cpufreq_verify_current_freq(policy, false)))) 2578 goto unlock; 2579 2580 refresh_frequency_limits(policy); 2581 2582unlock: 2583 cpufreq_cpu_release(policy); 2584} 2585EXPORT_SYMBOL(cpufreq_update_policy); 2586 2587/** 2588 * cpufreq_update_limits - Update policy limits for a given CPU. 2589 * @cpu: CPU to update the policy limits for. 2590 * 2591 * Invoke the driver's ->update_limits callback if present or call 2592 * cpufreq_update_policy() for @cpu. 2593 */ 2594void cpufreq_update_limits(unsigned int cpu) 2595{ 2596 if (cpufreq_driver->update_limits) 2597 cpufreq_driver->update_limits(cpu); 2598 else 2599 cpufreq_update_policy(cpu); 2600} 2601EXPORT_SYMBOL_GPL(cpufreq_update_limits); 2602 2603/********************************************************************* 2604 * BOOST * 2605 *********************************************************************/ 2606static int cpufreq_boost_set_sw(struct cpufreq_policy *policy, int state) 2607{ 2608 int ret; 2609 2610 if (!policy->freq_table) 2611 return -ENXIO; 2612 2613 ret = cpufreq_frequency_table_cpuinfo(policy, policy->freq_table); 2614 if (ret) { 2615 pr_err("%s: Policy frequency update failed\n", __func__); 2616 return ret; 2617 } 2618 2619 ret = freq_qos_update_request(policy->max_freq_req, policy->max); 2620 if (ret < 0) 2621 return ret; 2622 2623 return 0; 2624} 2625 2626int cpufreq_boost_trigger_state(int state) 2627{ 2628 struct cpufreq_policy *policy; 2629 unsigned long flags; 2630 int ret = 0; 2631 2632 if (cpufreq_driver->boost_enabled == state) 2633 return 0; 2634 2635 write_lock_irqsave(&cpufreq_driver_lock, flags); 2636 cpufreq_driver->boost_enabled = state; 2637 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2638 2639 get_online_cpus(); 2640 for_each_active_policy(policy) { 2641 ret = cpufreq_driver->set_boost(policy, state); 2642 if (ret) 2643 goto err_reset_state; 2644 } 2645 put_online_cpus(); 2646 2647 return 0; 2648 2649err_reset_state: 2650 put_online_cpus(); 2651 2652 write_lock_irqsave(&cpufreq_driver_lock, flags); 2653 cpufreq_driver->boost_enabled = !state; 2654 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2655 2656 pr_err("%s: Cannot %s BOOST\n", 2657 __func__, state ? "enable" : "disable"); 2658 2659 return ret; 2660} 2661 2662static bool cpufreq_boost_supported(void) 2663{ 2664 return cpufreq_driver->set_boost; 2665} 2666 2667static int create_boost_sysfs_file(void) 2668{ 2669 int ret; 2670 2671 ret = sysfs_create_file(cpufreq_global_kobject, &boost.attr); 2672 if (ret) 2673 pr_err("%s: cannot register global BOOST sysfs file\n", 2674 __func__); 2675 2676 return ret; 2677} 2678 2679static void remove_boost_sysfs_file(void) 2680{ 2681 if (cpufreq_boost_supported()) 2682 sysfs_remove_file(cpufreq_global_kobject, &boost.attr); 2683} 2684 2685int cpufreq_enable_boost_support(void) 2686{ 2687 if (!cpufreq_driver) 2688 return -EINVAL; 2689 2690 if (cpufreq_boost_supported()) 2691 return 0; 2692 2693 cpufreq_driver->set_boost = cpufreq_boost_set_sw; 2694 2695 /* This will get removed on driver unregister */ 2696 return create_boost_sysfs_file(); 2697} 2698EXPORT_SYMBOL_GPL(cpufreq_enable_boost_support); 2699 2700int cpufreq_boost_enabled(void) 2701{ 2702 return cpufreq_driver->boost_enabled; 2703} 2704EXPORT_SYMBOL_GPL(cpufreq_boost_enabled); 2705 2706/********************************************************************* 2707 * REGISTER / UNREGISTER CPUFREQ DRIVER * 2708 *********************************************************************/ 2709static enum cpuhp_state hp_online; 2710 2711static int cpuhp_cpufreq_online(unsigned int cpu) 2712{ 2713 cpufreq_online(cpu); 2714 2715 return 0; 2716} 2717 2718static int cpuhp_cpufreq_offline(unsigned int cpu) 2719{ 2720 cpufreq_offline(cpu); 2721 2722 return 0; 2723} 2724 2725/** 2726 * cpufreq_register_driver - register a CPU Frequency driver 2727 * @driver_data: A struct cpufreq_driver containing the values# 2728 * submitted by the CPU Frequency driver. 2729 * 2730 * Registers a CPU Frequency driver to this core code. This code 2731 * returns zero on success, -EEXIST when another driver got here first 2732 * (and isn't unregistered in the meantime). 2733 * 2734 */ 2735int cpufreq_register_driver(struct cpufreq_driver *driver_data) 2736{ 2737 unsigned long flags; 2738 int ret; 2739 2740 if (cpufreq_disabled()) 2741 return -ENODEV; 2742 2743 /* 2744 * The cpufreq core depends heavily on the availability of device 2745 * structure, make sure they are available before proceeding further. 2746 */ 2747 if (!get_cpu_device(0)) 2748 return -EPROBE_DEFER; 2749 2750 if (!driver_data || !driver_data->verify || !driver_data->init || 2751 !(driver_data->setpolicy || driver_data->target_index || 2752 driver_data->target) || 2753 (driver_data->setpolicy && (driver_data->target_index || 2754 driver_data->target)) || 2755 (!driver_data->get_intermediate != !driver_data->target_intermediate) || 2756 (!driver_data->online != !driver_data->offline)) 2757 return -EINVAL; 2758 2759 pr_debug("trying to register driver %s\n", driver_data->name); 2760 2761 /* Protect against concurrent CPU online/offline. */ 2762 cpus_read_lock(); 2763 2764 write_lock_irqsave(&cpufreq_driver_lock, flags); 2765 if (cpufreq_driver) { 2766 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2767 ret = -EEXIST; 2768 goto out; 2769 } 2770 cpufreq_driver = driver_data; 2771 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2772 2773 /* 2774 * Mark support for the scheduler's frequency invariance engine for 2775 * drivers that implement target(), target_index() or fast_switch(). 2776 */ 2777 if (!cpufreq_driver->setpolicy) { 2778 static_branch_enable_cpuslocked(&cpufreq_freq_invariance); 2779 pr_debug("supports frequency invariance"); 2780 } 2781 2782 if (driver_data->setpolicy) 2783 driver_data->flags |= CPUFREQ_CONST_LOOPS; 2784 2785 if (cpufreq_boost_supported()) { 2786 ret = create_boost_sysfs_file(); 2787 if (ret) 2788 goto err_null_driver; 2789 } 2790 2791 ret = subsys_interface_register(&cpufreq_interface); 2792 if (ret) 2793 goto err_boost_unreg; 2794 2795 if (!(cpufreq_driver->flags & CPUFREQ_STICKY) && 2796 list_empty(&cpufreq_policy_list)) { 2797 /* if all ->init() calls failed, unregister */ 2798 ret = -ENODEV; 2799 pr_debug("%s: No CPU initialized for driver %s\n", __func__, 2800 driver_data->name); 2801 goto err_if_unreg; 2802 } 2803 2804 ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN, 2805 "cpufreq:online", 2806 cpuhp_cpufreq_online, 2807 cpuhp_cpufreq_offline); 2808 if (ret < 0) 2809 goto err_if_unreg; 2810 hp_online = ret; 2811 ret = 0; 2812 2813 pr_debug("driver %s up and running\n", driver_data->name); 2814 goto out; 2815 2816err_if_unreg: 2817 subsys_interface_unregister(&cpufreq_interface); 2818err_boost_unreg: 2819 remove_boost_sysfs_file(); 2820err_null_driver: 2821 write_lock_irqsave(&cpufreq_driver_lock, flags); 2822 cpufreq_driver = NULL; 2823 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2824out: 2825 cpus_read_unlock(); 2826 return ret; 2827} 2828EXPORT_SYMBOL_GPL(cpufreq_register_driver); 2829 2830/* 2831 * cpufreq_unregister_driver - unregister the current CPUFreq driver 2832 * 2833 * Unregister the current CPUFreq driver. Only call this if you have 2834 * the right to do so, i.e. if you have succeeded in initialising before! 2835 * Returns zero if successful, and -EINVAL if the cpufreq_driver is 2836 * currently not initialised. 2837 */ 2838int cpufreq_unregister_driver(struct cpufreq_driver *driver) 2839{ 2840 unsigned long flags; 2841 2842 if (!cpufreq_driver || (driver != cpufreq_driver)) 2843 return -EINVAL; 2844 2845 pr_debug("unregistering driver %s\n", driver->name); 2846 2847 /* Protect against concurrent cpu hotplug */ 2848 cpus_read_lock(); 2849 subsys_interface_unregister(&cpufreq_interface); 2850 remove_boost_sysfs_file(); 2851 static_branch_disable_cpuslocked(&cpufreq_freq_invariance); 2852 cpuhp_remove_state_nocalls_cpuslocked(hp_online); 2853 2854 write_lock_irqsave(&cpufreq_driver_lock, flags); 2855 2856 cpufreq_driver = NULL; 2857 2858 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2859 cpus_read_unlock(); 2860 2861 return 0; 2862} 2863EXPORT_SYMBOL_GPL(cpufreq_unregister_driver); 2864 2865static int __init cpufreq_core_init(void) 2866{ 2867 struct cpufreq_governor *gov = cpufreq_default_governor(); 2868 2869 if (cpufreq_disabled()) 2870 return -ENODEV; 2871 2872 cpufreq_global_kobject = kobject_create_and_add("cpufreq", &cpu_subsys.dev_root->kobj); 2873 BUG_ON(!cpufreq_global_kobject); 2874 2875 if (!strlen(default_governor)) 2876 strncpy(default_governor, gov->name, CPUFREQ_NAME_LEN); 2877 2878 return 0; 2879} 2880module_param(off, int, 0444); 2881module_param_string(default_governor, default_governor, CPUFREQ_NAME_LEN, 0444); 2882core_initcall(cpufreq_core_init); 2883