1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * intel_idle.c - native hardware idle loop for modern Intel processors 4 * 5 * Copyright (c) 2013 - 2020, Intel Corporation. 6 * Len Brown <len.brown@intel.com> 7 * Rafael J. Wysocki <rafael.j.wysocki@intel.com> 8 */ 9 10/* 11 * intel_idle is a cpuidle driver that loads on all Intel CPUs with MWAIT 12 * in lieu of the legacy ACPI processor_idle driver. The intent is to 13 * make Linux more efficient on these processors, as intel_idle knows 14 * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. 15 */ 16 17/* 18 * Design Assumptions 19 * 20 * All CPUs have same idle states as boot CPU 21 * 22 * Chipset BM_STS (bus master status) bit is a NOP 23 * for preventing entry into deep C-states 24 * 25 * CPU will flush caches as needed when entering a C-state via MWAIT 26 * (in contrast to entering ACPI C3, in which case the WBINVD 27 * instruction needs to be executed to flush the caches) 28 */ 29 30/* 31 * Known limitations 32 * 33 * ACPI has a .suspend hack to turn off deep c-statees during suspend 34 * to avoid complications with the lapic timer workaround. 35 * Have not seen issues with suspend, but may need same workaround here. 36 * 37 */ 38 39/* un-comment DEBUG to enable pr_debug() statements */ 40/* #define DEBUG */ 41 42#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 43 44#include <linux/acpi.h> 45#include <linux/kernel.h> 46#include <linux/cpuidle.h> 47#include <linux/tick.h> 48#include <trace/events/power.h> 49#include <linux/sched.h> 50#include <linux/sched/smt.h> 51#include <linux/notifier.h> 52#include <linux/cpu.h> 53#include <linux/moduleparam.h> 54#include <asm/cpu_device_id.h> 55#include <asm/intel-family.h> 56#include <asm/nospec-branch.h> 57#include <asm/mwait.h> 58#include <asm/msr.h> 59#include <asm/fpu/api.h> 60 61#define INTEL_IDLE_VERSION "0.5.1" 62 63static struct cpuidle_driver intel_idle_driver = { 64 .name = "intel_idle", 65 .owner = THIS_MODULE, 66}; 67/* intel_idle.max_cstate=0 disables driver */ 68static int max_cstate = CPUIDLE_STATE_MAX - 1; 69static unsigned int disabled_states_mask __read_mostly; 70static unsigned int preferred_states_mask __read_mostly; 71static bool force_irq_on __read_mostly; 72 73static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; 74 75static unsigned long auto_demotion_disable_flags; 76 77static enum { 78 C1E_PROMOTION_PRESERVE, 79 C1E_PROMOTION_ENABLE, 80 C1E_PROMOTION_DISABLE 81} c1e_promotion = C1E_PROMOTION_PRESERVE; 82 83struct idle_cpu { 84 struct cpuidle_state *state_table; 85 86 /* 87 * Hardware C-state auto-demotion may not always be optimal. 88 * Indicate which enable bits to clear here. 89 */ 90 unsigned long auto_demotion_disable_flags; 91 bool byt_auto_demotion_disable_flag; 92 bool disable_promotion_to_c1e; 93 bool use_acpi; 94}; 95 96static const struct idle_cpu *icpu __initdata; 97static struct cpuidle_state *cpuidle_state_table __initdata; 98 99static unsigned int mwait_substates __initdata; 100 101/* 102 * Enable interrupts before entering the C-state. On some platforms and for 103 * some C-states, this may measurably decrease interrupt latency. 104 */ 105#define CPUIDLE_FLAG_IRQ_ENABLE BIT(14) 106 107/* 108 * Enable this state by default even if the ACPI _CST does not list it. 109 */ 110#define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15) 111 112/* 113 * Disable IBRS across idle (when KERNEL_IBRS), is exclusive vs IRQ_ENABLE 114 * above. 115 */ 116#define CPUIDLE_FLAG_IBRS BIT(16) 117 118/* 119 * Initialize large xstate for the C6-state entrance. 120 */ 121#define CPUIDLE_FLAG_INIT_XSTATE BIT(17) 122 123/* 124 * MWAIT takes an 8-bit "hint" in EAX "suggesting" 125 * the C-state (top nibble) and sub-state (bottom nibble) 126 * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc. 127 * 128 * We store the hint at the top of our "flags" for each state. 129 */ 130#define flg2MWAIT(flags) (((flags) >> 24) & 0xFF) 131#define MWAIT2flg(eax) ((eax & 0xFF) << 24) 132 133static __always_inline int __intel_idle(struct cpuidle_device *dev, 134 struct cpuidle_driver *drv, 135 int index, bool irqoff) 136{ 137 struct cpuidle_state *state = &drv->states[index]; 138 unsigned long eax = flg2MWAIT(state->flags); 139 unsigned long ecx = 1*irqoff; /* break on interrupt flag */ 140 141 mwait_idle_with_hints(eax, ecx); 142 143 return index; 144} 145 146/** 147 * intel_idle - Ask the processor to enter the given idle state. 148 * @dev: cpuidle device of the target CPU. 149 * @drv: cpuidle driver (assumed to point to intel_idle_driver). 150 * @index: Target idle state index. 151 * 152 * Use the MWAIT instruction to notify the processor that the CPU represented by 153 * @dev is idle and it can try to enter the idle state corresponding to @index. 154 * 155 * If the local APIC timer is not known to be reliable in the target idle state, 156 * enable one-shot tick broadcasting for the target CPU before executing MWAIT. 157 * 158 * Must be called under local_irq_disable(). 159 */ 160static __cpuidle int intel_idle(struct cpuidle_device *dev, 161 struct cpuidle_driver *drv, int index) 162{ 163 return __intel_idle(dev, drv, index, true); 164} 165 166static __cpuidle int intel_idle_irq(struct cpuidle_device *dev, 167 struct cpuidle_driver *drv, int index) 168{ 169 return __intel_idle(dev, drv, index, false); 170} 171 172static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev, 173 struct cpuidle_driver *drv, int index) 174{ 175 bool smt_active = sched_smt_active(); 176 u64 spec_ctrl = spec_ctrl_current(); 177 int ret; 178 179 if (smt_active) 180 native_wrmsrl(MSR_IA32_SPEC_CTRL, 0); 181 182 ret = __intel_idle(dev, drv, index, true); 183 184 if (smt_active) 185 native_wrmsrl(MSR_IA32_SPEC_CTRL, spec_ctrl); 186 187 return ret; 188} 189 190static __cpuidle int intel_idle_xstate(struct cpuidle_device *dev, 191 struct cpuidle_driver *drv, int index) 192{ 193 fpu_idle_fpregs(); 194 return __intel_idle(dev, drv, index, true); 195} 196 197/** 198 * intel_idle_s2idle - Ask the processor to enter the given idle state. 199 * @dev: cpuidle device of the target CPU. 200 * @drv: cpuidle driver (assumed to point to intel_idle_driver). 201 * @index: Target idle state index. 202 * 203 * Use the MWAIT instruction to notify the processor that the CPU represented by 204 * @dev is idle and it can try to enter the idle state corresponding to @index. 205 * 206 * Invoked as a suspend-to-idle callback routine with frozen user space, frozen 207 * scheduler tick and suspended scheduler clock on the target CPU. 208 */ 209static __cpuidle int intel_idle_s2idle(struct cpuidle_device *dev, 210 struct cpuidle_driver *drv, int index) 211{ 212 unsigned long ecx = 1; /* break on interrupt flag */ 213 struct cpuidle_state *state = &drv->states[index]; 214 unsigned long eax = flg2MWAIT(state->flags); 215 216 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) 217 fpu_idle_fpregs(); 218 219 mwait_idle_with_hints(eax, ecx); 220 221 return 0; 222} 223 224/* 225 * States are indexed by the cstate number, 226 * which is also the index into the MWAIT hint array. 227 * Thus C0 is a dummy. 228 */ 229static struct cpuidle_state nehalem_cstates[] __initdata = { 230 { 231 .name = "C1", 232 .desc = "MWAIT 0x00", 233 .flags = MWAIT2flg(0x00), 234 .exit_latency = 3, 235 .target_residency = 6, 236 .enter = &intel_idle, 237 .enter_s2idle = intel_idle_s2idle, }, 238 { 239 .name = "C1E", 240 .desc = "MWAIT 0x01", 241 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 242 .exit_latency = 10, 243 .target_residency = 20, 244 .enter = &intel_idle, 245 .enter_s2idle = intel_idle_s2idle, }, 246 { 247 .name = "C3", 248 .desc = "MWAIT 0x10", 249 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 250 .exit_latency = 20, 251 .target_residency = 80, 252 .enter = &intel_idle, 253 .enter_s2idle = intel_idle_s2idle, }, 254 { 255 .name = "C6", 256 .desc = "MWAIT 0x20", 257 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 258 .exit_latency = 200, 259 .target_residency = 800, 260 .enter = &intel_idle, 261 .enter_s2idle = intel_idle_s2idle, }, 262 { 263 .enter = NULL } 264}; 265 266static struct cpuidle_state snb_cstates[] __initdata = { 267 { 268 .name = "C1", 269 .desc = "MWAIT 0x00", 270 .flags = MWAIT2flg(0x00), 271 .exit_latency = 2, 272 .target_residency = 2, 273 .enter = &intel_idle, 274 .enter_s2idle = intel_idle_s2idle, }, 275 { 276 .name = "C1E", 277 .desc = "MWAIT 0x01", 278 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 279 .exit_latency = 10, 280 .target_residency = 20, 281 .enter = &intel_idle, 282 .enter_s2idle = intel_idle_s2idle, }, 283 { 284 .name = "C3", 285 .desc = "MWAIT 0x10", 286 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 287 .exit_latency = 80, 288 .target_residency = 211, 289 .enter = &intel_idle, 290 .enter_s2idle = intel_idle_s2idle, }, 291 { 292 .name = "C6", 293 .desc = "MWAIT 0x20", 294 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 295 .exit_latency = 104, 296 .target_residency = 345, 297 .enter = &intel_idle, 298 .enter_s2idle = intel_idle_s2idle, }, 299 { 300 .name = "C7", 301 .desc = "MWAIT 0x30", 302 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 303 .exit_latency = 109, 304 .target_residency = 345, 305 .enter = &intel_idle, 306 .enter_s2idle = intel_idle_s2idle, }, 307 { 308 .enter = NULL } 309}; 310 311static struct cpuidle_state byt_cstates[] __initdata = { 312 { 313 .name = "C1", 314 .desc = "MWAIT 0x00", 315 .flags = MWAIT2flg(0x00), 316 .exit_latency = 1, 317 .target_residency = 1, 318 .enter = &intel_idle, 319 .enter_s2idle = intel_idle_s2idle, }, 320 { 321 .name = "C6N", 322 .desc = "MWAIT 0x58", 323 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 324 .exit_latency = 300, 325 .target_residency = 275, 326 .enter = &intel_idle, 327 .enter_s2idle = intel_idle_s2idle, }, 328 { 329 .name = "C6S", 330 .desc = "MWAIT 0x52", 331 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 332 .exit_latency = 500, 333 .target_residency = 560, 334 .enter = &intel_idle, 335 .enter_s2idle = intel_idle_s2idle, }, 336 { 337 .name = "C7", 338 .desc = "MWAIT 0x60", 339 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 340 .exit_latency = 1200, 341 .target_residency = 4000, 342 .enter = &intel_idle, 343 .enter_s2idle = intel_idle_s2idle, }, 344 { 345 .name = "C7S", 346 .desc = "MWAIT 0x64", 347 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 348 .exit_latency = 10000, 349 .target_residency = 20000, 350 .enter = &intel_idle, 351 .enter_s2idle = intel_idle_s2idle, }, 352 { 353 .enter = NULL } 354}; 355 356static struct cpuidle_state cht_cstates[] __initdata = { 357 { 358 .name = "C1", 359 .desc = "MWAIT 0x00", 360 .flags = MWAIT2flg(0x00), 361 .exit_latency = 1, 362 .target_residency = 1, 363 .enter = &intel_idle, 364 .enter_s2idle = intel_idle_s2idle, }, 365 { 366 .name = "C6N", 367 .desc = "MWAIT 0x58", 368 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 369 .exit_latency = 80, 370 .target_residency = 275, 371 .enter = &intel_idle, 372 .enter_s2idle = intel_idle_s2idle, }, 373 { 374 .name = "C6S", 375 .desc = "MWAIT 0x52", 376 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 377 .exit_latency = 200, 378 .target_residency = 560, 379 .enter = &intel_idle, 380 .enter_s2idle = intel_idle_s2idle, }, 381 { 382 .name = "C7", 383 .desc = "MWAIT 0x60", 384 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 385 .exit_latency = 1200, 386 .target_residency = 4000, 387 .enter = &intel_idle, 388 .enter_s2idle = intel_idle_s2idle, }, 389 { 390 .name = "C7S", 391 .desc = "MWAIT 0x64", 392 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 393 .exit_latency = 10000, 394 .target_residency = 20000, 395 .enter = &intel_idle, 396 .enter_s2idle = intel_idle_s2idle, }, 397 { 398 .enter = NULL } 399}; 400 401static struct cpuidle_state ivb_cstates[] __initdata = { 402 { 403 .name = "C1", 404 .desc = "MWAIT 0x00", 405 .flags = MWAIT2flg(0x00), 406 .exit_latency = 1, 407 .target_residency = 1, 408 .enter = &intel_idle, 409 .enter_s2idle = intel_idle_s2idle, }, 410 { 411 .name = "C1E", 412 .desc = "MWAIT 0x01", 413 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 414 .exit_latency = 10, 415 .target_residency = 20, 416 .enter = &intel_idle, 417 .enter_s2idle = intel_idle_s2idle, }, 418 { 419 .name = "C3", 420 .desc = "MWAIT 0x10", 421 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 422 .exit_latency = 59, 423 .target_residency = 156, 424 .enter = &intel_idle, 425 .enter_s2idle = intel_idle_s2idle, }, 426 { 427 .name = "C6", 428 .desc = "MWAIT 0x20", 429 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 430 .exit_latency = 80, 431 .target_residency = 300, 432 .enter = &intel_idle, 433 .enter_s2idle = intel_idle_s2idle, }, 434 { 435 .name = "C7", 436 .desc = "MWAIT 0x30", 437 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 438 .exit_latency = 87, 439 .target_residency = 300, 440 .enter = &intel_idle, 441 .enter_s2idle = intel_idle_s2idle, }, 442 { 443 .enter = NULL } 444}; 445 446static struct cpuidle_state ivt_cstates[] __initdata = { 447 { 448 .name = "C1", 449 .desc = "MWAIT 0x00", 450 .flags = MWAIT2flg(0x00), 451 .exit_latency = 1, 452 .target_residency = 1, 453 .enter = &intel_idle, 454 .enter_s2idle = intel_idle_s2idle, }, 455 { 456 .name = "C1E", 457 .desc = "MWAIT 0x01", 458 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 459 .exit_latency = 10, 460 .target_residency = 80, 461 .enter = &intel_idle, 462 .enter_s2idle = intel_idle_s2idle, }, 463 { 464 .name = "C3", 465 .desc = "MWAIT 0x10", 466 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 467 .exit_latency = 59, 468 .target_residency = 156, 469 .enter = &intel_idle, 470 .enter_s2idle = intel_idle_s2idle, }, 471 { 472 .name = "C6", 473 .desc = "MWAIT 0x20", 474 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 475 .exit_latency = 82, 476 .target_residency = 300, 477 .enter = &intel_idle, 478 .enter_s2idle = intel_idle_s2idle, }, 479 { 480 .enter = NULL } 481}; 482 483static struct cpuidle_state ivt_cstates_4s[] __initdata = { 484 { 485 .name = "C1", 486 .desc = "MWAIT 0x00", 487 .flags = MWAIT2flg(0x00), 488 .exit_latency = 1, 489 .target_residency = 1, 490 .enter = &intel_idle, 491 .enter_s2idle = intel_idle_s2idle, }, 492 { 493 .name = "C1E", 494 .desc = "MWAIT 0x01", 495 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 496 .exit_latency = 10, 497 .target_residency = 250, 498 .enter = &intel_idle, 499 .enter_s2idle = intel_idle_s2idle, }, 500 { 501 .name = "C3", 502 .desc = "MWAIT 0x10", 503 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 504 .exit_latency = 59, 505 .target_residency = 300, 506 .enter = &intel_idle, 507 .enter_s2idle = intel_idle_s2idle, }, 508 { 509 .name = "C6", 510 .desc = "MWAIT 0x20", 511 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 512 .exit_latency = 84, 513 .target_residency = 400, 514 .enter = &intel_idle, 515 .enter_s2idle = intel_idle_s2idle, }, 516 { 517 .enter = NULL } 518}; 519 520static struct cpuidle_state ivt_cstates_8s[] __initdata = { 521 { 522 .name = "C1", 523 .desc = "MWAIT 0x00", 524 .flags = MWAIT2flg(0x00), 525 .exit_latency = 1, 526 .target_residency = 1, 527 .enter = &intel_idle, 528 .enter_s2idle = intel_idle_s2idle, }, 529 { 530 .name = "C1E", 531 .desc = "MWAIT 0x01", 532 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 533 .exit_latency = 10, 534 .target_residency = 500, 535 .enter = &intel_idle, 536 .enter_s2idle = intel_idle_s2idle, }, 537 { 538 .name = "C3", 539 .desc = "MWAIT 0x10", 540 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 541 .exit_latency = 59, 542 .target_residency = 600, 543 .enter = &intel_idle, 544 .enter_s2idle = intel_idle_s2idle, }, 545 { 546 .name = "C6", 547 .desc = "MWAIT 0x20", 548 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 549 .exit_latency = 88, 550 .target_residency = 700, 551 .enter = &intel_idle, 552 .enter_s2idle = intel_idle_s2idle, }, 553 { 554 .enter = NULL } 555}; 556 557static struct cpuidle_state hsw_cstates[] __initdata = { 558 { 559 .name = "C1", 560 .desc = "MWAIT 0x00", 561 .flags = MWAIT2flg(0x00), 562 .exit_latency = 2, 563 .target_residency = 2, 564 .enter = &intel_idle, 565 .enter_s2idle = intel_idle_s2idle, }, 566 { 567 .name = "C1E", 568 .desc = "MWAIT 0x01", 569 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 570 .exit_latency = 10, 571 .target_residency = 20, 572 .enter = &intel_idle, 573 .enter_s2idle = intel_idle_s2idle, }, 574 { 575 .name = "C3", 576 .desc = "MWAIT 0x10", 577 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 578 .exit_latency = 33, 579 .target_residency = 100, 580 .enter = &intel_idle, 581 .enter_s2idle = intel_idle_s2idle, }, 582 { 583 .name = "C6", 584 .desc = "MWAIT 0x20", 585 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 586 .exit_latency = 133, 587 .target_residency = 400, 588 .enter = &intel_idle, 589 .enter_s2idle = intel_idle_s2idle, }, 590 { 591 .name = "C7s", 592 .desc = "MWAIT 0x32", 593 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 594 .exit_latency = 166, 595 .target_residency = 500, 596 .enter = &intel_idle, 597 .enter_s2idle = intel_idle_s2idle, }, 598 { 599 .name = "C8", 600 .desc = "MWAIT 0x40", 601 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 602 .exit_latency = 300, 603 .target_residency = 900, 604 .enter = &intel_idle, 605 .enter_s2idle = intel_idle_s2idle, }, 606 { 607 .name = "C9", 608 .desc = "MWAIT 0x50", 609 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 610 .exit_latency = 600, 611 .target_residency = 1800, 612 .enter = &intel_idle, 613 .enter_s2idle = intel_idle_s2idle, }, 614 { 615 .name = "C10", 616 .desc = "MWAIT 0x60", 617 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 618 .exit_latency = 2600, 619 .target_residency = 7700, 620 .enter = &intel_idle, 621 .enter_s2idle = intel_idle_s2idle, }, 622 { 623 .enter = NULL } 624}; 625static struct cpuidle_state bdw_cstates[] __initdata = { 626 { 627 .name = "C1", 628 .desc = "MWAIT 0x00", 629 .flags = MWAIT2flg(0x00), 630 .exit_latency = 2, 631 .target_residency = 2, 632 .enter = &intel_idle, 633 .enter_s2idle = intel_idle_s2idle, }, 634 { 635 .name = "C1E", 636 .desc = "MWAIT 0x01", 637 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 638 .exit_latency = 10, 639 .target_residency = 20, 640 .enter = &intel_idle, 641 .enter_s2idle = intel_idle_s2idle, }, 642 { 643 .name = "C3", 644 .desc = "MWAIT 0x10", 645 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 646 .exit_latency = 40, 647 .target_residency = 100, 648 .enter = &intel_idle, 649 .enter_s2idle = intel_idle_s2idle, }, 650 { 651 .name = "C6", 652 .desc = "MWAIT 0x20", 653 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 654 .exit_latency = 133, 655 .target_residency = 400, 656 .enter = &intel_idle, 657 .enter_s2idle = intel_idle_s2idle, }, 658 { 659 .name = "C7s", 660 .desc = "MWAIT 0x32", 661 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 662 .exit_latency = 166, 663 .target_residency = 500, 664 .enter = &intel_idle, 665 .enter_s2idle = intel_idle_s2idle, }, 666 { 667 .name = "C8", 668 .desc = "MWAIT 0x40", 669 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 670 .exit_latency = 300, 671 .target_residency = 900, 672 .enter = &intel_idle, 673 .enter_s2idle = intel_idle_s2idle, }, 674 { 675 .name = "C9", 676 .desc = "MWAIT 0x50", 677 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 678 .exit_latency = 600, 679 .target_residency = 1800, 680 .enter = &intel_idle, 681 .enter_s2idle = intel_idle_s2idle, }, 682 { 683 .name = "C10", 684 .desc = "MWAIT 0x60", 685 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 686 .exit_latency = 2600, 687 .target_residency = 7700, 688 .enter = &intel_idle, 689 .enter_s2idle = intel_idle_s2idle, }, 690 { 691 .enter = NULL } 692}; 693 694static struct cpuidle_state skl_cstates[] __initdata = { 695 { 696 .name = "C1", 697 .desc = "MWAIT 0x00", 698 .flags = MWAIT2flg(0x00), 699 .exit_latency = 2, 700 .target_residency = 2, 701 .enter = &intel_idle, 702 .enter_s2idle = intel_idle_s2idle, }, 703 { 704 .name = "C1E", 705 .desc = "MWAIT 0x01", 706 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 707 .exit_latency = 10, 708 .target_residency = 20, 709 .enter = &intel_idle, 710 .enter_s2idle = intel_idle_s2idle, }, 711 { 712 .name = "C3", 713 .desc = "MWAIT 0x10", 714 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 715 .exit_latency = 70, 716 .target_residency = 100, 717 .enter = &intel_idle, 718 .enter_s2idle = intel_idle_s2idle, }, 719 { 720 .name = "C6", 721 .desc = "MWAIT 0x20", 722 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 723 .exit_latency = 85, 724 .target_residency = 200, 725 .enter = &intel_idle, 726 .enter_s2idle = intel_idle_s2idle, }, 727 { 728 .name = "C7s", 729 .desc = "MWAIT 0x33", 730 .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 731 .exit_latency = 124, 732 .target_residency = 800, 733 .enter = &intel_idle, 734 .enter_s2idle = intel_idle_s2idle, }, 735 { 736 .name = "C8", 737 .desc = "MWAIT 0x40", 738 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 739 .exit_latency = 200, 740 .target_residency = 800, 741 .enter = &intel_idle, 742 .enter_s2idle = intel_idle_s2idle, }, 743 { 744 .name = "C9", 745 .desc = "MWAIT 0x50", 746 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 747 .exit_latency = 480, 748 .target_residency = 5000, 749 .enter = &intel_idle, 750 .enter_s2idle = intel_idle_s2idle, }, 751 { 752 .name = "C10", 753 .desc = "MWAIT 0x60", 754 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 755 .exit_latency = 890, 756 .target_residency = 5000, 757 .enter = &intel_idle, 758 .enter_s2idle = intel_idle_s2idle, }, 759 { 760 .enter = NULL } 761}; 762 763static struct cpuidle_state skx_cstates[] __initdata = { 764 { 765 .name = "C1", 766 .desc = "MWAIT 0x00", 767 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE, 768 .exit_latency = 2, 769 .target_residency = 2, 770 .enter = &intel_idle, 771 .enter_s2idle = intel_idle_s2idle, }, 772 { 773 .name = "C1E", 774 .desc = "MWAIT 0x01", 775 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 776 .exit_latency = 10, 777 .target_residency = 20, 778 .enter = &intel_idle, 779 .enter_s2idle = intel_idle_s2idle, }, 780 { 781 .name = "C6", 782 .desc = "MWAIT 0x20", 783 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 784 .exit_latency = 133, 785 .target_residency = 600, 786 .enter = &intel_idle, 787 .enter_s2idle = intel_idle_s2idle, }, 788 { 789 .enter = NULL } 790}; 791 792static struct cpuidle_state icx_cstates[] __initdata = { 793 { 794 .name = "C1", 795 .desc = "MWAIT 0x00", 796 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE, 797 .exit_latency = 1, 798 .target_residency = 1, 799 .enter = &intel_idle, 800 .enter_s2idle = intel_idle_s2idle, }, 801 { 802 .name = "C1E", 803 .desc = "MWAIT 0x01", 804 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 805 .exit_latency = 4, 806 .target_residency = 4, 807 .enter = &intel_idle, 808 .enter_s2idle = intel_idle_s2idle, }, 809 { 810 .name = "C6", 811 .desc = "MWAIT 0x20", 812 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 813 .exit_latency = 170, 814 .target_residency = 600, 815 .enter = &intel_idle, 816 .enter_s2idle = intel_idle_s2idle, }, 817 { 818 .enter = NULL } 819}; 820 821/* 822 * On AlderLake C1 has to be disabled if C1E is enabled, and vice versa. 823 * C1E is enabled only if "C1E promotion" bit is set in MSR_IA32_POWER_CTL. 824 * But in this case there is effectively no C1, because C1 requests are 825 * promoted to C1E. If the "C1E promotion" bit is cleared, then both C1 826 * and C1E requests end up with C1, so there is effectively no C1E. 827 * 828 * By default we enable C1E and disable C1 by marking it with 829 * 'CPUIDLE_FLAG_UNUSABLE'. 830 */ 831static struct cpuidle_state adl_cstates[] __initdata = { 832 { 833 .name = "C1", 834 .desc = "MWAIT 0x00", 835 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 836 .exit_latency = 1, 837 .target_residency = 1, 838 .enter = &intel_idle, 839 .enter_s2idle = intel_idle_s2idle, }, 840 { 841 .name = "C1E", 842 .desc = "MWAIT 0x01", 843 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 844 .exit_latency = 2, 845 .target_residency = 4, 846 .enter = &intel_idle, 847 .enter_s2idle = intel_idle_s2idle, }, 848 { 849 .name = "C6", 850 .desc = "MWAIT 0x20", 851 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 852 .exit_latency = 220, 853 .target_residency = 600, 854 .enter = &intel_idle, 855 .enter_s2idle = intel_idle_s2idle, }, 856 { 857 .name = "C8", 858 .desc = "MWAIT 0x40", 859 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 860 .exit_latency = 280, 861 .target_residency = 800, 862 .enter = &intel_idle, 863 .enter_s2idle = intel_idle_s2idle, }, 864 { 865 .name = "C10", 866 .desc = "MWAIT 0x60", 867 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 868 .exit_latency = 680, 869 .target_residency = 2000, 870 .enter = &intel_idle, 871 .enter_s2idle = intel_idle_s2idle, }, 872 { 873 .enter = NULL } 874}; 875 876static struct cpuidle_state adl_l_cstates[] __initdata = { 877 { 878 .name = "C1", 879 .desc = "MWAIT 0x00", 880 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 881 .exit_latency = 1, 882 .target_residency = 1, 883 .enter = &intel_idle, 884 .enter_s2idle = intel_idle_s2idle, }, 885 { 886 .name = "C1E", 887 .desc = "MWAIT 0x01", 888 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 889 .exit_latency = 2, 890 .target_residency = 4, 891 .enter = &intel_idle, 892 .enter_s2idle = intel_idle_s2idle, }, 893 { 894 .name = "C6", 895 .desc = "MWAIT 0x20", 896 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 897 .exit_latency = 170, 898 .target_residency = 500, 899 .enter = &intel_idle, 900 .enter_s2idle = intel_idle_s2idle, }, 901 { 902 .name = "C8", 903 .desc = "MWAIT 0x40", 904 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 905 .exit_latency = 200, 906 .target_residency = 600, 907 .enter = &intel_idle, 908 .enter_s2idle = intel_idle_s2idle, }, 909 { 910 .name = "C10", 911 .desc = "MWAIT 0x60", 912 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 913 .exit_latency = 230, 914 .target_residency = 700, 915 .enter = &intel_idle, 916 .enter_s2idle = intel_idle_s2idle, }, 917 { 918 .enter = NULL } 919}; 920 921static struct cpuidle_state gmt_cstates[] __initdata = { 922 { 923 .name = "C1", 924 .desc = "MWAIT 0x00", 925 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 926 .exit_latency = 1, 927 .target_residency = 1, 928 .enter = &intel_idle, 929 .enter_s2idle = intel_idle_s2idle, }, 930 { 931 .name = "C1E", 932 .desc = "MWAIT 0x01", 933 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 934 .exit_latency = 2, 935 .target_residency = 4, 936 .enter = &intel_idle, 937 .enter_s2idle = intel_idle_s2idle, }, 938 { 939 .name = "C6", 940 .desc = "MWAIT 0x20", 941 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 942 .exit_latency = 195, 943 .target_residency = 585, 944 .enter = &intel_idle, 945 .enter_s2idle = intel_idle_s2idle, }, 946 { 947 .name = "C8", 948 .desc = "MWAIT 0x40", 949 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 950 .exit_latency = 260, 951 .target_residency = 1040, 952 .enter = &intel_idle, 953 .enter_s2idle = intel_idle_s2idle, }, 954 { 955 .name = "C10", 956 .desc = "MWAIT 0x60", 957 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 958 .exit_latency = 660, 959 .target_residency = 1980, 960 .enter = &intel_idle, 961 .enter_s2idle = intel_idle_s2idle, }, 962 { 963 .enter = NULL } 964}; 965 966static struct cpuidle_state spr_cstates[] __initdata = { 967 { 968 .name = "C1", 969 .desc = "MWAIT 0x00", 970 .flags = MWAIT2flg(0x00), 971 .exit_latency = 1, 972 .target_residency = 1, 973 .enter = &intel_idle, 974 .enter_s2idle = intel_idle_s2idle, }, 975 { 976 .name = "C1E", 977 .desc = "MWAIT 0x01", 978 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 979 .exit_latency = 2, 980 .target_residency = 4, 981 .enter = &intel_idle, 982 .enter_s2idle = intel_idle_s2idle, }, 983 { 984 .name = "C6", 985 .desc = "MWAIT 0x20", 986 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | 987 CPUIDLE_FLAG_INIT_XSTATE, 988 .exit_latency = 290, 989 .target_residency = 800, 990 .enter = &intel_idle, 991 .enter_s2idle = intel_idle_s2idle, }, 992 { 993 .enter = NULL } 994}; 995 996static struct cpuidle_state atom_cstates[] __initdata = { 997 { 998 .name = "C1E", 999 .desc = "MWAIT 0x00", 1000 .flags = MWAIT2flg(0x00), 1001 .exit_latency = 10, 1002 .target_residency = 20, 1003 .enter = &intel_idle, 1004 .enter_s2idle = intel_idle_s2idle, }, 1005 { 1006 .name = "C2", 1007 .desc = "MWAIT 0x10", 1008 .flags = MWAIT2flg(0x10), 1009 .exit_latency = 20, 1010 .target_residency = 80, 1011 .enter = &intel_idle, 1012 .enter_s2idle = intel_idle_s2idle, }, 1013 { 1014 .name = "C4", 1015 .desc = "MWAIT 0x30", 1016 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 1017 .exit_latency = 100, 1018 .target_residency = 400, 1019 .enter = &intel_idle, 1020 .enter_s2idle = intel_idle_s2idle, }, 1021 { 1022 .name = "C6", 1023 .desc = "MWAIT 0x52", 1024 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 1025 .exit_latency = 140, 1026 .target_residency = 560, 1027 .enter = &intel_idle, 1028 .enter_s2idle = intel_idle_s2idle, }, 1029 { 1030 .enter = NULL } 1031}; 1032static struct cpuidle_state tangier_cstates[] __initdata = { 1033 { 1034 .name = "C1", 1035 .desc = "MWAIT 0x00", 1036 .flags = MWAIT2flg(0x00), 1037 .exit_latency = 1, 1038 .target_residency = 4, 1039 .enter = &intel_idle, 1040 .enter_s2idle = intel_idle_s2idle, }, 1041 { 1042 .name = "C4", 1043 .desc = "MWAIT 0x30", 1044 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 1045 .exit_latency = 100, 1046 .target_residency = 400, 1047 .enter = &intel_idle, 1048 .enter_s2idle = intel_idle_s2idle, }, 1049 { 1050 .name = "C6", 1051 .desc = "MWAIT 0x52", 1052 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 1053 .exit_latency = 140, 1054 .target_residency = 560, 1055 .enter = &intel_idle, 1056 .enter_s2idle = intel_idle_s2idle, }, 1057 { 1058 .name = "C7", 1059 .desc = "MWAIT 0x60", 1060 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 1061 .exit_latency = 1200, 1062 .target_residency = 4000, 1063 .enter = &intel_idle, 1064 .enter_s2idle = intel_idle_s2idle, }, 1065 { 1066 .name = "C9", 1067 .desc = "MWAIT 0x64", 1068 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 1069 .exit_latency = 10000, 1070 .target_residency = 20000, 1071 .enter = &intel_idle, 1072 .enter_s2idle = intel_idle_s2idle, }, 1073 { 1074 .enter = NULL } 1075}; 1076static struct cpuidle_state avn_cstates[] __initdata = { 1077 { 1078 .name = "C1", 1079 .desc = "MWAIT 0x00", 1080 .flags = MWAIT2flg(0x00), 1081 .exit_latency = 2, 1082 .target_residency = 2, 1083 .enter = &intel_idle, 1084 .enter_s2idle = intel_idle_s2idle, }, 1085 { 1086 .name = "C6", 1087 .desc = "MWAIT 0x51", 1088 .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED, 1089 .exit_latency = 15, 1090 .target_residency = 45, 1091 .enter = &intel_idle, 1092 .enter_s2idle = intel_idle_s2idle, }, 1093 { 1094 .enter = NULL } 1095}; 1096static struct cpuidle_state knl_cstates[] __initdata = { 1097 { 1098 .name = "C1", 1099 .desc = "MWAIT 0x00", 1100 .flags = MWAIT2flg(0x00), 1101 .exit_latency = 1, 1102 .target_residency = 2, 1103 .enter = &intel_idle, 1104 .enter_s2idle = intel_idle_s2idle }, 1105 { 1106 .name = "C6", 1107 .desc = "MWAIT 0x10", 1108 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 1109 .exit_latency = 120, 1110 .target_residency = 500, 1111 .enter = &intel_idle, 1112 .enter_s2idle = intel_idle_s2idle }, 1113 { 1114 .enter = NULL } 1115}; 1116 1117static struct cpuidle_state bxt_cstates[] __initdata = { 1118 { 1119 .name = "C1", 1120 .desc = "MWAIT 0x00", 1121 .flags = MWAIT2flg(0x00), 1122 .exit_latency = 2, 1123 .target_residency = 2, 1124 .enter = &intel_idle, 1125 .enter_s2idle = intel_idle_s2idle, }, 1126 { 1127 .name = "C1E", 1128 .desc = "MWAIT 0x01", 1129 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1130 .exit_latency = 10, 1131 .target_residency = 20, 1132 .enter = &intel_idle, 1133 .enter_s2idle = intel_idle_s2idle, }, 1134 { 1135 .name = "C6", 1136 .desc = "MWAIT 0x20", 1137 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1138 .exit_latency = 133, 1139 .target_residency = 133, 1140 .enter = &intel_idle, 1141 .enter_s2idle = intel_idle_s2idle, }, 1142 { 1143 .name = "C7s", 1144 .desc = "MWAIT 0x31", 1145 .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED, 1146 .exit_latency = 155, 1147 .target_residency = 155, 1148 .enter = &intel_idle, 1149 .enter_s2idle = intel_idle_s2idle, }, 1150 { 1151 .name = "C8", 1152 .desc = "MWAIT 0x40", 1153 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 1154 .exit_latency = 1000, 1155 .target_residency = 1000, 1156 .enter = &intel_idle, 1157 .enter_s2idle = intel_idle_s2idle, }, 1158 { 1159 .name = "C9", 1160 .desc = "MWAIT 0x50", 1161 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 1162 .exit_latency = 2000, 1163 .target_residency = 2000, 1164 .enter = &intel_idle, 1165 .enter_s2idle = intel_idle_s2idle, }, 1166 { 1167 .name = "C10", 1168 .desc = "MWAIT 0x60", 1169 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 1170 .exit_latency = 10000, 1171 .target_residency = 10000, 1172 .enter = &intel_idle, 1173 .enter_s2idle = intel_idle_s2idle, }, 1174 { 1175 .enter = NULL } 1176}; 1177 1178static struct cpuidle_state dnv_cstates[] __initdata = { 1179 { 1180 .name = "C1", 1181 .desc = "MWAIT 0x00", 1182 .flags = MWAIT2flg(0x00), 1183 .exit_latency = 2, 1184 .target_residency = 2, 1185 .enter = &intel_idle, 1186 .enter_s2idle = intel_idle_s2idle, }, 1187 { 1188 .name = "C1E", 1189 .desc = "MWAIT 0x01", 1190 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1191 .exit_latency = 10, 1192 .target_residency = 20, 1193 .enter = &intel_idle, 1194 .enter_s2idle = intel_idle_s2idle, }, 1195 { 1196 .name = "C6", 1197 .desc = "MWAIT 0x20", 1198 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1199 .exit_latency = 50, 1200 .target_residency = 500, 1201 .enter = &intel_idle, 1202 .enter_s2idle = intel_idle_s2idle, }, 1203 { 1204 .enter = NULL } 1205}; 1206 1207/* 1208 * Note, depending on HW and FW revision, SnowRidge SoC may or may not support 1209 * C6, and this is indicated in the CPUID mwait leaf. 1210 */ 1211static struct cpuidle_state snr_cstates[] __initdata = { 1212 { 1213 .name = "C1", 1214 .desc = "MWAIT 0x00", 1215 .flags = MWAIT2flg(0x00), 1216 .exit_latency = 2, 1217 .target_residency = 2, 1218 .enter = &intel_idle, 1219 .enter_s2idle = intel_idle_s2idle, }, 1220 { 1221 .name = "C1E", 1222 .desc = "MWAIT 0x01", 1223 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1224 .exit_latency = 15, 1225 .target_residency = 25, 1226 .enter = &intel_idle, 1227 .enter_s2idle = intel_idle_s2idle, }, 1228 { 1229 .name = "C6", 1230 .desc = "MWAIT 0x20", 1231 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1232 .exit_latency = 130, 1233 .target_residency = 500, 1234 .enter = &intel_idle, 1235 .enter_s2idle = intel_idle_s2idle, }, 1236 { 1237 .enter = NULL } 1238}; 1239 1240static const struct idle_cpu idle_cpu_nehalem __initconst = { 1241 .state_table = nehalem_cstates, 1242 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 1243 .disable_promotion_to_c1e = true, 1244}; 1245 1246static const struct idle_cpu idle_cpu_nhx __initconst = { 1247 .state_table = nehalem_cstates, 1248 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 1249 .disable_promotion_to_c1e = true, 1250 .use_acpi = true, 1251}; 1252 1253static const struct idle_cpu idle_cpu_atom __initconst = { 1254 .state_table = atom_cstates, 1255}; 1256 1257static const struct idle_cpu idle_cpu_tangier __initconst = { 1258 .state_table = tangier_cstates, 1259}; 1260 1261static const struct idle_cpu idle_cpu_lincroft __initconst = { 1262 .state_table = atom_cstates, 1263 .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE, 1264}; 1265 1266static const struct idle_cpu idle_cpu_snb __initconst = { 1267 .state_table = snb_cstates, 1268 .disable_promotion_to_c1e = true, 1269}; 1270 1271static const struct idle_cpu idle_cpu_snx __initconst = { 1272 .state_table = snb_cstates, 1273 .disable_promotion_to_c1e = true, 1274 .use_acpi = true, 1275}; 1276 1277static const struct idle_cpu idle_cpu_byt __initconst = { 1278 .state_table = byt_cstates, 1279 .disable_promotion_to_c1e = true, 1280 .byt_auto_demotion_disable_flag = true, 1281}; 1282 1283static const struct idle_cpu idle_cpu_cht __initconst = { 1284 .state_table = cht_cstates, 1285 .disable_promotion_to_c1e = true, 1286 .byt_auto_demotion_disable_flag = true, 1287}; 1288 1289static const struct idle_cpu idle_cpu_ivb __initconst = { 1290 .state_table = ivb_cstates, 1291 .disable_promotion_to_c1e = true, 1292}; 1293 1294static const struct idle_cpu idle_cpu_ivt __initconst = { 1295 .state_table = ivt_cstates, 1296 .disable_promotion_to_c1e = true, 1297 .use_acpi = true, 1298}; 1299 1300static const struct idle_cpu idle_cpu_hsw __initconst = { 1301 .state_table = hsw_cstates, 1302 .disable_promotion_to_c1e = true, 1303}; 1304 1305static const struct idle_cpu idle_cpu_hsx __initconst = { 1306 .state_table = hsw_cstates, 1307 .disable_promotion_to_c1e = true, 1308 .use_acpi = true, 1309}; 1310 1311static const struct idle_cpu idle_cpu_bdw __initconst = { 1312 .state_table = bdw_cstates, 1313 .disable_promotion_to_c1e = true, 1314}; 1315 1316static const struct idle_cpu idle_cpu_bdx __initconst = { 1317 .state_table = bdw_cstates, 1318 .disable_promotion_to_c1e = true, 1319 .use_acpi = true, 1320}; 1321 1322static const struct idle_cpu idle_cpu_skl __initconst = { 1323 .state_table = skl_cstates, 1324 .disable_promotion_to_c1e = true, 1325}; 1326 1327static const struct idle_cpu idle_cpu_skx __initconst = { 1328 .state_table = skx_cstates, 1329 .disable_promotion_to_c1e = true, 1330 .use_acpi = true, 1331}; 1332 1333static const struct idle_cpu idle_cpu_icx __initconst = { 1334 .state_table = icx_cstates, 1335 .disable_promotion_to_c1e = true, 1336 .use_acpi = true, 1337}; 1338 1339static const struct idle_cpu idle_cpu_adl __initconst = { 1340 .state_table = adl_cstates, 1341}; 1342 1343static const struct idle_cpu idle_cpu_adl_l __initconst = { 1344 .state_table = adl_l_cstates, 1345}; 1346 1347static const struct idle_cpu idle_cpu_gmt __initconst = { 1348 .state_table = gmt_cstates, 1349}; 1350 1351static const struct idle_cpu idle_cpu_spr __initconst = { 1352 .state_table = spr_cstates, 1353 .disable_promotion_to_c1e = true, 1354 .use_acpi = true, 1355}; 1356 1357static const struct idle_cpu idle_cpu_avn __initconst = { 1358 .state_table = avn_cstates, 1359 .disable_promotion_to_c1e = true, 1360 .use_acpi = true, 1361}; 1362 1363static const struct idle_cpu idle_cpu_knl __initconst = { 1364 .state_table = knl_cstates, 1365 .use_acpi = true, 1366}; 1367 1368static const struct idle_cpu idle_cpu_bxt __initconst = { 1369 .state_table = bxt_cstates, 1370 .disable_promotion_to_c1e = true, 1371}; 1372 1373static const struct idle_cpu idle_cpu_dnv __initconst = { 1374 .state_table = dnv_cstates, 1375 .disable_promotion_to_c1e = true, 1376 .use_acpi = true, 1377}; 1378 1379static const struct idle_cpu idle_cpu_snr __initconst = { 1380 .state_table = snr_cstates, 1381 .disable_promotion_to_c1e = true, 1382 .use_acpi = true, 1383}; 1384 1385static const struct x86_cpu_id intel_idle_ids[] __initconst = { 1386 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EP, &idle_cpu_nhx), 1387 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM, &idle_cpu_nehalem), 1388 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_G, &idle_cpu_nehalem), 1389 X86_MATCH_INTEL_FAM6_MODEL(WESTMERE, &idle_cpu_nehalem), 1390 X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EP, &idle_cpu_nhx), 1391 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EX, &idle_cpu_nhx), 1392 X86_MATCH_INTEL_FAM6_MODEL(ATOM_BONNELL, &idle_cpu_atom), 1393 X86_MATCH_INTEL_FAM6_MODEL(ATOM_BONNELL_MID, &idle_cpu_lincroft), 1394 X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EX, &idle_cpu_nhx), 1395 X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE, &idle_cpu_snb), 1396 X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE_X, &idle_cpu_snx), 1397 X86_MATCH_INTEL_FAM6_MODEL(ATOM_SALTWELL, &idle_cpu_atom), 1398 X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT, &idle_cpu_byt), 1399 X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_MID, &idle_cpu_tangier), 1400 X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT, &idle_cpu_cht), 1401 X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE, &idle_cpu_ivb), 1402 X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X, &idle_cpu_ivt), 1403 X86_MATCH_INTEL_FAM6_MODEL(HASWELL, &idle_cpu_hsw), 1404 X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, &idle_cpu_hsx), 1405 X86_MATCH_INTEL_FAM6_MODEL(HASWELL_L, &idle_cpu_hsw), 1406 X86_MATCH_INTEL_FAM6_MODEL(HASWELL_G, &idle_cpu_hsw), 1407 X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_D, &idle_cpu_avn), 1408 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL, &idle_cpu_bdw), 1409 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G, &idle_cpu_bdw), 1410 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, &idle_cpu_bdx), 1411 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, &idle_cpu_bdx), 1412 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_L, &idle_cpu_skl), 1413 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE, &idle_cpu_skl), 1414 X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE_L, &idle_cpu_skl), 1415 X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE, &idle_cpu_skl), 1416 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, &idle_cpu_skx), 1417 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, &idle_cpu_icx), 1418 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, &idle_cpu_icx), 1419 X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE, &idle_cpu_adl), 1420 X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, &idle_cpu_adl_l), 1421 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GRACEMONT, &idle_cpu_gmt), 1422 X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &idle_cpu_spr), 1423 X86_MATCH_INTEL_FAM6_MODEL(EMERALDRAPIDS_X, &idle_cpu_spr), 1424 X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &idle_cpu_knl), 1425 X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &idle_cpu_knl), 1426 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT, &idle_cpu_bxt), 1427 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_PLUS, &idle_cpu_bxt), 1428 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_D, &idle_cpu_dnv), 1429 X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, &idle_cpu_snr), 1430 {} 1431}; 1432 1433static const struct x86_cpu_id intel_mwait_ids[] __initconst = { 1434 X86_MATCH_VENDOR_FAM_FEATURE(INTEL, 6, X86_FEATURE_MWAIT, NULL), 1435 {} 1436}; 1437 1438static bool __init intel_idle_max_cstate_reached(int cstate) 1439{ 1440 if (cstate + 1 > max_cstate) { 1441 pr_info("max_cstate %d reached\n", max_cstate); 1442 return true; 1443 } 1444 return false; 1445} 1446 1447static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state) 1448{ 1449 unsigned long eax = flg2MWAIT(state->flags); 1450 1451 if (boot_cpu_has(X86_FEATURE_ARAT)) 1452 return false; 1453 1454 /* 1455 * Switch over to one-shot tick broadcast if the target C-state 1456 * is deeper than C1. 1457 */ 1458 return !!((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK); 1459} 1460 1461#ifdef CONFIG_ACPI_PROCESSOR_CSTATE 1462#include <acpi/processor.h> 1463 1464static bool no_acpi __read_mostly; 1465module_param(no_acpi, bool, 0444); 1466MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list"); 1467 1468static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */ 1469module_param_named(use_acpi, force_use_acpi, bool, 0444); 1470MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list"); 1471 1472static struct acpi_processor_power acpi_state_table __initdata; 1473 1474/** 1475 * intel_idle_cst_usable - Check if the _CST information can be used. 1476 * 1477 * Check if all of the C-states listed by _CST in the max_cstate range are 1478 * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT. 1479 */ 1480static bool __init intel_idle_cst_usable(void) 1481{ 1482 int cstate, limit; 1483 1484 limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1), 1485 acpi_state_table.count); 1486 1487 for (cstate = 1; cstate < limit; cstate++) { 1488 struct acpi_processor_cx *cx = &acpi_state_table.states[cstate]; 1489 1490 if (cx->entry_method != ACPI_CSTATE_FFH) 1491 return false; 1492 } 1493 1494 return true; 1495} 1496 1497static bool __init intel_idle_acpi_cst_extract(void) 1498{ 1499 unsigned int cpu; 1500 1501 if (no_acpi) { 1502 pr_debug("Not allowed to use ACPI _CST\n"); 1503 return false; 1504 } 1505 1506 for_each_possible_cpu(cpu) { 1507 struct acpi_processor *pr = per_cpu(processors, cpu); 1508 1509 if (!pr) 1510 continue; 1511 1512 if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table)) 1513 continue; 1514 1515 acpi_state_table.count++; 1516 1517 if (!intel_idle_cst_usable()) 1518 continue; 1519 1520 if (!acpi_processor_claim_cst_control()) 1521 break; 1522 1523 return true; 1524 } 1525 1526 acpi_state_table.count = 0; 1527 pr_debug("ACPI _CST not found or not usable\n"); 1528 return false; 1529} 1530 1531static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) 1532{ 1533 int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 1534 1535 /* 1536 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 1537 * the interesting states are ACPI_CSTATE_FFH. 1538 */ 1539 for (cstate = 1; cstate < limit; cstate++) { 1540 struct acpi_processor_cx *cx; 1541 struct cpuidle_state *state; 1542 1543 if (intel_idle_max_cstate_reached(cstate - 1)) 1544 break; 1545 1546 cx = &acpi_state_table.states[cstate]; 1547 1548 state = &drv->states[drv->state_count++]; 1549 1550 snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate); 1551 strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); 1552 state->exit_latency = cx->latency; 1553 /* 1554 * For C1-type C-states use the same number for both the exit 1555 * latency and target residency, because that is the case for 1556 * C1 in the majority of the static C-states tables above. 1557 * For the other types of C-states, however, set the target 1558 * residency to 3 times the exit latency which should lead to 1559 * a reasonable balance between energy-efficiency and 1560 * performance in the majority of interesting cases. 1561 */ 1562 state->target_residency = cx->latency; 1563 if (cx->type > ACPI_STATE_C1) 1564 state->target_residency *= 3; 1565 1566 state->flags = MWAIT2flg(cx->address); 1567 if (cx->type > ACPI_STATE_C2) 1568 state->flags |= CPUIDLE_FLAG_TLB_FLUSHED; 1569 1570 if (disabled_states_mask & BIT(cstate)) 1571 state->flags |= CPUIDLE_FLAG_OFF; 1572 1573 if (intel_idle_state_needs_timer_stop(state)) 1574 state->flags |= CPUIDLE_FLAG_TIMER_STOP; 1575 1576 state->enter = intel_idle; 1577 state->enter_s2idle = intel_idle_s2idle; 1578 } 1579} 1580 1581static bool __init intel_idle_off_by_default(u32 mwait_hint) 1582{ 1583 int cstate, limit; 1584 1585 /* 1586 * If there are no _CST C-states, do not disable any C-states by 1587 * default. 1588 */ 1589 if (!acpi_state_table.count) 1590 return false; 1591 1592 limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 1593 /* 1594 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 1595 * the interesting states are ACPI_CSTATE_FFH. 1596 */ 1597 for (cstate = 1; cstate < limit; cstate++) { 1598 if (acpi_state_table.states[cstate].address == mwait_hint) 1599 return false; 1600 } 1601 return true; 1602} 1603#else /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 1604#define force_use_acpi (false) 1605 1606static inline bool intel_idle_acpi_cst_extract(void) { return false; } 1607static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { } 1608static inline bool intel_idle_off_by_default(u32 mwait_hint) { return false; } 1609#endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 1610 1611/** 1612 * ivt_idle_state_table_update - Tune the idle states table for Ivy Town. 1613 * 1614 * Tune IVT multi-socket targets. 1615 * Assumption: num_sockets == (max_package_num + 1). 1616 */ 1617static void __init ivt_idle_state_table_update(void) 1618{ 1619 /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ 1620 int cpu, package_num, num_sockets = 1; 1621 1622 for_each_online_cpu(cpu) { 1623 package_num = topology_physical_package_id(cpu); 1624 if (package_num + 1 > num_sockets) { 1625 num_sockets = package_num + 1; 1626 1627 if (num_sockets > 4) { 1628 cpuidle_state_table = ivt_cstates_8s; 1629 return; 1630 } 1631 } 1632 } 1633 1634 if (num_sockets > 2) 1635 cpuidle_state_table = ivt_cstates_4s; 1636 1637 /* else, 1 and 2 socket systems use default ivt_cstates */ 1638} 1639 1640/** 1641 * irtl_2_usec - IRTL to microseconds conversion. 1642 * @irtl: IRTL MSR value. 1643 * 1644 * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds. 1645 */ 1646static unsigned long long __init irtl_2_usec(unsigned long long irtl) 1647{ 1648 static const unsigned int irtl_ns_units[] __initconst = { 1649 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 1650 }; 1651 unsigned long long ns; 1652 1653 if (!irtl) 1654 return 0; 1655 1656 ns = irtl_ns_units[(irtl >> 10) & 0x7]; 1657 1658 return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC); 1659} 1660 1661/** 1662 * bxt_idle_state_table_update - Fix up the Broxton idle states table. 1663 * 1664 * On BXT, trust the IRTL (Interrupt Response Time Limit) MSR to show the 1665 * definitive maximum latency and use the same value for target_residency. 1666 */ 1667static void __init bxt_idle_state_table_update(void) 1668{ 1669 unsigned long long msr; 1670 unsigned int usec; 1671 1672 rdmsrl(MSR_PKGC6_IRTL, msr); 1673 usec = irtl_2_usec(msr); 1674 if (usec) { 1675 bxt_cstates[2].exit_latency = usec; 1676 bxt_cstates[2].target_residency = usec; 1677 } 1678 1679 rdmsrl(MSR_PKGC7_IRTL, msr); 1680 usec = irtl_2_usec(msr); 1681 if (usec) { 1682 bxt_cstates[3].exit_latency = usec; 1683 bxt_cstates[3].target_residency = usec; 1684 } 1685 1686 rdmsrl(MSR_PKGC8_IRTL, msr); 1687 usec = irtl_2_usec(msr); 1688 if (usec) { 1689 bxt_cstates[4].exit_latency = usec; 1690 bxt_cstates[4].target_residency = usec; 1691 } 1692 1693 rdmsrl(MSR_PKGC9_IRTL, msr); 1694 usec = irtl_2_usec(msr); 1695 if (usec) { 1696 bxt_cstates[5].exit_latency = usec; 1697 bxt_cstates[5].target_residency = usec; 1698 } 1699 1700 rdmsrl(MSR_PKGC10_IRTL, msr); 1701 usec = irtl_2_usec(msr); 1702 if (usec) { 1703 bxt_cstates[6].exit_latency = usec; 1704 bxt_cstates[6].target_residency = usec; 1705 } 1706 1707} 1708 1709/** 1710 * sklh_idle_state_table_update - Fix up the Sky Lake idle states table. 1711 * 1712 * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled. 1713 */ 1714static void __init sklh_idle_state_table_update(void) 1715{ 1716 unsigned long long msr; 1717 unsigned int eax, ebx, ecx, edx; 1718 1719 1720 /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */ 1721 if (max_cstate <= 7) 1722 return; 1723 1724 /* if PC10 not present in CPUID.MWAIT.EDX */ 1725 if ((mwait_substates & (0xF << 28)) == 0) 1726 return; 1727 1728 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 1729 1730 /* PC10 is not enabled in PKG C-state limit */ 1731 if ((msr & 0xF) != 8) 1732 return; 1733 1734 ecx = 0; 1735 cpuid(7, &eax, &ebx, &ecx, &edx); 1736 1737 /* if SGX is present */ 1738 if (ebx & (1 << 2)) { 1739 1740 rdmsrl(MSR_IA32_FEAT_CTL, msr); 1741 1742 /* if SGX is enabled */ 1743 if (msr & (1 << 18)) 1744 return; 1745 } 1746 1747 skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */ 1748 skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */ 1749} 1750 1751/** 1752 * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake 1753 * idle states table. 1754 */ 1755static void __init skx_idle_state_table_update(void) 1756{ 1757 unsigned long long msr; 1758 1759 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 1760 1761 /* 1762 * 000b: C0/C1 (no package C-state support) 1763 * 001b: C2 1764 * 010b: C6 (non-retention) 1765 * 011b: C6 (retention) 1766 * 111b: No Package C state limits. 1767 */ 1768 if ((msr & 0x7) < 2) { 1769 /* 1770 * Uses the CC6 + PC0 latency and 3 times of 1771 * latency for target_residency if the PC6 1772 * is disabled in BIOS. This is consistent 1773 * with how intel_idle driver uses _CST 1774 * to set the target_residency. 1775 */ 1776 skx_cstates[2].exit_latency = 92; 1777 skx_cstates[2].target_residency = 276; 1778 } 1779} 1780 1781/** 1782 * adl_idle_state_table_update - Adjust AlderLake idle states table. 1783 */ 1784static void __init adl_idle_state_table_update(void) 1785{ 1786 /* Check if user prefers C1 over C1E. */ 1787 if (preferred_states_mask & BIT(1) && !(preferred_states_mask & BIT(2))) { 1788 cpuidle_state_table[0].flags &= ~CPUIDLE_FLAG_UNUSABLE; 1789 cpuidle_state_table[1].flags |= CPUIDLE_FLAG_UNUSABLE; 1790 1791 /* Disable C1E by clearing the "C1E promotion" bit. */ 1792 c1e_promotion = C1E_PROMOTION_DISABLE; 1793 return; 1794 } 1795 1796 /* Make sure C1E is enabled by default */ 1797 c1e_promotion = C1E_PROMOTION_ENABLE; 1798} 1799 1800/** 1801 * spr_idle_state_table_update - Adjust Sapphire Rapids idle states table. 1802 */ 1803static void __init spr_idle_state_table_update(void) 1804{ 1805 unsigned long long msr; 1806 1807 /* 1808 * By default, the C6 state assumes the worst-case scenario of package 1809 * C6. However, if PC6 is disabled, we update the numbers to match 1810 * core C6. 1811 */ 1812 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 1813 1814 /* Limit value 2 and above allow for PC6. */ 1815 if ((msr & 0x7) < 2) { 1816 spr_cstates[2].exit_latency = 190; 1817 spr_cstates[2].target_residency = 600; 1818 } 1819} 1820 1821static bool __init intel_idle_verify_cstate(unsigned int mwait_hint) 1822{ 1823 unsigned int mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint) + 1; 1824 unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) & 1825 MWAIT_SUBSTATE_MASK; 1826 1827 /* Ignore the C-state if there are NO sub-states in CPUID for it. */ 1828 if (num_substates == 0) 1829 return false; 1830 1831 if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 1832 mark_tsc_unstable("TSC halts in idle states deeper than C2"); 1833 1834 return true; 1835} 1836 1837static void state_update_enter_method(struct cpuidle_state *state, int cstate) 1838{ 1839 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) { 1840 /* 1841 * Combining with XSTATE with IBRS or IRQ_ENABLE flags 1842 * is not currently supported but this driver. 1843 */ 1844 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS); 1845 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); 1846 state->enter = intel_idle_xstate; 1847 return; 1848 } 1849 1850 if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) && 1851 state->flags & CPUIDLE_FLAG_IBRS) { 1852 /* 1853 * IBRS mitigation requires that C-states are entered 1854 * with interrupts disabled. 1855 */ 1856 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); 1857 state->enter = intel_idle_ibrs; 1858 return; 1859 } 1860 1861 if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) { 1862 state->enter = intel_idle_irq; 1863 return; 1864 } 1865 1866 if (force_irq_on) { 1867 pr_info("forced intel_idle_irq for state %d\n", cstate); 1868 state->enter = intel_idle_irq; 1869 } 1870} 1871 1872static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) 1873{ 1874 int cstate; 1875 1876 switch (boot_cpu_data.x86_model) { 1877 case INTEL_FAM6_IVYBRIDGE_X: 1878 ivt_idle_state_table_update(); 1879 break; 1880 case INTEL_FAM6_ATOM_GOLDMONT: 1881 case INTEL_FAM6_ATOM_GOLDMONT_PLUS: 1882 bxt_idle_state_table_update(); 1883 break; 1884 case INTEL_FAM6_SKYLAKE: 1885 sklh_idle_state_table_update(); 1886 break; 1887 case INTEL_FAM6_SKYLAKE_X: 1888 skx_idle_state_table_update(); 1889 break; 1890 case INTEL_FAM6_SAPPHIRERAPIDS_X: 1891 case INTEL_FAM6_EMERALDRAPIDS_X: 1892 spr_idle_state_table_update(); 1893 break; 1894 case INTEL_FAM6_ALDERLAKE: 1895 case INTEL_FAM6_ALDERLAKE_L: 1896 case INTEL_FAM6_ATOM_GRACEMONT: 1897 adl_idle_state_table_update(); 1898 break; 1899 } 1900 1901 for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { 1902 struct cpuidle_state *state; 1903 unsigned int mwait_hint; 1904 1905 if (intel_idle_max_cstate_reached(cstate)) 1906 break; 1907 1908 if (!cpuidle_state_table[cstate].enter && 1909 !cpuidle_state_table[cstate].enter_s2idle) 1910 break; 1911 1912 /* If marked as unusable, skip this state. */ 1913 if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) { 1914 pr_debug("state %s is disabled\n", 1915 cpuidle_state_table[cstate].name); 1916 continue; 1917 } 1918 1919 mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); 1920 if (!intel_idle_verify_cstate(mwait_hint)) 1921 continue; 1922 1923 /* Structure copy. */ 1924 drv->states[drv->state_count] = cpuidle_state_table[cstate]; 1925 state = &drv->states[drv->state_count]; 1926 1927 state_update_enter_method(state, cstate); 1928 1929 1930 if ((disabled_states_mask & BIT(drv->state_count)) || 1931 ((icpu->use_acpi || force_use_acpi) && 1932 intel_idle_off_by_default(mwait_hint) && 1933 !(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE))) 1934 state->flags |= CPUIDLE_FLAG_OFF; 1935 1936 if (intel_idle_state_needs_timer_stop(state)) 1937 state->flags |= CPUIDLE_FLAG_TIMER_STOP; 1938 1939 drv->state_count++; 1940 } 1941 1942 if (icpu->byt_auto_demotion_disable_flag) { 1943 wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); 1944 wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); 1945 } 1946} 1947 1948/** 1949 * intel_idle_cpuidle_driver_init - Create the list of available idle states. 1950 * @drv: cpuidle driver structure to initialize. 1951 */ 1952static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv) 1953{ 1954 cpuidle_poll_state_init(drv); 1955 1956 if (disabled_states_mask & BIT(0)) 1957 drv->states[0].flags |= CPUIDLE_FLAG_OFF; 1958 1959 drv->state_count = 1; 1960 1961 if (icpu) 1962 intel_idle_init_cstates_icpu(drv); 1963 else 1964 intel_idle_init_cstates_acpi(drv); 1965} 1966 1967static void auto_demotion_disable(void) 1968{ 1969 unsigned long long msr_bits; 1970 1971 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 1972 msr_bits &= ~auto_demotion_disable_flags; 1973 wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 1974} 1975 1976static void c1e_promotion_enable(void) 1977{ 1978 unsigned long long msr_bits; 1979 1980 rdmsrl(MSR_IA32_POWER_CTL, msr_bits); 1981 msr_bits |= 0x2; 1982 wrmsrl(MSR_IA32_POWER_CTL, msr_bits); 1983} 1984 1985static void c1e_promotion_disable(void) 1986{ 1987 unsigned long long msr_bits; 1988 1989 rdmsrl(MSR_IA32_POWER_CTL, msr_bits); 1990 msr_bits &= ~0x2; 1991 wrmsrl(MSR_IA32_POWER_CTL, msr_bits); 1992} 1993 1994/** 1995 * intel_idle_cpu_init - Register the target CPU with the cpuidle core. 1996 * @cpu: CPU to initialize. 1997 * 1998 * Register a cpuidle device object for @cpu and update its MSRs in accordance 1999 * with the processor model flags. 2000 */ 2001static int intel_idle_cpu_init(unsigned int cpu) 2002{ 2003 struct cpuidle_device *dev; 2004 2005 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 2006 dev->cpu = cpu; 2007 2008 if (cpuidle_register_device(dev)) { 2009 pr_debug("cpuidle_register_device %d failed!\n", cpu); 2010 return -EIO; 2011 } 2012 2013 if (auto_demotion_disable_flags) 2014 auto_demotion_disable(); 2015 2016 if (c1e_promotion == C1E_PROMOTION_ENABLE) 2017 c1e_promotion_enable(); 2018 else if (c1e_promotion == C1E_PROMOTION_DISABLE) 2019 c1e_promotion_disable(); 2020 2021 return 0; 2022} 2023 2024static int intel_idle_cpu_online(unsigned int cpu) 2025{ 2026 struct cpuidle_device *dev; 2027 2028 if (!boot_cpu_has(X86_FEATURE_ARAT)) 2029 tick_broadcast_enable(); 2030 2031 /* 2032 * Some systems can hotplug a cpu at runtime after 2033 * the kernel has booted, we have to initialize the 2034 * driver in this case 2035 */ 2036 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 2037 if (!dev->registered) 2038 return intel_idle_cpu_init(cpu); 2039 2040 return 0; 2041} 2042 2043/** 2044 * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices. 2045 */ 2046static void __init intel_idle_cpuidle_devices_uninit(void) 2047{ 2048 int i; 2049 2050 for_each_online_cpu(i) 2051 cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i)); 2052} 2053 2054static int __init intel_idle_init(void) 2055{ 2056 const struct x86_cpu_id *id; 2057 unsigned int eax, ebx, ecx; 2058 int retval; 2059 2060 /* Do not load intel_idle at all for now if idle= is passed */ 2061 if (boot_option_idle_override != IDLE_NO_OVERRIDE) 2062 return -ENODEV; 2063 2064 if (max_cstate == 0) { 2065 pr_debug("disabled\n"); 2066 return -EPERM; 2067 } 2068 2069 id = x86_match_cpu(intel_idle_ids); 2070 if (id) { 2071 if (!boot_cpu_has(X86_FEATURE_MWAIT)) { 2072 pr_debug("Please enable MWAIT in BIOS SETUP\n"); 2073 return -ENODEV; 2074 } 2075 } else { 2076 id = x86_match_cpu(intel_mwait_ids); 2077 if (!id) 2078 return -ENODEV; 2079 } 2080 2081 if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF) 2082 return -ENODEV; 2083 2084 cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates); 2085 2086 if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || 2087 !(ecx & CPUID5_ECX_INTERRUPT_BREAK) || 2088 !mwait_substates) 2089 return -ENODEV; 2090 2091 pr_debug("MWAIT substates: 0x%x\n", mwait_substates); 2092 2093 icpu = (const struct idle_cpu *)id->driver_data; 2094 if (icpu) { 2095 cpuidle_state_table = icpu->state_table; 2096 auto_demotion_disable_flags = icpu->auto_demotion_disable_flags; 2097 if (icpu->disable_promotion_to_c1e) 2098 c1e_promotion = C1E_PROMOTION_DISABLE; 2099 if (icpu->use_acpi || force_use_acpi) 2100 intel_idle_acpi_cst_extract(); 2101 } else if (!intel_idle_acpi_cst_extract()) { 2102 return -ENODEV; 2103 } 2104 2105 pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n", 2106 boot_cpu_data.x86_model); 2107 2108 intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); 2109 if (!intel_idle_cpuidle_devices) 2110 return -ENOMEM; 2111 2112 intel_idle_cpuidle_driver_init(&intel_idle_driver); 2113 2114 retval = cpuidle_register_driver(&intel_idle_driver); 2115 if (retval) { 2116 struct cpuidle_driver *drv = cpuidle_get_driver(); 2117 printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"), 2118 drv ? drv->name : "none"); 2119 goto init_driver_fail; 2120 } 2121 2122 retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online", 2123 intel_idle_cpu_online, NULL); 2124 if (retval < 0) 2125 goto hp_setup_fail; 2126 2127 pr_debug("Local APIC timer is reliable in %s\n", 2128 boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1"); 2129 2130 return 0; 2131 2132hp_setup_fail: 2133 intel_idle_cpuidle_devices_uninit(); 2134 cpuidle_unregister_driver(&intel_idle_driver); 2135init_driver_fail: 2136 free_percpu(intel_idle_cpuidle_devices); 2137 return retval; 2138 2139} 2140device_initcall(intel_idle_init); 2141 2142/* 2143 * We are not really modular, but we used to support that. Meaning we also 2144 * support "intel_idle.max_cstate=..." at boot and also a read-only export of 2145 * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param 2146 * is the easiest way (currently) to continue doing that. 2147 */ 2148module_param(max_cstate, int, 0444); 2149/* 2150 * The positions of the bits that are set in this number are the indices of the 2151 * idle states to be disabled by default (as reflected by the names of the 2152 * corresponding idle state directories in sysfs, "state0", "state1" ... 2153 * "state<i>" ..., where <i> is the index of the given state). 2154 */ 2155module_param_named(states_off, disabled_states_mask, uint, 0444); 2156MODULE_PARM_DESC(states_off, "Mask of disabled idle states"); 2157/* 2158 * Some platforms come with mutually exclusive C-states, so that if one is 2159 * enabled, the other C-states must not be used. Example: C1 and C1E on 2160 * Sapphire Rapids platform. This parameter allows for selecting the 2161 * preferred C-states among the groups of mutually exclusive C-states - the 2162 * selected C-states will be registered, the other C-states from the mutually 2163 * exclusive group won't be registered. If the platform has no mutually 2164 * exclusive C-states, this parameter has no effect. 2165 */ 2166module_param_named(preferred_cstates, preferred_states_mask, uint, 0444); 2167MODULE_PARM_DESC(preferred_cstates, "Mask of preferred idle states"); 2168/* 2169 * Debugging option that forces the driver to enter all C-states with 2170 * interrupts enabled. Does not apply to C-states with 2171 * 'CPUIDLE_FLAG_INIT_XSTATE' and 'CPUIDLE_FLAG_IBRS' flags. 2172 */ 2173module_param(force_irq_on, bool, 0444); 2174