Lines Matching refs:coupled
3 * coupled.c - helper functions to enter the same idle state on multiple cpus
30 * WFI), and one or more "coupled" power states that affect blocks
32 * sometimes the whole SoC). Entering a coupled power state must
36 * WFI state until all cpus are ready to enter a coupled state, at
37 * which point the coupled state function will be called on all
46 * ready counter matches the number of online coupled cpus. If any
50 * requested_state stores the deepest coupled idle state each cpu
56 * and only read after all the cpus are ready for the coupled idle
60 * of cpus in the coupled set that are currently or soon will be
62 * the waiting loop, in the ready loop, or in the coupled idle state.
64 * or in the coupled idle state.
66 * To use coupled cpuidle states, a cpuidle driver must:
69 * coupled cpus, usually the same as cpu_possible_mask if all cpus
74 * coupled state. This is usually WFI.
88 * struct cpuidle_coupled - data for set of cpus that share a coupled idle state
89 * @coupled_cpus: mask of cpus that are part of the coupled set
90 * @requested_state: array of requested states for cpus in the coupled set
95 * @prevent: flag to prevent coupled idle while a cpu is hotplugging
132 * cpuidle_coupled_parallel_barrier - synchronize all online coupled cpus
137 * cpus in the same coupled group have called this function. Once any caller
144 * Must only be called from within a coupled idle state handler
151 int n = dev->coupled->online_count;
169 * cpuidle_state_is_coupled - check if a state is part of a coupled set
173 * Returns true if the target state is coupled with cpus besides this one
181 * cpuidle_coupled_state_verify - check if the coupled states are correctly set.
185 * * -EINVAL if any coupled state(safe_state_index) is wrongly set.
204 * @coupled: the struct coupled that contains the current cpu
206 static inline void cpuidle_coupled_set_ready(struct cpuidle_coupled *coupled)
208 atomic_add(MAX_WAITING_CPUS, &coupled->ready_waiting_counts);
213 * @coupled: the struct coupled that contains the current cpu
219 * down from the number of online cpus without going through the coupled idle
226 inline int cpuidle_coupled_set_not_ready(struct cpuidle_coupled *coupled)
231 all = coupled->online_count | (coupled->online_count << WAITING_BITS);
232 ret = atomic_add_unless(&coupled->ready_waiting_counts,
239 * cpuidle_coupled_no_cpus_ready - check if no cpus in a coupled set are ready
240 * @coupled: the struct coupled that contains the current cpu
242 * Returns true if all of the cpus in a coupled set are out of the ready loop.
244 static inline int cpuidle_coupled_no_cpus_ready(struct cpuidle_coupled *coupled)
246 int r = atomic_read(&coupled->ready_waiting_counts) >> WAITING_BITS;
251 * cpuidle_coupled_cpus_ready - check if all cpus in a coupled set are ready
252 * @coupled: the struct coupled that contains the current cpu
254 * Returns true if all cpus coupled to this target state are in the ready loop
256 static inline bool cpuidle_coupled_cpus_ready(struct cpuidle_coupled *coupled)
258 int r = atomic_read(&coupled->ready_waiting_counts) >> WAITING_BITS;
259 return r == coupled->online_count;
263 * cpuidle_coupled_cpus_waiting - check if all cpus in a coupled set are waiting
264 * @coupled: the struct coupled that contains the current cpu
266 * Returns true if all cpus coupled to this target state are in the wait loop
268 static inline bool cpuidle_coupled_cpus_waiting(struct cpuidle_coupled *coupled)
270 int w = atomic_read(&coupled->ready_waiting_counts) & WAITING_MASK;
271 return w == coupled->online_count;
275 * cpuidle_coupled_no_cpus_waiting - check if no cpus in coupled set are waiting
276 * @coupled: the struct coupled that contains the current cpu
278 * Returns true if all of the cpus in a coupled set are out of the waiting loop.
280 static inline int cpuidle_coupled_no_cpus_waiting(struct cpuidle_coupled *coupled)
282 int w = atomic_read(&coupled->ready_waiting_counts) & WAITING_MASK;
289 * @coupled: the struct coupled that contains the current cpu
291 * Returns the deepest idle state that all coupled cpus can enter
294 struct cpuidle_coupled *coupled)
306 for_each_cpu(i, &coupled->coupled_cpus)
307 if (cpu_online(i) && coupled->requested_state[i] < state)
308 state = coupled->requested_state[i];
343 * @coupled: the struct coupled that contains the current cpu
348 struct cpuidle_coupled *coupled)
352 for_each_cpu(cpu, &coupled->coupled_cpus)
360 * @coupled: the struct coupled that contains the current cpu
367 struct cpuidle_coupled *coupled, int next_state)
369 coupled->requested_state[cpu] = next_state;
375 return atomic_inc_return(&coupled->ready_waiting_counts) & WAITING_MASK;
381 * @coupled: the struct coupled that contains the current cpu
386 struct cpuidle_coupled *coupled)
394 atomic_dec(&coupled->ready_waiting_counts);
396 coupled->requested_state[cpu] = CPUIDLE_COUPLED_NOT_IDLE;
402 * @coupled: the struct coupled that contains the current cpu
408 static void cpuidle_coupled_set_done(int cpu, struct cpuidle_coupled *coupled)
410 cpuidle_coupled_set_not_waiting(cpu, coupled);
411 atomic_sub(MAX_WAITING_CPUS, &coupled->ready_waiting_counts);
440 static bool cpuidle_coupled_any_pokes_pending(struct cpuidle_coupled *coupled)
445 cpumask_and(&cpus, cpu_online_mask, &coupled->coupled_cpus);
452 * cpuidle_enter_state_coupled - attempt to enter a state with coupled cpus
457 * Coordinate with coupled cpus to enter the target state. This is a two
462 * all the other cpus to call this function. Once all coupled cpus are idle,
463 * the second stage will start. Each coupled cpu will spin until all cpus have
474 struct cpuidle_coupled *coupled = dev->coupled;
477 if (!coupled)
480 while (coupled->prevent) {
497 w = cpuidle_coupled_set_waiting(dev->cpu, coupled, next_state);
505 if (w == coupled->online_count) {
507 cpuidle_coupled_poke_others(dev->cpu, coupled);
512 * Wait for all coupled cpus to be idle, using the deepest state
519 while (!cpuidle_coupled_cpus_waiting(coupled) ||
525 cpuidle_coupled_set_not_waiting(dev->cpu, coupled);
529 if (coupled->prevent) {
530 cpuidle_coupled_set_not_waiting(dev->cpu, coupled);
541 cpuidle_coupled_set_not_waiting(dev->cpu, coupled);
552 * All coupled cpus are probably idle. There is a small chance that
554 * and spin until all coupled cpus have incremented the counter. Once a
560 cpuidle_coupled_set_ready(coupled);
561 while (!cpuidle_coupled_cpus_ready(coupled)) {
563 if (!cpuidle_coupled_cpus_waiting(coupled))
564 if (!cpuidle_coupled_set_not_ready(coupled))
583 * coupled idle state of all cpus and retry.
585 if (cpuidle_coupled_any_pokes_pending(coupled)) {
586 cpuidle_coupled_set_done(dev->cpu, coupled);
588 cpuidle_coupled_parallel_barrier(dev, &coupled->abort_barrier);
592 /* all cpus have acked the coupled state */
593 next_state = cpuidle_coupled_get_state(dev, coupled);
597 cpuidle_coupled_set_done(dev->cpu, coupled);
610 * Calling local_irq_enable here allows coupled states to return with
617 * Wait until all coupled cpus have exited idle. There is no risk that
621 while (!cpuidle_coupled_no_cpus_ready(coupled))
627 static void cpuidle_coupled_update_online_cpus(struct cpuidle_coupled *coupled)
630 cpumask_and(&cpus, cpu_online_mask, &coupled->coupled_cpus);
631 coupled->online_count = cpumask_weight(&cpus);
635 * cpuidle_coupled_register_device - register a coupled cpuidle device
638 * Called from cpuidle_register_device to handle coupled idle init. Finds the
639 * cpuidle_coupled struct for this set of coupled cpus, or creates one if none
647 struct cpuidle_coupled *coupled;
654 if (other_dev && other_dev->coupled) {
655 coupled = other_dev->coupled;
660 /* No existing coupled info found, create a new one */
661 coupled = kzalloc(sizeof(struct cpuidle_coupled), GFP_KERNEL);
662 if (!coupled)
665 coupled->coupled_cpus = dev->coupled_cpus;
668 dev->coupled = coupled;
669 if (WARN_ON(!cpumask_equal(&dev->coupled_cpus, &coupled->coupled_cpus)))
670 coupled->prevent++;
672 cpuidle_coupled_update_online_cpus(coupled);
674 coupled->refcnt++;
683 * cpuidle_coupled_unregister_device - unregister a coupled cpuidle device
686 * Called from cpuidle_unregister_device to tear down coupled idle. Removes the
687 * cpu from the coupled idle set, and frees the cpuidle_coupled_info struct if
692 struct cpuidle_coupled *coupled = dev->coupled;
697 if (--coupled->refcnt)
698 kfree(coupled);
699 dev->coupled = NULL;
703 * cpuidle_coupled_prevent_idle - prevent cpus from entering a coupled state
704 * @coupled: the struct coupled that contains the cpu that is changing state
706 * Disables coupled cpuidle on a coupled set of cpus. Used to ensure that
707 * cpu_online_mask doesn't change while cpus are coordinating coupled idle.
709 static void cpuidle_coupled_prevent_idle(struct cpuidle_coupled *coupled)
714 coupled->prevent++;
715 cpuidle_coupled_poke_others(cpu, coupled);
717 while (!cpuidle_coupled_no_cpus_waiting(coupled))
722 * cpuidle_coupled_allow_idle - allows cpus to enter a coupled state
723 * @coupled: the struct coupled that contains the cpu that is changing state
725 * Enables coupled cpuidle on a coupled set of cpus. Used to ensure that
726 * cpu_online_mask doesn't change while cpus are coordinating coupled idle.
728 static void cpuidle_coupled_allow_idle(struct cpuidle_coupled *coupled)
737 coupled->prevent--;
739 cpuidle_coupled_poke_others(cpu, coupled);
750 if (dev && dev->coupled) {
751 cpuidle_coupled_update_online_cpus(dev->coupled);
752 cpuidle_coupled_allow_idle(dev->coupled);
766 if (dev && dev->coupled)
767 cpuidle_coupled_prevent_idle(dev->coupled);
778 "cpuidle/coupled:prepare",
784 "cpuidle/coupled:online",