Lines Matching refs:cpu
17 #include <linux/cpu.h>
50 * cpuhp_cpu_state - Per cpu hotplug state storage
51 * @state: The current cpu state
60 * @done_up: Signal completion to the issuer of the task for cpu-up
61 * @done_down: Signal completion to the issuer of the task for cpu-down
124 int (*single)(unsigned int cpu);
125 int (*multi)(unsigned int cpu, struct hlist_node *node);
128 int (*single)(unsigned int cpu);
129 int (*multi)(unsigned int cpu, struct hlist_node *node);
146 * @cpu: The cpu for which the callback should be invoked
152 * Called from cpu hotplug and from the state register machinery.
154 static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state, bool bringup, struct hlist_node *node,
157 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
159 int (*cbm)(unsigned int cpu, struct hlist_node *node);
160 int (*cb)(unsigned int cpu);
179 trace_cpuhp_enter(cpu, st->target, state, cb);
180 ret = cb(cpu);
181 trace_cpuhp_exit(cpu, st->state, state, ret);
192 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
193 ret = cbm(cpu, node);
194 trace_cpuhp_exit(cpu, st->state, state, ret);
206 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
207 ret = cbm(cpu, node);
208 trace_cpuhp_exit(cpu, st->state, state, ret);
236 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
237 ret = cbm(cpu, node);
238 trace_cpuhp_exit(cpu, st->state, state, ret);
378 if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n")) {
448 static inline bool cpu_smt_allowed(unsigned int cpu)
454 if (topology_is_primary_thread(cpu)) {
464 return !cpumask_test_cpu(cpu, &cpus_booted_once_mask);
474 static inline bool cpu_smt_allowed(unsigned int cpu)
547 static int bringup_wait_for_ap(unsigned int cpu)
549 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
553 if (WARN_ON_ONCE((!cpu_online(cpu)))) {
557 /* Unpark the hotplug thread of the target cpu */
567 if (!cpu_smt_allowed(cpu)) {
578 static int bringup_cpu(unsigned int cpu)
580 struct task_struct *idle = idle_thread_get(cpu);
591 * setup the vector space for the cpu which comes online.
597 ret = __cpu_up(cpu, idle);
602 return bringup_wait_for_ap(cpu);
605 static int finish_cpu(unsigned int cpu)
607 struct task_struct *idle = idle_thread_get(cpu);
625 static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st)
628 cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
647 static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st, enum cpuhp_state target)
654 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
658 undo_cpu_up(cpu, st);
667 * The cpu hotplug threads manage the bringup and teardown of the cpus
669 static void cpuhp_create(unsigned int cpu)
671 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
677 static int cpuhp_should_run(unsigned int cpu)
685 * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
698 static void cpuhp_thread_fun(unsigned int cpu)
743 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
751 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
772 /* Invoke a single callback on a remote cpu */
773 static int cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup, struct hlist_node *node)
775 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
778 if (!cpu_online(cpu)) {
793 return cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
824 static int cpuhp_kick_ap_work(unsigned int cpu)
826 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
836 trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);
838 trace_cpuhp_exit(cpu, st->state, prev_state, ret);
907 #define arch_clear_mm_cpumask_cpu(cpu, mm) cpumask_clear_cpu(cpu, mm_cpumask(mm))
912 * @cpu: a CPU id
922 void clear_tasks_mm_cpumask(int cpu)
927 * This function is called after the cpu is taken down and marked
928 * offline, so its not like new tasks will ever get this cpu set in
933 WARN_ON(cpu_online(cpu));
947 arch_clear_mm_cpumask_cpu(cpu, t->mm);
958 int err, cpu = smp_processor_id();
975 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
985 tick_offline_cpu(cpu);
987 stop_machine_park(cpu);
991 static int takedown_cpu(unsigned int cpu)
993 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
997 kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread);
1000 * Prevent irq alloc/free while the dying cpu reorganizes the
1008 err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu));
1013 kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread);
1016 BUG_ON(cpu_online(cpu));
1028 /* Interrupts are moved away from the dying cpu, reenable alloc/free */
1031 hotplug_cpu__broadcast_tick_pull(cpu);
1033 __cpu_die(cpu);
1035 tick_cleanup_dead_cpu(cpu);
1036 rcutree_migrate_callbacks(cpu);
1056 * to an online cpu.
1061 static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st)
1064 cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
1068 static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st, enum cpuhp_state target)
1074 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
1078 undo_cpu_down(cpu, st);
1087 static int __ref _cpu_down(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
1089 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1092 if (num_active_cpus() == 1 && cpu_active(cpu)) {
1096 if (!cpu_present(cpu)) {
1101 if (!tasks_frozen && !cpu_isolated(cpu) && num_online_uniso_cpus() == 1) {
1117 ret = cpuhp_kick_ap_work(cpu);
1140 ret = cpuhp_down_callbacks(cpu, st, target);
1158 static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
1163 return _cpu_down(cpu, 0, target);
1166 static int cpu_down(unsigned int cpu, enum cpuhp_state target)
1171 err = cpu_down_maps_locked(cpu, target);
1177 * cpu_device_down - Bring down a cpu device
1178 * @dev: Pointer to the cpu device to offline
1180 * This function is meant to be used by device core cpu subsystem only.
1189 int remove_cpu(unsigned int cpu)
1194 ret = device_offline(get_cpu_device(cpu));
1203 unsigned int cpu;
1209 * Make certain the cpu I'm about to reboot on is online.
1217 for_each_online_cpu(cpu)
1219 if (cpu == primary_cpu) {
1223 error = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
1225 pr_err("Failed to offline CPU%d - error=%d", cpu, error);
1250 * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
1251 * @cpu: cpu that just started
1253 * It must be called by the arch code on the new cpu, before the new cpu
1254 * enables interrupts and before the "boot" cpu returns from __cpu_up().
1256 void notify_cpu_starting(unsigned int cpu)
1258 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1262 rcu_cpu_starting(cpu); /* Enables RCU usage on this CPU. */
1263 cpumask_set_cpu(cpu, &cpus_booted_once_mask);
1266 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
1283 /* Happens for the boot cpu */
1299 static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
1301 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1307 if (!cpu_present(cpu)) {
1321 /* Let it fail before we try to bring the cpu up */
1322 idle = idle_thread_get(cpu);
1337 ret = cpuhp_kick_ap_work(cpu);
1353 ret = cpuhp_up_callbacks(cpu, st, target);
1361 static int cpu_up(unsigned int cpu, enum cpuhp_state target)
1365 if (!cpu_possible(cpu)) {
1366 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n", cpu);
1373 err = try_online_node(cpu_to_node(cpu));
1384 if (!cpu_smt_allowed(cpu)) {
1389 err = _cpu_up(cpu, 0, target);
1396 * cpu_device_up - Bring up a cpu device
1397 * @dev: Pointer to the cpu device to online
1399 * This function is meant to be used by device core cpu subsystem only.
1408 int add_cpu(unsigned int cpu)
1413 ret = device_online(get_cpu_device(cpu));
1422 * @sleep_cpu: The cpu we hibernated on and should be brought up.
1445 unsigned int cpu;
1447 for_each_present_cpu(cpu)
1452 if (!cpu_online(cpu)) {
1453 cpu_up(cpu, CPUHP_ONLINE);
1463 int cpu, error = 0;
1484 for_each_online_cpu(cpu)
1486 if (cpu == primary) {
1496 trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
1497 error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
1498 trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
1500 cpumask_set_cpu(cpu, frozen_cpus);
1502 pr_err("Error taking CPU%d down: %d\n", cpu, error);
1534 int cpu, error;
1547 for_each_cpu(cpu, frozen_cpus)
1549 trace_suspend_resume(TPS("CPU_ON"), cpu, true);
1550 error = _cpu_up(cpu, 1, CPUHP_ONLINE);
1551 trace_suspend_resume(TPS("CPU_ON"), cpu, false);
1553 pr_info("CPU%d is up\n", cpu);
1556 pr_warn("Error taking CPU%d up: %d\n", cpu, error);
1611 * to disable cpu hotplug to avoid cpu hotplug race.
1699 /* Kicks the plugged cpu into life */
1702 .name = "cpu:bringup",
1752 .name = "cpu:teardown",
1805 /* Last state is scheduler control setting the cpu active */
1864 static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name, int (*startup)(unsigned int cpu),
1865 int (*teardown)(unsigned int cpu), bool multi_instance)
1867 /* (Un)Install the callbacks for further cpu hotplug operations */
1909 static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup, struct hlist_node *node)
1927 ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node);
1929 ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1932 ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1945 int cpu;
1948 for_each_present_cpu(cpu)
1950 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1953 if (cpu >= failedcpu) {
1957 /* Did we invoke the startup call on that cpu ? */
1959 cpuhp_issue_call(cpu, state, false, node);
1967 int cpu;
1984 * Try to call the startup callback for each present cpu
1985 * depending on the hotplug state of the cpu.
1987 for_each_present_cpu(cpu)
1989 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1996 ret = cpuhp_issue_call(cpu, state, true, node);
1999 cpuhp_rollback_install(cpu, state, node);
2027 * cpu state >= @state
2041 int (*startup)(unsigned int cpu), int (*teardown)(unsigned int cpu),
2044 int cpu, ret = 0;
2068 * Try to call the startup callback for each present cpu
2069 * depending on the hotplug state of the cpu.
2071 for_each_present_cpu(cpu)
2073 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2080 ret = cpuhp_issue_call(cpu, state, true, NULL);
2083 cpuhp_rollback_install(cpu, state, NULL);
2102 int __cpuhp_setup_state(enum cpuhp_state state, const char *name, bool invoke, int (*startup)(unsigned int cpu),
2103 int (*teardown)(unsigned int cpu), bool multi_instance)
2117 int cpu;
2132 * Call the teardown callback for each present cpu depending
2133 * on the hotplug state of the cpu. This function is not
2136 for_each_present_cpu(cpu)
2138 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2142 cpuhp_issue_call(cpu, state, false, node);
2159 * cpu state >= @state
2168 int cpu;
2185 * Call the teardown callback for each present cpu depending
2186 * on the hotplug state of the cpu. This function is not
2189 for_each_present_cpu(cpu)
2191 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2195 cpuhp_issue_call(cpu, state, false, NULL);
2213 static void cpuhp_offline_cpu_device(unsigned int cpu)
2215 struct device *dev = get_cpu_device(cpu);
2222 static void cpuhp_online_cpu_device(unsigned int cpu)
2224 struct device *dev = get_cpu_device(cpu);
2233 int cpu, ret = 0;
2236 for_each_online_cpu(cpu)
2238 if (topology_is_primary_thread(cpu)) {
2241 ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
2246 * As this needs to hold the cpu maps lock it's impossible
2248 * cpu_down() which takes cpu maps lock. cpu maps lock
2258 cpuhp_offline_cpu_device(cpu);
2269 int cpu, ret = 0;
2273 for_each_present_cpu(cpu)
2276 if (cpu_online(cpu) || !node_online(cpu_to_node(cpu))) {
2279 ret = _cpu_up(cpu, 0, CPUHP_ONLINE);
2284 cpuhp_online_cpu_device(cpu);
2520 int cpu, ret;
2532 for_each_possible_cpu(cpu)
2534 struct device *dev = get_cpu_device(cpu);
2621 void set_cpu_online(unsigned int cpu, bool online)
2634 if (!cpumask_test_and_set_cpu(cpu, &__cpu_online_mask)) {
2638 if (cpumask_test_and_clear_cpu(cpu, &__cpu_online_mask)) {
2649 int cpu = smp_processor_id();
2651 /* Mark the boot cpu "present", "online" etc for SMP and UP case */
2652 set_cpu_online(cpu, true);
2653 set_cpu_active(cpu, true);
2654 set_cpu_present(cpu, true);
2655 set_cpu_possible(cpu, true);
2658 __boot_cpu_id = cpu;