Lines Matching refs:env
2111 static void update_numa_stats(struct task_numa_env *env,
2132 !cpumask_test_cpu(cpu, env->p->cpus_ptr))
2145 ns->node_type = numa_classify(env->imbalance_pct, ns);
2151 static void task_numa_assign(struct task_numa_env *env,
2154 struct rq *rq = cpu_rq(env->dst_cpu);
2157 if (env->best_cpu != env->dst_cpu && xchg(&rq->numa_migrate_on, 1)) {
2159 int start = env->dst_cpu;
2162 for_each_cpu_wrap(cpu, cpumask_of_node(env->dst_nid), start + 1) {
2163 if (cpu == env->best_cpu || !idle_cpu(cpu) ||
2164 !cpumask_test_cpu(cpu, env->p->cpus_ptr)) {
2168 env->dst_cpu = cpu;
2169 rq = cpu_rq(env->dst_cpu);
2183 if (env->best_cpu != -1 && env->best_cpu != env->dst_cpu) {
2184 rq = cpu_rq(env->best_cpu);
2188 if (env->best_task)
2189 put_task_struct(env->best_task);
2193 env->best_task = p;
2194 env->best_imp = imp;
2195 env->best_cpu = env->dst_cpu;
2199 struct task_numa_env *env)
2212 src_capacity = env->src_stats.compute_capacity;
2213 dst_capacity = env->dst_stats.compute_capacity;
2217 orig_src_load = env->src_stats.load;
2218 orig_dst_load = env->dst_stats.load;
2239 static bool task_numa_compare(struct task_numa_env *env,
2242 struct numa_group *cur_ng, *p_ng = deref_curr_numa_group(env->p);
2243 struct rq *dst_rq = cpu_rq(env->dst_cpu);
2247 int dist = env->dist;
2262 * end try selecting ourselves (current == env->p) as a swap candidate.
2264 if (cur == env->p) {
2270 if (maymove && moveimp >= env->best_imp)
2277 if (!cpumask_test_cpu(env->src_cpu, cur->cpus_ptr))
2284 if (env->best_task &&
2285 env->best_task->numa_preferred_nid == env->src_nid &&
2286 cur->numa_preferred_nid != env->src_nid) {
2308 if (env->dst_stats.node_type == node_has_spare)
2311 imp = taskimp + task_weight(cur, env->src_nid, dist) -
2312 task_weight(cur, env->dst_nid, dist);
2325 imp += group_weight(cur, env->src_nid, dist) -
2326 group_weight(cur, env->dst_nid, dist);
2328 imp += task_weight(cur, env->src_nid, dist) -
2329 task_weight(cur, env->dst_nid, dist);
2333 if (cur->numa_preferred_nid == env->dst_nid)
2342 if (cur->numa_preferred_nid == env->src_nid)
2345 if (maymove && moveimp > imp && moveimp > env->best_imp) {
2355 if (env->best_task && cur->numa_preferred_nid == env->src_nid &&
2356 env->best_task->numa_preferred_nid != env->src_nid) {
2366 if (imp < SMALLIMP || imp <= env->best_imp + SMALLIMP / 2)
2372 load = task_h_load(env->p) - task_h_load(cur);
2376 dst_load = env->dst_stats.load + load;
2377 src_load = env->src_stats.load - load;
2379 if (load_too_imbalanced(src_load, dst_load, env))
2385 int cpu = env->dst_stats.idle_cpu;
2389 cpu = env->dst_cpu;
2395 if (!idle_cpu(cpu) && env->best_cpu >= 0 &&
2396 idle_cpu(env->best_cpu)) {
2397 cpu = env->best_cpu;
2400 env->dst_cpu = cpu;
2403 task_numa_assign(env, cur, imp);
2410 if (maymove && !cur && env->best_cpu >= 0 && idle_cpu(env->best_cpu))
2417 if (!maymove && env->best_task &&
2418 env->best_task->numa_preferred_nid == env->src_nid) {
2427 static void task_numa_find_cpu(struct task_numa_env *env,
2437 if (env->dst_stats.node_type == node_has_spare) {
2447 src_running = env->src_stats.nr_running - 1;
2448 dst_running = env->dst_stats.nr_running + 1;
2451 env->imb_numa_nr);
2456 if (env->dst_stats.idle_cpu >= 0) {
2457 env->dst_cpu = env->dst_stats.idle_cpu;
2458 task_numa_assign(env, NULL, 0);
2465 * If the improvement from just moving env->p direction is better
2468 load = task_h_load(env->p);
2469 dst_load = env->dst_stats.load + load;
2470 src_load = env->src_stats.load - load;
2471 maymove = !load_too_imbalanced(src_load, dst_load, env);
2474 for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) {
2476 if (!cpumask_test_cpu(cpu, env->p->cpus_ptr))
2479 env->dst_cpu = cpu;
2480 if (task_numa_compare(env, taskimp, groupimp, maymove))
2487 struct task_numa_env env = {
2515 sd = rcu_dereference(per_cpu(sd_numa, env.src_cpu));
2517 env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2;
2518 env.imb_numa_nr = sd->imb_numa_nr;
2533 env.dst_nid = p->numa_preferred_nid;
2534 dist = env.dist = node_distance(env.src_nid, env.dst_nid);
2535 taskweight = task_weight(p, env.src_nid, dist);
2536 groupweight = group_weight(p, env.src_nid, dist);
2537 update_numa_stats(&env, &env.src_stats, env.src_nid, false);
2538 taskimp = task_weight(p, env.dst_nid, dist) - taskweight;
2539 groupimp = group_weight(p, env.dst_nid, dist) - groupweight;
2540 update_numa_stats(&env, &env.dst_stats, env.dst_nid, true);
2543 task_numa_find_cpu(&env, taskimp, groupimp);
2553 if (env.best_cpu == -1 || (ng && ng->active_nodes > 1)) {
2555 if (nid == env.src_nid || nid == p->numa_preferred_nid)
2558 dist = node_distance(env.src_nid, env.dst_nid);
2560 dist != env.dist) {
2561 taskweight = task_weight(p, env.src_nid, dist);
2562 groupweight = group_weight(p, env.src_nid, dist);
2571 env.dist = dist;
2572 env.dst_nid = nid;
2573 update_numa_stats(&env, &env.dst_stats, env.dst_nid, true);
2574 task_numa_find_cpu(&env, taskimp, groupimp);
2587 if (env.best_cpu == -1)
2588 nid = env.src_nid;
2590 nid = cpu_to_node(env.best_cpu);
2597 if (env.best_cpu == -1) {
2598 trace_sched_stick_numa(p, env.src_cpu, NULL, -1);
2602 best_rq = cpu_rq(env.best_cpu);
2603 if (env.best_task == NULL) {
2604 ret = migrate_task_to(p, env.best_cpu);
2607 trace_sched_stick_numa(p, env.src_cpu, NULL, env.best_cpu);
2611 ret = migrate_swap(p, env.best_task, env.best_cpu, env.src_cpu);
2615 trace_sched_stick_numa(p, env.src_cpu, env.best_task, env.best_cpu);
2616 put_task_struct(env.best_task);
9019 static int task_hot(struct task_struct *p, struct lb_env *env)
9023 lockdep_assert_rq_held(env->src_rq);
9032 if (env->sd->flags & SD_SHARE_CPUCAPACITY)
9038 if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running &&
9049 if (!sched_core_cookie_match(cpu_rq(env->dst_cpu), p))
9055 delta = rq_clock_task(env->src_rq) - p->se.exec_start;
9066 static int migrate_degrades_locality(struct task_struct *p, struct lb_env *env)
9075 if (!p->numa_faults || !(env->sd->flags & SD_NUMA))
9078 src_nid = cpu_to_node(env->src_cpu);
9079 dst_nid = cpu_to_node(env->dst_cpu);
9086 if (env->src_rq->nr_running > env->src_rq->nr_preferred_running)
9097 if (env->idle == CPU_IDLE)
9114 struct lb_env *env)
9124 int can_migrate_task(struct task_struct *p, struct lb_env *env)
9128 lockdep_assert_rq_held(env->src_rq);
9137 if (throttled_lb_pair(task_group(p), env->src_cpu, env->dst_cpu))
9144 if (!cpumask_test_cpu(env->dst_cpu, p->cpus_ptr)) {
9149 env->flags |= LBF_SOME_PINNED;
9161 if (env->idle == CPU_NEWLY_IDLE ||
9162 env->flags & (LBF_DST_PINNED | LBF_ACTIVE_LB))
9165 /* Prevent to re-select dst_cpu via env's CPUs: */
9166 for_each_cpu_and(cpu, env->dst_grpmask, env->cpus) {
9168 env->flags |= LBF_DST_PINNED;
9169 env->new_dst_cpu = cpu;
9178 env->flags &= ~LBF_ALL_PINNED;
9182 if (env->flags & LBF_IGNORE_PREFERRED_CLUSTER_TASKS &&
9183 !preferred_cluster(cpu_rq(env->dst_cpu)->cluster, p))
9187 if (task_on_cpu(env->src_rq, p)) {
9199 if (env->flags & LBF_ACTIVE_LB)
9202 tsk_cache_hot = migrate_degrades_locality(p, env);
9204 tsk_cache_hot = task_hot(p, env);
9207 env->sd->nr_balance_failed > env->sd->cache_nice_tries) {
9209 schedstat_inc(env->sd->lb_hot_gained[env->idle]);
9220 * detach_task() -- detach the task for the migration specified in env
9222 static void detach_task(struct task_struct *p, struct lb_env *env)
9224 lockdep_assert_rq_held(env->src_rq);
9226 deactivate_task(env->src_rq, p, DEQUEUE_NOCLOCK);
9228 double_lock_balance(env->src_rq, env->dst_rq);
9229 if (!(env->src_rq->clock_update_flags & RQCF_UPDATED))
9230 update_rq_clock(env->src_rq);
9232 set_task_cpu(p, env->dst_cpu);
9234 double_unlock_balance(env->src_rq, env->dst_rq);
9239 * detach_one_task() -- tries to dequeue exactly one task from env->src_rq, as
9244 static struct task_struct *detach_one_task(struct lb_env *env)
9248 lockdep_assert_rq_held(env->src_rq);
9251 &env->src_rq->cfs_tasks, se.group_node) {
9252 if (!can_migrate_task(p, env))
9255 detach_task(p, env);
9259 * lb_gained[env->idle] is updated (other is detach_tasks)
9263 schedstat_inc(env->sd->lb_gained[env->idle]);
9275 static int detach_tasks(struct lb_env *env)
9277 struct list_head *tasks = &env->src_rq->cfs_tasks;
9282 int orig_loop = env->loop;
9285 lockdep_assert_rq_held(env->src_rq);
9291 if (env->src_rq->nr_running <= 1) {
9292 env->flags &= ~LBF_ALL_PINNED;
9296 if (env->imbalance <= 0)
9300 if (!same_cluster(env->dst_cpu, env->src_cpu))
9301 env->flags |= LBF_IGNORE_PREFERRED_CLUSTER_TASKS;
9310 if (env->idle != CPU_NOT_IDLE && env->src_rq->nr_running <= 1)
9313 env->loop++;
9318 if (env->loop > env->loop_max &&
9319 !(env->flags & LBF_ALL_PINNED))
9323 if (env->loop > env->loop_break) {
9324 env->loop_break += SCHED_NR_MIGRATE_BREAK;
9325 env->flags |= LBF_NEED_BREAK;
9331 if (!can_migrate_task(p, env))
9334 switch (env->migration_type) {
9339 * value. Make sure that env->imbalance decreases
9346 load < 16 && !env->sd->nr_balance_failed)
9355 if (shr_bound(load, env->sd->nr_balance_failed) > env->imbalance)
9358 env->imbalance -= load;
9364 if (util > env->imbalance)
9367 env->imbalance -= util;
9371 env->imbalance--;
9376 if (task_fits_cpu(p, env->src_cpu))
9379 env->imbalance = 0;
9383 detach_task(p, env);
9384 list_add(&p->se.group_node, &env->tasks);
9394 if (env->idle == CPU_NEWLY_IDLE)
9402 if (env->imbalance <= 0)
9411 if (env->flags & LBF_IGNORE_PREFERRED_CLUSTER_TASKS && !detached) {
9412 tasks = &env->src_rq->cfs_tasks;
9413 env->flags &= ~LBF_IGNORE_PREFERRED_CLUSTER_TASKS;
9414 env->loop = orig_loop;
9424 schedstat_add(env->sd->lb_gained[env->idle], detached);
9459 static void attach_tasks(struct lb_env *env)
9461 struct list_head *tasks = &env->tasks;
9465 rq_lock(env->dst_rq, &rf);
9466 update_rq_clock(env->dst_rq);
9472 attach_task(env->dst_rq, p);
9475 rq_unlock(env->dst_rq, &rf);
10014 * @env: The load balancing environment
10019 * @env::dst_cpu can do asym_packing if it has higher priority than the
10022 * SMT is a special case. If we are balancing load between cores, @env::dst_cpu
10030 * Return: true if @env::dst_cpu can do with asym_packing load balance. False
10034 sched_asym(struct lb_env *env, struct sd_lb_stats *sds, struct sg_lb_stats *sgs,
10038 if (!sched_use_asym_prio(env->sd, env->dst_cpu))
10050 return sched_asym_prefer(env->dst_cpu, group->asym_prefer_cpu);
10064 static inline bool smt_balance(struct lb_env *env, struct sg_lb_stats *sgs,
10067 if (env->idle == CPU_NOT_IDLE)
10083 static inline long sibling_imbalance(struct lb_env *env,
10091 if (env->idle == CPU_NOT_IDLE || !busiest->sum_nr_running)
10133 * @env: The load balancing environment.
10139 static inline void update_sg_lb_stats(struct lb_env *env,
10151 for_each_cpu_and(i, sched_group_span(group), env->cpus) {
10187 if (env->sd->flags & SD_ASYM_CPUCAPACITY) {
10193 } else if ((env->idle != CPU_NOT_IDLE) &&
10194 sched_reduced_capacity(rq, env->sd)) {
10215 if (!local_group && env->sd->flags & SD_ASYM_PACKING &&
10216 env->idle != CPU_NOT_IDLE && sgs->sum_h_nr_running &&
10217 sched_asym(env, sds, sgs, group)) {
10222 if (!local_group && smt_balance(env, sgs, group))
10225 sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs);
10235 * @env: The load balancing environment.
10246 static bool update_sd_pick_busiest(struct lb_env *env,
10263 if ((env->sd->flags & SD_ASYM_CPUCAPACITY) &&
10265 (!capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) ||
10382 if ((env->sd->flags & SD_ASYM_CPUCAPACITY) &&
10384 (capacity_greater(sg->sgc->min_capacity, capacity_of(env->dst_cpu))))
10763 static void update_idle_cpu_scan(struct lb_env *env,
10777 if (!sched_feat(SIS_UTIL) || env->idle == CPU_NEWLY_IDLE)
10780 llc_weight = per_cpu(sd_llc_size, env->dst_cpu);
10781 if (env->sd->span_weight != llc_weight)
10784 sd_share = rcu_dereference(per_cpu(sd_llc_shared, env->dst_cpu));
10821 pct = env->sd->imbalance_pct;
10836 * @env: The load balancing environment.
10840 static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds)
10842 struct sched_group *sg = env->sd->groups;
10852 local_group = cpumask_test_cpu(env->dst_cpu, sched_group_span(sg));
10857 if (env->idle != CPU_NEWLY_IDLE ||
10859 update_group_capacity(env->sd, env->dst_cpu);
10862 update_sg_lb_stats(env, sds, sg, sgs, &sg_status);
10868 if (update_sd_pick_busiest(env, sds, sg, sgs)) {
10880 } while (sg != env->sd->groups);
10891 if (env->sd->flags & SD_NUMA)
10892 env->fbq_type = fbq_classify_group(&sds->busiest_stat);
10894 if (!env->sd->parent) {
10895 struct root_domain *rd = env->dst_rq->rd;
10904 struct root_domain *rd = env->dst_rq->rd;
10910 update_idle_cpu_scan(env, sum_util);
10916 * @env: load balance environment
10919 static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds)
10927 if (env->sd->flags & SD_ASYM_CPUCAPACITY) {
10929 env->migration_type = migrate_misfit;
10930 env->imbalance = 1;
10936 env->migration_type = migrate_load;
10937 env->imbalance = busiest->group_misfit_task_load;
10947 env->migration_type = migrate_task;
10948 env->imbalance = busiest->sum_h_nr_running;
10954 env->migration_type = migrate_task;
10955 env->imbalance = 1;
10966 env->migration_type = migrate_task;
10967 env->imbalance = 1;
10977 !(env->sd->flags & SD_SHARE_PKG_RESOURCES)) {
10986 env->migration_type = migrate_util;
10987 env->imbalance = max(local->group_capacity, local->group_util) -
10997 if (env->idle != CPU_NOT_IDLE && env->imbalance == 0) {
10998 env->migration_type = migrate_task;
10999 env->imbalance = 1;
11010 env->migration_type = migrate_task;
11011 env->imbalance = sibling_imbalance(env, sds, busiest, local);
11018 env->migration_type = migrate_task;
11019 env->imbalance = max_t(long, 0,
11025 if (env->sd->flags & SD_NUMA) {
11026 env->imbalance = adjust_numa_imbalance(env->imbalance,
11028 env->sd->imb_numa_nr);
11033 env->imbalance >>= 1;
11056 env->imbalance = 0;
11068 env->imbalance = 0;
11082 env->migration_type = migrate_load;
11083 env->imbalance = min(
11114 * @env: The load balancing environment.
11121 static struct sched_group *find_busiest_group(struct lb_env *env)
11132 update_sd_lb_stats(env, &sds);
11145 struct root_domain *rd = env->dst_rq->rd;
11199 env->sd->imbalance_pct * local->avg_load)
11208 sibling_imbalance(env, &sds, busiest, local) > 1)
11212 if (env->idle == CPU_NOT_IDLE) {
11251 calculate_imbalance(env, &sds);
11252 return env->imbalance ? sds.busiest : NULL;
11255 env->imbalance = 0;
11262 static struct rq *find_busiest_queue(struct lb_env *env,
11270 for_each_cpu_and(i, sched_group_span(group), env->cpus) {
11297 if (rt > env->fbq_type)
11315 if (env->sd->flags & SD_ASYM_CPUCAPACITY &&
11316 !capacity_greater(capacity_of(env->dst_cpu), capacity) &&
11327 if ((env->sd->flags & SD_ASYM_PACKING) &&
11328 sched_use_asym_prio(env->sd, i) &&
11329 sched_asym_prefer(i, env->dst_cpu) &&
11333 switch (env->migration_type) {
11341 if (nr_running == 1 && load > env->imbalance &&
11342 !check_cpu_capacity(rq, env->sd))
11414 asym_active_balance(struct lb_env *env)
11422 * If @env::src_cpu is an SMT core with busy siblings, let
11423 * the lower priority @env::dst_cpu help it. Do not follow
11426 return env->idle != CPU_NOT_IDLE && (env->sd->flags & SD_ASYM_PACKING) &&
11427 sched_use_asym_prio(env->sd, env->dst_cpu) &&
11428 (sched_asym_prefer(env->dst_cpu, env->src_cpu) ||
11429 !sched_use_asym_prio(env->sd, env->src_cpu));
11433 imbalanced_active_balance(struct lb_env *env)
11435 struct sched_domain *sd = env->sd;
11442 if ((env->migration_type == migrate_task) &&
11449 static int need_active_balance(struct lb_env *env)
11451 struct sched_domain *sd = env->sd;
11453 if (asym_active_balance(env))
11456 if (imbalanced_active_balance(env))
11465 if ((env->idle != CPU_NOT_IDLE) &&
11466 (env->src_rq->cfs.h_nr_running == 1)) {
11467 if ((check_cpu_capacity(env->src_rq, sd)) &&
11468 (capacity_of(env->src_cpu)*sd->imbalance_pct < capacity_of(env->dst_cpu)*100))
11472 if (env->migration_type == migrate_misfit)
11490 static int should_we_balance(struct lb_env *env)
11493 struct sched_group *sg = env->sd->groups;
11500 if (!cpumask_test_cpu(env->dst_cpu, env->cpus))
11510 if (env->idle == CPU_NEWLY_IDLE) {
11511 if (env->dst_rq->nr_running > 0 || env->dst_rq->ttwu_pending)
11518 for_each_cpu_and(cpu, swb_cpus, env->cpus) {
11527 if (!(env->sd->flags & SD_SHARE_CPUCAPACITY) && !is_core_idle(cpu)) {
11545 return cpu == env->dst_cpu;
11550 return idle_smt == env->dst_cpu;
11553 return group_balance_cpu_not_isolated(sg) == env->dst_cpu;
11570 struct lb_env env = {
11579 .tasks = LIST_HEAD_INIT(env.tasks),
11587 if (!should_we_balance(&env)) {
11592 group = find_busiest_group(&env);
11598 busiest = find_busiest_queue(&env, group);
11604 WARN_ON_ONCE(busiest == env.dst_rq);
11606 schedstat_add(sd->lb_imbalance[idle], env.imbalance);
11608 env.src_cpu = busiest->cpu;
11609 env.src_rq = busiest;
11613 env.flags |= LBF_ALL_PINNED;
11621 env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running);
11631 cur_ld_moved = detach_tasks(&env);
11644 attach_tasks(&env);
11650 if (env.flags & LBF_NEED_BREAK) {
11651 env.flags &= ~LBF_NEED_BREAK;
11653 if (env.loop < busiest->nr_running)
11676 if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) {
11678 /* Prevent to re-select dst_cpu via env's CPUs */
11679 __cpumask_clear_cpu(env.dst_cpu, env.cpus);
11681 env.dst_rq = cpu_rq(env.new_dst_cpu);
11682 env.dst_cpu = env.new_dst_cpu;
11683 env.flags &= ~LBF_DST_PINNED;
11684 env.loop = 0;
11685 env.loop_break = SCHED_NR_MIGRATE_BREAK;
11700 if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0)
11705 if (unlikely(env.flags & LBF_ALL_PINNED)) {
11715 if (!cpumask_subset(cpus, env.dst_grpmask)) {
11716 env.loop = 0;
11717 env.loop_break = SCHED_NR_MIGRATE_BREAK;
11735 if (need_active_balance(&env)) {
11751 env.flags &= ~LBF_ALL_PINNED;
11778 if (likely(!active_balance) || need_active_balance(&env)) {
11791 if (sd_parent && !(env.flags & LBF_ALL_PINNED)) {
11817 if (env.idle == CPU_NEWLY_IDLE)
11821 if ((env.flags & LBF_ALL_PINNED &&
11914 struct lb_env env = {
11929 detach_task(push_task, &env);
11944 struct lb_env env = {
11957 p = detach_one_task(&env);
12090 * env->dst_cpu, so we can't know our idle