Lines Matching refs:now
561 u64 now;
963 static void ioc_refresh_vrate(struct ioc *ioc, struct ioc_now *now)
965 s64 pleft = ioc->period_at + ioc->period_us - now->now;
1042 static void ioc_now(struct ioc *ioc, struct ioc_now *now)
1047 now->now_ns = ktime_get();
1048 now->now = ktime_to_us(now->now_ns);
1061 now->vnow = ioc->period_at_vtime +
1062 (now->now - ioc->period_at) * vrate;
1066 static void ioc_start_period(struct ioc *ioc, struct ioc_now *now)
1071 ioc->period_at = now->now;
1072 ioc->period_at_vtime = now->vnow;
1085 bool save, struct ioc_now *now)
1106 iocg->saved_margin = now->vnow - atomic64_read(&iocg->vtime);
1160 bool save, struct ioc_now *now)
1162 __propagate_weights(iocg, active, inuse, save, now);
1246 static void weight_updated(struct ioc_gq *iocg, struct ioc_now *now)
1257 propagate_weights(iocg, weight, iocg->inuse, true, now);
1261 static bool iocg_activate(struct ioc_gq *iocg, struct ioc_now *now)
1273 ioc_now(ioc, now);
1286 ioc_now(ioc, now);
1307 vtarget = now->vnow - ioc->margins.target;
1323 iocg->last_inuse ?: iocg->weight, true, now);
1325 TRACE_IOCG_PATH(iocg_activate, iocg, now,
1328 iocg->activated_at = now->now;
1332 ioc->dfgv_period_at = now->now;
1334 ioc_start_period(ioc, now);
1346 static bool iocg_kick_delay(struct ioc_gq *iocg, struct ioc_now *now)
1360 if (time_before64(now->now, iocg->delay_at))
1364 tdelta = now->now - iocg->delay_at;
1373 abs_cost_to_cost(iocg->abs_vdebt, hwa) - now->vnow;
1390 iocg->delay_at = now->now;
1396 iocg->indelay_since = now->now;
1401 iocg->stat.indelay_us += now->now - iocg->indelay_since;
1411 struct ioc_now *now)
1424 iocg->indebt_since = now->now;
1425 propagate_weights(iocg, iocg->active, 0, false, now);
1436 struct ioc_now *now)
1449 iocg->stat.indebt_us += now->now - iocg->indebt_since;
1453 false, now);
1490 struct ioc_now *now)
1501 vbudget = now->vnow - atomic64_read(&iocg->vtime);
1513 iocg_pay_debt(iocg, abs_vpay, now);
1518 iocg_kick_delay(iocg, now);
1543 iocg->stat.wait_us += now->now - iocg->wait_since;
1550 iocg->wait_since = now->now;
1557 expires = now->now_ns +
1576 struct ioc_now now;
1579 ioc_now(iocg->ioc, &now);
1582 iocg_kick_waitq(iocg, pay_debt, &now);
1692 static void iocg_flush_stat_leaf(struct ioc_gq *iocg, struct ioc_now *now)
1716 static void iocg_flush_stat(struct list_head *target_iocgs, struct ioc_now *now)
1723 iocg_flush_stat_leaf(iocg, now);
1740 u32 usage, struct ioc_now *now)
1752 time_after64(vtime, now->vnow - ioc->margins.min))
1756 excess = now->vnow - vtime - ioc->margins.target;
1780 delta = div64_s64(WEIGHT_ONE * (now->vnow - vtime),
1781 now->vnow - ioc->period_at_vtime);
1845 static void transfer_surpluses(struct list_head *surpluses, struct ioc_now *now)
2023 * All inner nodes now have ->hweight_inuse and ->child_adjusted_sum and
2047 TRACE_IOCG_PATH(inuse_transfer, iocg, now,
2052 __propagate_weights(iocg, iocg->active, inuse, true, now);
2072 struct ioc_now *now)
2079 ioc->dfgv_period_at = now->now;
2095 if (time_before64(now->now, ioc->dfgv_period_at + DFGV_PERIOD))
2102 dur = now->now - ioc->dfgv_period_at;
2105 ioc->dfgv_period_at = now->now;
2142 iocg_kick_waitq(iocg, true, now);
2144 TRACE_IOCG_PATH(iocg_forgive_debt, iocg, now, usage_pct,
2162 static int ioc_check_iocgs(struct ioc *ioc, struct ioc_now *now)
2176 iocg->stat.wait_us += now->now - iocg->wait_since;
2177 iocg->wait_since = now->now;
2181 now->now - iocg->indebt_since;
2182 iocg->indebt_since = now->now;
2186 now->now - iocg->indelay_since;
2187 iocg->indelay_since = now->now;
2193 iocg_kick_waitq(iocg, true, now);
2207 excess = now->vnow - vtime - ioc->margins.target;
2216 TRACE_IOCG_PATH(iocg_idle, iocg, now,
2219 __propagate_weights(iocg, 0, 0, false, now);
2234 struct ioc_now now;
2252 ioc_now(ioc, &now);
2254 period_vtime = now.vnow - ioc->period_at_vtime;
2260 nr_debtors = ioc_check_iocgs(ioc, &now);
2266 iocg_flush_stat(&ioc->active_iocgs, &now);
2290 time_after64(vtime, now.vnow -
2292 time_before64(vdone, now.vnow - period_vtime))
2306 time_before64(vtime, now.vnow - ioc->margins.low))) {
2320 usage_dur = max_t(u64, now.now - iocg->activated_at, 1);
2322 usage_dur = max_t(u64, now.now - ioc->period_at, 1);
2336 usage, &now);
2359 TRACE_IOCG_PATH(inuse_shortage, iocg, &now,
2364 iocg->active, true, &now);
2374 transfer_surpluses(&surpluses, &now);
2433 ioc_forgive_debts(ioc, usage_us_sum, nr_debtors, &now);
2443 ioc_start_period(ioc, &now);
2450 ioc_refresh_vrate(ioc, &now);
2457 u64 abs_cost, struct ioc_now *now)
2470 margin = now->vnow - vtime - cost;
2503 propagate_weights(iocg, iocg->active, new_inuse, true, now);
2506 } while (time_after64(vtime + cost, now->vnow) &&
2511 TRACE_IOCG_PATH(inuse_adjust, iocg, now,
2600 struct ioc_now now;
2615 if (!iocg_activate(iocg, &now))
2620 cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now);
2628 time_before_eq64(vtime + cost, now.vnow)) {
2676 iocg_incur_debt(iocg, abs_cost, &now);
2677 if (iocg_kick_delay(iocg, &now))
2692 &now);
2715 iocg_kick_waitq(iocg, ioc_locked, &now);
2736 struct ioc_now now;
2748 ioc_now(ioc, &now);
2751 cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now);
2763 time_before_eq64(atomic64_read(&iocg->vtime) + cost, now.vnow)) {
2777 iocg_incur_debt(iocg, abs_cost, &now);
2778 if (iocg_kick_delay(iocg, &now))
2977 struct ioc_now now;
2981 ioc_now(ioc, &now);
2984 atomic64_set(&iocg->vtime, now.vnow);
2985 atomic64_set(&iocg->done_vtime, now.vnow);
3005 weight_updated(iocg, &now);
3019 struct ioc_now now;
3021 ioc_now(ioc, &now);
3022 propagate_weights(iocg, 0, 0, false, &now);
3090 struct ioc_now now;
3111 ioc_now(iocg->ioc, &now);
3112 weight_updated(iocg, &now);
3140 ioc_now(iocg->ioc, &now);
3141 weight_updated(iocg, &now);