Lines Matching refs:now
564 u64 now;
955 static void ioc_refresh_vrate(struct ioc *ioc, struct ioc_now *now)
957 s64 pleft = ioc->period_at + ioc->period_us - now->now;
986 static void ioc_now(struct ioc *ioc, struct ioc_now *now)
990 now->now_ns = ktime_get();
991 now->now = ktime_to_us(now->now_ns);
992 now->vrate = atomic64_read(&ioc->vtime_rate);
1004 now->vnow = ioc->period_at_vtime +
1005 (now->now - ioc->period_at) * now->vrate;
1009 static void ioc_start_period(struct ioc *ioc, struct ioc_now *now)
1014 ioc->period_at = now->now;
1015 ioc->period_at_vtime = now->vnow;
1028 bool save, struct ioc_now *now)
1049 iocg->saved_margin = now->vnow - atomic64_read(&iocg->vtime);
1103 bool save, struct ioc_now *now)
1105 __propagate_weights(iocg, active, inuse, save, now);
1189 static void weight_updated(struct ioc_gq *iocg, struct ioc_now *now)
1200 propagate_weights(iocg, weight, iocg->inuse, true, now);
1204 static bool iocg_activate(struct ioc_gq *iocg, struct ioc_now *now)
1216 ioc_now(ioc, now);
1229 ioc_now(ioc, now);
1250 vtarget = now->vnow - ioc->margins.target;
1266 iocg->last_inuse ?: iocg->weight, true, now);
1268 TRACE_IOCG_PATH(iocg_activate, iocg, now,
1271 iocg->activated_at = now->now;
1275 ioc->dfgv_period_at = now->now;
1277 ioc_start_period(ioc, now);
1289 static bool iocg_kick_delay(struct ioc_gq *iocg, struct ioc_now *now)
1303 if (time_before64(now->now, iocg->delay_at))
1307 tdelta = now->now - iocg->delay_at;
1316 abs_cost_to_cost(iocg->abs_vdebt, hwa) - now->vnow;
1333 iocg->delay_at = now->now;
1339 iocg->indelay_since = now->now;
1344 iocg->local_stat.indelay_us += now->now - iocg->indelay_since;
1354 struct ioc_now *now)
1367 iocg->indebt_since = now->now;
1368 propagate_weights(iocg, iocg->active, 0, false, now);
1379 struct ioc_now *now)
1392 iocg->local_stat.indebt_us += now->now - iocg->indebt_since;
1396 false, now);
1433 struct ioc_now *now)
1444 vbudget = now->vnow - atomic64_read(&iocg->vtime);
1456 iocg_pay_debt(iocg, abs_vpay, now);
1461 iocg_kick_delay(iocg, now);
1486 iocg->local_stat.wait_us += now->now - iocg->wait_since;
1493 iocg->wait_since = now->now;
1500 expires = now->now_ns +
1519 struct ioc_now now;
1522 ioc_now(iocg->ioc, &now);
1525 iocg_kick_waitq(iocg, pay_debt, &now);
1615 static void iocg_flush_stat_one(struct ioc_gq *iocg, struct ioc_now *now)
1665 static void iocg_flush_stat(struct list_head *target_iocgs, struct ioc_now *now)
1672 iocg_flush_stat_one(iocg, now);
1678 iocg_flush_stat_one(iocg, now);
1689 u32 usage, struct ioc_now *now)
1701 time_after64(vtime, now->vnow - ioc->margins.min))
1705 excess = now->vnow - vtime - ioc->margins.target;
1729 delta = div64_s64(WEIGHT_ONE * (now->vnow - vtime),
1730 now->vnow - ioc->period_at_vtime);
1794 static void transfer_surpluses(struct list_head *surpluses, struct ioc_now *now)
1972 * All inner nodes now have ->hweight_inuse and ->child_adjusted_sum and
1996 TRACE_IOCG_PATH(inuse_transfer, iocg, now,
2001 __propagate_weights(iocg, iocg->active, inuse, true, now);
2021 struct ioc_now *now)
2028 ioc->dfgv_period_at = now->now;
2044 if (time_before64(now->now, ioc->dfgv_period_at + DFGV_PERIOD))
2051 dur = now->now - ioc->dfgv_period_at;
2054 ioc->dfgv_period_at = now->now;
2091 iocg_kick_waitq(iocg, true, now);
2093 TRACE_IOCG_PATH(iocg_forgive_debt, iocg, now, usage_pct,
2105 struct ioc_now now;
2121 ioc_now(ioc, &now);
2123 period_vtime = now.vnow - ioc->period_at_vtime;
2144 iocg->local_stat.wait_us += now.now - iocg->wait_since;
2145 iocg->wait_since = now.now;
2149 now.now - iocg->indebt_since;
2150 iocg->indebt_since = now.now;
2154 now.now - iocg->indelay_since;
2155 iocg->indelay_since = now.now;
2161 iocg_kick_waitq(iocg, true, &now);
2175 excess = now.vnow - vtime - ioc->margins.target;
2184 __propagate_weights(iocg, 0, 0, false, &now);
2196 iocg_flush_stat(&ioc->active_iocgs, &now);
2220 time_after64(vtime, now.vnow -
2222 time_before64(vdone, now.vnow - period_vtime))
2241 usage_dur = max_t(u64, now.now - iocg->activated_at, 1);
2243 usage_dur = max_t(u64, now.now - ioc->period_at, 1);
2254 time_before64(vtime, now.vnow - ioc->margins.low))) {
2264 usage, &now);
2287 TRACE_IOCG_PATH(inuse_shortage, iocg, &now,
2292 iocg->active, true, &now);
2302 transfer_surpluses(&surpluses, &now);
2404 ioc_forgive_debts(ioc, usage_us_sum, nr_debtors, &now);
2414 ioc_start_period(ioc, &now);
2421 ioc_refresh_vrate(ioc, &now);
2428 u64 abs_cost, struct ioc_now *now)
2441 margin = now->vnow - vtime - cost;
2474 propagate_weights(iocg, iocg->active, new_inuse, true, now);
2477 } while (time_after64(vtime + cost, now->vnow) &&
2482 TRACE_IOCG_PATH(inuse_adjust, iocg, now,
2567 struct ioc_now now;
2582 if (!iocg_activate(iocg, &now))
2587 cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now);
2595 time_before_eq64(vtime + cost, now.vnow)) {
2643 iocg_incur_debt(iocg, abs_cost, &now);
2644 if (iocg_kick_delay(iocg, &now))
2659 &now);
2682 iocg_kick_waitq(iocg, ioc_locked, &now);
2703 struct ioc_now now;
2715 ioc_now(ioc, &now);
2718 cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now);
2730 time_before_eq64(atomic64_read(&iocg->vtime) + cost, now.vnow)) {
2744 iocg_incur_debt(iocg, abs_cost, &now);
2745 if (iocg_kick_delay(iocg, &now))
2943 struct ioc_now now;
2947 ioc_now(ioc, &now);
2950 atomic64_set(&iocg->vtime, now.vnow);
2951 atomic64_set(&iocg->done_vtime, now.vnow);
2971 weight_updated(iocg, &now);
2985 struct ioc_now now;
2987 ioc_now(ioc, &now);
2988 propagate_weights(iocg, 0, 0, false, &now);
3062 struct ioc_now now;
3083 ioc_now(iocg->ioc, &now);
3084 weight_updated(iocg, &now);
3110 ioc_now(iocg->ioc, &now);
3111 weight_updated(iocg, &now);