Lines Matching defs:vtime
51 * The device virtual time (vtime) is used as the primary control metric.
73 * A given cgroup's vtime runs slower in inverse proportion to its hweight.
75 * against the device vtime - an IO which takes 10ms on the underlying
79 * vtime is running at a rate determined by its hweight. A cgroup tracks
80 * the vtime consumed by past IOs and can issue a new IO if doing so
81 * wouldn't outrun the current device vtime. Otherwise, the IO is
82 * suspended until the vtime has progressed enough to cover it.
97 * To slow down, we lower the vrate - the rate at which the device vtime
98 * passes compared to the wall clock. For example, if the vtime is running
144 * snapback mechanism in case the cgroup needs more IO vtime for itself.
147 * accelerating vtime for other cgroups and work conservation can also be
166 * - cur_per : Internal wall and device vtime clock
220 * iocg->vtime is targeted at 50% behind the device vtime, which
222 * immediately canceled if the vtime margin runs below 10%.
239 * As vtime is used to calculate the cost of each IO, it needs to
246 * 1s worth of vtime is 2^37. This gives us both sub-nanosecond
430 u64 period_at_vtime; /* vtime starttime */
495 * `vtime` is this iocg's vtime cursor which progresses as IOs are
496 * issued. If lagging behind device vtime, the delta represents
501 * than issue. The delta behind `vtime` represents the cost of
504 atomic64_t vtime;
514 * and invalidating `vtime`.
649 * vtime credit shortage and down on device saturation.
723 atomic64_add(cost, &iocg->vtime);
957 * When an iocg accumulates too much vtime or gets deactivated, we throw away
958 * some vtime, which lowers the overall device utilization. As the exact amount
960 * vrate accordingly so that the extra vtime generated in the current period
1052 * The current vtime is
1054 * vtime at period start + (wallclock time since the start) * vrate
1106 iocg->saved_margin = now->vnow - atomic64_read(&iocg->vtime);
1265 u64 vtime, vtarget;
1308 vtime = atomic64_read(&iocg->vtime);
1310 atomic64_add(vtarget - vtime, &iocg->vtime);
1311 atomic64_add(vtarget - vtime, &iocg->done_vtime);
1312 vtime = vtarget;
1326 last_period, cur_period, vtime);
1372 vover = atomic64_read(&iocg->vtime) +
1501 vbudget = now->vnow - atomic64_read(&iocg->vtime);
1511 atomic64_add(vpay, &iocg->vtime);
1532 * Wake up the ones which are due and see how much vtime we'll need for
1638 if (atomic64_read(&iocg->done_vtime) != atomic64_read(&iocg->vtime))
1743 u64 vtime = atomic64_read(&iocg->vtime);
1752 time_after64(vtime, now->vnow - ioc->margins.min))
1756 excess = now->vnow - vtime - ioc->margins.target;
1758 atomic64_add(excess, &iocg->vtime);
1760 vtime += excess;
1780 delta = div64_s64(WEIGHT_ONE * (now->vnow - vtime),
2192 /* might be oversleeping vtime / hweight changes, kick */
2198 u64 vtime = atomic64_read(&iocg->vtime);
2207 excess = now->vnow - vtime - ioc->margins.target;
2218 atomic64_read(&ioc->cur_period), vtime);
2270 u64 vdone, vtime, usage_us;
2274 * Collect unused and wind vtime closer to vnow to prevent
2278 vtime = atomic64_read(&iocg->vtime);
2289 time_after64(vtime, vdone) &&
2290 time_after64(vtime, now.vnow -
2302 /* see whether there's surplus vtime */
2306 time_before64(vtime, now.vnow - ioc->margins.low))) {
2310 if (vdone != vtime) {
2312 cost_to_abs_cost(vtime - vdone, hw_inuse),
2368 /* genuinely short on vtime */
2384 * too much IO and should lower vtime rate. If we're not missing
2386 * and should increase vtime rate.
2456 static u64 adjust_inuse_and_calc_cost(struct ioc_gq *iocg, u64 vtime,
2470 margin = now->vnow - vtime - cost;
2506 } while (time_after64(vtime + cost, now->vnow) &&
2602 u64 abs_cost, cost, vtime;
2610 /* calculate the absolute vtime cost */
2619 vtime = atomic64_read(&iocg->vtime);
2620 cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now);
2628 time_before_eq64(vtime + cost, now.vnow)) {
2660 * the abs_cost instead of advancing vtime. iocg_kick_waitq() will pay
2664 * actual budget available to the cgroup. If we just wound vtime, we
2698 * current vrate. vtime and hweight changes can make it too short
2737 u64 vtime, abs_cost, cost;
2750 vtime = atomic64_read(&iocg->vtime);
2751 cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now);
2759 * Charge if there's enough vtime budget and the existing request has
2763 time_before_eq64(atomic64_read(&iocg->vtime) + cost, now.vnow)) {
2984 atomic64_set(&iocg->vtime, now.vnow);