Lines Matching refs:iocg
195 #define TRACE_IOCG_PATH(type, iocg, ...) \
200 cgroup_path(iocg_to_blkg(iocg)->blkcg->css.cgroup, \
202 trace_iocost_##type(iocg, trace_iocg_path, \
209 #define TRACE_IOCG_PATH(type, iocg, ...) do { } while (0)
220 * iocg->vtime is targeted at 50% behind the device vtime, which
469 * A iocg can get its weight from two sources - an explicit
480 * `last_inuse` remembers `inuse` while an iocg is idle to persist
497 * `vtime` is this iocg's vtime cursor which progresses as IOs are
515 * The period this iocg was last active in. Used for deactivation
551 /* this iocg's depth in the hierarchy and ancestors including self */
577 struct ioc_gq *iocg;
697 static struct blkcg_gq *iocg_to_blkg(struct ioc_gq *iocg)
699 return pd_to_blkg(&iocg->pd);
725 static void iocg_commit_bio(struct ioc_gq *iocg, struct bio *bio,
731 atomic64_add(cost, &iocg->vtime);
733 gcs = get_cpu_ptr(iocg->pcpu_stat);
738 static void iocg_lock(struct ioc_gq *iocg, bool lock_ioc, unsigned long *flags)
741 spin_lock_irqsave(&iocg->ioc->lock, *flags);
742 spin_lock(&iocg->waitq.lock);
744 spin_lock_irqsave(&iocg->waitq.lock, *flags);
748 static void iocg_unlock(struct ioc_gq *iocg, bool unlock_ioc, unsigned long *flags)
751 spin_unlock(&iocg->waitq.lock);
752 spin_unlock_irqrestore(&iocg->ioc->lock, *flags);
754 spin_unlock_irqrestore(&iocg->waitq.lock, *flags);
949 * When an iocg accumulates too much vtime or gets deactivated, we throw away
1023 * Update @iocg's `active` and `inuse` to @active and @inuse, update level
1027 static void __propagate_weights(struct ioc_gq *iocg, u32 active, u32 inuse,
1030 struct ioc *ioc = iocg->ioc;
1040 if (list_empty(&iocg->active_list) && iocg->child_active_sum) {
1041 inuse = DIV64_U64_ROUND_UP(active * iocg->child_inuse_sum,
1042 iocg->child_active_sum);
1047 iocg->last_inuse = iocg->inuse;
1049 iocg->saved_margin = now->vnow - atomic64_read(&iocg->vtime);
1051 if (active == iocg->active && inuse == iocg->inuse)
1054 for (lvl = iocg->level - 1; lvl >= 0; lvl--) {
1055 struct ioc_gq *parent = iocg->ancestors[lvl];
1056 struct ioc_gq *child = iocg->ancestors[lvl + 1];
1102 static void propagate_weights(struct ioc_gq *iocg, u32 active, u32 inuse,
1105 __propagate_weights(iocg, active, inuse, save, now);
1106 commit_weights(iocg->ioc);
1109 static void current_hweight(struct ioc_gq *iocg, u32 *hw_activep, u32 *hw_inusep)
1111 struct ioc *ioc = iocg->ioc;
1118 if (ioc_gen == iocg->hweight_gen)
1134 for (lvl = 0; lvl <= iocg->level - 1; lvl++) {
1135 struct ioc_gq *parent = iocg->ancestors[lvl];
1136 struct ioc_gq *child = iocg->ancestors[lvl + 1];
1153 iocg->hweight_active = max_t(u32, hwa, 1);
1154 iocg->hweight_inuse = max_t(u32, hwi, 1);
1155 iocg->hweight_gen = ioc_gen;
1158 *hw_activep = iocg->hweight_active;
1160 *hw_inusep = iocg->hweight_inuse;
1164 * Calculate the hweight_inuse @iocg would get with max @inuse assuming all the
1167 static u32 current_hweight_max(struct ioc_gq *iocg)
1170 u32 inuse = iocg->active;
1174 lockdep_assert_held(&iocg->ioc->lock);
1176 for (lvl = iocg->level - 1; lvl >= 0; lvl--) {
1177 struct ioc_gq *parent = iocg->ancestors[lvl];
1178 struct ioc_gq *child = iocg->ancestors[lvl + 1];
1189 static void weight_updated(struct ioc_gq *iocg, struct ioc_now *now)
1191 struct ioc *ioc = iocg->ioc;
1192 struct blkcg_gq *blkg = iocg_to_blkg(iocg);
1198 weight = iocg->cfg_weight ?: iocc->dfl_weight;
1199 if (weight != iocg->weight && iocg->active)
1200 propagate_weights(iocg, weight, iocg->inuse, true, now);
1201 iocg->weight = weight;
1204 static bool iocg_activate(struct ioc_gq *iocg, struct ioc_now *now)
1206 struct ioc *ioc = iocg->ioc;
1215 if (!list_empty(&iocg->active_list)) {
1218 if (atomic64_read(&iocg->active_period) != cur_period)
1219 atomic64_set(&iocg->active_period, cur_period);
1224 if (iocg->child_active_sum)
1233 last_period = atomic64_read(&iocg->active_period);
1234 atomic64_set(&iocg->active_period, cur_period);
1237 if (!list_empty(&iocg->active_list))
1239 for (i = iocg->level - 1; i > 0; i--)
1240 if (!list_empty(&iocg->ancestors[i]->active_list))
1243 if (iocg->child_active_sum)
1251 vtime = atomic64_read(&iocg->vtime);
1253 atomic64_add(vtarget - vtime, &iocg->vtime);
1254 atomic64_add(vtarget - vtime, &iocg->done_vtime);
1262 iocg->hweight_gen = atomic_read(&ioc->hweight_gen) - 1;
1263 list_add(&iocg->active_list, &ioc->active_iocgs);
1265 propagate_weights(iocg, iocg->weight,
1266 iocg->last_inuse ?: iocg->weight, true, now);
1268 TRACE_IOCG_PATH(iocg_activate, iocg, now,
1271 iocg->activated_at = now->now;
1289 static bool iocg_kick_delay(struct ioc_gq *iocg, struct ioc_now *now)
1291 struct ioc *ioc = iocg->ioc;
1292 struct blkcg_gq *blkg = iocg_to_blkg(iocg);
1297 lockdep_assert_held(&iocg->waitq.lock);
1303 if (time_before64(now->now, iocg->delay_at))
1307 tdelta = now->now - iocg->delay_at;
1308 if (iocg->delay)
1309 delay = iocg->delay >> div64_u64(tdelta, USEC_PER_SEC);
1314 current_hweight(iocg, &hwa, NULL);
1315 vover = atomic64_read(&iocg->vtime) +
1316 abs_cost_to_cost(iocg->abs_vdebt, hwa) - now->vnow;
1332 iocg->delay = new_delay;
1333 iocg->delay_at = now->now;
1338 if (!iocg->indelay_since)
1339 iocg->indelay_since = now->now;
1343 if (iocg->indelay_since) {
1344 iocg->local_stat.indelay_us += now->now - iocg->indelay_since;
1345 iocg->indelay_since = 0;
1347 iocg->delay = 0;
1353 static void iocg_incur_debt(struct ioc_gq *iocg, u64 abs_cost,
1358 lockdep_assert_held(&iocg->ioc->lock);
1359 lockdep_assert_held(&iocg->waitq.lock);
1360 WARN_ON_ONCE(list_empty(&iocg->active_list));
1363 * Once in debt, debt handling owns inuse. @iocg stays at the minimum
1366 if (!iocg->abs_vdebt && abs_cost) {
1367 iocg->indebt_since = now->now;
1368 propagate_weights(iocg, iocg->active, 0, false, now);
1371 iocg->abs_vdebt += abs_cost;
1373 gcs = get_cpu_ptr(iocg->pcpu_stat);
1378 static void iocg_pay_debt(struct ioc_gq *iocg, u64 abs_vpay,
1381 lockdep_assert_held(&iocg->ioc->lock);
1382 lockdep_assert_held(&iocg->waitq.lock);
1384 /* make sure that nobody messed with @iocg */
1385 WARN_ON_ONCE(list_empty(&iocg->active_list));
1386 WARN_ON_ONCE(iocg->inuse > 1);
1388 iocg->abs_vdebt -= min(abs_vpay, iocg->abs_vdebt);
1391 if (!iocg->abs_vdebt) {
1392 iocg->local_stat.indebt_us += now->now - iocg->indebt_since;
1393 iocg->indebt_since = 0;
1395 propagate_weights(iocg, iocg->active, iocg->last_inuse,
1412 iocg_commit_bio(ctx->iocg, wait->bio, wait->abs_cost, cost);
1430 * addition to iocg->waitq.lock.
1432 static void iocg_kick_waitq(struct ioc_gq *iocg, bool pay_debt,
1435 struct ioc *ioc = iocg->ioc;
1436 struct iocg_wake_ctx ctx = { .iocg = iocg };
1441 lockdep_assert_held(&iocg->waitq.lock);
1443 current_hweight(iocg, &hwa, NULL);
1444 vbudget = now->vnow - atomic64_read(&iocg->vtime);
1447 if (pay_debt && iocg->abs_vdebt && vbudget > 0) {
1449 u64 abs_vpay = min_t(u64, abs_vbudget, iocg->abs_vdebt);
1454 atomic64_add(vpay, &iocg->vtime);
1455 atomic64_add(vpay, &iocg->done_vtime);
1456 iocg_pay_debt(iocg, abs_vpay, now);
1460 if (iocg->abs_vdebt || iocg->delay)
1461 iocg_kick_delay(iocg, now);
1469 if (iocg->abs_vdebt) {
1470 s64 vdebt = abs_cost_to_cost(iocg->abs_vdebt, hwa);
1480 current_hweight(iocg, NULL, &ctx.hw_inuse);
1482 __wake_up_locked_key(&iocg->waitq, TASK_NORMAL, &ctx);
1484 if (!waitqueue_active(&iocg->waitq)) {
1485 if (iocg->wait_since) {
1486 iocg->local_stat.wait_us += now->now - iocg->wait_since;
1487 iocg->wait_since = 0;
1492 if (!iocg->wait_since)
1493 iocg->wait_since = now->now;
1506 oexpires = ktime_to_ns(hrtimer_get_softexpires(&iocg->waitq_timer));
1507 if (hrtimer_is_queued(&iocg->waitq_timer) &&
1511 hrtimer_start_range_ns(&iocg->waitq_timer, ns_to_ktime(expires),
1517 struct ioc_gq *iocg = container_of(timer, struct ioc_gq, waitq_timer);
1518 bool pay_debt = READ_ONCE(iocg->abs_vdebt);
1522 ioc_now(iocg->ioc, &now);
1524 iocg_lock(iocg, pay_debt, &flags);
1525 iocg_kick_waitq(iocg, pay_debt, &now);
1526 iocg_unlock(iocg, pay_debt, &flags);
1570 /* was iocg idle this period? */
1571 static bool iocg_is_idle(struct ioc_gq *iocg)
1573 struct ioc *ioc = iocg->ioc;
1576 if (atomic64_read(&iocg->active_period) ==
1581 if (atomic64_read(&iocg->done_vtime) != atomic64_read(&iocg->vtime))
1588 * Call this function on the target leaf @iocg's to build pre-order traversal
1592 static void iocg_build_inner_walk(struct ioc_gq *iocg,
1597 WARN_ON_ONCE(!list_empty(&iocg->walk_list));
1600 for (lvl = iocg->level - 1; lvl >= 0; lvl--) {
1601 if (!list_empty(&iocg->ancestors[lvl]->walk_list))
1606 while (++lvl <= iocg->level - 1) {
1607 struct ioc_gq *inner = iocg->ancestors[lvl];
1615 static void iocg_flush_stat_one(struct ioc_gq *iocg, struct ioc_now *now)
1617 struct ioc *ioc = iocg->ioc;
1623 lockdep_assert_held(&iocg->ioc->lock);
1628 per_cpu_ptr(&iocg->pcpu_stat->abs_vusage, cpu));
1630 vusage_delta = abs_vusage - iocg->last_stat_abs_vusage;
1631 iocg->last_stat_abs_vusage = abs_vusage;
1633 iocg->usage_delta_us = div64_u64(vusage_delta, ioc->vtime_base_rate);
1634 iocg->local_stat.usage_us += iocg->usage_delta_us;
1638 iocg->local_stat.usage_us + iocg->desc_stat.usage_us;
1640 iocg->local_stat.wait_us + iocg->desc_stat.wait_us;
1642 iocg->local_stat.indebt_us + iocg->desc_stat.indebt_us;
1644 iocg->local_stat.indelay_us + iocg->desc_stat.indelay_us;
1647 if (iocg->level > 0) {
1649 &iocg->ancestors[iocg->level - 1]->desc_stat;
1652 new_stat.usage_us - iocg->last_stat.usage_us;
1654 new_stat.wait_us - iocg->last_stat.wait_us;
1656 new_stat.indebt_us - iocg->last_stat.indebt_us;
1658 new_stat.indelay_us - iocg->last_stat.indelay_us;
1661 iocg->last_stat = new_stat;
1668 struct ioc_gq *iocg, *tiocg;
1671 list_for_each_entry(iocg, target_iocgs, active_list) {
1672 iocg_flush_stat_one(iocg, now);
1673 iocg_build_inner_walk(iocg, &inner_walk);
1677 list_for_each_entry_safe_reverse(iocg, tiocg, &inner_walk, walk_list) {
1678 iocg_flush_stat_one(iocg, now);
1679 list_del_init(&iocg->walk_list);
1684 * Determine what @iocg's hweight_inuse should be after donating unused
1686 * function also throws away @iocg's excess budget.
1688 static u32 hweight_after_donation(struct ioc_gq *iocg, u32 old_hwi, u32 hwm,
1691 struct ioc *ioc = iocg->ioc;
1692 u64 vtime = atomic64_read(&iocg->vtime);
1696 if (iocg->abs_vdebt)
1700 if (waitqueue_active(&iocg->waitq) ||
1707 atomic64_add(excess, &iocg->vtime);
1708 atomic64_add(excess, &iocg->done_vtime);
1714 * Let's say the distance between iocg's and device's vtimes as a
1715 * fraction of period duration is delta. Assuming that the iocg will
1738 * For work-conservation, an iocg which isn't using all of its share should
1740 * bumping up vrate accordingly 2. lowering the donating iocg's inuse weight.
1743 * global hweight_inuse updates when idle iocg's get activated or inuse weights
1747 * #2 is inherently safe with local operations. The donating iocg can easily
1750 * iocg activations safe. The only effect activations have is decreasing
1754 * So, we go with #2. The challenge is calculating how each donating iocg's
1774 * Combining the above two, we can determine how each leaf iocg's inuse should
1798 struct ioc_gq *iocg, *tiocg, *root_iocg;
1810 list_for_each_entry(iocg, surpluses, surplus_list) {
1813 current_hweight(iocg, &hwa, NULL);
1814 after_sum += iocg->hweight_after_donation;
1816 if (iocg->hweight_after_donation > hwa) {
1817 over_sum += iocg->hweight_after_donation;
1818 list_add(&iocg->walk_list, &over_hwa);
1834 list_for_each_entry_safe(iocg, tiocg, &over_hwa, walk_list) {
1836 iocg->hweight_after_donation =
1837 div_u64((u64)iocg->hweight_after_donation *
1839 list_del_init(&iocg->walk_list);
1846 list_for_each_entry(iocg, surpluses, surplus_list) {
1847 iocg_build_inner_walk(iocg, &inner_walk);
1853 list_for_each_entry(iocg, &inner_walk, walk_list) {
1854 iocg->child_adjusted_sum = 0;
1855 iocg->hweight_donating = 0;
1856 iocg->hweight_after_donation = 0;
1863 list_for_each_entry(iocg, surpluses, surplus_list) {
1864 struct ioc_gq *parent = iocg->ancestors[iocg->level - 1];
1866 parent->hweight_donating += iocg->hweight_donating;
1867 parent->hweight_after_donation += iocg->hweight_after_donation;
1870 list_for_each_entry_reverse(iocg, &inner_walk, walk_list) {
1871 if (iocg->level > 0) {
1872 struct ioc_gq *parent = iocg->ancestors[iocg->level - 1];
1874 parent->hweight_donating += iocg->hweight_donating;
1875 parent->hweight_after_donation += iocg->hweight_after_donation;
1884 list_for_each_entry(iocg, &inner_walk, walk_list) {
1885 if (iocg->level) {
1886 struct ioc_gq *parent = iocg->ancestors[iocg->level - 1];
1888 iocg->hweight_active = DIV64_U64_ROUND_UP(
1889 (u64)parent->hweight_active * iocg->active,
1894 iocg->hweight_donating = min(iocg->hweight_donating,
1895 iocg->hweight_active);
1896 iocg->hweight_after_donation = min(iocg->hweight_after_donation,
1897 iocg->hweight_donating - 1);
1898 if (WARN_ON_ONCE(iocg->hweight_active <= 1 ||
1899 iocg->hweight_donating <= 1 ||
1900 iocg->hweight_after_donation == 0)) {
1901 pr_warn("iocg: invalid donation weights in ");
1902 pr_cont_cgroup_path(iocg_to_blkg(iocg)->blkcg->css.cgroup);
1904 iocg->hweight_active, iocg->hweight_donating,
1905 iocg->hweight_after_donation);
1931 list_for_each_entry(iocg, &inner_walk, walk_list) {
1936 if (iocg->level == 0) {
1938 iocg->child_adjusted_sum = DIV64_U64_ROUND_UP(
1939 iocg->child_active_sum * (WEIGHT_ONE - iocg->hweight_donating),
1940 WEIGHT_ONE - iocg->hweight_after_donation);
1944 parent = iocg->ancestors[iocg->level - 1];
1947 iocg->hweight_inuse = DIV64_U64_ROUND_UP(
1948 (u64)gamma * (iocg->hweight_active - iocg->hweight_donating),
1949 WEIGHT_ONE) + iocg->hweight_after_donation;
1953 (u64)parent->child_adjusted_sum * iocg->hweight_inuse,
1958 iocg->child_active_sum * iocg->hweight_donating,
1959 iocg->hweight_active);
1960 sf = iocg->child_active_sum - st;
1962 (u64)iocg->active * iocg->hweight_donating,
1963 iocg->hweight_active);
1965 (u64)inuse * iocg->hweight_after_donation,
1966 iocg->hweight_inuse);
1968 iocg->child_adjusted_sum = sf + DIV64_U64_ROUND_UP(st * wptp, wpt);
1975 list_for_each_entry(iocg, surpluses, surplus_list) {
1976 struct ioc_gq *parent = iocg->ancestors[iocg->level - 1];
1983 * @iocg->inuse stay at the minimum and we don't wanna
1986 if (iocg->abs_vdebt) {
1987 WARN_ON_ONCE(iocg->inuse > 1);
1993 parent->child_adjusted_sum * iocg->hweight_after_donation,
1996 TRACE_IOCG_PATH(inuse_transfer, iocg, now,
1997 iocg->inuse, inuse,
1998 iocg->hweight_inuse,
1999 iocg->hweight_after_donation);
2001 __propagate_weights(iocg, iocg->active, inuse, true, now);
2005 list_for_each_entry_safe(iocg, tiocg, &inner_walk, walk_list)
2006 list_del_init(&iocg->walk_list);
2010 * A low weight iocg can amass a large amount of debt, for example, when
2013 * more. If there are no other subsequent IO issuers, the in-debt iocg may end
2023 struct ioc_gq *iocg;
2075 list_for_each_entry(iocg, &ioc->active_iocgs, active_list) {
2078 if (!iocg->abs_vdebt && !iocg->delay)
2081 spin_lock(&iocg->waitq.lock);
2083 old_debt = iocg->abs_vdebt;
2084 old_delay = iocg->delay;
2086 if (iocg->abs_vdebt)
2087 iocg->abs_vdebt = iocg->abs_vdebt >> nr_cycles ?: 1;
2088 if (iocg->delay)
2089 iocg->delay = iocg->delay >> nr_cycles ?: 1;
2091 iocg_kick_waitq(iocg, true, now);
2093 TRACE_IOCG_PATH(iocg_forgive_debt, iocg, now, usage_pct,
2094 old_debt, iocg->abs_vdebt,
2095 old_delay, iocg->delay);
2097 spin_unlock(&iocg->waitq.lock);
2104 struct ioc_gq *iocg, *tiocg;
2135 list_for_each_entry_safe(iocg, tiocg, &ioc->active_iocgs, active_list) {
2136 if (!waitqueue_active(&iocg->waitq) && !iocg->abs_vdebt &&
2137 !iocg->delay && !iocg_is_idle(iocg))
2140 spin_lock(&iocg->waitq.lock);
2143 if (iocg->wait_since) {
2144 iocg->local_stat.wait_us += now.now - iocg->wait_since;
2145 iocg->wait_since = now.now;
2147 if (iocg->indebt_since) {
2148 iocg->local_stat.indebt_us +=
2149 now.now - iocg->indebt_since;
2150 iocg->indebt_since = now.now;
2152 if (iocg->indelay_since) {
2153 iocg->local_stat.indelay_us +=
2154 now.now - iocg->indelay_since;
2155 iocg->indelay_since = now.now;
2158 if (waitqueue_active(&iocg->waitq) || iocg->abs_vdebt ||
2159 iocg->delay) {
2161 iocg_kick_waitq(iocg, true, &now);
2162 if (iocg->abs_vdebt || iocg->delay)
2164 } else if (iocg_is_idle(iocg)) {
2166 u64 vtime = atomic64_read(&iocg->vtime);
2170 * @iocg has been inactive for a full duration and will
2179 current_hweight(iocg, NULL, &old_hwi);
2184 __propagate_weights(iocg, 0, 0, false, &now);
2185 list_del_init(&iocg->active_list);
2188 spin_unlock(&iocg->waitq.lock);
2199 list_for_each_entry(iocg, &ioc->active_iocgs, active_list) {
2207 vdone = atomic64_read(&iocg->done_vtime);
2208 vtime = atomic64_read(&iocg->vtime);
2209 current_hweight(iocg, &hw_active, &hw_inuse);
2218 !atomic_read(&iocg_to_blkg(iocg)->use_delay) &&
2229 usage_us = iocg->usage_delta_us;
2240 if (time_after64(iocg->activated_at, ioc->period_at))
2241 usage_dur = max_t(u64, now.now - iocg->activated_at, 1);
2251 WARN_ON_ONCE(!list_empty(&iocg->surplus_list));
2253 (!waitqueue_active(&iocg->waitq) &&
2261 current_hweight(iocg, &hwa, &old_hwi);
2262 hwm = current_hweight_max(iocg);
2263 new_hwi = hweight_after_donation(iocg, old_hwi, hwm,
2273 iocg->hweight_donating = hwa;
2274 iocg->hweight_after_donation = new_hwi;
2275 list_add(&iocg->surplus_list, &surpluses);
2276 } else if (!iocg->abs_vdebt) {
2278 * @iocg doesn't have enough to donate. Reset
2284 * as @iocg doesn't have a meaningful amount of
2287 TRACE_IOCG_PATH(inuse_shortage, iocg, &now,
2288 iocg->inuse, iocg->active,
2289 iocg->hweight_inuse, new_hwi);
2291 __propagate_weights(iocg, iocg->active,
2292 iocg->active, true, &now);
2307 list_for_each_entry_safe(iocg, tiocg, &surpluses, surplus_list)
2308 list_del_init(&iocg->surplus_list);
2427 static u64 adjust_inuse_and_calc_cost(struct ioc_gq *iocg, u64 vtime,
2430 struct ioc *ioc = iocg->ioc;
2432 u32 __maybe_unused old_inuse = iocg->inuse, __maybe_unused old_hwi;
2438 current_hweight(iocg, NULL, &hwi);
2444 if (iocg->abs_vdebt)
2451 if (margin >= iocg->saved_margin || margin >= margins->low ||
2452 iocg->inuse == iocg->active)
2457 /* we own inuse only when @iocg is in the normal active state */
2458 if (iocg->abs_vdebt || list_empty(&iocg->active_list)) {
2467 * be reading 0 iocg->active before ioc->lock which will lead to
2470 new_inuse = iocg->inuse;
2471 adj_step = DIV_ROUND_UP(iocg->active * INUSE_ADJ_STEP_PCT, 100);
2474 propagate_weights(iocg, iocg->active, new_inuse, true, now);
2475 current_hweight(iocg, NULL, &hwi);
2478 iocg->inuse != iocg->active);
2482 TRACE_IOCG_PATH(inuse_adjust, iocg, now,
2483 old_inuse, iocg->inuse, old_hwi, hwi);
2488 static void calc_vtime_cost_builtin(struct bio *bio, struct ioc_gq *iocg,
2491 struct ioc *ioc = iocg->ioc;
2512 if (iocg->cursor) {
2513 seek_pages = abs(bio->bi_iter.bi_sector - iocg->cursor);
2529 static u64 calc_vtime_cost(struct bio *bio, struct ioc_gq *iocg, bool is_merge)
2533 calc_vtime_cost_builtin(bio, iocg, is_merge, &cost);
2566 struct ioc_gq *iocg = blkg_to_iocg(blkg);
2574 if (!ioc->enabled || !iocg || !iocg->level)
2578 abs_cost = calc_vtime_cost(bio, iocg, false);
2582 if (!iocg_activate(iocg, &now))
2585 iocg->cursor = bio_end_sector(bio);
2586 vtime = atomic64_read(&iocg->vtime);
2587 cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now);
2594 if (!waitqueue_active(&iocg->waitq) && !iocg->abs_vdebt &&
2596 iocg_commit_bio(iocg, bio, abs_cost, cost);
2603 * debt. Otherwise, the issuer is blocked on @iocg->waitq. Debt handling
2604 * requires @ioc->lock, waitq handling @iocg->waitq.lock. Determine
2608 ioc_locked = use_debt || READ_ONCE(iocg->abs_vdebt);
2610 iocg_lock(iocg, ioc_locked, &flags);
2613 * @iocg must stay activated for debt and waitq handling. Deactivation
2619 if (unlikely(list_empty(&iocg->active_list))) {
2620 iocg_unlock(iocg, ioc_locked, &flags);
2621 iocg_commit_bio(iocg, bio, abs_cost, cost);
2636 * An iocg with vdebt should stay online so that the timer can keep
2639 * clear them and leave @iocg inactive w/ dangling use_delay heavily
2643 iocg_incur_debt(iocg, abs_cost, &now);
2644 if (iocg_kick_delay(iocg, &now))
2647 iocg_unlock(iocg, ioc_locked, &flags);
2652 if (!iocg->abs_vdebt && iocg->inuse != iocg->active) {
2654 iocg_unlock(iocg, false, &flags);
2658 propagate_weights(iocg, iocg->active, iocg->active, true,
2672 * All waiters are on iocg->waitq and the wait states are
2681 __add_wait_queue_entry_tail(&iocg->waitq, &wait.wait);
2682 iocg_kick_waitq(iocg, ioc_locked, &now);
2684 iocg_unlock(iocg, ioc_locked, &flags);
2694 finish_wait(&iocg->waitq, &wait.wait);
2700 struct ioc_gq *iocg = blkg_to_iocg(bio->bi_blkg);
2708 if (!ioc->enabled || !iocg || !iocg->level)
2711 abs_cost = calc_vtime_cost(bio, iocg, true);
2717 vtime = atomic64_read(&iocg->vtime);
2718 cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now);
2722 blk_rq_pos(rq) + blk_rq_sectors(rq) == iocg->cursor)
2723 iocg->cursor = bio_end;
2730 time_before_eq64(atomic64_read(&iocg->vtime) + cost, now.vnow)) {
2731 iocg_commit_bio(iocg, bio, abs_cost, cost);
2736 * Otherwise, account it as debt if @iocg is online, which it should
2741 spin_lock(&iocg->waitq.lock);
2743 if (likely(!list_empty(&iocg->active_list))) {
2744 iocg_incur_debt(iocg, abs_cost, &now);
2745 if (iocg_kick_delay(iocg, &now))
2749 iocg_commit_bio(iocg, bio, abs_cost, cost);
2752 spin_unlock(&iocg->waitq.lock);
2758 struct ioc_gq *iocg = blkg_to_iocg(bio->bi_blkg);
2760 if (iocg && bio->bi_iocost_cost)
2761 atomic64_add(bio->bi_iocost_cost, &iocg->done_vtime);
2889 * target bio has an iocg associated and need to test for NULL iocg.
2923 struct ioc_gq *iocg;
2925 iocg = kzalloc_node(struct_size(iocg, ancestors, levels), gfp, q->node);
2926 if (!iocg)
2929 iocg->pcpu_stat = alloc_percpu_gfp(struct iocg_pcpu_stat, gfp);
2930 if (!iocg->pcpu_stat) {
2931 kfree(iocg);
2935 return &iocg->pd;
2940 struct ioc_gq *iocg = pd_to_iocg(pd);
2941 struct blkcg_gq *blkg = pd_to_blkg(&iocg->pd);
2949 iocg->ioc = ioc;
2950 atomic64_set(&iocg->vtime, now.vnow);
2951 atomic64_set(&iocg->done_vtime, now.vnow);
2952 atomic64_set(&iocg->active_period, atomic64_read(&ioc->cur_period));
2953 INIT_LIST_HEAD(&iocg->active_list);
2954 INIT_LIST_HEAD(&iocg->walk_list);
2955 INIT_LIST_HEAD(&iocg->surplus_list);
2956 iocg->hweight_active = WEIGHT_ONE;
2957 iocg->hweight_inuse = WEIGHT_ONE;
2959 init_waitqueue_head(&iocg->waitq);
2960 hrtimer_init(&iocg->waitq_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
2961 iocg->waitq_timer.function = iocg_waitq_timer_fn;
2963 iocg->level = blkg->blkcg->css.cgroup->level;
2967 iocg->ancestors[tiocg->level] = tiocg;
2971 weight_updated(iocg, &now);
2977 struct ioc_gq *iocg = pd_to_iocg(pd);
2978 struct ioc *ioc = iocg->ioc;
2984 if (!list_empty(&iocg->active_list)) {
2988 propagate_weights(iocg, 0, 0, false, &now);
2989 list_del_init(&iocg->active_list);
2992 WARN_ON_ONCE(!list_empty(&iocg->walk_list));
2993 WARN_ON_ONCE(!list_empty(&iocg->surplus_list));
2997 hrtimer_cancel(&iocg->waitq_timer);
2999 free_percpu(iocg->pcpu_stat);
3000 kfree(iocg);
3005 struct ioc_gq *iocg = pd_to_iocg(pd);
3006 struct ioc *ioc = iocg->ioc;
3012 if (iocg->level == 0) {
3021 iocg->last_stat.usage_us);
3026 iocg->last_stat.wait_us,
3027 iocg->last_stat.indebt_us,
3028 iocg->last_stat.indelay_us);
3037 struct ioc_gq *iocg = pd_to_iocg(pd);
3039 if (dname && iocg->cfg_weight)
3040 seq_printf(sf, "%s %u\n", dname, iocg->cfg_weight / WEIGHT_ONE);
3063 struct ioc_gq *iocg;
3079 struct ioc_gq *iocg = blkg_to_iocg(blkg);
3081 if (iocg) {
3082 spin_lock(&iocg->ioc->lock);
3083 ioc_now(iocg->ioc, &now);
3084 weight_updated(iocg, &now);
3085 spin_unlock(&iocg->ioc->lock);
3097 iocg = blkg_to_iocg(ctx.blkg);
3108 spin_lock(&iocg->ioc->lock);
3109 iocg->cfg_weight = v * WEIGHT_ONE;
3110 ioc_now(iocg->ioc, &now);
3111 weight_updated(iocg, &now);
3112 spin_unlock(&iocg->ioc->lock);