Lines Matching refs:iocg
195 #define TRACE_IOCG_PATH(type, iocg, ...) \
200 cgroup_path(iocg_to_blkg(iocg)->blkcg->css.cgroup, \
202 trace_iocost_##type(iocg, trace_iocg_path, \
209 #define TRACE_IOCG_PATH(type, iocg, ...) do { } while (0)
220 * iocg->vtime is targeted at 50% behind the device vtime, which
467 * A iocg can get its weight from two sources - an explicit
478 * `last_inuse` remembers `inuse` while an iocg is idle to persist
495 * `vtime` is this iocg's vtime cursor which progresses as IOs are
513 * The period this iocg was last active in. Used for deactivation
548 /* this iocg's depth in the hierarchy and ancestors including self */
573 struct ioc_gq *iocg;
689 static struct blkcg_gq *iocg_to_blkg(struct ioc_gq *iocg)
691 return pd_to_blkg(&iocg->pd);
717 static void iocg_commit_bio(struct ioc_gq *iocg, struct bio *bio,
723 atomic64_add(cost, &iocg->vtime);
725 gcs = get_cpu_ptr(iocg->pcpu_stat);
730 static void iocg_lock(struct ioc_gq *iocg, bool lock_ioc, unsigned long *flags)
733 spin_lock_irqsave(&iocg->ioc->lock, *flags);
734 spin_lock(&iocg->waitq.lock);
736 spin_lock_irqsave(&iocg->waitq.lock, *flags);
740 static void iocg_unlock(struct ioc_gq *iocg, bool unlock_ioc, unsigned long *flags)
743 spin_unlock(&iocg->waitq.lock);
744 spin_unlock_irqrestore(&iocg->ioc->lock, *flags);
746 spin_unlock_irqrestore(&iocg->waitq.lock, *flags);
957 * When an iocg accumulates too much vtime or gets deactivated, we throw away
1080 * Update @iocg's `active` and `inuse` to @active and @inuse, update level
1084 static void __propagate_weights(struct ioc_gq *iocg, u32 active, u32 inuse,
1087 struct ioc *ioc = iocg->ioc;
1097 if (list_empty(&iocg->active_list) && iocg->child_active_sum) {
1098 inuse = DIV64_U64_ROUND_UP(active * iocg->child_inuse_sum,
1099 iocg->child_active_sum);
1104 iocg->last_inuse = iocg->inuse;
1106 iocg->saved_margin = now->vnow - atomic64_read(&iocg->vtime);
1108 if (active == iocg->active && inuse == iocg->inuse)
1111 for (lvl = iocg->level - 1; lvl >= 0; lvl--) {
1112 struct ioc_gq *parent = iocg->ancestors[lvl];
1113 struct ioc_gq *child = iocg->ancestors[lvl + 1];
1159 static void propagate_weights(struct ioc_gq *iocg, u32 active, u32 inuse,
1162 __propagate_weights(iocg, active, inuse, save, now);
1163 commit_weights(iocg->ioc);
1166 static void current_hweight(struct ioc_gq *iocg, u32 *hw_activep, u32 *hw_inusep)
1168 struct ioc *ioc = iocg->ioc;
1175 if (ioc_gen == iocg->hweight_gen)
1191 for (lvl = 0; lvl <= iocg->level - 1; lvl++) {
1192 struct ioc_gq *parent = iocg->ancestors[lvl];
1193 struct ioc_gq *child = iocg->ancestors[lvl + 1];
1210 iocg->hweight_active = max_t(u32, hwa, 1);
1211 iocg->hweight_inuse = max_t(u32, hwi, 1);
1212 iocg->hweight_gen = ioc_gen;
1215 *hw_activep = iocg->hweight_active;
1217 *hw_inusep = iocg->hweight_inuse;
1221 * Calculate the hweight_inuse @iocg would get with max @inuse assuming all the
1224 static u32 current_hweight_max(struct ioc_gq *iocg)
1227 u32 inuse = iocg->active;
1231 lockdep_assert_held(&iocg->ioc->lock);
1233 for (lvl = iocg->level - 1; lvl >= 0; lvl--) {
1234 struct ioc_gq *parent = iocg->ancestors[lvl];
1235 struct ioc_gq *child = iocg->ancestors[lvl + 1];
1246 static void weight_updated(struct ioc_gq *iocg, struct ioc_now *now)
1248 struct ioc *ioc = iocg->ioc;
1249 struct blkcg_gq *blkg = iocg_to_blkg(iocg);
1255 weight = iocg->cfg_weight ?: iocc->dfl_weight;
1256 if (weight != iocg->weight && iocg->active)
1257 propagate_weights(iocg, weight, iocg->inuse, true, now);
1258 iocg->weight = weight;
1261 static bool iocg_activate(struct ioc_gq *iocg, struct ioc_now *now)
1263 struct ioc *ioc = iocg->ioc;
1272 if (!list_empty(&iocg->active_list)) {
1275 if (atomic64_read(&iocg->active_period) != cur_period)
1276 atomic64_set(&iocg->active_period, cur_period);
1281 if (iocg->child_active_sum)
1290 last_period = atomic64_read(&iocg->active_period);
1291 atomic64_set(&iocg->active_period, cur_period);
1294 if (!list_empty(&iocg->active_list))
1296 for (i = iocg->level - 1; i > 0; i--)
1297 if (!list_empty(&iocg->ancestors[i]->active_list))
1300 if (iocg->child_active_sum)
1308 vtime = atomic64_read(&iocg->vtime);
1310 atomic64_add(vtarget - vtime, &iocg->vtime);
1311 atomic64_add(vtarget - vtime, &iocg->done_vtime);
1319 iocg->hweight_gen = atomic_read(&ioc->hweight_gen) - 1;
1320 list_add(&iocg->active_list, &ioc->active_iocgs);
1322 propagate_weights(iocg, iocg->weight,
1323 iocg->last_inuse ?: iocg->weight, true, now);
1325 TRACE_IOCG_PATH(iocg_activate, iocg, now,
1328 iocg->activated_at = now->now;
1346 static bool iocg_kick_delay(struct ioc_gq *iocg, struct ioc_now *now)
1348 struct ioc *ioc = iocg->ioc;
1349 struct blkcg_gq *blkg = iocg_to_blkg(iocg);
1354 lockdep_assert_held(&iocg->waitq.lock);
1360 if (time_before64(now->now, iocg->delay_at))
1364 tdelta = now->now - iocg->delay_at;
1365 if (iocg->delay)
1366 delay = iocg->delay >> div64_u64(tdelta, USEC_PER_SEC);
1371 current_hweight(iocg, &hwa, NULL);
1372 vover = atomic64_read(&iocg->vtime) +
1373 abs_cost_to_cost(iocg->abs_vdebt, hwa) - now->vnow;
1389 iocg->delay = new_delay;
1390 iocg->delay_at = now->now;
1395 if (!iocg->indelay_since)
1396 iocg->indelay_since = now->now;
1400 if (iocg->indelay_since) {
1401 iocg->stat.indelay_us += now->now - iocg->indelay_since;
1402 iocg->indelay_since = 0;
1404 iocg->delay = 0;
1410 static void iocg_incur_debt(struct ioc_gq *iocg, u64 abs_cost,
1415 lockdep_assert_held(&iocg->ioc->lock);
1416 lockdep_assert_held(&iocg->waitq.lock);
1417 WARN_ON_ONCE(list_empty(&iocg->active_list));
1420 * Once in debt, debt handling owns inuse. @iocg stays at the minimum
1423 if (!iocg->abs_vdebt && abs_cost) {
1424 iocg->indebt_since = now->now;
1425 propagate_weights(iocg, iocg->active, 0, false, now);
1428 iocg->abs_vdebt += abs_cost;
1430 gcs = get_cpu_ptr(iocg->pcpu_stat);
1435 static void iocg_pay_debt(struct ioc_gq *iocg, u64 abs_vpay,
1438 lockdep_assert_held(&iocg->ioc->lock);
1439 lockdep_assert_held(&iocg->waitq.lock);
1441 /* make sure that nobody messed with @iocg */
1442 WARN_ON_ONCE(list_empty(&iocg->active_list));
1443 WARN_ON_ONCE(iocg->inuse > 1);
1445 iocg->abs_vdebt -= min(abs_vpay, iocg->abs_vdebt);
1448 if (!iocg->abs_vdebt) {
1449 iocg->stat.indebt_us += now->now - iocg->indebt_since;
1450 iocg->indebt_since = 0;
1452 propagate_weights(iocg, iocg->active, iocg->last_inuse,
1469 iocg_commit_bio(ctx->iocg, wait->bio, wait->abs_cost, cost);
1487 * addition to iocg->waitq.lock.
1489 static void iocg_kick_waitq(struct ioc_gq *iocg, bool pay_debt,
1492 struct ioc *ioc = iocg->ioc;
1493 struct iocg_wake_ctx ctx = { .iocg = iocg };
1498 lockdep_assert_held(&iocg->waitq.lock);
1500 current_hweight(iocg, &hwa, NULL);
1501 vbudget = now->vnow - atomic64_read(&iocg->vtime);
1504 if (pay_debt && iocg->abs_vdebt && vbudget > 0) {
1506 u64 abs_vpay = min_t(u64, abs_vbudget, iocg->abs_vdebt);
1511 atomic64_add(vpay, &iocg->vtime);
1512 atomic64_add(vpay, &iocg->done_vtime);
1513 iocg_pay_debt(iocg, abs_vpay, now);
1517 if (iocg->abs_vdebt || iocg->delay)
1518 iocg_kick_delay(iocg, now);
1526 if (iocg->abs_vdebt) {
1527 s64 vdebt = abs_cost_to_cost(iocg->abs_vdebt, hwa);
1537 current_hweight(iocg, NULL, &ctx.hw_inuse);
1539 __wake_up_locked_key(&iocg->waitq, TASK_NORMAL, &ctx);
1541 if (!waitqueue_active(&iocg->waitq)) {
1542 if (iocg->wait_since) {
1543 iocg->stat.wait_us += now->now - iocg->wait_since;
1544 iocg->wait_since = 0;
1549 if (!iocg->wait_since)
1550 iocg->wait_since = now->now;
1563 oexpires = ktime_to_ns(hrtimer_get_softexpires(&iocg->waitq_timer));
1564 if (hrtimer_is_queued(&iocg->waitq_timer) &&
1568 hrtimer_start_range_ns(&iocg->waitq_timer, ns_to_ktime(expires),
1574 struct ioc_gq *iocg = container_of(timer, struct ioc_gq, waitq_timer);
1575 bool pay_debt = READ_ONCE(iocg->abs_vdebt);
1579 ioc_now(iocg->ioc, &now);
1581 iocg_lock(iocg, pay_debt, &flags);
1582 iocg_kick_waitq(iocg, pay_debt, &now);
1583 iocg_unlock(iocg, pay_debt, &flags);
1627 /* was iocg idle this period? */
1628 static bool iocg_is_idle(struct ioc_gq *iocg)
1630 struct ioc *ioc = iocg->ioc;
1633 if (atomic64_read(&iocg->active_period) ==
1638 if (atomic64_read(&iocg->done_vtime) != atomic64_read(&iocg->vtime))
1645 * Call this function on the target leaf @iocg's to build pre-order traversal
1649 static void iocg_build_inner_walk(struct ioc_gq *iocg,
1654 WARN_ON_ONCE(!list_empty(&iocg->walk_list));
1657 for (lvl = iocg->level - 1; lvl >= 0; lvl--) {
1658 if (!list_empty(&iocg->ancestors[lvl]->walk_list))
1663 while (++lvl <= iocg->level - 1) {
1664 struct ioc_gq *inner = iocg->ancestors[lvl];
1672 static void iocg_flush_stat_upward(struct ioc_gq *iocg)
1674 if (iocg->level > 0) {
1676 &iocg->ancestors[iocg->level - 1]->stat;
1679 iocg->stat.usage_us - iocg->last_stat.usage_us;
1681 iocg->stat.wait_us - iocg->last_stat.wait_us;
1683 iocg->stat.indebt_us - iocg->last_stat.indebt_us;
1685 iocg->stat.indelay_us - iocg->last_stat.indelay_us;
1688 iocg->last_stat = iocg->stat;
1692 static void iocg_flush_stat_leaf(struct ioc_gq *iocg, struct ioc_now *now)
1694 struct ioc *ioc = iocg->ioc;
1699 lockdep_assert_held(&iocg->ioc->lock);
1704 per_cpu_ptr(&iocg->pcpu_stat->abs_vusage, cpu));
1706 vusage_delta = abs_vusage - iocg->last_stat_abs_vusage;
1707 iocg->last_stat_abs_vusage = abs_vusage;
1709 iocg->usage_delta_us = div64_u64(vusage_delta, ioc->vtime_base_rate);
1710 iocg->stat.usage_us += iocg->usage_delta_us;
1712 iocg_flush_stat_upward(iocg);
1719 struct ioc_gq *iocg, *tiocg;
1722 list_for_each_entry(iocg, target_iocgs, active_list) {
1723 iocg_flush_stat_leaf(iocg, now);
1724 iocg_build_inner_walk(iocg, &inner_walk);
1728 list_for_each_entry_safe_reverse(iocg, tiocg, &inner_walk, walk_list) {
1729 iocg_flush_stat_upward(iocg);
1730 list_del_init(&iocg->walk_list);
1735 * Determine what @iocg's hweight_inuse should be after donating unused
1737 * function also throws away @iocg's excess budget.
1739 static u32 hweight_after_donation(struct ioc_gq *iocg, u32 old_hwi, u32 hwm,
1742 struct ioc *ioc = iocg->ioc;
1743 u64 vtime = atomic64_read(&iocg->vtime);
1747 if (iocg->abs_vdebt)
1751 if (waitqueue_active(&iocg->waitq) ||
1758 atomic64_add(excess, &iocg->vtime);
1759 atomic64_add(excess, &iocg->done_vtime);
1765 * Let's say the distance between iocg's and device's vtimes as a
1766 * fraction of period duration is delta. Assuming that the iocg will
1789 * For work-conservation, an iocg which isn't using all of its share should
1791 * bumping up vrate accordingly 2. lowering the donating iocg's inuse weight.
1794 * global hweight_inuse updates when idle iocg's get activated or inuse weights
1798 * #2 is inherently safe with local operations. The donating iocg can easily
1801 * iocg activations safe. The only effect activations have is decreasing
1805 * So, we go with #2. The challenge is calculating how each donating iocg's
1825 * Combining the above two, we can determine how each leaf iocg's inuse should
1849 struct ioc_gq *iocg, *tiocg, *root_iocg;
1861 list_for_each_entry(iocg, surpluses, surplus_list) {
1864 current_hweight(iocg, &hwa, NULL);
1865 after_sum += iocg->hweight_after_donation;
1867 if (iocg->hweight_after_donation > hwa) {
1868 over_sum += iocg->hweight_after_donation;
1869 list_add(&iocg->walk_list, &over_hwa);
1885 list_for_each_entry_safe(iocg, tiocg, &over_hwa, walk_list) {
1887 iocg->hweight_after_donation =
1888 div_u64((u64)iocg->hweight_after_donation *
1890 list_del_init(&iocg->walk_list);
1897 list_for_each_entry(iocg, surpluses, surplus_list) {
1898 iocg_build_inner_walk(iocg, &inner_walk);
1904 list_for_each_entry(iocg, &inner_walk, walk_list) {
1905 iocg->child_adjusted_sum = 0;
1906 iocg->hweight_donating = 0;
1907 iocg->hweight_after_donation = 0;
1914 list_for_each_entry(iocg, surpluses, surplus_list) {
1915 struct ioc_gq *parent = iocg->ancestors[iocg->level - 1];
1917 parent->hweight_donating += iocg->hweight_donating;
1918 parent->hweight_after_donation += iocg->hweight_after_donation;
1921 list_for_each_entry_reverse(iocg, &inner_walk, walk_list) {
1922 if (iocg->level > 0) {
1923 struct ioc_gq *parent = iocg->ancestors[iocg->level - 1];
1925 parent->hweight_donating += iocg->hweight_donating;
1926 parent->hweight_after_donation += iocg->hweight_after_donation;
1935 list_for_each_entry(iocg, &inner_walk, walk_list) {
1936 if (iocg->level) {
1937 struct ioc_gq *parent = iocg->ancestors[iocg->level - 1];
1939 iocg->hweight_active = DIV64_U64_ROUND_UP(
1940 (u64)parent->hweight_active * iocg->active,
1945 iocg->hweight_donating = min(iocg->hweight_donating,
1946 iocg->hweight_active);
1947 iocg->hweight_after_donation = min(iocg->hweight_after_donation,
1948 iocg->hweight_donating - 1);
1949 if (WARN_ON_ONCE(iocg->hweight_active <= 1 ||
1950 iocg->hweight_donating <= 1 ||
1951 iocg->hweight_after_donation == 0)) {
1952 pr_warn("iocg: invalid donation weights in ");
1953 pr_cont_cgroup_path(iocg_to_blkg(iocg)->blkcg->css.cgroup);
1955 iocg->hweight_active, iocg->hweight_donating,
1956 iocg->hweight_after_donation);
1982 list_for_each_entry(iocg, &inner_walk, walk_list) {
1987 if (iocg->level == 0) {
1989 iocg->child_adjusted_sum = DIV64_U64_ROUND_UP(
1990 iocg->child_active_sum * (WEIGHT_ONE - iocg->hweight_donating),
1991 WEIGHT_ONE - iocg->hweight_after_donation);
1995 parent = iocg->ancestors[iocg->level - 1];
1998 iocg->hweight_inuse = DIV64_U64_ROUND_UP(
1999 (u64)gamma * (iocg->hweight_active - iocg->hweight_donating),
2000 WEIGHT_ONE) + iocg->hweight_after_donation;
2004 (u64)parent->child_adjusted_sum * iocg->hweight_inuse,
2009 iocg->child_active_sum * iocg->hweight_donating,
2010 iocg->hweight_active);
2011 sf = iocg->child_active_sum - st;
2013 (u64)iocg->active * iocg->hweight_donating,
2014 iocg->hweight_active);
2016 (u64)inuse * iocg->hweight_after_donation,
2017 iocg->hweight_inuse);
2019 iocg->child_adjusted_sum = sf + DIV64_U64_ROUND_UP(st * wptp, wpt);
2026 list_for_each_entry(iocg, surpluses, surplus_list) {
2027 struct ioc_gq *parent = iocg->ancestors[iocg->level - 1];
2034 * @iocg->inuse stay at the minimum and we don't wanna
2037 if (iocg->abs_vdebt) {
2038 WARN_ON_ONCE(iocg->inuse > 1);
2044 parent->child_adjusted_sum * iocg->hweight_after_donation,
2047 TRACE_IOCG_PATH(inuse_transfer, iocg, now,
2048 iocg->inuse, inuse,
2049 iocg->hweight_inuse,
2050 iocg->hweight_after_donation);
2052 __propagate_weights(iocg, iocg->active, inuse, true, now);
2056 list_for_each_entry_safe(iocg, tiocg, &inner_walk, walk_list)
2057 list_del_init(&iocg->walk_list);
2061 * A low weight iocg can amass a large amount of debt, for example, when
2064 * more. If there are no other subsequent IO issuers, the in-debt iocg may end
2074 struct ioc_gq *iocg;
2126 list_for_each_entry(iocg, &ioc->active_iocgs, active_list) {
2129 if (!iocg->abs_vdebt && !iocg->delay)
2132 spin_lock(&iocg->waitq.lock);
2134 old_debt = iocg->abs_vdebt;
2135 old_delay = iocg->delay;
2137 if (iocg->abs_vdebt)
2138 iocg->abs_vdebt = iocg->abs_vdebt >> nr_cycles ?: 1;
2139 if (iocg->delay)
2140 iocg->delay = iocg->delay >> nr_cycles ?: 1;
2142 iocg_kick_waitq(iocg, true, now);
2144 TRACE_IOCG_PATH(iocg_forgive_debt, iocg, now, usage_pct,
2145 old_debt, iocg->abs_vdebt,
2146 old_delay, iocg->delay);
2148 spin_unlock(&iocg->waitq.lock);
2165 struct ioc_gq *iocg, *tiocg;
2167 list_for_each_entry_safe(iocg, tiocg, &ioc->active_iocgs, active_list) {
2168 if (!waitqueue_active(&iocg->waitq) && !iocg->abs_vdebt &&
2169 !iocg->delay && !iocg_is_idle(iocg))
2172 spin_lock(&iocg->waitq.lock);
2175 if (iocg->wait_since) {
2176 iocg->stat.wait_us += now->now - iocg->wait_since;
2177 iocg->wait_since = now->now;
2179 if (iocg->indebt_since) {
2180 iocg->stat.indebt_us +=
2181 now->now - iocg->indebt_since;
2182 iocg->indebt_since = now->now;
2184 if (iocg->indelay_since) {
2185 iocg->stat.indelay_us +=
2186 now->now - iocg->indelay_since;
2187 iocg->indelay_since = now->now;
2190 if (waitqueue_active(&iocg->waitq) || iocg->abs_vdebt ||
2191 iocg->delay) {
2193 iocg_kick_waitq(iocg, true, now);
2194 if (iocg->abs_vdebt || iocg->delay)
2196 } else if (iocg_is_idle(iocg)) {
2198 u64 vtime = atomic64_read(&iocg->vtime);
2202 * @iocg has been inactive for a full duration and will
2211 current_hweight(iocg, NULL, &old_hwi);
2216 TRACE_IOCG_PATH(iocg_idle, iocg, now,
2217 atomic64_read(&iocg->active_period),
2219 __propagate_weights(iocg, 0, 0, false, now);
2220 list_del_init(&iocg->active_list);
2223 spin_unlock(&iocg->waitq.lock);
2233 struct ioc_gq *iocg, *tiocg;
2269 list_for_each_entry(iocg, &ioc->active_iocgs, active_list) {
2277 vdone = atomic64_read(&iocg->done_vtime);
2278 vtime = atomic64_read(&iocg->vtime);
2279 current_hweight(iocg, &hw_active, &hw_inuse);
2288 !atomic_read(&iocg_to_blkg(iocg)->use_delay) &&
2299 usage_us = iocg->usage_delta_us;
2303 WARN_ON_ONCE(!list_empty(&iocg->surplus_list));
2305 (!waitqueue_active(&iocg->waitq) &&
2319 if (time_after64(iocg->activated_at, ioc->period_at))
2320 usage_dur = max_t(u64, now.now - iocg->activated_at, 1);
2333 current_hweight(iocg, &hwa, &old_hwi);
2334 hwm = current_hweight_max(iocg);
2335 new_hwi = hweight_after_donation(iocg, old_hwi, hwm,
2345 iocg->hweight_donating = hwa;
2346 iocg->hweight_after_donation = new_hwi;
2347 list_add(&iocg->surplus_list, &surpluses);
2348 } else if (!iocg->abs_vdebt) {
2350 * @iocg doesn't have enough to donate. Reset
2356 * as @iocg doesn't have a meaningful amount of
2359 TRACE_IOCG_PATH(inuse_shortage, iocg, &now,
2360 iocg->inuse, iocg->active,
2361 iocg->hweight_inuse, new_hwi);
2363 __propagate_weights(iocg, iocg->active,
2364 iocg->active, true, &now);
2379 list_for_each_entry_safe(iocg, tiocg, &surpluses, surplus_list)
2380 list_del_init(&iocg->surplus_list);
2456 static u64 adjust_inuse_and_calc_cost(struct ioc_gq *iocg, u64 vtime,
2459 struct ioc *ioc = iocg->ioc;
2461 u32 __maybe_unused old_inuse = iocg->inuse, __maybe_unused old_hwi;
2467 current_hweight(iocg, NULL, &hwi);
2473 if (iocg->abs_vdebt)
2480 if (margin >= iocg->saved_margin || margin >= margins->low ||
2481 iocg->inuse == iocg->active)
2486 /* we own inuse only when @iocg is in the normal active state */
2487 if (iocg->abs_vdebt || list_empty(&iocg->active_list)) {
2496 * be reading 0 iocg->active before ioc->lock which will lead to
2499 new_inuse = iocg->inuse;
2500 adj_step = DIV_ROUND_UP(iocg->active * INUSE_ADJ_STEP_PCT, 100);
2503 propagate_weights(iocg, iocg->active, new_inuse, true, now);
2504 current_hweight(iocg, NULL, &hwi);
2507 iocg->inuse != iocg->active);
2511 TRACE_IOCG_PATH(inuse_adjust, iocg, now,
2512 old_inuse, iocg->inuse, old_hwi, hwi);
2517 static void calc_vtime_cost_builtin(struct bio *bio, struct ioc_gq *iocg,
2520 struct ioc *ioc = iocg->ioc;
2545 if (iocg->cursor) {
2546 seek_pages = abs(bio->bi_iter.bi_sector - iocg->cursor);
2562 static u64 calc_vtime_cost(struct bio *bio, struct ioc_gq *iocg, bool is_merge)
2566 calc_vtime_cost_builtin(bio, iocg, is_merge, &cost);
2599 struct ioc_gq *iocg = blkg_to_iocg(blkg);
2607 if (!ioc->enabled || !iocg || !iocg->level)
2611 abs_cost = calc_vtime_cost(bio, iocg, false);
2615 if (!iocg_activate(iocg, &now))
2618 iocg->cursor = bio_end_sector(bio);
2619 vtime = atomic64_read(&iocg->vtime);
2620 cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now);
2627 if (!waitqueue_active(&iocg->waitq) && !iocg->abs_vdebt &&
2629 iocg_commit_bio(iocg, bio, abs_cost, cost);
2636 * debt. Otherwise, the issuer is blocked on @iocg->waitq. Debt handling
2637 * requires @ioc->lock, waitq handling @iocg->waitq.lock. Determine
2641 ioc_locked = use_debt || READ_ONCE(iocg->abs_vdebt);
2643 iocg_lock(iocg, ioc_locked, &flags);
2646 * @iocg must stay activated for debt and waitq handling. Deactivation
2652 if (unlikely(list_empty(&iocg->active_list))) {
2653 iocg_unlock(iocg, ioc_locked, &flags);
2654 iocg_commit_bio(iocg, bio, abs_cost, cost);
2669 * An iocg with vdebt should stay online so that the timer can keep
2672 * clear them and leave @iocg inactive w/ dangling use_delay heavily
2676 iocg_incur_debt(iocg, abs_cost, &now);
2677 if (iocg_kick_delay(iocg, &now))
2680 iocg_unlock(iocg, ioc_locked, &flags);
2685 if (!iocg->abs_vdebt && iocg->inuse != iocg->active) {
2687 iocg_unlock(iocg, false, &flags);
2691 propagate_weights(iocg, iocg->active, iocg->active, true,
2705 * All waiters are on iocg->waitq and the wait states are
2714 __add_wait_queue_entry_tail(&iocg->waitq, &wait.wait);
2715 iocg_kick_waitq(iocg, ioc_locked, &now);
2717 iocg_unlock(iocg, ioc_locked, &flags);
2727 finish_wait(&iocg->waitq, &wait.wait);
2733 struct ioc_gq *iocg = blkg_to_iocg(bio->bi_blkg);
2741 if (!ioc->enabled || !iocg || !iocg->level)
2744 abs_cost = calc_vtime_cost(bio, iocg, true);
2750 vtime = atomic64_read(&iocg->vtime);
2751 cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now);
2755 blk_rq_pos(rq) + blk_rq_sectors(rq) == iocg->cursor)
2756 iocg->cursor = bio_end;
2763 time_before_eq64(atomic64_read(&iocg->vtime) + cost, now.vnow)) {
2764 iocg_commit_bio(iocg, bio, abs_cost, cost);
2769 * Otherwise, account it as debt if @iocg is online, which it should
2774 spin_lock(&iocg->waitq.lock);
2776 if (likely(!list_empty(&iocg->active_list))) {
2777 iocg_incur_debt(iocg, abs_cost, &now);
2778 if (iocg_kick_delay(iocg, &now))
2782 iocg_commit_bio(iocg, bio, abs_cost, cost);
2785 spin_unlock(&iocg->waitq.lock);
2791 struct ioc_gq *iocg = blkg_to_iocg(bio->bi_blkg);
2793 if (iocg && bio->bi_iocost_cost)
2794 atomic64_add(bio->bi_iocost_cost, &iocg->done_vtime);
2916 * target bio has an iocg associated and need to test for NULL iocg.
2956 struct ioc_gq *iocg;
2958 iocg = kzalloc_node(struct_size(iocg, ancestors, levels), gfp,
2960 if (!iocg)
2963 iocg->pcpu_stat = alloc_percpu_gfp(struct iocg_pcpu_stat, gfp);
2964 if (!iocg->pcpu_stat) {
2965 kfree(iocg);
2969 return &iocg->pd;
2974 struct ioc_gq *iocg = pd_to_iocg(pd);
2975 struct blkcg_gq *blkg = pd_to_blkg(&iocg->pd);
2983 iocg->ioc = ioc;
2984 atomic64_set(&iocg->vtime, now.vnow);
2985 atomic64_set(&iocg->done_vtime, now.vnow);
2986 atomic64_set(&iocg->active_period, atomic64_read(&ioc->cur_period));
2987 INIT_LIST_HEAD(&iocg->active_list);
2988 INIT_LIST_HEAD(&iocg->walk_list);
2989 INIT_LIST_HEAD(&iocg->surplus_list);
2990 iocg->hweight_active = WEIGHT_ONE;
2991 iocg->hweight_inuse = WEIGHT_ONE;
2993 init_waitqueue_head(&iocg->waitq);
2994 hrtimer_init(&iocg->waitq_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
2995 iocg->waitq_timer.function = iocg_waitq_timer_fn;
2997 iocg->level = blkg->blkcg->css.cgroup->level;
3001 iocg->ancestors[tiocg->level] = tiocg;
3005 weight_updated(iocg, &now);
3011 struct ioc_gq *iocg = pd_to_iocg(pd);
3012 struct ioc *ioc = iocg->ioc;
3018 if (!list_empty(&iocg->active_list)) {
3022 propagate_weights(iocg, 0, 0, false, &now);
3023 list_del_init(&iocg->active_list);
3026 WARN_ON_ONCE(!list_empty(&iocg->walk_list));
3027 WARN_ON_ONCE(!list_empty(&iocg->surplus_list));
3031 hrtimer_cancel(&iocg->waitq_timer);
3033 free_percpu(iocg->pcpu_stat);
3034 kfree(iocg);
3039 struct ioc_gq *iocg = pd_to_iocg(pd);
3040 struct ioc *ioc = iocg->ioc;
3045 if (iocg->level == 0) {
3052 seq_printf(s, " cost.usage=%llu", iocg->last_stat.usage_us);
3056 iocg->last_stat.wait_us,
3057 iocg->last_stat.indebt_us,
3058 iocg->last_stat.indelay_us);
3065 struct ioc_gq *iocg = pd_to_iocg(pd);
3067 if (dname && iocg->cfg_weight)
3068 seq_printf(sf, "%s %u\n", dname, iocg->cfg_weight / WEIGHT_ONE);
3091 struct ioc_gq *iocg;
3107 struct ioc_gq *iocg = blkg_to_iocg(blkg);
3109 if (iocg) {
3110 spin_lock(&iocg->ioc->lock);
3111 ioc_now(iocg->ioc, &now);
3112 weight_updated(iocg, &now);
3113 spin_unlock(&iocg->ioc->lock);
3127 iocg = blkg_to_iocg(ctx.blkg);
3138 spin_lock(&iocg->ioc->lock);
3139 iocg->cfg_weight = v * WEIGHT_ONE;
3140 ioc_now(iocg->ioc, &now);
3141 weight_updated(iocg, &now);
3142 spin_unlock(&iocg->ioc->lock);