1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * net/sched/sch_mqprio.c 4 * 5 * Copyright (c) 2010 John Fastabend <john.r.fastabend@intel.com> 6 */ 7 8#include <linux/types.h> 9#include <linux/slab.h> 10#include <linux/kernel.h> 11#include <linux/string.h> 12#include <linux/errno.h> 13#include <linux/skbuff.h> 14#include <linux/module.h> 15#include <net/netlink.h> 16#include <net/pkt_sched.h> 17#include <net/sch_generic.h> 18#include <net/pkt_cls.h> 19 20struct mqprio_sched { 21 struct Qdisc **qdiscs; 22 u16 mode; 23 u16 shaper; 24 int hw_offload; 25 u32 flags; 26 u64 min_rate[TC_QOPT_MAX_QUEUE]; 27 u64 max_rate[TC_QOPT_MAX_QUEUE]; 28}; 29 30static void mqprio_destroy(struct Qdisc *sch) 31{ 32 struct net_device *dev = qdisc_dev(sch); 33 struct mqprio_sched *priv = qdisc_priv(sch); 34 unsigned int ntx; 35 36 if (priv->qdiscs) { 37 for (ntx = 0; 38 ntx < dev->num_tx_queues && priv->qdiscs[ntx]; 39 ntx++) 40 qdisc_put(priv->qdiscs[ntx]); 41 kfree(priv->qdiscs); 42 } 43 44 if (priv->hw_offload && dev->netdev_ops->ndo_setup_tc) { 45 struct tc_mqprio_qopt_offload mqprio = { { 0 } }; 46 47 switch (priv->mode) { 48 case TC_MQPRIO_MODE_DCB: 49 case TC_MQPRIO_MODE_CHANNEL: 50 dev->netdev_ops->ndo_setup_tc(dev, 51 TC_SETUP_QDISC_MQPRIO, 52 &mqprio); 53 break; 54 default: 55 return; 56 } 57 } else { 58 netdev_set_num_tc(dev, 0); 59 } 60} 61 62static int mqprio_parse_opt(struct net_device *dev, struct tc_mqprio_qopt *qopt) 63{ 64 int i, j; 65 66 /* Verify num_tc is not out of max range */ 67 if (qopt->num_tc > TC_MAX_QUEUE) 68 return -EINVAL; 69 70 /* Verify priority mapping uses valid tcs */ 71 for (i = 0; i < TC_BITMASK + 1; i++) { 72 if (qopt->prio_tc_map[i] >= qopt->num_tc) 73 return -EINVAL; 74 } 75 76 /* Limit qopt->hw to maximum supported offload value. Drivers have 77 * the option of overriding this later if they don't support the a 78 * given offload type. 79 */ 80 if (qopt->hw > TC_MQPRIO_HW_OFFLOAD_MAX) 81 qopt->hw = TC_MQPRIO_HW_OFFLOAD_MAX; 82 83 /* If hardware offload is requested we will leave it to the device 84 * to either populate the queue counts itself or to validate the 85 * provided queue counts. If ndo_setup_tc is not present then 86 * hardware doesn't support offload and we should return an error. 87 */ 88 if (qopt->hw) 89 return dev->netdev_ops->ndo_setup_tc ? 0 : -EINVAL; 90 91 for (i = 0; i < qopt->num_tc; i++) { 92 unsigned int last = qopt->offset[i] + qopt->count[i]; 93 94 /* Verify the queue count is in tx range being equal to the 95 * real_num_tx_queues indicates the last queue is in use. 96 */ 97 if (qopt->offset[i] >= dev->real_num_tx_queues || 98 !qopt->count[i] || 99 last > dev->real_num_tx_queues) 100 return -EINVAL; 101 102 /* Verify that the offset and counts do not overlap */ 103 for (j = i + 1; j < qopt->num_tc; j++) { 104 if (last > qopt->offset[j]) 105 return -EINVAL; 106 } 107 } 108 109 return 0; 110} 111 112static const struct nla_policy mqprio_policy[TCA_MQPRIO_MAX + 1] = { 113 [TCA_MQPRIO_MODE] = { .len = sizeof(u16) }, 114 [TCA_MQPRIO_SHAPER] = { .len = sizeof(u16) }, 115 [TCA_MQPRIO_MIN_RATE64] = { .type = NLA_NESTED }, 116 [TCA_MQPRIO_MAX_RATE64] = { .type = NLA_NESTED }, 117}; 118 119static int parse_attr(struct nlattr *tb[], int maxtype, struct nlattr *nla, 120 const struct nla_policy *policy, int len) 121{ 122 int nested_len = nla_len(nla) - NLA_ALIGN(len); 123 124 if (nested_len >= nla_attr_size(0)) 125 return nla_parse_deprecated(tb, maxtype, 126 nla_data(nla) + NLA_ALIGN(len), 127 nested_len, policy, NULL); 128 129 memset(tb, 0, sizeof(struct nlattr *) * (maxtype + 1)); 130 return 0; 131} 132 133static int mqprio_parse_nlattr(struct Qdisc *sch, struct tc_mqprio_qopt *qopt, 134 struct nlattr *opt, 135 struct netlink_ext_ack *extack) 136{ 137 struct mqprio_sched *priv = qdisc_priv(sch); 138 struct nlattr *tb[TCA_MQPRIO_MAX + 1]; 139 struct nlattr *attr; 140 int i, rem, err; 141 142 err = parse_attr(tb, TCA_MQPRIO_MAX, opt, mqprio_policy, 143 sizeof(*qopt)); 144 if (err < 0) 145 return err; 146 147 if (!qopt->hw) { 148 NL_SET_ERR_MSG(extack, 149 "mqprio TCA_OPTIONS can only contain netlink attributes in hardware mode"); 150 return -EINVAL; 151 } 152 153 if (tb[TCA_MQPRIO_MODE]) { 154 priv->flags |= TC_MQPRIO_F_MODE; 155 priv->mode = *(u16 *)nla_data(tb[TCA_MQPRIO_MODE]); 156 } 157 158 if (tb[TCA_MQPRIO_SHAPER]) { 159 priv->flags |= TC_MQPRIO_F_SHAPER; 160 priv->shaper = *(u16 *)nla_data(tb[TCA_MQPRIO_SHAPER]); 161 } 162 163 if (tb[TCA_MQPRIO_MIN_RATE64]) { 164 if (priv->shaper != TC_MQPRIO_SHAPER_BW_RATE) { 165 NL_SET_ERR_MSG_ATTR(extack, tb[TCA_MQPRIO_MIN_RATE64], 166 "min_rate accepted only when shaper is in bw_rlimit mode"); 167 return -EINVAL; 168 } 169 i = 0; 170 nla_for_each_nested(attr, tb[TCA_MQPRIO_MIN_RATE64], 171 rem) { 172 if (nla_type(attr) != TCA_MQPRIO_MIN_RATE64) { 173 NL_SET_ERR_MSG_ATTR(extack, attr, 174 "Attribute type expected to be TCA_MQPRIO_MIN_RATE64"); 175 return -EINVAL; 176 } 177 178 if (nla_len(attr) != sizeof(u64)) { 179 NL_SET_ERR_MSG_ATTR(extack, attr, 180 "Attribute TCA_MQPRIO_MIN_RATE64 expected to have 8 bytes length"); 181 return -EINVAL; 182 } 183 184 if (i >= qopt->num_tc) 185 break; 186 priv->min_rate[i] = *(u64 *)nla_data(attr); 187 i++; 188 } 189 priv->flags |= TC_MQPRIO_F_MIN_RATE; 190 } 191 192 if (tb[TCA_MQPRIO_MAX_RATE64]) { 193 if (priv->shaper != TC_MQPRIO_SHAPER_BW_RATE) { 194 NL_SET_ERR_MSG_ATTR(extack, tb[TCA_MQPRIO_MAX_RATE64], 195 "max_rate accepted only when shaper is in bw_rlimit mode"); 196 return -EINVAL; 197 } 198 i = 0; 199 nla_for_each_nested(attr, tb[TCA_MQPRIO_MAX_RATE64], 200 rem) { 201 if (nla_type(attr) != TCA_MQPRIO_MAX_RATE64) { 202 NL_SET_ERR_MSG_ATTR(extack, attr, 203 "Attribute type expected to be TCA_MQPRIO_MAX_RATE64"); 204 return -EINVAL; 205 } 206 207 if (nla_len(attr) != sizeof(u64)) { 208 NL_SET_ERR_MSG_ATTR(extack, attr, 209 "Attribute TCA_MQPRIO_MAX_RATE64 expected to have 8 bytes length"); 210 return -EINVAL; 211 } 212 213 if (i >= qopt->num_tc) 214 break; 215 priv->max_rate[i] = *(u64 *)nla_data(attr); 216 i++; 217 } 218 priv->flags |= TC_MQPRIO_F_MAX_RATE; 219 } 220 221 return 0; 222} 223 224static int mqprio_init(struct Qdisc *sch, struct nlattr *opt, 225 struct netlink_ext_ack *extack) 226{ 227 struct net_device *dev = qdisc_dev(sch); 228 struct mqprio_sched *priv = qdisc_priv(sch); 229 struct netdev_queue *dev_queue; 230 struct Qdisc *qdisc; 231 int i, err = -EOPNOTSUPP; 232 struct tc_mqprio_qopt *qopt = NULL; 233 int len; 234 235 BUILD_BUG_ON(TC_MAX_QUEUE != TC_QOPT_MAX_QUEUE); 236 BUILD_BUG_ON(TC_BITMASK != TC_QOPT_BITMASK); 237 238 if (sch->parent != TC_H_ROOT) 239 return -EOPNOTSUPP; 240 241 if (!netif_is_multiqueue(dev)) 242 return -EOPNOTSUPP; 243 244 /* make certain can allocate enough classids to handle queues */ 245 if (dev->num_tx_queues >= TC_H_MIN_PRIORITY) 246 return -ENOMEM; 247 248 if (!opt || nla_len(opt) < sizeof(*qopt)) 249 return -EINVAL; 250 251 qopt = nla_data(opt); 252 if (mqprio_parse_opt(dev, qopt)) 253 return -EINVAL; 254 255 len = nla_len(opt) - NLA_ALIGN(sizeof(*qopt)); 256 if (len > 0) { 257 err = mqprio_parse_nlattr(sch, qopt, opt, extack); 258 if (err) 259 return err; 260 } 261 262 /* pre-allocate qdisc, attachment can't fail */ 263 priv->qdiscs = kcalloc(dev->num_tx_queues, sizeof(priv->qdiscs[0]), 264 GFP_KERNEL); 265 if (!priv->qdiscs) 266 return -ENOMEM; 267 268 for (i = 0; i < dev->num_tx_queues; i++) { 269 dev_queue = netdev_get_tx_queue(dev, i); 270 qdisc = qdisc_create_dflt(dev_queue, 271 get_default_qdisc_ops(dev, i), 272 TC_H_MAKE(TC_H_MAJ(sch->handle), 273 TC_H_MIN(i + 1)), extack); 274 if (!qdisc) 275 return -ENOMEM; 276 277 priv->qdiscs[i] = qdisc; 278 qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT; 279 } 280 281 /* If the mqprio options indicate that hardware should own 282 * the queue mapping then run ndo_setup_tc otherwise use the 283 * supplied and verified mapping 284 */ 285 if (qopt->hw) { 286 struct tc_mqprio_qopt_offload mqprio = {.qopt = *qopt}; 287 288 switch (priv->mode) { 289 case TC_MQPRIO_MODE_DCB: 290 if (priv->shaper != TC_MQPRIO_SHAPER_DCB) 291 return -EINVAL; 292 break; 293 case TC_MQPRIO_MODE_CHANNEL: 294 mqprio.flags = priv->flags; 295 if (priv->flags & TC_MQPRIO_F_MODE) 296 mqprio.mode = priv->mode; 297 if (priv->flags & TC_MQPRIO_F_SHAPER) 298 mqprio.shaper = priv->shaper; 299 if (priv->flags & TC_MQPRIO_F_MIN_RATE) 300 for (i = 0; i < mqprio.qopt.num_tc; i++) 301 mqprio.min_rate[i] = priv->min_rate[i]; 302 if (priv->flags & TC_MQPRIO_F_MAX_RATE) 303 for (i = 0; i < mqprio.qopt.num_tc; i++) 304 mqprio.max_rate[i] = priv->max_rate[i]; 305 break; 306 default: 307 return -EINVAL; 308 } 309 err = dev->netdev_ops->ndo_setup_tc(dev, 310 TC_SETUP_QDISC_MQPRIO, 311 &mqprio); 312 if (err) 313 return err; 314 315 priv->hw_offload = mqprio.qopt.hw; 316 } else { 317 netdev_set_num_tc(dev, qopt->num_tc); 318 for (i = 0; i < qopt->num_tc; i++) 319 netdev_set_tc_queue(dev, i, 320 qopt->count[i], qopt->offset[i]); 321 } 322 323 /* Always use supplied priority mappings */ 324 for (i = 0; i < TC_BITMASK + 1; i++) 325 netdev_set_prio_tc_map(dev, i, qopt->prio_tc_map[i]); 326 327 sch->flags |= TCQ_F_MQROOT; 328 return 0; 329} 330 331static void mqprio_attach(struct Qdisc *sch) 332{ 333 struct net_device *dev = qdisc_dev(sch); 334 struct mqprio_sched *priv = qdisc_priv(sch); 335 struct Qdisc *qdisc, *old; 336 unsigned int ntx; 337 338 /* Attach underlying qdisc */ 339 for (ntx = 0; ntx < dev->num_tx_queues; ntx++) { 340 qdisc = priv->qdiscs[ntx]; 341 old = dev_graft_qdisc(qdisc->dev_queue, qdisc); 342 if (old) 343 qdisc_put(old); 344 if (ntx < dev->real_num_tx_queues) 345 qdisc_hash_add(qdisc, false); 346 } 347 kfree(priv->qdiscs); 348 priv->qdiscs = NULL; 349} 350 351static void mqprio_change_real_num_tx(struct Qdisc *sch, 352 unsigned int new_real_tx) 353{ 354 struct net_device *dev = qdisc_dev(sch); 355 struct Qdisc *qdisc; 356 unsigned int i; 357 358 for (i = new_real_tx; i < dev->real_num_tx_queues; i++) { 359 qdisc = netdev_get_tx_queue(dev, i)->qdisc_sleeping; 360 /* Only update the default qdiscs we created, 361 * qdiscs with handles are always hashed. 362 */ 363 if (qdisc != &noop_qdisc && !qdisc->handle) 364 qdisc_hash_del(qdisc); 365 } 366 for (i = dev->real_num_tx_queues; i < new_real_tx; i++) { 367 qdisc = netdev_get_tx_queue(dev, i)->qdisc_sleeping; 368 if (qdisc != &noop_qdisc && !qdisc->handle) 369 qdisc_hash_add(qdisc, false); 370 } 371} 372 373static struct netdev_queue *mqprio_queue_get(struct Qdisc *sch, 374 unsigned long cl) 375{ 376 struct net_device *dev = qdisc_dev(sch); 377 unsigned long ntx = cl - 1; 378 379 if (ntx >= dev->num_tx_queues) 380 return NULL; 381 return netdev_get_tx_queue(dev, ntx); 382} 383 384static int mqprio_graft(struct Qdisc *sch, unsigned long cl, struct Qdisc *new, 385 struct Qdisc **old, struct netlink_ext_ack *extack) 386{ 387 struct net_device *dev = qdisc_dev(sch); 388 struct netdev_queue *dev_queue = mqprio_queue_get(sch, cl); 389 390 if (!dev_queue) 391 return -EINVAL; 392 393 if (dev->flags & IFF_UP) 394 dev_deactivate(dev); 395 396 *old = dev_graft_qdisc(dev_queue, new); 397 398 if (new) 399 new->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT; 400 401 if (dev->flags & IFF_UP) 402 dev_activate(dev); 403 404 return 0; 405} 406 407static int dump_rates(struct mqprio_sched *priv, 408 struct tc_mqprio_qopt *opt, struct sk_buff *skb) 409{ 410 struct nlattr *nest; 411 int i; 412 413 if (priv->flags & TC_MQPRIO_F_MIN_RATE) { 414 nest = nla_nest_start_noflag(skb, TCA_MQPRIO_MIN_RATE64); 415 if (!nest) 416 goto nla_put_failure; 417 418 for (i = 0; i < opt->num_tc; i++) { 419 if (nla_put(skb, TCA_MQPRIO_MIN_RATE64, 420 sizeof(priv->min_rate[i]), 421 &priv->min_rate[i])) 422 goto nla_put_failure; 423 } 424 nla_nest_end(skb, nest); 425 } 426 427 if (priv->flags & TC_MQPRIO_F_MAX_RATE) { 428 nest = nla_nest_start_noflag(skb, TCA_MQPRIO_MAX_RATE64); 429 if (!nest) 430 goto nla_put_failure; 431 432 for (i = 0; i < opt->num_tc; i++) { 433 if (nla_put(skb, TCA_MQPRIO_MAX_RATE64, 434 sizeof(priv->max_rate[i]), 435 &priv->max_rate[i])) 436 goto nla_put_failure; 437 } 438 nla_nest_end(skb, nest); 439 } 440 return 0; 441 442nla_put_failure: 443 nla_nest_cancel(skb, nest); 444 return -1; 445} 446 447static int mqprio_dump(struct Qdisc *sch, struct sk_buff *skb) 448{ 449 struct net_device *dev = qdisc_dev(sch); 450 struct mqprio_sched *priv = qdisc_priv(sch); 451 struct nlattr *nla = (struct nlattr *)skb_tail_pointer(skb); 452 struct tc_mqprio_qopt opt = { 0 }; 453 struct Qdisc *qdisc; 454 unsigned int ntx, tc; 455 456 sch->q.qlen = 0; 457 memset(&sch->bstats, 0, sizeof(sch->bstats)); 458 memset(&sch->qstats, 0, sizeof(sch->qstats)); 459 460 /* MQ supports lockless qdiscs. However, statistics accounting needs 461 * to account for all, none, or a mix of locked and unlocked child 462 * qdiscs. Percpu stats are added to counters in-band and locking 463 * qdisc totals are added at end. 464 */ 465 for (ntx = 0; ntx < dev->num_tx_queues; ntx++) { 466 qdisc = netdev_get_tx_queue(dev, ntx)->qdisc_sleeping; 467 spin_lock_bh(qdisc_lock(qdisc)); 468 469 if (qdisc_is_percpu_stats(qdisc)) { 470 __u32 qlen = qdisc_qlen_sum(qdisc); 471 472 __gnet_stats_copy_basic(NULL, &sch->bstats, 473 qdisc->cpu_bstats, 474 &qdisc->bstats); 475 __gnet_stats_copy_queue(&sch->qstats, 476 qdisc->cpu_qstats, 477 &qdisc->qstats, qlen); 478 sch->q.qlen += qlen; 479 } else { 480 sch->q.qlen += qdisc->q.qlen; 481 sch->bstats.bytes += qdisc->bstats.bytes; 482 sch->bstats.packets += qdisc->bstats.packets; 483 sch->qstats.backlog += qdisc->qstats.backlog; 484 sch->qstats.drops += qdisc->qstats.drops; 485 sch->qstats.requeues += qdisc->qstats.requeues; 486 sch->qstats.overlimits += qdisc->qstats.overlimits; 487 } 488 489 spin_unlock_bh(qdisc_lock(qdisc)); 490 } 491 492 opt.num_tc = netdev_get_num_tc(dev); 493 memcpy(opt.prio_tc_map, dev->prio_tc_map, sizeof(opt.prio_tc_map)); 494 opt.hw = priv->hw_offload; 495 496 for (tc = 0; tc < netdev_get_num_tc(dev); tc++) { 497 opt.count[tc] = dev->tc_to_txq[tc].count; 498 opt.offset[tc] = dev->tc_to_txq[tc].offset; 499 } 500 501 if (nla_put(skb, TCA_OPTIONS, sizeof(opt), &opt)) 502 goto nla_put_failure; 503 504 if ((priv->flags & TC_MQPRIO_F_MODE) && 505 nla_put_u16(skb, TCA_MQPRIO_MODE, priv->mode)) 506 goto nla_put_failure; 507 508 if ((priv->flags & TC_MQPRIO_F_SHAPER) && 509 nla_put_u16(skb, TCA_MQPRIO_SHAPER, priv->shaper)) 510 goto nla_put_failure; 511 512 if ((priv->flags & TC_MQPRIO_F_MIN_RATE || 513 priv->flags & TC_MQPRIO_F_MAX_RATE) && 514 (dump_rates(priv, &opt, skb) != 0)) 515 goto nla_put_failure; 516 517 return nla_nest_end(skb, nla); 518nla_put_failure: 519 nlmsg_trim(skb, nla); 520 return -1; 521} 522 523static struct Qdisc *mqprio_leaf(struct Qdisc *sch, unsigned long cl) 524{ 525 struct netdev_queue *dev_queue = mqprio_queue_get(sch, cl); 526 527 if (!dev_queue) 528 return NULL; 529 530 return dev_queue->qdisc_sleeping; 531} 532 533static unsigned long mqprio_find(struct Qdisc *sch, u32 classid) 534{ 535 struct net_device *dev = qdisc_dev(sch); 536 unsigned int ntx = TC_H_MIN(classid); 537 538 /* There are essentially two regions here that have valid classid 539 * values. The first region will have a classid value of 1 through 540 * num_tx_queues. All of these are backed by actual Qdiscs. 541 */ 542 if (ntx < TC_H_MIN_PRIORITY) 543 return (ntx <= dev->num_tx_queues) ? ntx : 0; 544 545 /* The second region represents the hardware traffic classes. These 546 * are represented by classid values of TC_H_MIN_PRIORITY through 547 * TC_H_MIN_PRIORITY + netdev_get_num_tc - 1 548 */ 549 return ((ntx - TC_H_MIN_PRIORITY) < netdev_get_num_tc(dev)) ? ntx : 0; 550} 551 552static int mqprio_dump_class(struct Qdisc *sch, unsigned long cl, 553 struct sk_buff *skb, struct tcmsg *tcm) 554{ 555 if (cl < TC_H_MIN_PRIORITY) { 556 struct netdev_queue *dev_queue = mqprio_queue_get(sch, cl); 557 struct net_device *dev = qdisc_dev(sch); 558 int tc = netdev_txq_to_tc(dev, cl - 1); 559 560 tcm->tcm_parent = (tc < 0) ? 0 : 561 TC_H_MAKE(TC_H_MAJ(sch->handle), 562 TC_H_MIN(tc + TC_H_MIN_PRIORITY)); 563 tcm->tcm_info = dev_queue->qdisc_sleeping->handle; 564 } else { 565 tcm->tcm_parent = TC_H_ROOT; 566 tcm->tcm_info = 0; 567 } 568 tcm->tcm_handle |= TC_H_MIN(cl); 569 return 0; 570} 571 572static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl, 573 struct gnet_dump *d) 574 __releases(d->lock) 575 __acquires(d->lock) 576{ 577 if (cl >= TC_H_MIN_PRIORITY) { 578 int i; 579 __u32 qlen = 0; 580 struct gnet_stats_queue qstats = {0}; 581 struct gnet_stats_basic_packed bstats = {0}; 582 struct net_device *dev = qdisc_dev(sch); 583 struct netdev_tc_txq tc = dev->tc_to_txq[cl & TC_BITMASK]; 584 585 /* Drop lock here it will be reclaimed before touching 586 * statistics this is required because the d->lock we 587 * hold here is the look on dev_queue->qdisc_sleeping 588 * also acquired below. 589 */ 590 if (d->lock) 591 spin_unlock_bh(d->lock); 592 593 for (i = tc.offset; i < tc.offset + tc.count; i++) { 594 struct netdev_queue *q = netdev_get_tx_queue(dev, i); 595 struct Qdisc *qdisc = rtnl_dereference(q->qdisc); 596 597 spin_lock_bh(qdisc_lock(qdisc)); 598 599 if (qdisc_is_percpu_stats(qdisc)) { 600 qlen = qdisc_qlen_sum(qdisc); 601 602 __gnet_stats_copy_basic(NULL, &bstats, 603 qdisc->cpu_bstats, 604 &qdisc->bstats); 605 __gnet_stats_copy_queue(&qstats, 606 qdisc->cpu_qstats, 607 &qdisc->qstats, 608 qlen); 609 } else { 610 qlen += qdisc->q.qlen; 611 bstats.bytes += qdisc->bstats.bytes; 612 bstats.packets += qdisc->bstats.packets; 613 qstats.backlog += qdisc->qstats.backlog; 614 qstats.drops += qdisc->qstats.drops; 615 qstats.requeues += qdisc->qstats.requeues; 616 qstats.overlimits += qdisc->qstats.overlimits; 617 } 618 spin_unlock_bh(qdisc_lock(qdisc)); 619 } 620 621 /* Reclaim root sleeping lock before completing stats */ 622 if (d->lock) 623 spin_lock_bh(d->lock); 624 if (gnet_stats_copy_basic(NULL, d, NULL, &bstats) < 0 || 625 gnet_stats_copy_queue(d, NULL, &qstats, qlen) < 0) 626 return -1; 627 } else { 628 struct netdev_queue *dev_queue = mqprio_queue_get(sch, cl); 629 630 sch = dev_queue->qdisc_sleeping; 631 if (gnet_stats_copy_basic(qdisc_root_sleeping_running(sch), d, 632 sch->cpu_bstats, &sch->bstats) < 0 || 633 qdisc_qstats_copy(d, sch) < 0) 634 return -1; 635 } 636 return 0; 637} 638 639static void mqprio_walk(struct Qdisc *sch, struct qdisc_walker *arg) 640{ 641 struct net_device *dev = qdisc_dev(sch); 642 unsigned long ntx; 643 644 if (arg->stop) 645 return; 646 647 /* Walk hierarchy with a virtual class per tc */ 648 arg->count = arg->skip; 649 for (ntx = arg->skip; ntx < netdev_get_num_tc(dev); ntx++) { 650 if (arg->fn(sch, ntx + TC_H_MIN_PRIORITY, arg) < 0) { 651 arg->stop = 1; 652 return; 653 } 654 arg->count++; 655 } 656 657 /* Pad the values and skip over unused traffic classes */ 658 if (ntx < TC_MAX_QUEUE) { 659 arg->count = TC_MAX_QUEUE; 660 ntx = TC_MAX_QUEUE; 661 } 662 663 /* Reset offset, sort out remaining per-queue qdiscs */ 664 for (ntx -= TC_MAX_QUEUE; ntx < dev->num_tx_queues; ntx++) { 665 if (arg->fn(sch, ntx + 1, arg) < 0) { 666 arg->stop = 1; 667 return; 668 } 669 arg->count++; 670 } 671} 672 673static struct netdev_queue *mqprio_select_queue(struct Qdisc *sch, 674 struct tcmsg *tcm) 675{ 676 return mqprio_queue_get(sch, TC_H_MIN(tcm->tcm_parent)); 677} 678 679static const struct Qdisc_class_ops mqprio_class_ops = { 680 .graft = mqprio_graft, 681 .leaf = mqprio_leaf, 682 .find = mqprio_find, 683 .walk = mqprio_walk, 684 .dump = mqprio_dump_class, 685 .dump_stats = mqprio_dump_class_stats, 686 .select_queue = mqprio_select_queue, 687}; 688 689static struct Qdisc_ops mqprio_qdisc_ops __read_mostly = { 690 .cl_ops = &mqprio_class_ops, 691 .id = "mqprio", 692 .priv_size = sizeof(struct mqprio_sched), 693 .init = mqprio_init, 694 .destroy = mqprio_destroy, 695 .attach = mqprio_attach, 696 .change_real_num_tx = mqprio_change_real_num_tx, 697 .dump = mqprio_dump, 698 .owner = THIS_MODULE, 699}; 700 701static int __init mqprio_module_init(void) 702{ 703 return register_qdisc(&mqprio_qdisc_ops); 704} 705 706static void __exit mqprio_module_exit(void) 707{ 708 unregister_qdisc(&mqprio_qdisc_ops); 709} 710 711module_init(mqprio_module_init); 712module_exit(mqprio_module_exit); 713 714MODULE_LICENSE("GPL"); 715