Lines Matching refs:qdisc
30 #include <trace/events/qdisc.h>
62 * qdisc_lock(qdisc) spinlock.
65 * - enqueue, dequeue are serialized via qdisc root lock
66 * - ingress filtering is also serialized via qdisc root lock
307 * required. Owning qdisc running bit guarantees that only one CPU
321 /* And release qdisc */
373 * this qdisc at a time. qdisc_lock(q) serializes queue accesses for
628 static int noop_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
635 static struct sk_buff *noop_dequeue(struct Qdisc *qdisc)
650 RCU_POINTER_INITIALIZER(qdisc, &noop_qdisc),
677 static int noqueue_init(struct Qdisc *qdisc, struct nlattr *opt,
683 qdisc->enqueue = NULL;
721 static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
725 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
733 if (qdisc_is_percpu_stats(qdisc))
734 return qdisc_drop_cpu(skb, qdisc, to_free);
736 return qdisc_drop(skb, qdisc, to_free);
739 qdisc_update_stats_at_enqueue(qdisc, pkt_len);
743 static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc)
745 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
760 qdisc_update_stats_at_dequeue(qdisc, skb);
762 READ_ONCE(qdisc->state) & QDISC_STATE_NON_EMPTY) {
768 clear_bit(__QDISC_STATE_MISSED, &qdisc->state);
769 clear_bit(__QDISC_STATE_DRAINING, &qdisc->state);
784 static struct sk_buff *pfifo_fast_peek(struct Qdisc *qdisc)
786 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
799 static void pfifo_fast_reset(struct Qdisc *qdisc)
802 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
818 if (qdisc_is_percpu_stats(qdisc)) {
822 q = per_cpu_ptr(qdisc->cpu_qstats, i);
829 static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb)
842 static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt,
845 unsigned int qlen = qdisc_dev(qdisc)->tx_queue_len;
846 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
863 qdisc->flags |= TCQ_F_CAN_BYPASS;
964 /* seqlock has the same scope of busylock, for NOLOCK qdisc */
1013 /* Under qdisc_lock(qdisc) and BH! */
1015 void qdisc_reset(struct Qdisc *qdisc)
1017 const struct Qdisc_ops *ops = qdisc->ops;
1019 trace_qdisc_reset(qdisc);
1022 ops->reset(qdisc);
1024 __skb_queue_purge(&qdisc->gso_skb);
1025 __skb_queue_purge(&qdisc->skb_bad_txq);
1027 qdisc->q.qlen = 0;
1028 qdisc->qstats.backlog = 0;
1032 void qdisc_free(struct Qdisc *qdisc)
1034 if (qdisc_is_percpu_stats(qdisc)) {
1035 free_percpu(qdisc->cpu_bstats);
1036 free_percpu(qdisc->cpu_qstats);
1039 kfree(qdisc);
1049 static void __qdisc_destroy(struct Qdisc *qdisc)
1051 const struct Qdisc_ops *ops = qdisc->ops;
1054 qdisc_hash_del(qdisc);
1056 qdisc_put_stab(rtnl_dereference(qdisc->stab));
1058 gen_kill_estimator(&qdisc->rate_est);
1060 qdisc_reset(qdisc);
1063 ops->destroy(qdisc);
1066 netdev_put(qdisc_dev(qdisc), &qdisc->dev_tracker);
1068 trace_qdisc_destroy(qdisc);
1070 call_rcu(&qdisc->rcu, qdisc_free_cb);
1073 void qdisc_destroy(struct Qdisc *qdisc)
1075 if (qdisc->flags & TCQ_F_BUILTIN)
1078 __qdisc_destroy(qdisc);
1081 void qdisc_put(struct Qdisc *qdisc)
1083 if (!qdisc)
1086 if (qdisc->flags & TCQ_F_BUILTIN ||
1087 !refcount_dec_and_test(&qdisc->refcnt))
1090 __qdisc_destroy(qdisc);
1096 * qdisc reference counter reached zero.
1099 void qdisc_put_unlocked(struct Qdisc *qdisc)
1101 if (qdisc->flags & TCQ_F_BUILTIN ||
1102 !refcount_dec_and_rtnl_lock(&qdisc->refcnt))
1105 __qdisc_destroy(qdisc);
1110 /* Attach toplevel qdisc to device queue. */
1112 struct Qdisc *qdisc)
1121 if (qdisc == NULL)
1122 qdisc = &noop_qdisc;
1123 rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc);
1124 rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc);
1136 struct Qdisc *qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);
1139 if (qdisc) {
1140 rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
1143 qdisc_put(qdisc);
1151 struct Qdisc *qdisc;
1159 qdisc = qdisc_create_dflt(dev_queue, ops, TC_H_ROOT, NULL);
1160 if (!qdisc)
1164 qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT;
1165 rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc);
1171 struct Qdisc *qdisc;
1178 qdisc = rtnl_dereference(txq->qdisc_sleeping);
1179 rcu_assign_pointer(dev->qdisc, qdisc);
1180 qdisc_refcount_inc(qdisc);
1182 qdisc = qdisc_create_dflt(txq, &mq_qdisc_ops, TC_H_ROOT, NULL);
1183 if (qdisc) {
1184 rcu_assign_pointer(dev->qdisc, qdisc);
1185 qdisc->ops->attach(qdisc);
1188 qdisc = rtnl_dereference(dev->qdisc);
1190 /* Detect default qdisc setup/init failed and fallback to "noqueue" */
1191 if (qdisc == &noop_qdisc) {
1192 netdev_warn(dev, "default qdisc (%s) fail, fallback to %s\n",
1197 qdisc = rtnl_dereference(txq->qdisc_sleeping);
1198 rcu_assign_pointer(dev->qdisc, qdisc);
1199 qdisc_refcount_inc(qdisc);
1204 if (qdisc != &noop_qdisc)
1205 qdisc_hash_add(qdisc, false);
1219 rcu_assign_pointer(dev_queue->qdisc, new_qdisc);
1235 if (rtnl_dereference(dev->qdisc) == &noop_qdisc)
1254 static void qdisc_deactivate(struct Qdisc *qdisc)
1256 if (qdisc->flags & TCQ_F_BUILTIN)
1259 set_bit(__QDISC_STATE_DEACTIVATED, &qdisc->state);
1267 struct Qdisc *qdisc;
1269 qdisc = rtnl_dereference(dev_queue->qdisc);
1270 if (qdisc) {
1271 qdisc_deactivate(qdisc);
1272 rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
1280 struct Qdisc *qdisc;
1283 qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);
1284 if (!qdisc)
1287 nolock = qdisc->flags & TCQ_F_NOLOCK;
1290 spin_lock_bh(&qdisc->seqlock);
1291 spin_lock_bh(qdisc_lock(qdisc));
1293 qdisc_reset(qdisc);
1295 spin_unlock_bh(qdisc_lock(qdisc));
1297 clear_bit(__QDISC_STATE_MISSED, &qdisc->state);
1298 clear_bit(__QDISC_STATE_DRAINING, &qdisc->state);
1299 spin_unlock_bh(&qdisc->seqlock);
1351 /* Wait for outstanding qdisc-less dev_queue_xmit calls or
1352 * outstanding qdisc enqueuing calls.
1390 struct Qdisc *qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);
1391 const struct Qdisc_ops *ops = qdisc->ops;
1394 return ops->change_tx_queue_len(qdisc, dev->tx_queue_len);
1401 struct Qdisc *qdisc = rtnl_dereference(dev->qdisc);
1403 if (qdisc->ops->change_real_num_tx)
1404 qdisc->ops->change_real_num_tx(qdisc, new_real_tx);
1411 struct Qdisc *qdisc;
1415 qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc_sleeping);
1419 if (qdisc != &noop_qdisc && !qdisc->handle)
1420 qdisc_hash_del(qdisc);
1423 qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc_sleeping);
1424 if (qdisc != &noop_qdisc && !qdisc->handle)
1425 qdisc_hash_add(qdisc, false);
1457 struct Qdisc *qdisc = _qdisc;
1459 rcu_assign_pointer(dev_queue->qdisc, qdisc);
1460 rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc);
1465 rcu_assign_pointer(dev->qdisc, &noop_qdisc);
1478 qdisc_put(rtnl_dereference(dev->qdisc));
1479 rcu_assign_pointer(dev->qdisc, &noop_qdisc);
1593 void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc,
1596 miniqp->miniq1.cpu_bstats = qdisc->cpu_bstats;
1597 miniqp->miniq1.cpu_qstats = qdisc->cpu_qstats;
1598 miniqp->miniq2.cpu_bstats = qdisc->cpu_bstats;
1599 miniqp->miniq2.cpu_qstats = qdisc->cpu_qstats;