Lines Matching refs:qdisc
30 #include <trace/events/qdisc.h>
60 * qdisc_lock(qdisc) spinlock.
63 * - enqueue, dequeue are serialized via qdisc root lock
64 * - ingress filtering is also serialized via qdisc root lock
315 /* And release qdisc */
367 * this qdisc at a time. qdisc_lock(q) serializes queue accesses for
548 static int noop_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
555 static struct sk_buff *noop_dequeue(struct Qdisc *qdisc)
570 RCU_POINTER_INITIALIZER(qdisc, &noop_qdisc),
598 static int noqueue_init(struct Qdisc *qdisc, struct nlattr *opt,
604 qdisc->enqueue = NULL;
642 static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
646 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
654 if (qdisc_is_percpu_stats(qdisc))
655 return qdisc_drop_cpu(skb, qdisc, to_free);
657 return qdisc_drop(skb, qdisc, to_free);
660 qdisc_update_stats_at_enqueue(qdisc, pkt_len);
664 static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc)
666 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
681 qdisc_update_stats_at_dequeue(qdisc, skb);
683 test_bit(__QDISC_STATE_MISSED, &qdisc->state)) {
689 clear_bit(__QDISC_STATE_MISSED, &qdisc->state);
700 WRITE_ONCE(qdisc->empty, true);
706 static struct sk_buff *pfifo_fast_peek(struct Qdisc *qdisc)
708 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
721 static void pfifo_fast_reset(struct Qdisc *qdisc)
724 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
740 if (qdisc_is_percpu_stats(qdisc)) {
744 q = per_cpu_ptr(qdisc->cpu_qstats, i);
751 static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb)
764 static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt,
767 unsigned int qlen = qdisc_dev(qdisc)->tx_queue_len;
768 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
785 qdisc->flags |= TCQ_F_CAN_BYPASS;
887 /* seqlock has the same scope of busylock, for NOLOCK qdisc */
941 /* Under qdisc_lock(qdisc) and BH! */
943 void qdisc_reset(struct Qdisc *qdisc)
945 const struct Qdisc_ops *ops = qdisc->ops;
948 trace_qdisc_reset(qdisc);
951 ops->reset(qdisc);
953 skb_queue_walk_safe(&qdisc->gso_skb, skb, tmp) {
954 __skb_unlink(skb, &qdisc->gso_skb);
958 skb_queue_walk_safe(&qdisc->skb_bad_txq, skb, tmp) {
959 __skb_unlink(skb, &qdisc->skb_bad_txq);
963 qdisc->q.qlen = 0;
964 qdisc->qstats.backlog = 0;
968 void qdisc_free(struct Qdisc *qdisc)
970 if (qdisc_is_percpu_stats(qdisc)) {
971 free_percpu(qdisc->cpu_bstats);
972 free_percpu(qdisc->cpu_qstats);
975 kfree(qdisc);
985 static void qdisc_destroy(struct Qdisc *qdisc)
987 const struct Qdisc_ops *ops = qdisc->ops;
990 qdisc_hash_del(qdisc);
992 qdisc_put_stab(rtnl_dereference(qdisc->stab));
994 gen_kill_estimator(&qdisc->rate_est);
996 qdisc_reset(qdisc);
999 ops->destroy(qdisc);
1002 dev_put(qdisc_dev(qdisc));
1004 trace_qdisc_destroy(qdisc);
1006 call_rcu(&qdisc->rcu, qdisc_free_cb);
1009 void qdisc_put(struct Qdisc *qdisc)
1011 if (!qdisc)
1014 if (qdisc->flags & TCQ_F_BUILTIN ||
1015 !refcount_dec_and_test(&qdisc->refcnt))
1018 qdisc_destroy(qdisc);
1024 * qdisc reference counter reached zero.
1027 void qdisc_put_unlocked(struct Qdisc *qdisc)
1029 if (qdisc->flags & TCQ_F_BUILTIN ||
1030 !refcount_dec_and_rtnl_lock(&qdisc->refcnt))
1033 qdisc_destroy(qdisc);
1038 /* Attach toplevel qdisc to device queue. */
1040 struct Qdisc *qdisc)
1049 if (qdisc == NULL)
1050 qdisc = &noop_qdisc;
1051 dev_queue->qdisc_sleeping = qdisc;
1052 rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc);
1064 struct Qdisc *qdisc = dev_queue->qdisc_sleeping;
1067 if (qdisc) {
1068 rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
1071 qdisc_put(qdisc);
1079 struct Qdisc *qdisc;
1087 qdisc = qdisc_create_dflt(dev_queue, ops, TC_H_ROOT, NULL);
1088 if (!qdisc)
1092 qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT;
1093 dev_queue->qdisc_sleeping = qdisc;
1099 struct Qdisc *qdisc;
1106 qdisc = txq->qdisc_sleeping;
1107 rcu_assign_pointer(dev->qdisc, qdisc);
1108 qdisc_refcount_inc(qdisc);
1110 qdisc = qdisc_create_dflt(txq, &mq_qdisc_ops, TC_H_ROOT, NULL);
1111 if (qdisc) {
1112 rcu_assign_pointer(dev->qdisc, qdisc);
1113 qdisc->ops->attach(qdisc);
1116 qdisc = rtnl_dereference(dev->qdisc);
1118 /* Detect default qdisc setup/init failed and fallback to "noqueue" */
1119 if (qdisc == &noop_qdisc) {
1120 netdev_warn(dev, "default qdisc (%s) fail, fallback to %s\n",
1125 qdisc = txq->qdisc_sleeping;
1126 rcu_assign_pointer(dev->qdisc, qdisc);
1127 qdisc_refcount_inc(qdisc);
1132 if (qdisc != &noop_qdisc)
1133 qdisc_hash_add(qdisc, false);
1147 rcu_assign_pointer(dev_queue->qdisc, new_qdisc);
1163 if (rtnl_dereference(dev->qdisc) == &noop_qdisc)
1182 static void qdisc_deactivate(struct Qdisc *qdisc)
1184 if (qdisc->flags & TCQ_F_BUILTIN)
1187 set_bit(__QDISC_STATE_DEACTIVATED, &qdisc->state);
1195 struct Qdisc *qdisc;
1197 qdisc = rtnl_dereference(dev_queue->qdisc);
1198 if (qdisc) {
1199 qdisc_deactivate(qdisc);
1200 rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
1208 struct Qdisc *qdisc;
1211 qdisc = dev_queue->qdisc_sleeping;
1212 if (!qdisc)
1215 nolock = qdisc->flags & TCQ_F_NOLOCK;
1218 spin_lock_bh(&qdisc->seqlock);
1219 spin_lock_bh(qdisc_lock(qdisc));
1221 qdisc_reset(qdisc);
1223 spin_unlock_bh(qdisc_lock(qdisc));
1225 clear_bit(__QDISC_STATE_MISSED, &qdisc->state);
1226 spin_unlock_bh(&qdisc->seqlock);
1278 /* Wait for outstanding qdisc-less dev_queue_xmit calls or
1279 * outstanding qdisc enqueuing calls.
1317 struct Qdisc *qdisc = dev_queue->qdisc_sleeping;
1318 const struct Qdisc_ops *ops = qdisc->ops;
1321 return ops->change_tx_queue_len(qdisc, dev->tx_queue_len);
1328 struct Qdisc *qdisc = rtnl_dereference(dev->qdisc);
1330 if (qdisc->ops->change_real_num_tx)
1331 qdisc->ops->change_real_num_tx(qdisc, new_real_tx);
1360 struct Qdisc *qdisc = _qdisc;
1362 rcu_assign_pointer(dev_queue->qdisc, qdisc);
1363 dev_queue->qdisc_sleeping = qdisc;
1368 rcu_assign_pointer(dev->qdisc, &noop_qdisc);
1381 qdisc_put(rtnl_dereference(dev->qdisc));
1382 rcu_assign_pointer(dev->qdisc, &noop_qdisc);
1473 void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc,
1476 miniqp->miniq1.cpu_bstats = qdisc->cpu_bstats;
1477 miniqp->miniq1.cpu_qstats = qdisc->cpu_qstats;
1478 miniqp->miniq2.cpu_bstats = qdisc->cpu_bstats;
1479 miniqp->miniq2.cpu_qstats = qdisc->cpu_qstats;