45 	/* Only for !TCQ_F_NOLOCK qdisc. Never access it directly.
73 struct Qdisc {  struct
75 					   struct Qdisc *sch,  argument
77 	struct sk_buff *	(*dequeue)(struct Qdisc *sch);  argument
96 #define TCQ_F_NOLOCK		0x100 /* qdisc does not require locking */
97 #define TCQ_F_OFFLOADED		0x200 /* qdisc is offloaded to HW */
122 	unsigned long		state2; /* must be written under qdisc spinlock */
123 	struct Qdisc            *next_sched;  argument
136 static inline void qdisc_refcount_inc(struct Qdisc *qdisc)  in qdisc_refcount_inc()  argument
138 	if (qdisc->flags & TCQ_F_BUILTIN)  in qdisc_refcount_inc()
140 	refcount_inc(&qdisc->refcnt);  in qdisc_refcount_inc()
143 static inline bool qdisc_refcount_dec_if_one(struct Qdisc *qdisc)  in qdisc_refcount_dec_if_one()  argument
145 	if (qdisc->flags & TCQ_F_BUILTIN)  in qdisc_refcount_dec_if_one()
147 	return refcount_dec_if_one(&qdisc->refcnt);  in qdisc_refcount_dec_if_one()
150 /* Intended to be used by unlocked users, when concurrent qdisc release is
154 static inline struct Qdisc *qdisc_refcount_inc_nz(struct Qdisc *qdisc)  in qdisc_refcount_inc_nz()  argument
156 	if (qdisc->flags & TCQ_F_BUILTIN)  in qdisc_refcount_inc_nz()
157 		return qdisc;  in qdisc_refcount_inc_nz()
158 	if (refcount_inc_not_zero(&qdisc->refcnt))  in qdisc_refcount_inc_nz()
159 		return qdisc;  in qdisc_refcount_inc_nz()
163 /* For !TCQ_F_NOLOCK qdisc: callers must either call this within a qdisc
167 static inline bool qdisc_is_running(struct Qdisc *qdisc)  in qdisc_is_running()  argument
169 	if (qdisc->flags & TCQ_F_NOLOCK)  in qdisc_is_running()
170 		return spin_is_locked(&qdisc->seqlock);  in qdisc_is_running()
171 	return test_bit(__QDISC_STATE2_RUNNING, &qdisc->state2);  in qdisc_is_running()
174 static inline bool nolock_qdisc_is_empty(const struct Qdisc *qdisc)  in nolock_qdisc_is_empty()  argument
176 	return !(READ_ONCE(qdisc->state) & QDISC_STATE_NON_EMPTY);  in nolock_qdisc_is_empty()
179 static inline bool qdisc_is_percpu_stats(const struct Qdisc *q)  in qdisc_is_percpu_stats()
184 static inline bool qdisc_is_empty(const struct Qdisc *qdisc)  in qdisc_is_empty()  argument
186 	if (qdisc_is_percpu_stats(qdisc))  in qdisc_is_empty()
187 		return nolock_qdisc_is_empty(qdisc);  in qdisc_is_empty()
188 	return !READ_ONCE(qdisc->q.qlen);  in qdisc_is_empty()
191 /* For !TCQ_F_NOLOCK qdisc, qdisc_run_begin/end() must be invoked with
192  * the qdisc root lock acquired.
194 static inline bool qdisc_run_begin(struct Qdisc *qdisc)  in qdisc_run_begin()  argument
196 	if (qdisc->flags & TCQ_F_NOLOCK) {  in qdisc_run_begin()
197 		if (spin_trylock(&qdisc->seqlock))  in qdisc_run_begin()
205 		if (test_and_set_bit(__QDISC_STATE_MISSED, &qdisc->state))  in qdisc_run_begin()
212 		return spin_trylock(&qdisc->seqlock);  in qdisc_run_begin()
214 	return !__test_and_set_bit(__QDISC_STATE2_RUNNING, &qdisc->state2);  in qdisc_run_begin()
217 static inline void qdisc_run_end(struct Qdisc *qdisc)  in qdisc_run_end()  argument
219 	if (qdisc->flags & TCQ_F_NOLOCK) {  in qdisc_run_end()
220 		spin_unlock(&qdisc->seqlock);  in qdisc_run_end()
229 				      &qdisc->state)))  in qdisc_run_end()
230 			__netif_schedule(qdisc);  in qdisc_run_end()
232 		__clear_bit(__QDISC_STATE2_RUNNING, &qdisc->state2);  in qdisc_run_end()
236 static inline bool qdisc_may_bulk(const struct Qdisc *qdisc)  in qdisc_may_bulk()  argument
238 	return qdisc->flags & TCQ_F_ONETXQUEUE;  in qdisc_may_bulk()
248 	/* Child qdisc manipulation */
249 	struct netdev_queue *	(*select_queue)(struct Qdisc *, struct tcmsg *);
250 	int			(*graft)(struct Qdisc *, unsigned long cl,
251 					struct Qdisc *, struct Qdisc **,
253 	struct Qdisc *		(*leaf)(struct Qdisc *, unsigned long cl);
254 	void			(*qlen_notify)(struct Qdisc *, unsigned long);
257 	unsigned long		(*find)(struct Qdisc *, u32 classid);
258 	int			(*change)(struct Qdisc *, u32, u32,
261 	int			(*delete)(struct Qdisc *, unsigned long,
263 	void			(*walk)(struct Qdisc *, struct qdisc_walker * arg);
266 	struct tcf_block *	(*tcf_block)(struct Qdisc *sch,
269 	unsigned long		(*bind_tcf)(struct Qdisc *, unsigned long,
271 	void			(*unbind_tcf)(struct Qdisc *, unsigned long);
274 	int			(*dump)(struct Qdisc *, unsigned long,
276 	int			(*dump_stats)(struct Qdisc *, unsigned long,
295 					   struct Qdisc *sch,
297 	struct sk_buff *	(*dequeue)(struct Qdisc *);
298 	struct sk_buff *	(*peek)(struct Qdisc *);
300 	int			(*init)(struct Qdisc *sch, struct nlattr *arg,
302 	void			(*reset)(struct Qdisc *);
303 	void			(*destroy)(struct Qdisc *);
304 	int			(*change)(struct Qdisc *sch,
307 	void			(*attach)(struct Qdisc *sch);
308 	int			(*change_tx_queue_len)(struct Qdisc *, unsigned int);
309 	void			(*change_real_num_tx)(struct Qdisc *sch,
312 	int			(*dump)(struct Qdisc *, struct sk_buff *);
313 	int			(*dump_stats)(struct Qdisc *, struct gnet_dump *);
315 	void			(*ingress_block_set)(struct Qdisc *sch,
317 	void			(*egress_block_set)(struct Qdisc *sch,
319 	u32			(*ingress_block_get)(struct Qdisc *sch);
320 	u32			(*egress_block_get)(struct Qdisc *sch);
473 	struct Qdisc *q;
517 static inline int qdisc_qlen(const struct Qdisc *q)  in qdisc_qlen()
522 static inline int qdisc_qlen_sum(const struct Qdisc *q)  in qdisc_qlen_sum()
542 static inline spinlock_t *qdisc_lock(struct Qdisc *qdisc)  in qdisc_lock()  argument
544 	return &qdisc->q.lock;  in qdisc_lock()
547 static inline struct Qdisc *qdisc_root(const struct Qdisc *qdisc)  in qdisc_root()  argument
549 	struct Qdisc *q = rcu_dereference_rtnl(qdisc->dev_queue->qdisc);  in qdisc_root()
554 static inline struct Qdisc *qdisc_root_bh(const struct Qdisc *qdisc)  in qdisc_root_bh()  argument
556 	return rcu_dereference_bh(qdisc->dev_queue->qdisc);  in qdisc_root_bh()
559 static inline struct Qdisc *qdisc_root_sleeping(const struct Qdisc *qdisc)  in qdisc_root_sleeping()  argument
561 	return rcu_dereference_rtnl(qdisc->dev_queue->qdisc_sleeping);  in qdisc_root_sleeping()
564 static inline spinlock_t *qdisc_root_sleeping_lock(const struct Qdisc *qdisc)  in qdisc_root_sleeping_lock()  argument
566 	struct Qdisc *root = qdisc_root_sleeping(qdisc);  in qdisc_root_sleeping_lock()
572 static inline struct net_device *qdisc_dev(const struct Qdisc *qdisc)  in qdisc_dev()  argument
574 	return qdisc->dev_queue->dev;  in qdisc_dev()
577 static inline void sch_tree_lock(struct Qdisc *q)  in sch_tree_lock()
585 static inline void sch_tree_unlock(struct Qdisc *q)  in sch_tree_unlock()
593 extern struct Qdisc noop_qdisc;
654 		WARN(1, "Qdisc class overflow");  in qdisc_class_get()
664 		WARN(1, "Qdisc class underflow");  in qdisc_class_put()
681 void qdisc_class_hash_grow(struct Qdisc *, struct Qdisc_class_hash *);
692 struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue,
693 			      struct Qdisc *qdisc);
694 void qdisc_reset(struct Qdisc *qdisc);
695 void qdisc_destroy(struct Qdisc *qdisc);
696 void qdisc_put(struct Qdisc *qdisc);
697 void qdisc_put_unlocked(struct Qdisc *qdisc);
698 void qdisc_tree_reduce_backlog(struct Qdisc *qdisc, int n, int len);
700 int qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type,
702 void qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch,
703 				struct Qdisc *new, struct Qdisc *old,
708 qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type,  in qdisc_offload_dump_helper()
716 qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch,  in qdisc_offload_graft_helper()
717 			   struct Qdisc *new, struct Qdisc *old,  in qdisc_offload_graft_helper()
726 struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
729 void qdisc_free(struct Qdisc *qdisc);
730 struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue,
760 	struct Qdisc *qdisc;  in qdisc_reset_all_tx_gt()  local
763 		qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc);  in qdisc_reset_all_tx_gt()
764 		if (qdisc) {  in qdisc_reset_all_tx_gt()
765 			spin_lock_bh(qdisc_lock(qdisc));  in qdisc_reset_all_tx_gt()
766 			qdisc_reset(qdisc);  in qdisc_reset_all_tx_gt()
767 			spin_unlock_bh(qdisc_lock(qdisc));  in qdisc_reset_all_tx_gt()
780 		const struct Qdisc *q = rcu_dereference(txq->qdisc);  in qdisc_all_tx_empty()
799 		if (rcu_access_pointer(txq->qdisc) !=  in qdisc_tx_changing()
806 /* Is the device using the noop qdisc on all queues?  */
813 		if (rcu_access_pointer(txq->qdisc) != &noop_qdisc)  in qdisc_tx_is_noop()
824 /* additional qdisc xmit flags (NET_XMIT_MASK in linux/netdevice.h) */
837 					   const struct Qdisc *sch)  in qdisc_calculate_pkt_len()
847 static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch,  in qdisc_enqueue()
870 static inline void qdisc_bstats_cpu_update(struct Qdisc *sch,  in qdisc_bstats_cpu_update()
876 static inline void qdisc_bstats_update(struct Qdisc *sch,  in qdisc_bstats_update()
882 static inline void qdisc_qstats_backlog_dec(struct Qdisc *sch,  in qdisc_qstats_backlog_dec()
888 static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch,  in qdisc_qstats_cpu_backlog_dec()
894 static inline void qdisc_qstats_backlog_inc(struct Qdisc *sch,  in qdisc_qstats_backlog_inc()
900 static inline void qdisc_qstats_cpu_backlog_inc(struct Qdisc *sch,  in qdisc_qstats_cpu_backlog_inc()
906 static inline void qdisc_qstats_cpu_qlen_inc(struct Qdisc *sch)  in qdisc_qstats_cpu_qlen_inc()
911 static inline void qdisc_qstats_cpu_qlen_dec(struct Qdisc *sch)  in qdisc_qstats_cpu_qlen_dec()
916 static inline void qdisc_qstats_cpu_requeues_inc(struct Qdisc *sch)  in qdisc_qstats_cpu_requeues_inc()
921 static inline void __qdisc_qstats_drop(struct Qdisc *sch, int count)  in __qdisc_qstats_drop()
936 static inline void qdisc_qstats_drop(struct Qdisc *sch)  in qdisc_qstats_drop()
941 static inline void qdisc_qstats_cpu_drop(struct Qdisc *sch)  in qdisc_qstats_cpu_drop()
946 static inline void qdisc_qstats_overlimit(struct Qdisc *sch)  in qdisc_qstats_overlimit()
951 static inline int qdisc_qstats_copy(struct gnet_dump *d, struct Qdisc *sch)  in qdisc_qstats_copy()
958 static inline void qdisc_qstats_qlen_backlog(struct Qdisc *sch,  __u32 *qlen,  in qdisc_qstats_qlen_backlog()
968 static inline void qdisc_tree_flush_backlog(struct Qdisc *sch)  in qdisc_tree_flush_backlog()
976 static inline void qdisc_purge_queue(struct Qdisc *sch)  in qdisc_purge_queue()
1001 static inline int qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch)  in qdisc_enqueue_tail()
1034 static inline struct sk_buff *qdisc_dequeue_head(struct Qdisc *sch)  in qdisc_dequeue_head()
1077 /* Instead of calling kfree_skb() while root qdisc lock is held,
1096 static inline unsigned int __qdisc_queue_drop_head(struct Qdisc *sch,  in __qdisc_queue_drop_head()
1113 static inline struct sk_buff *qdisc_peek_head(struct Qdisc *sch)  in qdisc_peek_head()
1120 /* generic pseudo peek method for non-work-conserving qdisc */
1121 static inline struct sk_buff *qdisc_peek_dequeued(struct Qdisc *sch)  in qdisc_peek_dequeued()
1140 static inline void qdisc_update_stats_at_dequeue(struct Qdisc *sch,  in qdisc_update_stats_at_dequeue()
1154 static inline void qdisc_update_stats_at_enqueue(struct Qdisc *sch,  in qdisc_update_stats_at_enqueue()
1166 /* use instead of qdisc->dequeue() for all qdiscs queried with ->peek() */
1167 static inline struct sk_buff *qdisc_dequeue_peeked(struct Qdisc *sch)  in qdisc_dequeue_peeked()
1203 static inline void qdisc_reset_queue(struct Qdisc *sch)  in qdisc_reset_queue()
1208 static inline struct Qdisc *qdisc_replace(struct Qdisc *sch, struct Qdisc *new,  in qdisc_replace()
1209 					  struct Qdisc **pold)  in qdisc_replace()
1211 	struct Qdisc *old;  in qdisc_replace()
1223 static inline void rtnl_qdisc_drop(struct sk_buff *skb, struct Qdisc *sch)  in rtnl_qdisc_drop()
1229 static inline int qdisc_drop_cpu(struct sk_buff *skb, struct Qdisc *sch,  in qdisc_drop_cpu()
1238 static inline int qdisc_drop(struct sk_buff *skb, struct Qdisc *sch,  in qdisc_drop()
1247 static inline int qdisc_drop_reason(struct sk_buff *skb, struct Qdisc *sch,  in qdisc_drop_reason()
1255 static inline int qdisc_drop_all(struct sk_buff *skb, struct Qdisc *sch,  in qdisc_drop_all()
1297 	 * Qdisc using 64bit rate should add new attributes  in psched_ratecfg_getrate()
1321 /* Mini Qdisc serves for specific needs of ingress/clsact Qdisc.
1351 void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc,
1356 void mq_change_real_num_tx(struct Qdisc *sch, unsigned int new_real_tx);
1360 /* Make sure qdisc is no longer in SCHED state. */
1361 static inline void qdisc_synchronize(const struct Qdisc *q)  in qdisc_synchronize()