20 			/* Virtual address of skb_shinfo(skb)->frags[0].page + offset. */
36 	/* This indicates where we are processing relative to skb->data. */
39 	/* This is non-zero if the packet cannot be merged with the new skb. */
71 		/* Free the skb? */
103 #define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb)  argument
106 static inline int gro_recursion_inc_test(struct sk_buff *skb)  in gro_recursion_inc_test()  argument
108 	return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT;  in gro_recursion_inc_test()
114 					       struct sk_buff *skb)  in call_gro_receive()  argument
116 	if (unlikely(gro_recursion_inc_test(skb))) {  in call_gro_receive()
117 		NAPI_GRO_CB(skb)->flush |= 1;  in call_gro_receive()
121 	return cb(head, skb);  in call_gro_receive()
129 						  struct sk_buff *skb)  in call_gro_receive_sk()  argument
131 	if (unlikely(gro_recursion_inc_test(skb))) {  in call_gro_receive_sk()
132 		NAPI_GRO_CB(skb)->flush |= 1;  in call_gro_receive_sk()
136 	return cb(sk, head, skb);  in call_gro_receive_sk()
139 static inline unsigned int skb_gro_offset(const struct sk_buff *skb)  in skb_gro_offset()  argument
141 	return NAPI_GRO_CB(skb)->data_offset;  in skb_gro_offset()
144 static inline unsigned int skb_gro_len(const struct sk_buff *skb)  in skb_gro_len()  argument
146 	return skb->len - NAPI_GRO_CB(skb)->data_offset;  in skb_gro_len()
149 static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len)  in skb_gro_pull()  argument
151 	NAPI_GRO_CB(skb)->data_offset += len;  in skb_gro_pull()
154 static inline void *skb_gro_header_fast(const struct sk_buff *skb,  in skb_gro_header_fast()  argument
157 	return NAPI_GRO_CB(skb)->frag0 + offset;  in skb_gro_header_fast()
160 static inline bool skb_gro_may_pull(const struct sk_buff *skb,  in skb_gro_may_pull()  argument
163 	return likely(hlen <= NAPI_GRO_CB(skb)->frag0_len);  in skb_gro_may_pull()
166 static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen,  in skb_gro_header_slow()  argument
169 	if (!pskb_may_pull(skb, hlen))  in skb_gro_header_slow()
172 	return skb->data + offset;  in skb_gro_header_slow()
175 static inline void *skb_gro_header(struct sk_buff *skb, unsigned int hlen,  in skb_gro_header()  argument
180 	ptr = skb_gro_header_fast(skb, offset);  in skb_gro_header()
181 	if (!skb_gro_may_pull(skb, hlen))  in skb_gro_header()
182 		ptr = skb_gro_header_slow(skb, hlen, offset);  in skb_gro_header()
186 static inline int skb_gro_receive_network_offset(const struct sk_buff *skb)  in skb_gro_receive_network_offset()  argument
188 	return NAPI_GRO_CB(skb)->network_offsets[NAPI_GRO_CB(skb)->encap_mark];  in skb_gro_receive_network_offset()
191 static inline void *skb_gro_network_header(const struct sk_buff *skb)  in skb_gro_network_header()  argument
193 	if (skb_gro_may_pull(skb, skb_gro_offset(skb)))  in skb_gro_network_header()
194 		return skb_gro_header_fast(skb, skb_gro_receive_network_offset(skb));  in skb_gro_network_header()
196 	return skb->data + skb_gro_receive_network_offset(skb);  in skb_gro_network_header()
199 static inline __wsum inet_gro_compute_pseudo(const struct sk_buff *skb,  in inet_gro_compute_pseudo()  argument
202 	const struct iphdr *iph = skb_gro_network_header(skb);  in inet_gro_compute_pseudo()
205 				  skb_gro_len(skb), proto, 0);  in inet_gro_compute_pseudo()
208 static inline void skb_gro_postpull_rcsum(struct sk_buff *skb,  in skb_gro_postpull_rcsum()  argument
211 	if (NAPI_GRO_CB(skb)->csum_valid)  in skb_gro_postpull_rcsum()
212 		NAPI_GRO_CB(skb)->csum = wsum_negate(csum_partial(start, len,  in skb_gro_postpull_rcsum()
213 						wsum_negate(NAPI_GRO_CB(skb)->csum)));  in skb_gro_postpull_rcsum()
221 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb);
223 static inline bool skb_at_gro_remcsum_start(struct sk_buff *skb)  in skb_at_gro_remcsum_start()  argument
225 	return (NAPI_GRO_CB(skb)->gro_remcsum_start == skb_gro_offset(skb));  in skb_at_gro_remcsum_start()
228 static inline bool __skb_gro_checksum_validate_needed(struct sk_buff *skb,  in __skb_gro_checksum_validate_needed()  argument
232 	return ((skb->ip_summed != CHECKSUM_PARTIAL ||  in __skb_gro_checksum_validate_needed()
233 		skb_checksum_start_offset(skb) <  in __skb_gro_checksum_validate_needed()
234 		 skb_gro_offset(skb)) &&  in __skb_gro_checksum_validate_needed()
235 		!skb_at_gro_remcsum_start(skb) &&  in __skb_gro_checksum_validate_needed()
236 		NAPI_GRO_CB(skb)->csum_cnt == 0 &&  in __skb_gro_checksum_validate_needed()
240 static inline __sum16 __skb_gro_checksum_validate_complete(struct sk_buff *skb,  in __skb_gro_checksum_validate_complete()  argument
243 	if (NAPI_GRO_CB(skb)->csum_valid &&  in __skb_gro_checksum_validate_complete()
244 	    !csum_fold(csum_add(psum, NAPI_GRO_CB(skb)->csum)))  in __skb_gro_checksum_validate_complete()
247 	NAPI_GRO_CB(skb)->csum = psum;  in __skb_gro_checksum_validate_complete()
249 	return __skb_gro_checksum_complete(skb);  in __skb_gro_checksum_validate_complete()
252 static inline void skb_gro_incr_csum_unnecessary(struct sk_buff *skb)  in skb_gro_incr_csum_unnecessary()  argument
254 	if (NAPI_GRO_CB(skb)->csum_cnt > 0) {  in skb_gro_incr_csum_unnecessary()
256 		NAPI_GRO_CB(skb)->csum_cnt--;  in skb_gro_incr_csum_unnecessary()
258 		/* Update skb for CHECKSUM_UNNECESSARY and csum_level when we  in skb_gro_incr_csum_unnecessary()
262 		__skb_incr_checksum_unnecessary(skb);  in skb_gro_incr_csum_unnecessary()
266 #define __skb_gro_checksum_validate(skb, proto, zero_okay, check,	\  argument
270 	if (__skb_gro_checksum_validate_needed(skb, zero_okay, check))	\
271 		__ret = __skb_gro_checksum_validate_complete(skb,	\
272 				compute_pseudo(skb, proto));		\
274 		skb_gro_incr_csum_unnecessary(skb);			\
278 #define skb_gro_checksum_validate(skb, proto, compute_pseudo)		\  argument
279 	__skb_gro_checksum_validate(skb, proto, false, 0, compute_pseudo)
281 #define skb_gro_checksum_validate_zero_check(skb, proto, check,		\  argument
283 	__skb_gro_checksum_validate(skb, proto, true, check, compute_pseudo)
285 #define skb_gro_checksum_simple_validate(skb)				\  argument
286 	__skb_gro_checksum_validate(skb, 0, false, 0, null_compute_pseudo)
288 static inline bool __skb_gro_checksum_convert_check(struct sk_buff *skb)  in __skb_gro_checksum_convert_check()  argument
290 	return (NAPI_GRO_CB(skb)->csum_cnt == 0 &&  in __skb_gro_checksum_convert_check()
291 		!NAPI_GRO_CB(skb)->csum_valid);  in __skb_gro_checksum_convert_check()
294 static inline void __skb_gro_checksum_convert(struct sk_buff *skb,  in __skb_gro_checksum_convert()  argument
297 	NAPI_GRO_CB(skb)->csum = ~pseudo;  in __skb_gro_checksum_convert()
298 	NAPI_GRO_CB(skb)->csum_valid = 1;  in __skb_gro_checksum_convert()
301 #define skb_gro_checksum_try_convert(skb, proto, compute_pseudo)	\  argument
303 	if (__skb_gro_checksum_convert_check(skb))			\
304 		__skb_gro_checksum_convert(skb, 			\
305 					   compute_pseudo(skb, proto));	\
319 static inline void *skb_gro_remcsum_process(struct sk_buff *skb, void *ptr,  in skb_gro_remcsum_process()  argument
328 	BUG_ON(!NAPI_GRO_CB(skb)->csum_valid);  in skb_gro_remcsum_process()
331 		NAPI_GRO_CB(skb)->gro_remcsum_start = off + hdrlen + start;  in skb_gro_remcsum_process()
335 	ptr = skb_gro_header(skb, off + plen, off);  in skb_gro_remcsum_process()
339 	delta = remcsum_adjust(ptr + hdrlen, NAPI_GRO_CB(skb)->csum,  in skb_gro_remcsum_process()
342 	/* Adjust skb->csum since we changed the packet */  in skb_gro_remcsum_process()
343 	NAPI_GRO_CB(skb)->csum = csum_add(NAPI_GRO_CB(skb)->csum, delta);  in skb_gro_remcsum_process()
351 static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb,  in skb_gro_remcsum_cleanup()  argument
360 	ptr = skb_gro_header(skb, plen, grc->offset);  in skb_gro_remcsum_cleanup()
368 static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush)  in skb_gro_flush_final()  argument
371 		NAPI_GRO_CB(skb)->flush |= flush;  in skb_gro_flush_final()
373 static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb,  in skb_gro_flush_final_remcsum()  argument
379 		NAPI_GRO_CB(skb)->flush |= flush;  in skb_gro_flush_final_remcsum()
380 		skb_gro_remcsum_cleanup(skb, grc);  in skb_gro_flush_final_remcsum()
381 		skb->remcsum_offload = 0;  in skb_gro_flush_final_remcsum()
385 static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush)  in skb_gro_flush_final()  argument
387 	NAPI_GRO_CB(skb)->flush |= flush;  in skb_gro_flush_final()
389 static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb,  in skb_gro_flush_final_remcsum()  argument
394 	NAPI_GRO_CB(skb)->flush |= flush;  in skb_gro_flush_final_remcsum()
395 	skb_gro_remcsum_cleanup(skb, grc);  in skb_gro_flush_final_remcsum()
396 	skb->remcsum_offload = 0;  in skb_gro_flush_final_remcsum()
415 #define indirect_call_gro_receive_inet(cb, f2, f1, head, skb)	\  argument
417 	unlikely(gro_recursion_inc_test(skb)) ?			\
418 		NAPI_GRO_CB(skb)->flush |= 1, NULL :		\
419 		INDIRECT_CALL_INET(cb, f2, f1, head, skb);	\
422 struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb,
424 int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup);
426 static inline struct udphdr *udp_gro_udphdr(struct sk_buff *skb)  in udp_gro_udphdr()  argument
431 	off  = skb_gro_offset(skb);  in udp_gro_udphdr()
433 	uh   = skb_gro_header(skb, hlen, off);  in udp_gro_udphdr()
438 static inline __wsum ip6_gro_compute_pseudo(const struct sk_buff *skb,  in ip6_gro_compute_pseudo()  argument
441 	const struct ipv6hdr *iph = skb_gro_network_header(skb);  in ip6_gro_compute_pseudo()
444 					    skb_gro_len(skb), proto, 0));  in ip6_gro_compute_pseudo()
510 int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb);
511 int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb);
523 /* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded,
526 static inline void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, int segs)  in gro_normal_one()  argument
528 	list_add_tail(&skb->list, &napi->rx_list);  in gro_normal_one()
537  * The caller must verify skb_valid_dst(skb) is false and skb->dev is initialized.
540 static inline void inet_get_iif_sdif(const struct sk_buff *skb, int *iif, int *sdif)  in inet_get_iif_sdif()  argument
542 	*iif = inet_iif(skb) ?: skb->dev->ifindex;  in inet_get_iif_sdif()
546 	if (netif_is_l3_slave(skb->dev)) {  in inet_get_iif_sdif()
547 		struct net_device *master = netdev_master_upper_dev_get_rcu(skb->dev);  in inet_get_iif_sdif()
558  * The caller must verify skb_valid_dst(skb) is false and skb->dev is initialized.
561 static inline void inet6_get_iif_sdif(const struct sk_buff *skb, int *iif, int *sdif)  in inet6_get_iif_sdif()  argument
563 	/* using skb->dev->ifindex because skb_dst(skb) is not initialized */  in inet6_get_iif_sdif()
564 	*iif = skb->dev->ifindex;  in inet6_get_iif_sdif()
568 	if (netif_is_l3_slave(skb->dev)) {  in inet6_get_iif_sdif()
569 		struct net_device *master = netdev_master_upper_dev_get_rcu(skb->dev);  in inet6_get_iif_sdif()