41 	int (*finish)(struct net *net, struct sock *sk, struct sk_buff *skb);
103 static int xfrm_rcv_cb(struct sk_buff *skb, unsigned int family, u8 protocol,  in xfrm_rcv_cb()  argument
114 	ret = afinfo->callback(skb, protocol, err);  in xfrm_rcv_cb()
120 struct sec_path *secpath_set(struct sk_buff *skb)  in secpath_set()  argument
122 	struct sec_path *sp, *tmp = skb_ext_find(skb, SKB_EXT_SEC_PATH);  in secpath_set()
124 	sp = skb_ext_add(skb, SKB_EXT_SEC_PATH);  in secpath_set()
143 int xfrm_parse_spi(struct sk_buff *skb, u8 nexthdr, __be32 *spi, __be32 *seq)  in xfrm_parse_spi()  argument
160 		if (!pskb_may_pull(skb, sizeof(struct ip_comp_hdr)))  in xfrm_parse_spi()
162 		*spi = htonl(ntohs(*(__be16 *)(skb_transport_header(skb) + 2)));  in xfrm_parse_spi()
169 	if (!pskb_may_pull(skb, hlen))  in xfrm_parse_spi()
172 	*spi = *(__be32 *)(skb_transport_header(skb) + offset);  in xfrm_parse_spi()
173 	*seq = *(__be32 *)(skb_transport_header(skb) + offset_seq);  in xfrm_parse_spi()
178 static int xfrm4_remove_beet_encap(struct xfrm_state *x, struct sk_buff *skb)  in xfrm4_remove_beet_encap()  argument
184 	skb->protocol = htons(ETH_P_IP);  in xfrm4_remove_beet_encap()
186 	if (unlikely(XFRM_MODE_SKB_CB(skb)->protocol == IPPROTO_BEETPH)) {  in xfrm4_remove_beet_encap()
190 		if (!pskb_may_pull(skb, sizeof(*ph)))  in xfrm4_remove_beet_encap()
193 		ph = (struct ip_beet_phdr *)skb->data;  in xfrm4_remove_beet_encap()
200 		XFRM_MODE_SKB_CB(skb)->protocol = ph->nexthdr;  in xfrm4_remove_beet_encap()
202 		if (!pskb_may_pull(skb, phlen))  in xfrm4_remove_beet_encap()
204 		__skb_pull(skb, phlen);  in xfrm4_remove_beet_encap()
207 	skb_push(skb, sizeof(*iph));  in xfrm4_remove_beet_encap()
208 	skb_reset_network_header(skb);  in xfrm4_remove_beet_encap()
209 	skb_mac_header_rebuild(skb);  in xfrm4_remove_beet_encap()
211 	xfrm4_beet_make_header(skb);  in xfrm4_remove_beet_encap()
213 	iph = ip_hdr(skb);  in xfrm4_remove_beet_encap()
216 	iph->tot_len = htons(skb->len);  in xfrm4_remove_beet_encap()
220 	iph->check = ip_fast_csum(skb_network_header(skb), iph->ihl);  in xfrm4_remove_beet_encap()
226 static void ipip_ecn_decapsulate(struct sk_buff *skb)  in ipip_ecn_decapsulate()  argument
228 	struct iphdr *inner_iph = ipip_hdr(skb);  in ipip_ecn_decapsulate()
230 	if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos))  in ipip_ecn_decapsulate()
234 static int xfrm4_remove_tunnel_encap(struct xfrm_state *x, struct sk_buff *skb)  in xfrm4_remove_tunnel_encap()  argument
238 	skb->protocol = htons(ETH_P_IP);  in xfrm4_remove_tunnel_encap()
240 	if (!pskb_may_pull(skb, sizeof(struct iphdr)))  in xfrm4_remove_tunnel_encap()
243 	err = skb_unclone(skb, GFP_ATOMIC);  in xfrm4_remove_tunnel_encap()
248 		ipv4_copy_dscp(XFRM_MODE_SKB_CB(skb)->tos, ipip_hdr(skb));  in xfrm4_remove_tunnel_encap()
250 		ipip_ecn_decapsulate(skb);  in xfrm4_remove_tunnel_encap()
252 	skb_reset_network_header(skb);  in xfrm4_remove_tunnel_encap()
253 	skb_mac_header_rebuild(skb);  in xfrm4_remove_tunnel_encap()
254 	if (skb->mac_len)  in xfrm4_remove_tunnel_encap()
255 		eth_hdr(skb)->h_proto = skb->protocol;  in xfrm4_remove_tunnel_encap()
263 static void ipip6_ecn_decapsulate(struct sk_buff *skb)  in ipip6_ecn_decapsulate()  argument
265 	struct ipv6hdr *inner_iph = ipipv6_hdr(skb);  in ipip6_ecn_decapsulate()
267 	if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos))  in ipip6_ecn_decapsulate()
268 		IP6_ECN_set_ce(skb, inner_iph);  in ipip6_ecn_decapsulate()
271 static int xfrm6_remove_tunnel_encap(struct xfrm_state *x, struct sk_buff *skb)  in xfrm6_remove_tunnel_encap()  argument
275 	skb->protocol = htons(ETH_P_IPV6);  in xfrm6_remove_tunnel_encap()
277 	if (!pskb_may_pull(skb, sizeof(struct ipv6hdr)))  in xfrm6_remove_tunnel_encap()
280 	err = skb_unclone(skb, GFP_ATOMIC);  in xfrm6_remove_tunnel_encap()
285 		ipv6_copy_dscp(XFRM_MODE_SKB_CB(skb)->tos, ipipv6_hdr(skb));  in xfrm6_remove_tunnel_encap()
287 		ipip6_ecn_decapsulate(skb);  in xfrm6_remove_tunnel_encap()
289 	skb_reset_network_header(skb);  in xfrm6_remove_tunnel_encap()
290 	skb_mac_header_rebuild(skb);  in xfrm6_remove_tunnel_encap()
291 	if (skb->mac_len)  in xfrm6_remove_tunnel_encap()
292 		eth_hdr(skb)->h_proto = skb->protocol;  in xfrm6_remove_tunnel_encap()
300 static int xfrm6_remove_beet_encap(struct xfrm_state *x, struct sk_buff *skb)  in xfrm6_remove_beet_encap()  argument
306 	skb->protocol = htons(ETH_P_IPV6);  in xfrm6_remove_beet_encap()
308 	err = skb_cow_head(skb, size + skb->mac_len);  in xfrm6_remove_beet_encap()
312 	__skb_push(skb, size);  in xfrm6_remove_beet_encap()
313 	skb_reset_network_header(skb);  in xfrm6_remove_beet_encap()
314 	skb_mac_header_rebuild(skb);  in xfrm6_remove_beet_encap()
316 	xfrm6_beet_make_header(skb);  in xfrm6_remove_beet_encap()
318 	ip6h = ipv6_hdr(skb);  in xfrm6_remove_beet_encap()
319 	ip6h->payload_len = htons(skb->len - size);  in xfrm6_remove_beet_encap()
334  * header currently is.  skb->data shall point to the start of the
339 			     struct sk_buff *skb)  in xfrm_inner_mode_encap_remove()  argument
345 			return xfrm4_remove_beet_encap(x, skb);  in xfrm_inner_mode_encap_remove()
347 			return xfrm6_remove_beet_encap(x, skb);  in xfrm_inner_mode_encap_remove()
351 		switch (XFRM_MODE_SKB_CB(skb)->protocol) {  in xfrm_inner_mode_encap_remove()
353 			return xfrm4_remove_tunnel_encap(x, skb);  in xfrm_inner_mode_encap_remove()
355 			return xfrm6_remove_tunnel_encap(x, skb);  in xfrm_inner_mode_encap_remove()
365 static int xfrm_prepare_input(struct xfrm_state *x, struct sk_buff *skb)  in xfrm_prepare_input()  argument
369 		xfrm4_extract_header(skb);  in xfrm_prepare_input()
372 		xfrm6_extract_header(skb);  in xfrm_prepare_input()
379 	return xfrm_inner_mode_encap_remove(x, skb);  in xfrm_prepare_input()
388  * currently is.  skb->data shall point to the start of the payload.
390 static int xfrm4_transport_input(struct xfrm_state *x, struct sk_buff *skb)  in xfrm4_transport_input()  argument
392 	struct xfrm_offload *xo = xfrm_offload(skb);  in xfrm4_transport_input()
393 	int ihl = skb->data - skb_transport_header(skb);  in xfrm4_transport_input()
395 	if (skb->transport_header != skb->network_header) {  in xfrm4_transport_input()
396 		memmove(skb_transport_header(skb),  in xfrm4_transport_input()
397 			skb_network_header(skb), ihl);  in xfrm4_transport_input()
400 				skb_mac_header_was_set(skb) ? skb_mac_header_len(skb) : 0;  in xfrm4_transport_input()
401 		skb->network_header = skb->transport_header;  in xfrm4_transport_input()
403 	ip_hdr(skb)->tot_len = htons(skb->len + ihl);  in xfrm4_transport_input()
404 	skb_reset_transport_header(skb);  in xfrm4_transport_input()
408 static int xfrm6_transport_input(struct xfrm_state *x, struct sk_buff *skb)  in xfrm6_transport_input()  argument
411 	struct xfrm_offload *xo = xfrm_offload(skb);  in xfrm6_transport_input()
412 	int ihl = skb->data - skb_transport_header(skb);  in xfrm6_transport_input()
414 	if (skb->transport_header != skb->network_header) {  in xfrm6_transport_input()
415 		memmove(skb_transport_header(skb),  in xfrm6_transport_input()
416 			skb_network_header(skb), ihl);  in xfrm6_transport_input()
419 				skb_mac_header_was_set(skb) ? skb_mac_header_len(skb) : 0;  in xfrm6_transport_input()
420 		skb->network_header = skb->transport_header;  in xfrm6_transport_input()
422 	ipv6_hdr(skb)->payload_len = htons(skb->len + ihl -  in xfrm6_transport_input()
424 	skb_reset_transport_header(skb);  in xfrm6_transport_input()
433 				 struct sk_buff *skb)  in xfrm_inner_mode_input()  argument
438 		return xfrm_prepare_input(x, skb);  in xfrm_inner_mode_input()
441 			return xfrm4_transport_input(x, skb);  in xfrm_inner_mode_input()
443 			return xfrm6_transport_input(x, skb);  in xfrm_inner_mode_input()
450 			return x->mode_cbs->input(x, skb);  in xfrm_inner_mode_input()
463 int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type)  in xfrm_input()  argument
466 	struct net *net = dev_net(skb->dev);  in xfrm_input()
472 	u32 mark = skb->mark;  in xfrm_input()
478 	struct xfrm_offload *xo = xfrm_offload(skb);  in xfrm_input()
483 		x = xfrm_input_state(skb);  in xfrm_input()
493 				dev_put(skb->dev);  in xfrm_input()
506 			seq = XFRM_SKB_CB(skb)->seq.input.low;  in xfrm_input()
510 		seq = XFRM_SPI_SKB_CB(skb)->seq;  in xfrm_input()
514 			family = XFRM_SPI_SKB_CB(skb)->family;  in xfrm_input()
523 					xfrm_audit_state_icvfail(x, skb,  in xfrm_input()
539 			if (xfrm_parse_spi(skb, nexthdr, &spi, &seq)) {  in xfrm_input()
548 	family = XFRM_SPI_SKB_CB(skb)->family;  in xfrm_input()
550 	/* if tunnel is present override skb->mark value with tunnel i_key */  in xfrm_input()
553 		if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4)  in xfrm_input()
554 			mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4->parms.i_key);  in xfrm_input()
557 		if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6)  in xfrm_input()
558 			mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6->parms.i_key);  in xfrm_input()
562 	sp = secpath_set(skb);  in xfrm_input()
569 	if (!spi && xfrm_parse_spi(skb, nexthdr, &spi, &seq)) {  in xfrm_input()
570 		secpath_reset(skb);  in xfrm_input()
575 	daddr = (xfrm_address_t *)(skb_network_header(skb) +  in xfrm_input()
576 				   XFRM_SPI_SKB_CB(skb)->daddroff);  in xfrm_input()
578 		sp = skb_sec_path(skb);  in xfrm_input()
581 			secpath_reset(skb);  in xfrm_input()
588 			secpath_reset(skb);  in xfrm_input()
590 			xfrm_audit_state_notfound(skb, family, spi, seq);  in xfrm_input()
595 			secpath_reset(skb);  in xfrm_input()
597 			xfrm_audit_state_notfound(skb, family, spi, seq);  in xfrm_input()
603 		skb->mark = xfrm_smark_get(skb->mark, x);  in xfrm_input()
607 		skb_dst_force(skb);  in xfrm_input()
608 		if (!skb_dst(skb)) {  in xfrm_input()
630 		if (xfrm_replay_check(x, skb, seq)) {  in xfrm_input()
642 		if (xfrm_tunnel_check(skb, x, family)) {  in xfrm_input()
649 		XFRM_SKB_CB(skb)->seq.input.low = seq;  in xfrm_input()
650 		XFRM_SKB_CB(skb)->seq.input.hi = seq_hi;  in xfrm_input()
652 		dev_hold(skb->dev);  in xfrm_input()
655 			nexthdr = x->type_offload->input_tail(x, skb);  in xfrm_input()
657 			nexthdr = x->type->input(x, skb);  in xfrm_input()
662 		dev_put(skb->dev);  in xfrm_input()
667 				xfrm_audit_state_icvfail(x, skb,  in xfrm_input()
678 		if (xfrm_replay_recheck(x, skb, seq)) {  in xfrm_input()
685 		x->curlft.bytes += skb->len;  in xfrm_input()
691 		XFRM_MODE_SKB_CB(skb)->protocol = nexthdr;  in xfrm_input()
693 		err = xfrm_inner_mode_input(x, skb);  in xfrm_input()
713 		err = xfrm_parse_spi(skb, nexthdr, &spi, &seq);  in xfrm_input()
721 	err = xfrm_rcv_cb(skb, family, x->type->proto, 0);  in xfrm_input()
725 	nf_reset_ct(skb);  in xfrm_input()
728 		sp = skb_sec_path(skb);  in xfrm_input()
731 		if (skb_valid_dst(skb))  in xfrm_input()
732 			skb_dst_drop(skb);  in xfrm_input()
733 		gro_cells_receive(&gro_cells, skb);  in xfrm_input()
736 		xo = xfrm_offload(skb);  in xfrm_input()
744 			err = afinfo->transport_finish(skb, xfrm_gro || async);  in xfrm_input()
747 			sp = skb_sec_path(skb);  in xfrm_input()
750 			if (skb_valid_dst(skb))  in xfrm_input()
751 				skb_dst_drop(skb);  in xfrm_input()
752 			gro_cells_receive(&gro_cells, skb);  in xfrm_input()
762 	xfrm_rcv_cb(skb, family, x && x->type ? x->type->proto : nexthdr, -1);  in xfrm_input()
763 	kfree_skb(skb);  in xfrm_input()
768 int xfrm_input_resume(struct sk_buff *skb, int nexthdr)  in xfrm_input_resume()  argument
770 	return xfrm_input(skb, nexthdr, 0, -1);  in xfrm_input_resume()
778 	struct sk_buff *skb;  in xfrm_trans_reinject()  local
786 	while ((skb = __skb_dequeue(&queue)))  in xfrm_trans_reinject()
787 		XFRM_TRANS_SKB_CB(skb)->finish(XFRM_TRANS_SKB_CB(skb)->net,  in xfrm_trans_reinject()
788 					       NULL, skb);  in xfrm_trans_reinject()
792 int xfrm_trans_queue_net(struct net *net, struct sk_buff *skb,  in xfrm_trans_queue_net()  argument
803 	BUILD_BUG_ON(sizeof(struct xfrm_trans_cb) > sizeof(skb->cb));  in xfrm_trans_queue_net()
805 	XFRM_TRANS_SKB_CB(skb)->finish = finish;  in xfrm_trans_queue_net()
806 	XFRM_TRANS_SKB_CB(skb)->net = net;  in xfrm_trans_queue_net()
808 	__skb_queue_tail(&trans->queue, skb);  in xfrm_trans_queue_net()
815 int xfrm_trans_queue(struct sk_buff *skb,  in xfrm_trans_queue()  argument
819 	return xfrm_trans_queue_net(dev_net(skb->dev), skb, finish);  in xfrm_trans_queue()