1 // SPDX-License-Identifier: GPL-2.0-only
11  * 		Split up af-specific functions
36 	rcu_dereference_protected((table), lockdep_is_held(&(net)->xfrm.xfrm_state_lock))
38 	rcu_dereference_check((table), lockdep_is_held(&(net)->xfrm.xfrm_state_lock))
56 static inline bool xfrm_state_hold_rcu(struct xfrm_state __rcu *x)  in xfrm_state_hold_rcu()  argument
58 	return refcount_inc_not_zero(&x->refcnt);  in xfrm_state_hold_rcu()
67 	lockdep_assert_held(&net->xfrm.xfrm_state_lock);  in xfrm_dst_hash()
69 	return __xfrm_dst_hash(daddr, saddr, reqid, family, net->xfrm.state_hmask);  in xfrm_dst_hash()
77 	lockdep_assert_held(&net->xfrm.xfrm_state_lock);  in xfrm_src_hash()
79 	return __xfrm_src_hash(daddr, saddr, family, net->xfrm.state_hmask);  in xfrm_src_hash()
86 	lockdep_assert_held(&net->xfrm.xfrm_state_lock);  in xfrm_spi_hash()
88 	return __xfrm_spi_hash(daddr, spi, proto, family, net->xfrm.state_hmask);  in xfrm_spi_hash()
93 	lockdep_assert_held(&net->xfrm.xfrm_state_lock);  in xfrm_seq_hash()
95 	return __xfrm_seq_hash(seq, net->xfrm.state_hmask);  in xfrm_seq_hash()
104 				if (_x->xso.type == XFRM_DEV_OFFLOAD_PACKET) \
110 		if (!_x || _x->xso.type == XFRM_DEV_OFFLOAD_PACKET)        \
114 			hlist_add_before_rcu(_n, &_x->by);                 \
125 	struct xfrm_state *x;  in xfrm_hash_transfer()  local
127 	hlist_for_each_entry_safe(x, tmp, list, bydst) {  in xfrm_hash_transfer()
130 		h = __xfrm_dst_hash(&x->id.daddr, &x->props.saddr,  in xfrm_hash_transfer()
131 				    x->props.reqid, x->props.family,  in xfrm_hash_transfer()
133 		XFRM_STATE_INSERT(bydst, &x->bydst, ndsttable + h, x->xso.type);  in xfrm_hash_transfer()
135 		h = __xfrm_src_hash(&x->id.daddr, &x->props.saddr,  in xfrm_hash_transfer()
136 				    x->props.family,  in xfrm_hash_transfer()
138 		XFRM_STATE_INSERT(bysrc, &x->bysrc, nsrctable + h, x->xso.type);  in xfrm_hash_transfer()
140 		if (x->id.spi) {  in xfrm_hash_transfer()
141 			h = __xfrm_spi_hash(&x->id.daddr, x->id.spi,  in xfrm_hash_transfer()
142 					    x->id.proto, x->props.family,  in xfrm_hash_transfer()
144 			XFRM_STATE_INSERT(byspi, &x->byspi, nspitable + h,  in xfrm_hash_transfer()
145 					  x->xso.type);  in xfrm_hash_transfer()
148 		if (x->km.seq) {  in xfrm_hash_transfer()
149 			h = __xfrm_seq_hash(x->km.seq, nhashmask);  in xfrm_hash_transfer()
150 			XFRM_STATE_INSERT(byseq, &x->byseq, nseqtable + h,  in xfrm_hash_transfer()
151 					  x->xso.type);  in xfrm_hash_transfer()
169 	nsize = xfrm_hash_new_size(net->xfrm.state_hmask);  in xfrm_hash_resize()
192 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_hash_resize()
193 	write_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);  in xfrm_hash_resize()
195 	nhashmask = (nsize / sizeof(struct hlist_head)) - 1U;  in xfrm_hash_resize()
196 	odst = xfrm_state_deref_prot(net->xfrm.state_bydst, net);  in xfrm_hash_resize()
197 	for (i = net->xfrm.state_hmask; i >= 0; i--)  in xfrm_hash_resize()
200 	osrc = xfrm_state_deref_prot(net->xfrm.state_bysrc, net);  in xfrm_hash_resize()
201 	ospi = xfrm_state_deref_prot(net->xfrm.state_byspi, net);  in xfrm_hash_resize()
202 	oseq = xfrm_state_deref_prot(net->xfrm.state_byseq, net);  in xfrm_hash_resize()
203 	ohashmask = net->xfrm.state_hmask;  in xfrm_hash_resize()
205 	rcu_assign_pointer(net->xfrm.state_bydst, ndst);  in xfrm_hash_resize()
206 	rcu_assign_pointer(net->xfrm.state_bysrc, nsrc);  in xfrm_hash_resize()
207 	rcu_assign_pointer(net->xfrm.state_byspi, nspi);  in xfrm_hash_resize()
208 	rcu_assign_pointer(net->xfrm.state_byseq, nseq);  in xfrm_hash_resize()
209 	net->xfrm.state_hmask = nhashmask;  in xfrm_hash_resize()
211 	write_seqcount_end(&net->xfrm.xfrm_state_hash_generation);  in xfrm_hash_resize()
212 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_hash_resize()
230 int __xfrm_state_delete(struct xfrm_state *x);
232 int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol);
234 void km_state_expired(struct xfrm_state *x, int hard, u32 portid);
242 		return -EAFNOSUPPORT;  in xfrm_register_type()
244 #define X(afi, T, name) do {			\  in xfrm_register_type()  macro
245 		WARN_ON((afi)->type_ ## name);	\  in xfrm_register_type()
246 		(afi)->type_ ## name = (T);	\  in xfrm_register_type()
249 	switch (type->proto) {  in xfrm_register_type()
251 		X(afinfo, type, comp);  in xfrm_register_type()
254 		X(afinfo, type, ah);  in xfrm_register_type()
257 		X(afinfo, type, esp);  in xfrm_register_type()
260 		X(afinfo, type, ipip);  in xfrm_register_type()
263 		X(afinfo, type, dstopts);  in xfrm_register_type()
266 		X(afinfo, type, routing);  in xfrm_register_type()
269 		X(afinfo, type, ipip6);  in xfrm_register_type()
273 		err = -EPROTONOSUPPORT;  in xfrm_register_type()
276 #undef X  in xfrm_register_type()
289 #define X(afi, T, name) do {				\  in xfrm_unregister_type()  macro
290 		WARN_ON((afi)->type_ ## name != (T));	\  in xfrm_unregister_type()
291 		(afi)->type_ ## name = NULL;		\  in xfrm_unregister_type()
294 	switch (type->proto) {  in xfrm_unregister_type()
296 		X(afinfo, type, comp);  in xfrm_unregister_type()
299 		X(afinfo, type, ah);  in xfrm_unregister_type()
302 		X(afinfo, type, esp);  in xfrm_unregister_type()
305 		X(afinfo, type, ipip);  in xfrm_unregister_type()
308 		X(afinfo, type, dstopts);  in xfrm_unregister_type()
311 		X(afinfo, type, routing);  in xfrm_unregister_type()
314 		X(afinfo, type, ipip6);  in xfrm_unregister_type()
320 #undef X  in xfrm_unregister_type()
338 		type = afinfo->type_comp;  in xfrm_get_type()
341 		type = afinfo->type_ah;  in xfrm_get_type()
344 		type = afinfo->type_esp;  in xfrm_get_type()
347 		type = afinfo->type_ipip;  in xfrm_get_type()
350 		type = afinfo->type_dstopts;  in xfrm_get_type()
353 		type = afinfo->type_routing;  in xfrm_get_type()
356 		type = afinfo->type_ipip6;  in xfrm_get_type()
362 	if (unlikely(type && !try_module_get(type->owner)))  in xfrm_get_type()
368 		request_module("xfrm-type-%d-%d", family, proto);  in xfrm_get_type()
378 	module_put(type->owner);  in xfrm_put_type()
388 		return -EAFNOSUPPORT;  in xfrm_register_type_offload()
390 	switch (type->proto) {  in xfrm_register_type_offload()
392 		WARN_ON(afinfo->type_offload_esp);  in xfrm_register_type_offload()
393 		afinfo->type_offload_esp = type;  in xfrm_register_type_offload()
397 		err = -EPROTONOSUPPORT;  in xfrm_register_type_offload()
414 	switch (type->proto) {  in xfrm_unregister_type_offload()
416 		WARN_ON(afinfo->type_offload_esp != type);  in xfrm_unregister_type_offload()
417 		afinfo->type_offload_esp = NULL;  in xfrm_unregister_type_offload()
427 void xfrm_set_type_offload(struct xfrm_state *x)  in xfrm_set_type_offload()  argument
434 	afinfo = xfrm_state_get_afinfo(x->props.family);  in xfrm_set_type_offload()
438 	switch (x->id.proto) {  in xfrm_set_type_offload()
440 		type = afinfo->type_offload_esp;  in xfrm_set_type_offload()
446 	if ((type && !try_module_get(type->owner)))  in xfrm_set_type_offload()
452 		request_module("xfrm-offload-%d-%d", x->props.family,  in xfrm_set_type_offload()
453 			       x->id.proto);  in xfrm_set_type_offload()
459 	x->type_offload = type;  in xfrm_set_type_offload()
521 		if (mode->family == family)  in xfrm_get_mode()
526 		if (mode->family == family)  in xfrm_get_mode()
542 		return -EINVAL;  in xfrm_register_mode_cbs()
576 	if (cbs && !try_module_get(cbs->owner))  in xfrm_get_mode_cbs()
582 		request_module("xfrm-iptfs");  in xfrm_get_mode_cbs()
590 void xfrm_state_free(struct xfrm_state *x)  in xfrm_state_free()  argument
592 	kmem_cache_free(xfrm_state_cache, x);  in xfrm_state_free()
596 static void ___xfrm_state_destroy(struct xfrm_state *x)  in ___xfrm_state_destroy()  argument
598 	if (x->mode_cbs && x->mode_cbs->destroy_state)  in ___xfrm_state_destroy()
599 		x->mode_cbs->destroy_state(x);  in ___xfrm_state_destroy()
600 	hrtimer_cancel(&x->mtimer);  in ___xfrm_state_destroy()
601 	del_timer_sync(&x->rtimer);  in ___xfrm_state_destroy()
602 	kfree(x->aead);  in ___xfrm_state_destroy()
603 	kfree(x->aalg);  in ___xfrm_state_destroy()
604 	kfree(x->ealg);  in ___xfrm_state_destroy()
605 	kfree(x->calg);  in ___xfrm_state_destroy()
606 	kfree(x->encap);  in ___xfrm_state_destroy()
607 	kfree(x->coaddr);  in ___xfrm_state_destroy()
608 	kfree(x->replay_esn);  in ___xfrm_state_destroy()
609 	kfree(x->preplay_esn);  in ___xfrm_state_destroy()
610 	if (x->type) {  in ___xfrm_state_destroy()
611 		x->type->destructor(x);  in ___xfrm_state_destroy()
612 		xfrm_put_type(x->type);  in ___xfrm_state_destroy()
614 	if (x->xfrag.page)  in ___xfrm_state_destroy()
615 		put_page(x->xfrag.page);  in ___xfrm_state_destroy()
616 	xfrm_dev_state_free(x);  in ___xfrm_state_destroy()
617 	security_xfrm_state_free(x);  in ___xfrm_state_destroy()
618 	xfrm_state_free(x);  in ___xfrm_state_destroy()
623 	struct xfrm_state *x;  in xfrm_state_gc_task()  local
633 	hlist_for_each_entry_safe(x, tmp, &gc_list, gclist)  in xfrm_state_gc_task()
634 		___xfrm_state_destroy(x);  in xfrm_state_gc_task()
639 	struct xfrm_state *x = container_of(me, struct xfrm_state, mtimer);  in xfrm_timer_handler()  local
646 	spin_lock(&x->lock);  in xfrm_timer_handler()
647 	xfrm_dev_state_update_stats(x);  in xfrm_timer_handler()
649 	if (x->km.state == XFRM_STATE_DEAD)  in xfrm_timer_handler()
651 	if (x->km.state == XFRM_STATE_EXPIRED)  in xfrm_timer_handler()
653 	if (x->lft.hard_add_expires_seconds) {  in xfrm_timer_handler()
654 		time64_t tmo = x->lft.hard_add_expires_seconds +  in xfrm_timer_handler()
655 			x->curlft.add_time - now;  in xfrm_timer_handler()
657 			if (x->xflags & XFRM_SOFT_EXPIRE) {  in xfrm_timer_handler()
660 				 * workaround: fix x->curflt.add_time by below:  in xfrm_timer_handler()
662 				x->curlft.add_time = now - x->saved_tmo - 1;  in xfrm_timer_handler()
663 				tmo = x->lft.hard_add_expires_seconds - x->saved_tmo;  in xfrm_timer_handler()
670 	if (x->lft.hard_use_expires_seconds) {  in xfrm_timer_handler()
671 		time64_t tmo = x->lft.hard_use_expires_seconds +  in xfrm_timer_handler()
672 			(READ_ONCE(x->curlft.use_time) ? : now) - now;  in xfrm_timer_handler()
678 	if (x->km.dying)  in xfrm_timer_handler()
680 	if (x->lft.soft_add_expires_seconds) {  in xfrm_timer_handler()
681 		time64_t tmo = x->lft.soft_add_expires_seconds +  in xfrm_timer_handler()
682 			x->curlft.add_time - now;  in xfrm_timer_handler()
685 			x->xflags &= ~XFRM_SOFT_EXPIRE;  in xfrm_timer_handler()
688 			x->xflags |= XFRM_SOFT_EXPIRE;  in xfrm_timer_handler()
689 			x->saved_tmo = tmo;  in xfrm_timer_handler()
692 	if (x->lft.soft_use_expires_seconds) {  in xfrm_timer_handler()
693 		time64_t tmo = x->lft.soft_use_expires_seconds +  in xfrm_timer_handler()
694 			(READ_ONCE(x->curlft.use_time) ? : now) - now;  in xfrm_timer_handler()
701 	x->km.dying = warn;  in xfrm_timer_handler()
703 		km_state_expired(x, 0, 0);  in xfrm_timer_handler()
706 		hrtimer_forward_now(&x->mtimer, ktime_set(next, 0));  in xfrm_timer_handler()
713 	if (x->km.state == XFRM_STATE_ACQ && x->id.spi == 0)  in xfrm_timer_handler()
714 		x->km.state = XFRM_STATE_EXPIRED;  in xfrm_timer_handler()
716 	err = __xfrm_state_delete(x);  in xfrm_timer_handler()
718 		km_state_expired(x, 1, 0);  in xfrm_timer_handler()
720 	xfrm_audit_state_delete(x, err ? 0 : 1, true);  in xfrm_timer_handler()
723 	spin_unlock(&x->lock);  in xfrm_timer_handler()
731 	struct xfrm_state *x;  in xfrm_state_alloc()  local
733 	x = kmem_cache_zalloc(xfrm_state_cache, GFP_ATOMIC);  in xfrm_state_alloc()
735 	if (x) {  in xfrm_state_alloc()
736 		write_pnet(&x->xs_net, net);  in xfrm_state_alloc()
737 		refcount_set(&x->refcnt, 1);  in xfrm_state_alloc()
738 		atomic_set(&x->tunnel_users, 0);  in xfrm_state_alloc()
739 		INIT_LIST_HEAD(&x->km.all);  in xfrm_state_alloc()
740 		INIT_HLIST_NODE(&x->state_cache);  in xfrm_state_alloc()
741 		INIT_HLIST_NODE(&x->bydst);  in xfrm_state_alloc()
742 		INIT_HLIST_NODE(&x->bysrc);  in xfrm_state_alloc()
743 		INIT_HLIST_NODE(&x->byspi);  in xfrm_state_alloc()
744 		INIT_HLIST_NODE(&x->byseq);  in xfrm_state_alloc()
745 		hrtimer_init(&x->mtimer, CLOCK_BOOTTIME, HRTIMER_MODE_ABS_SOFT);  in xfrm_state_alloc()
746 		x->mtimer.function = xfrm_timer_handler;  in xfrm_state_alloc()
747 		timer_setup(&x->rtimer, xfrm_replay_timer_handler, 0);  in xfrm_state_alloc()
748 		x->curlft.add_time = ktime_get_real_seconds();  in xfrm_state_alloc()
749 		x->lft.soft_byte_limit = XFRM_INF;  in xfrm_state_alloc()
750 		x->lft.soft_packet_limit = XFRM_INF;  in xfrm_state_alloc()
751 		x->lft.hard_byte_limit = XFRM_INF;  in xfrm_state_alloc()
752 		x->lft.hard_packet_limit = XFRM_INF;  in xfrm_state_alloc()
753 		x->replay_maxage = 0;  in xfrm_state_alloc()
754 		x->replay_maxdiff = 0;  in xfrm_state_alloc()
755 		x->pcpu_num = UINT_MAX;  in xfrm_state_alloc()
756 		spin_lock_init(&x->lock);  in xfrm_state_alloc()
757 		x->mode_data = NULL;  in xfrm_state_alloc()
759 	return x;  in xfrm_state_alloc()
764 void xfrm_dev_state_delete(struct xfrm_state *x)  in xfrm_dev_state_delete()  argument
766 	struct xfrm_dev_offload *xso = &x->xso;  in xfrm_dev_state_delete()
767 	struct net_device *dev = READ_ONCE(xso->dev);  in xfrm_dev_state_delete()
770 		dev->xfrmdev_ops->xdo_dev_state_delete(x);  in xfrm_dev_state_delete()
772 		hlist_add_head(&x->dev_gclist, &xfrm_state_dev_gc_list);  in xfrm_dev_state_delete()
778 void xfrm_dev_state_free(struct xfrm_state *x)  in xfrm_dev_state_free()  argument
780 	struct xfrm_dev_offload *xso = &x->xso;  in xfrm_dev_state_free()
781 	struct net_device *dev = READ_ONCE(xso->dev);  in xfrm_dev_state_free()
783 	xfrm_unset_type_offload(x);  in xfrm_dev_state_free()
785 	if (dev && dev->xfrmdev_ops) {  in xfrm_dev_state_free()
787 		if (!hlist_unhashed(&x->dev_gclist))  in xfrm_dev_state_free()
788 			hlist_del(&x->dev_gclist);  in xfrm_dev_state_free()
791 		if (dev->xfrmdev_ops->xdo_dev_state_free)  in xfrm_dev_state_free()
792 			dev->xfrmdev_ops->xdo_dev_state_free(x);  in xfrm_dev_state_free()
793 		WRITE_ONCE(xso->dev, NULL);  in xfrm_dev_state_free()
794 		xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;  in xfrm_dev_state_free()
795 		netdev_put(dev, &xso->dev_tracker);  in xfrm_dev_state_free()
800 void __xfrm_state_destroy(struct xfrm_state *x, bool sync)  in __xfrm_state_destroy()  argument
802 	WARN_ON(x->km.state != XFRM_STATE_DEAD);  in __xfrm_state_destroy()
806 		___xfrm_state_destroy(x);  in __xfrm_state_destroy()
809 		hlist_add_head(&x->gclist, &xfrm_state_gc_list);  in __xfrm_state_destroy()
816 int __xfrm_state_delete(struct xfrm_state *x)  in __xfrm_state_delete()  argument
818 	struct net *net = xs_net(x);  in __xfrm_state_delete()
819 	int err = -ESRCH;  in __xfrm_state_delete()
821 	if (x->km.state != XFRM_STATE_DEAD) {  in __xfrm_state_delete()
822 		x->km.state = XFRM_STATE_DEAD;  in __xfrm_state_delete()
824 		spin_lock(&net->xfrm.xfrm_state_lock);  in __xfrm_state_delete()
825 		list_del(&x->km.all);  in __xfrm_state_delete()
826 		hlist_del_rcu(&x->bydst);  in __xfrm_state_delete()
827 		hlist_del_rcu(&x->bysrc);  in __xfrm_state_delete()
828 		if (x->km.seq)  in __xfrm_state_delete()
829 			hlist_del_rcu(&x->byseq);  in __xfrm_state_delete()
830 		if (!hlist_unhashed(&x->state_cache))  in __xfrm_state_delete()
831 			hlist_del_rcu(&x->state_cache);  in __xfrm_state_delete()
832 		if (!hlist_unhashed(&x->state_cache_input))  in __xfrm_state_delete()
833 			hlist_del_rcu(&x->state_cache_input);  in __xfrm_state_delete()
835 		if (x->id.spi)  in __xfrm_state_delete()
836 			hlist_del_rcu(&x->byspi);  in __xfrm_state_delete()
837 		net->xfrm.state_num--;  in __xfrm_state_delete()
838 		xfrm_nat_keepalive_state_updated(x);  in __xfrm_state_delete()
839 		spin_unlock(&net->xfrm.xfrm_state_lock);  in __xfrm_state_delete()
841 		if (x->encap_sk)  in __xfrm_state_delete()
842 			sock_put(rcu_dereference_raw(x->encap_sk));  in __xfrm_state_delete()
844 		xfrm_dev_state_delete(x);  in __xfrm_state_delete()
850 		xfrm_state_put(x);  in __xfrm_state_delete()
858 int xfrm_state_delete(struct xfrm_state *x)  in xfrm_state_delete()  argument
862 	spin_lock_bh(&x->lock);  in xfrm_state_delete()
863 	err = __xfrm_state_delete(x);  in xfrm_state_delete()
864 	spin_unlock_bh(&x->lock);  in xfrm_state_delete()
876 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_state_flush_secctx_check()
877 		struct xfrm_state *x;  in xfrm_state_flush_secctx_check()  local
879 		hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {  in xfrm_state_flush_secctx_check()
880 			if (xfrm_id_proto_match(x->id.proto, proto) &&  in xfrm_state_flush_secctx_check()
881 			   (err = security_xfrm_state_delete(x)) != 0) {  in xfrm_state_flush_secctx_check()
882 				xfrm_audit_state_delete(x, 0, task_valid);  in xfrm_state_flush_secctx_check()
896 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_dev_state_flush_secctx_check()
897 		struct xfrm_state *x;  in xfrm_dev_state_flush_secctx_check()  local
900 		hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {  in xfrm_dev_state_flush_secctx_check()
901 			xso = &x->xso;  in xfrm_dev_state_flush_secctx_check()
903 			if (xso->dev == dev &&  in xfrm_dev_state_flush_secctx_check()
904 			   (err = security_xfrm_state_delete(x)) != 0) {  in xfrm_dev_state_flush_secctx_check()
905 				xfrm_audit_state_delete(x, 0, task_valid);  in xfrm_dev_state_flush_secctx_check()
931 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_flush()
936 	err = -ESRCH;  in xfrm_state_flush()
937 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_state_flush()
938 		struct xfrm_state *x;  in xfrm_state_flush()  local
940 		hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {  in xfrm_state_flush()
941 			if (!xfrm_state_kern(x) &&  in xfrm_state_flush()
942 			    xfrm_id_proto_match(x->id.proto, proto)) {  in xfrm_state_flush()
943 				xfrm_state_hold(x);  in xfrm_state_flush()
944 				spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_flush()
946 				err = xfrm_state_delete(x);  in xfrm_state_flush()
947 				xfrm_audit_state_delete(x, err ? 0 : 1,  in xfrm_state_flush()
950 					xfrm_state_put_sync(x);  in xfrm_state_flush()
952 					xfrm_state_put(x);  in xfrm_state_flush()
956 				spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_flush()
962 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_flush()
972 	struct xfrm_state *x;  in xfrm_dev_state_flush()  local
977 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_dev_state_flush()
982 	err = -ESRCH;  in xfrm_dev_state_flush()
983 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_dev_state_flush()
985 		hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {  in xfrm_dev_state_flush()
986 			xso = &x->xso;  in xfrm_dev_state_flush()
988 			if (!xfrm_state_kern(x) && xso->dev == dev) {  in xfrm_dev_state_flush()
989 				xfrm_state_hold(x);  in xfrm_dev_state_flush()
990 				spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_dev_state_flush()
992 				err = xfrm_state_delete(x);  in xfrm_dev_state_flush()
993 				xfrm_dev_state_free(x);  in xfrm_dev_state_flush()
995 				xfrm_audit_state_delete(x, err ? 0 : 1,  in xfrm_dev_state_flush()
997 				xfrm_state_put(x);  in xfrm_dev_state_flush()
1001 				spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_dev_state_flush()
1010 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_dev_state_flush()
1014 	hlist_for_each_entry_safe(x, tmp, &xfrm_state_dev_gc_list, dev_gclist) {  in xfrm_dev_state_flush()
1015 		xso = &x->xso;  in xfrm_dev_state_flush()
1017 		if (xso->dev == dev) {  in xfrm_dev_state_flush()
1019 			xfrm_dev_state_free(x);  in xfrm_dev_state_flush()
1035 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_sad_getinfo()
1036 	si->sadcnt = net->xfrm.state_num;  in xfrm_sad_getinfo()
1037 	si->sadhcnt = net->xfrm.state_hmask + 1;  in xfrm_sad_getinfo()
1038 	si->sadhmcnt = xfrm_state_hashmax;  in xfrm_sad_getinfo()
1039 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_sad_getinfo()
1046 	const struct flowi4 *fl4 = &fl->u.ip4;  in __xfrm4_init_tempsel()
1048 	sel->daddr.a4 = fl4->daddr;  in __xfrm4_init_tempsel()
1049 	sel->saddr.a4 = fl4->saddr;  in __xfrm4_init_tempsel()
1050 	sel->dport = xfrm_flowi_dport(fl, &fl4->uli);  in __xfrm4_init_tempsel()
1051 	sel->dport_mask = htons(0xffff);  in __xfrm4_init_tempsel()
1052 	sel->sport = xfrm_flowi_sport(fl, &fl4->uli);  in __xfrm4_init_tempsel()
1053 	sel->sport_mask = htons(0xffff);  in __xfrm4_init_tempsel()
1054 	sel->family = AF_INET;  in __xfrm4_init_tempsel()
1055 	sel->prefixlen_d = 32;  in __xfrm4_init_tempsel()
1056 	sel->prefixlen_s = 32;  in __xfrm4_init_tempsel()
1057 	sel->proto = fl4->flowi4_proto;  in __xfrm4_init_tempsel()
1058 	sel->ifindex = fl4->flowi4_oif;  in __xfrm4_init_tempsel()
1064 	const struct flowi6 *fl6 = &fl->u.ip6;  in __xfrm6_init_tempsel()
1067 	*(struct in6_addr *)&sel->daddr = fl6->daddr;  in __xfrm6_init_tempsel()
1068 	*(struct in6_addr *)&sel->saddr = fl6->saddr;  in __xfrm6_init_tempsel()
1069 	sel->dport = xfrm_flowi_dport(fl, &fl6->uli);  in __xfrm6_init_tempsel()
1070 	sel->dport_mask = htons(0xffff);  in __xfrm6_init_tempsel()
1071 	sel->sport = xfrm_flowi_sport(fl, &fl6->uli);  in __xfrm6_init_tempsel()
1072 	sel->sport_mask = htons(0xffff);  in __xfrm6_init_tempsel()
1073 	sel->family = AF_INET6;  in __xfrm6_init_tempsel()
1074 	sel->prefixlen_d = 128;  in __xfrm6_init_tempsel()
1075 	sel->prefixlen_s = 128;  in __xfrm6_init_tempsel()
1076 	sel->proto = fl6->flowi6_proto;  in __xfrm6_init_tempsel()
1077 	sel->ifindex = fl6->flowi6_oif;  in __xfrm6_init_tempsel()
1081 xfrm_init_tempstate(struct xfrm_state *x, const struct flowi *fl,  in xfrm_init_tempstate()  argument
1088 		__xfrm4_init_tempsel(&x->sel, fl);  in xfrm_init_tempstate()
1091 		__xfrm6_init_tempsel(&x->sel, fl);  in xfrm_init_tempstate()
1095 	x->id = tmpl->id;  in xfrm_init_tempstate()
1097 	switch (tmpl->encap_family) {  in xfrm_init_tempstate()
1099 		if (x->id.daddr.a4 == 0)  in xfrm_init_tempstate()
1100 			x->id.daddr.a4 = daddr->a4;  in xfrm_init_tempstate()
1101 		x->props.saddr = tmpl->saddr;  in xfrm_init_tempstate()
1102 		if (x->props.saddr.a4 == 0)  in xfrm_init_tempstate()
1103 			x->props.saddr.a4 = saddr->a4;  in xfrm_init_tempstate()
1106 		if (ipv6_addr_any((struct in6_addr *)&x->id.daddr))  in xfrm_init_tempstate()
1107 			memcpy(&x->id.daddr, daddr, sizeof(x->sel.daddr));  in xfrm_init_tempstate()
1108 		memcpy(&x->props.saddr, &tmpl->saddr, sizeof(x->props.saddr));  in xfrm_init_tempstate()
1109 		if (ipv6_addr_any((struct in6_addr *)&x->props.saddr))  in xfrm_init_tempstate()
1110 			memcpy(&x->props.saddr, saddr, sizeof(x->props.saddr));  in xfrm_init_tempstate()
1114 	x->props.mode = tmpl->mode;  in xfrm_init_tempstate()
1115 	x->props.reqid = tmpl->reqid;  in xfrm_init_tempstate()
1116 	x->props.family = tmpl->encap_family;  in xfrm_init_tempstate()
1131 		sequence = read_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);  in xfrm_hash_ptrs_get()
1133 		ptrs->bydst = xfrm_state_deref_check(net->xfrm.state_bydst, net);  in xfrm_hash_ptrs_get()
1134 		ptrs->bysrc = xfrm_state_deref_check(net->xfrm.state_bysrc, net);  in xfrm_hash_ptrs_get()
1135 		ptrs->byspi = xfrm_state_deref_check(net->xfrm.state_byspi, net);  in xfrm_hash_ptrs_get()
1136 		ptrs->hmask = net->xfrm.state_hmask;  in xfrm_hash_ptrs_get()
1137 	} while (read_seqcount_retry(&net->xfrm.xfrm_state_hash_generation, sequence));  in xfrm_hash_ptrs_get()
1147 	unsigned int h = __xfrm_spi_hash(daddr, spi, proto, family, state_ptrs->hmask);  in __xfrm_state_lookup_all()
1148 	struct xfrm_state *x;  in __xfrm_state_lookup_all()  local
1150 	hlist_for_each_entry_rcu(x, state_ptrs->byspi + h, byspi) {  in __xfrm_state_lookup_all()
1152 		if (xdo->type == XFRM_DEV_OFFLOAD_PACKET) {  in __xfrm_state_lookup_all()
1153 			if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET)  in __xfrm_state_lookup_all()
1162 			if (xdo->dev != x->xso.dev)  in __xfrm_state_lookup_all()
1164 		} else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)  in __xfrm_state_lookup_all()
1168 		if (x->props.family != family ||  in __xfrm_state_lookup_all()
1169 		    x->id.spi       != spi ||  in __xfrm_state_lookup_all()
1170 		    x->id.proto     != proto ||  in __xfrm_state_lookup_all()
1171 		    !xfrm_addr_equal(&x->id.daddr, daddr, family))  in __xfrm_state_lookup_all()
1174 		if ((mark & x->mark.m) != x->mark.v)  in __xfrm_state_lookup_all()
1176 		if (!xfrm_state_hold_rcu(x))  in __xfrm_state_lookup_all()
1178 		return x;  in __xfrm_state_lookup_all()
1190 	unsigned int h = __xfrm_spi_hash(daddr, spi, proto, family, state_ptrs->hmask);  in __xfrm_state_lookup()
1191 	struct xfrm_state *x;  in __xfrm_state_lookup()  local
1193 	hlist_for_each_entry_rcu(x, state_ptrs->byspi + h, byspi) {  in __xfrm_state_lookup()
1194 		if (x->props.family != family ||  in __xfrm_state_lookup()
1195 		    x->id.spi       != spi ||  in __xfrm_state_lookup()
1196 		    x->id.proto     != proto ||  in __xfrm_state_lookup()
1197 		    !xfrm_addr_equal(&x->id.daddr, daddr, family))  in __xfrm_state_lookup()
1200 		if ((mark & x->mark.m) != x->mark.v)  in __xfrm_state_lookup()
1202 		if (!xfrm_state_hold_rcu(x))  in __xfrm_state_lookup()
1204 		return x;  in __xfrm_state_lookup()
1217 	struct xfrm_state *x = NULL;  in xfrm_input_state_lookup()  local
1219 	state_cache_input = raw_cpu_ptr(net->xfrm.state_cache_input);  in xfrm_input_state_lookup()
1222 	hlist_for_each_entry_rcu(x, state_cache_input, state_cache_input) {  in xfrm_input_state_lookup()
1223 		if (x->props.family != family ||  in xfrm_input_state_lookup()
1224 		    x->id.spi       != spi ||  in xfrm_input_state_lookup()
1225 		    x->id.proto     != proto ||  in xfrm_input_state_lookup()
1226 		    !xfrm_addr_equal(&x->id.daddr, daddr, family))  in xfrm_input_state_lookup()
1229 		if ((mark & x->mark.m) != x->mark.v)  in xfrm_input_state_lookup()
1231 		if (!xfrm_state_hold_rcu(x))  in xfrm_input_state_lookup()
1238 	x = __xfrm_state_lookup(&state_ptrs, mark, daddr, spi, proto, family);  in xfrm_input_state_lookup()
1240 	if (x && x->km.state == XFRM_STATE_VALID) {  in xfrm_input_state_lookup()
1241 		spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_input_state_lookup()
1242 		if (hlist_unhashed(&x->state_cache_input)) {  in xfrm_input_state_lookup()
1243 			hlist_add_head_rcu(&x->state_cache_input, state_cache_input);  in xfrm_input_state_lookup()
1245 			hlist_del_rcu(&x->state_cache_input);  in xfrm_input_state_lookup()
1246 			hlist_add_head_rcu(&x->state_cache_input, state_cache_input);  in xfrm_input_state_lookup()
1248 		spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_input_state_lookup()
1253 	return x;  in xfrm_input_state_lookup()
1263 	unsigned int h = __xfrm_src_hash(daddr, saddr, family, state_ptrs->hmask);  in __xfrm_state_lookup_byaddr()
1264 	struct xfrm_state *x;  in __xfrm_state_lookup_byaddr()  local
1266 	hlist_for_each_entry_rcu(x, state_ptrs->bysrc + h, bysrc) {  in __xfrm_state_lookup_byaddr()
1267 		if (x->props.family != family ||  in __xfrm_state_lookup_byaddr()
1268 		    x->id.proto     != proto ||  in __xfrm_state_lookup_byaddr()
1269 		    !xfrm_addr_equal(&x->id.daddr, daddr, family) ||  in __xfrm_state_lookup_byaddr()
1270 		    !xfrm_addr_equal(&x->props.saddr, saddr, family))  in __xfrm_state_lookup_byaddr()
1273 		if ((mark & x->mark.m) != x->mark.v)  in __xfrm_state_lookup_byaddr()
1275 		if (!xfrm_state_hold_rcu(x))  in __xfrm_state_lookup_byaddr()
1277 		return x;  in __xfrm_state_lookup_byaddr()
1284 __xfrm_state_locate(struct xfrm_state *x, int use_spi, int family)  in __xfrm_state_locate()  argument
1287 	struct net *net = xs_net(x);  in __xfrm_state_locate()
1288 	u32 mark = x->mark.v & x->mark.m;  in __xfrm_state_locate()
1293 		return __xfrm_state_lookup(&state_ptrs, mark, &x->id.daddr,  in __xfrm_state_locate()
1294 					   x->id.spi, x->id.proto, family);  in __xfrm_state_locate()
1297 						  &x->id.daddr,  in __xfrm_state_locate()
1298 						  &x->props.saddr,  in __xfrm_state_locate()
1299 						  x->id.proto, family);  in __xfrm_state_locate()
1305 	    (net->xfrm.state_hmask + 1) < xfrm_state_hashmax &&  in xfrm_hash_grow_check()
1306 	    net->xfrm.state_num > net->xfrm.state_hmask)  in xfrm_hash_grow_check()
1307 		schedule_work(&net->xfrm.state_hash_work);  in xfrm_hash_grow_check()
1310 static void xfrm_state_look_at(struct xfrm_policy *pol, struct xfrm_state *x,  in xfrm_state_look_at()  argument
1332 	if (x->km.state == XFRM_STATE_VALID) {  in xfrm_state_look_at()
1333 		if ((x->sel.family &&  in xfrm_state_look_at()
1334 		     (x->sel.family != family ||  in xfrm_state_look_at()
1335 		      !xfrm_selector_match(&x->sel, fl, family))) ||  in xfrm_state_look_at()
1336 		    !security_xfrm_state_pol_flow_match(x, pol,  in xfrm_state_look_at()
1337 							&fl->u.__fl_common))  in xfrm_state_look_at()
1340 		if (x->pcpu_num != UINT_MAX && x->pcpu_num != pcpu_id)  in xfrm_state_look_at()
1344 		    ((*best)->pcpu_num == UINT_MAX && x->pcpu_num == pcpu_id) ||  in xfrm_state_look_at()
1345 		    (*best)->km.dying > x->km.dying ||  in xfrm_state_look_at()
1346 		    ((*best)->km.dying == x->km.dying &&  in xfrm_state_look_at()
1347 		     (*best)->curlft.add_time < x->curlft.add_time))  in xfrm_state_look_at()
1348 			*best = x;  in xfrm_state_look_at()
1349 	} else if (x->km.state == XFRM_STATE_ACQ) {  in xfrm_state_look_at()
1350 		if (!*best || x->pcpu_num == pcpu_id)  in xfrm_state_look_at()
1352 	} else if (x->km.state == XFRM_STATE_ERROR ||  in xfrm_state_look_at()
1353 		   x->km.state == XFRM_STATE_EXPIRED) {  in xfrm_state_look_at()
1354 		if ((!x->sel.family ||  in xfrm_state_look_at()
1355 		     (x->sel.family == family &&  in xfrm_state_look_at()
1356 		      xfrm_selector_match(&x->sel, fl, family))) &&  in xfrm_state_look_at()
1357 		    security_xfrm_state_pol_flow_match(x, pol,  in xfrm_state_look_at()
1358 						       &fl->u.__fl_common))  in xfrm_state_look_at()
1359 			*error = -ESRCH;  in xfrm_state_look_at()
1373 	struct xfrm_state *x, *x0, *to_put;  in xfrm_state_find()  local
1377 	u32 mark = pol->mark.v & pol->mark.m;  in xfrm_state_find()
1378 	unsigned short encap_family = tmpl->encap_family;  in xfrm_state_find()
1392 	sequence = read_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);  in xfrm_state_find()
1395 	hlist_for_each_entry_rcu(x, &pol->state_cache_list, state_cache) {  in xfrm_state_find()
1396 		if (x->props.family == encap_family &&  in xfrm_state_find()
1397 		    x->props.reqid == tmpl->reqid &&  in xfrm_state_find()
1398 		    (mark & x->mark.m) == x->mark.v &&  in xfrm_state_find()
1399 		    x->if_id == if_id &&  in xfrm_state_find()
1400 		    !(x->props.flags & XFRM_STATE_WILDRECV) &&  in xfrm_state_find()
1401 		    xfrm_state_addr_check(x, daddr, saddr, encap_family) &&  in xfrm_state_find()
1402 		    tmpl->mode == x->props.mode &&  in xfrm_state_find()
1403 		    tmpl->id.proto == x->id.proto &&  in xfrm_state_find()
1404 		    (tmpl->id.spi == x->id.spi || !tmpl->id.spi))  in xfrm_state_find()
1405 			xfrm_state_look_at(pol, x, fl, encap_family,  in xfrm_state_find()
1412 	hlist_for_each_entry_rcu(x, &pol->state_cache_list, state_cache) {  in xfrm_state_find()
1413 		if (x->props.family == encap_family &&  in xfrm_state_find()
1414 		    x->props.reqid == tmpl->reqid &&  in xfrm_state_find()
1415 		    (mark & x->mark.m) == x->mark.v &&  in xfrm_state_find()
1416 		    x->if_id == if_id &&  in xfrm_state_find()
1417 		    !(x->props.flags & XFRM_STATE_WILDRECV) &&  in xfrm_state_find()
1418 		    xfrm_addr_equal(&x->id.daddr, daddr, encap_family) &&  in xfrm_state_find()
1419 		    tmpl->mode == x->props.mode &&  in xfrm_state_find()
1420 		    tmpl->id.proto == x->id.proto &&  in xfrm_state_find()
1421 		    (tmpl->id.spi == x->id.spi || !tmpl->id.spi))  in xfrm_state_find()
1422 			xfrm_state_look_at(pol, x, fl, family,  in xfrm_state_find()
1437 	h = __xfrm_dst_hash(daddr, saddr, tmpl->reqid, encap_family, state_ptrs.hmask);  in xfrm_state_find()
1438 	hlist_for_each_entry_rcu(x, state_ptrs.bydst + h, bydst) {  in xfrm_state_find()
1440 		if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {  in xfrm_state_find()
1441 			if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET)  in xfrm_state_find()
1450 			if (pol->xdo.dev != x->xso.dev)  in xfrm_state_find()
1452 		} else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)  in xfrm_state_find()
1456 		if (x->props.family == encap_family &&  in xfrm_state_find()
1457 		    x->props.reqid == tmpl->reqid &&  in xfrm_state_find()
1458 		    (mark & x->mark.m) == x->mark.v &&  in xfrm_state_find()
1459 		    x->if_id == if_id &&  in xfrm_state_find()
1460 		    !(x->props.flags & XFRM_STATE_WILDRECV) &&  in xfrm_state_find()
1461 		    xfrm_state_addr_check(x, daddr, saddr, encap_family) &&  in xfrm_state_find()
1462 		    tmpl->mode == x->props.mode &&  in xfrm_state_find()
1463 		    tmpl->id.proto == x->id.proto &&  in xfrm_state_find()
1464 		    (tmpl->id.spi == x->id.spi || !tmpl->id.spi))  in xfrm_state_find()
1465 			xfrm_state_look_at(pol, x, fl, family,  in xfrm_state_find()
1471 	h_wildcard = __xfrm_dst_hash(daddr, &saddr_wildcard, tmpl->reqid,  in xfrm_state_find()
1473 	hlist_for_each_entry_rcu(x, state_ptrs.bydst + h_wildcard, bydst) {  in xfrm_state_find()
1475 		if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {  in xfrm_state_find()
1476 			if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET)  in xfrm_state_find()
1485 			if (pol->xdo.dev != x->xso.dev)  in xfrm_state_find()
1487 		} else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)  in xfrm_state_find()
1491 		if (x->props.family == encap_family &&  in xfrm_state_find()
1492 		    x->props.reqid == tmpl->reqid &&  in xfrm_state_find()
1493 		    (mark & x->mark.m) == x->mark.v &&  in xfrm_state_find()
1494 		    x->if_id == if_id &&  in xfrm_state_find()
1495 		    !(x->props.flags & XFRM_STATE_WILDRECV) &&  in xfrm_state_find()
1496 		    xfrm_addr_equal(&x->id.daddr, daddr, encap_family) &&  in xfrm_state_find()
1497 		    tmpl->mode == x->props.mode &&  in xfrm_state_find()
1498 		    tmpl->id.proto == x->id.proto &&  in xfrm_state_find()
1499 		    (tmpl->id.spi == x->id.spi || !tmpl->id.spi))  in xfrm_state_find()
1500 			xfrm_state_look_at(pol, x, fl, family,  in xfrm_state_find()
1505 	if (!(pol->flags & XFRM_POLICY_CPU_ACQUIRE) ||  in xfrm_state_find()
1506 	    (best && (best->pcpu_num == pcpu_id)))  in xfrm_state_find()
1507 		x = best;  in xfrm_state_find()
1509 	if (!x && !error && !acquire_in_progress) {  in xfrm_state_find()
1510 		if (tmpl->id.spi &&  in xfrm_state_find()
1512 						  tmpl->id.spi, tmpl->id.proto,  in xfrm_state_find()
1514 						  &pol->xdo)) != NULL) {  in xfrm_state_find()
1516 			error = -EEXIST;  in xfrm_state_find()
1522 		 * for each and every packet - garbage collection might not  in xfrm_state_find()
1526 			error = -ESRCH;  in xfrm_state_find()
1530 		x = xfrm_state_alloc(net);  in xfrm_state_find()
1531 		if (x == NULL) {  in xfrm_state_find()
1532 			error = -ENOMEM;  in xfrm_state_find()
1537 		xfrm_init_tempstate(x, fl, tmpl, daddr, saddr, family);  in xfrm_state_find()
1538 		memcpy(&x->mark, &pol->mark, sizeof(x->mark));  in xfrm_state_find()
1539 		x->if_id = if_id;  in xfrm_state_find()
1540 		if ((pol->flags & XFRM_POLICY_CPU_ACQUIRE) && best)  in xfrm_state_find()
1541 			x->pcpu_num = pcpu_id;  in xfrm_state_find()
1543 		error = security_xfrm_state_alloc_acquire(x, pol->security, fl->flowi_secid);  in xfrm_state_find()
1545 			x->km.state = XFRM_STATE_DEAD;  in xfrm_state_find()
1546 			to_put = x;  in xfrm_state_find()
1547 			x = NULL;  in xfrm_state_find()
1551 		if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {  in xfrm_state_find()
1552 			struct xfrm_dev_offload *xdo = &pol->xdo;  in xfrm_state_find()
1553 			struct xfrm_dev_offload *xso = &x->xso;  in xfrm_state_find()
1555 			xso->type = XFRM_DEV_OFFLOAD_PACKET;  in xfrm_state_find()
1556 			xso->dir = xdo->dir;  in xfrm_state_find()
1557 			xso->dev = xdo->dev;  in xfrm_state_find()
1558 			xso->real_dev = xdo->real_dev;  in xfrm_state_find()
1559 			xso->flags = XFRM_DEV_OFFLOAD_FLAG_ACQ;  in xfrm_state_find()
1560 			netdev_hold(xso->dev, &xso->dev_tracker, GFP_ATOMIC);  in xfrm_state_find()
1561 			error = xso->dev->xfrmdev_ops->xdo_dev_state_add(x, NULL);  in xfrm_state_find()
1563 				xso->dir = 0;  in xfrm_state_find()
1564 				netdev_put(xso->dev, &xso->dev_tracker);  in xfrm_state_find()
1565 				xso->dev = NULL;  in xfrm_state_find()
1566 				xso->real_dev = NULL;  in xfrm_state_find()
1567 				xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;  in xfrm_state_find()
1568 				x->km.state = XFRM_STATE_DEAD;  in xfrm_state_find()
1569 				to_put = x;  in xfrm_state_find()
1570 				x = NULL;  in xfrm_state_find()
1575 		if (km_query(x, tmpl, pol) == 0) {  in xfrm_state_find()
1576 			spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_find()
1577 			x->km.state = XFRM_STATE_ACQ;  in xfrm_state_find()
1578 			x->dir = XFRM_SA_DIR_OUT;  in xfrm_state_find()
1579 			list_add(&x->km.all, &net->xfrm.state_all);  in xfrm_state_find()
1580 			h = xfrm_dst_hash(net, daddr, saddr, tmpl->reqid, encap_family);  in xfrm_state_find()
1581 			XFRM_STATE_INSERT(bydst, &x->bydst,  in xfrm_state_find()
1582 					  net->xfrm.state_bydst + h,  in xfrm_state_find()
1583 					  x->xso.type);  in xfrm_state_find()
1585 			XFRM_STATE_INSERT(bysrc, &x->bysrc,  in xfrm_state_find()
1586 					  net->xfrm.state_bysrc + h,  in xfrm_state_find()
1587 					  x->xso.type);  in xfrm_state_find()
1588 			INIT_HLIST_NODE(&x->state_cache);  in xfrm_state_find()
1589 			if (x->id.spi) {  in xfrm_state_find()
1590 				h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, encap_family);  in xfrm_state_find()
1591 				XFRM_STATE_INSERT(byspi, &x->byspi,  in xfrm_state_find()
1592 						  net->xfrm.state_byspi + h,  in xfrm_state_find()
1593 						  x->xso.type);  in xfrm_state_find()
1595 			if (x->km.seq) {  in xfrm_state_find()
1596 				h = xfrm_seq_hash(net, x->km.seq);  in xfrm_state_find()
1597 				XFRM_STATE_INSERT(byseq, &x->byseq,  in xfrm_state_find()
1598 						  net->xfrm.state_byseq + h,  in xfrm_state_find()
1599 						  x->xso.type);  in xfrm_state_find()
1601 			x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;  in xfrm_state_find()
1602 			hrtimer_start(&x->mtimer,  in xfrm_state_find()
1603 				      ktime_set(net->xfrm.sysctl_acq_expires, 0),  in xfrm_state_find()
1605 			net->xfrm.state_num++;  in xfrm_state_find()
1606 			xfrm_hash_grow_check(net, x->bydst.next != NULL);  in xfrm_state_find()
1607 			spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_find()
1610 			struct xfrm_dev_offload *xso = &x->xso;  in xfrm_state_find()
1612 			if (xso->type == XFRM_DEV_OFFLOAD_PACKET) {  in xfrm_state_find()
1613 				xfrm_dev_state_delete(x);  in xfrm_state_find()
1614 				xfrm_dev_state_free(x);  in xfrm_state_find()
1617 			x->km.state = XFRM_STATE_DEAD;  in xfrm_state_find()
1618 			to_put = x;  in xfrm_state_find()
1619 			x = NULL;  in xfrm_state_find()
1620 			error = -ESRCH;  in xfrm_state_find()
1623 		/* Use the already installed 'fallback' while the CPU-specific  in xfrm_state_find()
1626 			x = best;  in xfrm_state_find()
1629 	if (x) {  in xfrm_state_find()
1630 		if (!xfrm_state_hold_rcu(x)) {  in xfrm_state_find()
1631 			*err = -EAGAIN;  in xfrm_state_find()
1632 			x = NULL;  in xfrm_state_find()
1635 		*err = acquire_in_progress ? -EAGAIN : error;  in xfrm_state_find()
1638 	if (x && x->km.state == XFRM_STATE_VALID && !cached &&  in xfrm_state_find()
1639 	    (!(pol->flags & XFRM_POLICY_CPU_ACQUIRE) || x->pcpu_num == pcpu_id)) {  in xfrm_state_find()
1640 		spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_find()
1641 		if (hlist_unhashed(&x->state_cache))  in xfrm_state_find()
1642 			hlist_add_head_rcu(&x->state_cache, &pol->state_cache_list);  in xfrm_state_find()
1643 		spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_find()
1650 	if (read_seqcount_retry(&net->xfrm.xfrm_state_hash_generation, sequence)) {  in xfrm_state_find()
1651 		*err = -EAGAIN;  in xfrm_state_find()
1652 		if (x) {  in xfrm_state_find()
1653 			xfrm_state_put(x);  in xfrm_state_find()
1654 			x = NULL;  in xfrm_state_find()
1658 	return x;  in xfrm_state_find()
1667 	struct xfrm_state *rx = NULL, *x = NULL;  in xfrm_stateonly_find()  local
1669 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_stateonly_find()
1671 	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {  in xfrm_stateonly_find()
1672 		if (x->props.family == family &&  in xfrm_stateonly_find()
1673 		    x->props.reqid == reqid &&  in xfrm_stateonly_find()
1674 		    (mark & x->mark.m) == x->mark.v &&  in xfrm_stateonly_find()
1675 		    x->if_id == if_id &&  in xfrm_stateonly_find()
1676 		    !(x->props.flags & XFRM_STATE_WILDRECV) &&  in xfrm_stateonly_find()
1677 		    xfrm_state_addr_check(x, daddr, saddr, family) &&  in xfrm_stateonly_find()
1678 		    mode == x->props.mode &&  in xfrm_stateonly_find()
1679 		    proto == x->id.proto &&  in xfrm_stateonly_find()
1680 		    x->km.state == XFRM_STATE_VALID) {  in xfrm_stateonly_find()
1681 			rx = x;  in xfrm_stateonly_find()
1688 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_stateonly_find()
1698 	struct xfrm_state *x;  in xfrm_state_lookup_byspi()  local
1701 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byspi()
1702 	list_for_each_entry(w, &net->xfrm.state_all, all) {  in xfrm_state_lookup_byspi()
1703 		x = container_of(w, struct xfrm_state, km);  in xfrm_state_lookup_byspi()
1704 		if (x->props.family != family ||  in xfrm_state_lookup_byspi()
1705 			x->id.spi != spi)  in xfrm_state_lookup_byspi()
1708 		xfrm_state_hold(x);  in xfrm_state_lookup_byspi()
1709 		spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byspi()
1710 		return x;  in xfrm_state_lookup_byspi()
1712 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byspi()
1717 static void __xfrm_state_insert(struct xfrm_state *x)  in __xfrm_state_insert()  argument
1719 	struct net *net = xs_net(x);  in __xfrm_state_insert()
1722 	list_add(&x->km.all, &net->xfrm.state_all);  in __xfrm_state_insert()
1724 	h = xfrm_dst_hash(net, &x->id.daddr, &x->props.saddr,  in __xfrm_state_insert()
1725 			  x->props.reqid, x->props.family);  in __xfrm_state_insert()
1726 	XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h,  in __xfrm_state_insert()
1727 			  x->xso.type);  in __xfrm_state_insert()
1729 	h = xfrm_src_hash(net, &x->id.daddr, &x->props.saddr, x->props.family);  in __xfrm_state_insert()
1730 	XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h,  in __xfrm_state_insert()
1731 			  x->xso.type);  in __xfrm_state_insert()
1733 	if (x->id.spi) {  in __xfrm_state_insert()
1734 		h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto,  in __xfrm_state_insert()
1735 				  x->props.family);  in __xfrm_state_insert()
1737 		XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h,  in __xfrm_state_insert()
1738 				  x->xso.type);  in __xfrm_state_insert()
1741 	if (x->km.seq) {  in __xfrm_state_insert()
1742 		h = xfrm_seq_hash(net, x->km.seq);  in __xfrm_state_insert()
1744 		XFRM_STATE_INSERT(byseq, &x->byseq, net->xfrm.state_byseq + h,  in __xfrm_state_insert()
1745 				  x->xso.type);  in __xfrm_state_insert()
1748 	hrtimer_start(&x->mtimer, ktime_set(1, 0), HRTIMER_MODE_REL_SOFT);  in __xfrm_state_insert()
1749 	if (x->replay_maxage)  in __xfrm_state_insert()
1750 		mod_timer(&x->rtimer, jiffies + x->replay_maxage);  in __xfrm_state_insert()
1752 	net->xfrm.state_num++;  in __xfrm_state_insert()
1754 	xfrm_hash_grow_check(net, x->bydst.next != NULL);  in __xfrm_state_insert()
1755 	xfrm_nat_keepalive_state_updated(x);  in __xfrm_state_insert()
1758 /* net->xfrm.xfrm_state_lock is held */
1762 	unsigned short family = xnew->props.family;  in __xfrm_state_bump_genids()
1763 	u32 reqid = xnew->props.reqid;  in __xfrm_state_bump_genids()
1764 	struct xfrm_state *x;  in __xfrm_state_bump_genids()  local
1766 	u32 mark = xnew->mark.v & xnew->mark.m;  in __xfrm_state_bump_genids()
1767 	u32 if_id = xnew->if_id;  in __xfrm_state_bump_genids()
1768 	u32 cpu_id = xnew->pcpu_num;  in __xfrm_state_bump_genids()
1770 	h = xfrm_dst_hash(net, &xnew->id.daddr, &xnew->props.saddr, reqid, family);  in __xfrm_state_bump_genids()
1771 	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {  in __xfrm_state_bump_genids()
1772 		if (x->props.family	== family &&  in __xfrm_state_bump_genids()
1773 		    x->props.reqid	== reqid &&  in __xfrm_state_bump_genids()
1774 		    x->if_id		== if_id &&  in __xfrm_state_bump_genids()
1775 		    x->pcpu_num		== cpu_id &&  in __xfrm_state_bump_genids()
1776 		    (mark & x->mark.m) == x->mark.v &&  in __xfrm_state_bump_genids()
1777 		    xfrm_addr_equal(&x->id.daddr, &xnew->id.daddr, family) &&  in __xfrm_state_bump_genids()
1778 		    xfrm_addr_equal(&x->props.saddr, &xnew->props.saddr, family))  in __xfrm_state_bump_genids()
1779 			x->genid++;  in __xfrm_state_bump_genids()
1783 void xfrm_state_insert(struct xfrm_state *x)  in xfrm_state_insert()  argument
1785 	struct net *net = xs_net(x);  in xfrm_state_insert()
1787 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_insert()
1788 	__xfrm_state_bump_genids(x);  in xfrm_state_insert()
1789 	__xfrm_state_insert(x);  in xfrm_state_insert()
1790 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_insert()
1794 /* net->xfrm.xfrm_state_lock is held */
1804 	struct xfrm_state *x;  in __find_acq_core()  local
1805 	u32 mark = m->v & m->m;  in __find_acq_core()
1807 	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {  in __find_acq_core()
1808 		if (x->props.reqid  != reqid ||  in __find_acq_core()
1809 		    x->props.mode   != mode ||  in __find_acq_core()
1810 		    x->props.family != family ||  in __find_acq_core()
1811 		    x->km.state     != XFRM_STATE_ACQ ||  in __find_acq_core()
1812 		    x->id.spi       != 0 ||  in __find_acq_core()
1813 		    x->id.proto	    != proto ||  in __find_acq_core()
1814 		    (mark & x->mark.m) != x->mark.v ||  in __find_acq_core()
1815 		    x->pcpu_num != pcpu_num ||  in __find_acq_core()
1816 		    !xfrm_addr_equal(&x->id.daddr, daddr, family) ||  in __find_acq_core()
1817 		    !xfrm_addr_equal(&x->props.saddr, saddr, family))  in __find_acq_core()
1820 		xfrm_state_hold(x);  in __find_acq_core()
1821 		return x;  in __find_acq_core()
1827 	x = xfrm_state_alloc(net);  in __find_acq_core()
1828 	if (likely(x)) {  in __find_acq_core()
1831 			x->sel.daddr.a4 = daddr->a4;  in __find_acq_core()
1832 			x->sel.saddr.a4 = saddr->a4;  in __find_acq_core()
1833 			x->sel.prefixlen_d = 32;  in __find_acq_core()
1834 			x->sel.prefixlen_s = 32;  in __find_acq_core()
1835 			x->props.saddr.a4 = saddr->a4;  in __find_acq_core()
1836 			x->id.daddr.a4 = daddr->a4;  in __find_acq_core()
1840 			x->sel.daddr.in6 = daddr->in6;  in __find_acq_core()
1841 			x->sel.saddr.in6 = saddr->in6;  in __find_acq_core()
1842 			x->sel.prefixlen_d = 128;  in __find_acq_core()
1843 			x->sel.prefixlen_s = 128;  in __find_acq_core()
1844 			x->props.saddr.in6 = saddr->in6;  in __find_acq_core()
1845 			x->id.daddr.in6 = daddr->in6;  in __find_acq_core()
1849 		x->pcpu_num = pcpu_num;  in __find_acq_core()
1850 		x->km.state = XFRM_STATE_ACQ;  in __find_acq_core()
1851 		x->id.proto = proto;  in __find_acq_core()
1852 		x->props.family = family;  in __find_acq_core()
1853 		x->props.mode = mode;  in __find_acq_core()
1854 		x->props.reqid = reqid;  in __find_acq_core()
1855 		x->if_id = if_id;  in __find_acq_core()
1856 		x->mark.v = m->v;  in __find_acq_core()
1857 		x->mark.m = m->m;  in __find_acq_core()
1858 		x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;  in __find_acq_core()
1859 		xfrm_state_hold(x);  in __find_acq_core()
1860 		hrtimer_start(&x->mtimer,  in __find_acq_core()
1861 			      ktime_set(net->xfrm.sysctl_acq_expires, 0),  in __find_acq_core()
1863 		list_add(&x->km.all, &net->xfrm.state_all);  in __find_acq_core()
1864 		XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h,  in __find_acq_core()
1865 				  x->xso.type);  in __find_acq_core()
1867 		XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h,  in __find_acq_core()
1868 				  x->xso.type);  in __find_acq_core()
1870 		net->xfrm.state_num++;  in __find_acq_core()
1872 		xfrm_hash_grow_check(net, x->bydst.next != NULL);  in __find_acq_core()
1875 	return x;  in __find_acq_core()
1880 int xfrm_state_add(struct xfrm_state *x)  in xfrm_state_add()  argument
1882 	struct net *net = xs_net(x);  in xfrm_state_add()
1886 	u32 mark = x->mark.v & x->mark.m;  in xfrm_state_add()
1887 	int use_spi = xfrm_id_proto_match(x->id.proto, IPSEC_PROTO_ANY);  in xfrm_state_add()
1889 	family = x->props.family;  in xfrm_state_add()
1893 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_add()
1895 	x1 = __xfrm_state_locate(x, use_spi, family);  in xfrm_state_add()
1899 		err = -EEXIST;  in xfrm_state_add()
1903 	if (use_spi && x->km.seq) {  in xfrm_state_add()
1904 		x1 = __xfrm_find_acq_byseq(net, mark, x->km.seq, x->pcpu_num);  in xfrm_state_add()
1905 		if (x1 && ((x1->id.proto != x->id.proto) ||  in xfrm_state_add()
1906 		    !xfrm_addr_equal(&x1->id.daddr, &x->id.daddr, family))) {  in xfrm_state_add()
1913 		x1 = __find_acq_core(net, &x->mark, family, x->props.mode,  in xfrm_state_add()
1914 				     x->props.reqid, x->if_id, x->pcpu_num, x->id.proto,  in xfrm_state_add()
1915 				     &x->id.daddr, &x->props.saddr, 0);  in xfrm_state_add()
1917 	__xfrm_state_bump_genids(x);  in xfrm_state_add()
1918 	__xfrm_state_insert(x);  in xfrm_state_add()
1922 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_add()
1937 static inline int clone_security(struct xfrm_state *x, struct xfrm_sec_ctx *security)  in clone_security()  argument
1940 	int size = sizeof(*uctx) + security->ctx_len;  in clone_security()
1945 		return -ENOMEM;  in clone_security()
1947 	uctx->exttype = XFRMA_SEC_CTX;  in clone_security()
1948 	uctx->len = size;  in clone_security()
1949 	uctx->ctx_doi = security->ctx_doi;  in clone_security()
1950 	uctx->ctx_alg = security->ctx_alg;  in clone_security()
1951 	uctx->ctx_len = security->ctx_len;  in clone_security()
1952 	memcpy(uctx + 1, security->ctx_str, security->ctx_len);  in clone_security()
1953 	err = security_xfrm_state_alloc(x, uctx);  in clone_security()
1965 	struct xfrm_state *x = xfrm_state_alloc(net);  in xfrm_state_clone()  local
1966 	if (!x)  in xfrm_state_clone()
1969 	memcpy(&x->id, &orig->id, sizeof(x->id));  in xfrm_state_clone()
1970 	memcpy(&x->sel, &orig->sel, sizeof(x->sel));  in xfrm_state_clone()
1971 	memcpy(&x->lft, &orig->lft, sizeof(x->lft));  in xfrm_state_clone()
1972 	x->props.mode = orig->props.mode;  in xfrm_state_clone()
1973 	x->props.replay_window = orig->props.replay_window;  in xfrm_state_clone()
1974 	x->props.reqid = orig->props.reqid;  in xfrm_state_clone()
1975 	x->props.family = orig->props.family;  in xfrm_state_clone()
1976 	x->props.saddr = orig->props.saddr;  in xfrm_state_clone()
1978 	if (orig->aalg) {  in xfrm_state_clone()
1979 		x->aalg = xfrm_algo_auth_clone(orig->aalg);  in xfrm_state_clone()
1980 		if (!x->aalg)  in xfrm_state_clone()
1983 	x->props.aalgo = orig->props.aalgo;  in xfrm_state_clone()
1985 	if (orig->aead) {  in xfrm_state_clone()
1986 		x->aead = xfrm_algo_aead_clone(orig->aead);  in xfrm_state_clone()
1987 		x->geniv = orig->geniv;  in xfrm_state_clone()
1988 		if (!x->aead)  in xfrm_state_clone()
1991 	if (orig->ealg) {  in xfrm_state_clone()
1992 		x->ealg = xfrm_algo_clone(orig->ealg);  in xfrm_state_clone()
1993 		if (!x->ealg)  in xfrm_state_clone()
1996 	x->props.ealgo = orig->props.ealgo;  in xfrm_state_clone()
1998 	if (orig->calg) {  in xfrm_state_clone()
1999 		x->calg = xfrm_algo_clone(orig->calg);  in xfrm_state_clone()
2000 		if (!x->calg)  in xfrm_state_clone()
2003 	x->props.calgo = orig->props.calgo;  in xfrm_state_clone()
2005 	if (encap || orig->encap) {  in xfrm_state_clone()
2007 			x->encap = kmemdup(encap, sizeof(*x->encap),  in xfrm_state_clone()
2010 			x->encap = kmemdup(orig->encap, sizeof(*x->encap),  in xfrm_state_clone()
2013 		if (!x->encap)  in xfrm_state_clone()
2017 	if (orig->security)  in xfrm_state_clone()
2018 		if (clone_security(x, orig->security))  in xfrm_state_clone()
2021 	if (orig->coaddr) {  in xfrm_state_clone()
2022 		x->coaddr = kmemdup(orig->coaddr, sizeof(*x->coaddr),  in xfrm_state_clone()
2024 		if (!x->coaddr)  in xfrm_state_clone()
2028 	if (orig->replay_esn) {  in xfrm_state_clone()
2029 		if (xfrm_replay_clone(x, orig))  in xfrm_state_clone()
2033 	memcpy(&x->mark, &orig->mark, sizeof(x->mark));  in xfrm_state_clone()
2034 	memcpy(&x->props.smark, &orig->props.smark, sizeof(x->props.smark));  in xfrm_state_clone()
2036 	x->props.flags = orig->props.flags;  in xfrm_state_clone()
2037 	x->props.extra_flags = orig->props.extra_flags;  in xfrm_state_clone()
2039 	x->pcpu_num = orig->pcpu_num;  in xfrm_state_clone()
2040 	x->if_id = orig->if_id;  in xfrm_state_clone()
2041 	x->tfcpad = orig->tfcpad;  in xfrm_state_clone()
2042 	x->replay_maxdiff = orig->replay_maxdiff;  in xfrm_state_clone()
2043 	x->replay_maxage = orig->replay_maxage;  in xfrm_state_clone()
2044 	memcpy(&x->curlft, &orig->curlft, sizeof(x->curlft));  in xfrm_state_clone()
2045 	x->km.state = orig->km.state;  in xfrm_state_clone()
2046 	x->km.seq = orig->km.seq;  in xfrm_state_clone()
2047 	x->replay = orig->replay;  in xfrm_state_clone()
2048 	x->preplay = orig->preplay;  in xfrm_state_clone()
2049 	x->mapping_maxage = orig->mapping_maxage;  in xfrm_state_clone()
2050 	x->lastused = orig->lastused;  in xfrm_state_clone()
2051 	x->new_mapping = 0;  in xfrm_state_clone()
2052 	x->new_mapping_sport = 0;  in xfrm_state_clone()
2053 	x->dir = orig->dir;  in xfrm_state_clone()
2055 	x->mode_cbs = orig->mode_cbs;  in xfrm_state_clone()
2056 	if (x->mode_cbs && x->mode_cbs->clone_state) {  in xfrm_state_clone()
2057 		if (x->mode_cbs->clone_state(x, orig))  in xfrm_state_clone()
2061 	return x;  in xfrm_state_clone()
2064 	xfrm_state_put(x);  in xfrm_state_clone()
2073 	struct xfrm_state *x = NULL;  in xfrm_migrate_state_find()  local
2075 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_migrate_state_find()
2077 	if (m->reqid) {  in xfrm_migrate_state_find()
2078 		h = xfrm_dst_hash(net, &m->old_daddr, &m->old_saddr,  in xfrm_migrate_state_find()
2079 				  m->reqid, m->old_family);  in xfrm_migrate_state_find()
2080 		hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {  in xfrm_migrate_state_find()
2081 			if (x->props.mode != m->mode ||  in xfrm_migrate_state_find()
2082 			    x->id.proto != m->proto)  in xfrm_migrate_state_find()
2084 			if (m->reqid && x->props.reqid != m->reqid)  in xfrm_migrate_state_find()
2086 			if (if_id != 0 && x->if_id != if_id)  in xfrm_migrate_state_find()
2088 			if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr,  in xfrm_migrate_state_find()
2089 					     m->old_family) ||  in xfrm_migrate_state_find()
2090 			    !xfrm_addr_equal(&x->props.saddr, &m->old_saddr,  in xfrm_migrate_state_find()
2091 					     m->old_family))  in xfrm_migrate_state_find()
2093 			xfrm_state_hold(x);  in xfrm_migrate_state_find()
2097 		h = xfrm_src_hash(net, &m->old_daddr, &m->old_saddr,  in xfrm_migrate_state_find()
2098 				  m->old_family);  in xfrm_migrate_state_find()
2099 		hlist_for_each_entry(x, net->xfrm.state_bysrc+h, bysrc) {  in xfrm_migrate_state_find()
2100 			if (x->props.mode != m->mode ||  in xfrm_migrate_state_find()
2101 			    x->id.proto != m->proto)  in xfrm_migrate_state_find()
2103 			if (if_id != 0 && x->if_id != if_id)  in xfrm_migrate_state_find()
2105 			if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr,  in xfrm_migrate_state_find()
2106 					     m->old_family) ||  in xfrm_migrate_state_find()
2107 			    !xfrm_addr_equal(&x->props.saddr, &m->old_saddr,  in xfrm_migrate_state_find()
2108 					     m->old_family))  in xfrm_migrate_state_find()
2110 			xfrm_state_hold(x);  in xfrm_migrate_state_find()
2115 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_migrate_state_find()
2117 	return x;  in xfrm_migrate_state_find()
2121 struct xfrm_state *xfrm_state_migrate(struct xfrm_state *x,  in xfrm_state_migrate()  argument
2127 	xc = xfrm_state_clone(x, encap);  in xfrm_state_migrate()
2131 	xc->props.family = m->new_family;  in xfrm_state_migrate()
2136 	memcpy(&xc->id.daddr, &m->new_daddr, sizeof(xc->id.daddr));  in xfrm_state_migrate()
2137 	memcpy(&xc->props.saddr, &m->new_saddr, sizeof(xc->props.saddr));  in xfrm_state_migrate()
2140 	if (xfrm_addr_equal(&x->id.daddr, &m->new_daddr, m->new_family)) {  in xfrm_state_migrate()
2157 int xfrm_state_update(struct xfrm_state *x)  in xfrm_state_update()  argument
2161 	int use_spi = xfrm_id_proto_match(x->id.proto, IPSEC_PROTO_ANY);  in xfrm_state_update()
2162 	struct net *net = xs_net(x);  in xfrm_state_update()
2166 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update()
2167 	x1 = __xfrm_state_locate(x, use_spi, x->props.family);  in xfrm_state_update()
2169 	err = -ESRCH;  in xfrm_state_update()
2175 		err = -EEXIST;  in xfrm_state_update()
2179 	if (x1->km.state == XFRM_STATE_ACQ) {  in xfrm_state_update()
2180 		if (x->dir && x1->dir != x->dir)  in xfrm_state_update()
2183 		__xfrm_state_insert(x);  in xfrm_state_update()
2184 		x = NULL;  in xfrm_state_update()
2186 		if (x1->dir != x->dir)  in xfrm_state_update()
2192 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update()
2200 	if (!x) {  in xfrm_state_update()
2206 	err = -EINVAL;  in xfrm_state_update()
2207 	spin_lock_bh(&x1->lock);  in xfrm_state_update()
2208 	if (likely(x1->km.state == XFRM_STATE_VALID)) {  in xfrm_state_update()
2209 		if (x->encap && x1->encap &&  in xfrm_state_update()
2210 		    x->encap->encap_type == x1->encap->encap_type)  in xfrm_state_update()
2211 			memcpy(x1->encap, x->encap, sizeof(*x1->encap));  in xfrm_state_update()
2212 		else if (x->encap || x1->encap)  in xfrm_state_update()
2215 		if (x->coaddr && x1->coaddr) {  in xfrm_state_update()
2216 			memcpy(x1->coaddr, x->coaddr, sizeof(*x1->coaddr));  in xfrm_state_update()
2218 		if (!use_spi && memcmp(&x1->sel, &x->sel, sizeof(x1->sel)))  in xfrm_state_update()
2219 			memcpy(&x1->sel, &x->sel, sizeof(x1->sel));  in xfrm_state_update()
2220 		memcpy(&x1->lft, &x->lft, sizeof(x1->lft));  in xfrm_state_update()
2221 		x1->km.dying = 0;  in xfrm_state_update()
2223 		hrtimer_start(&x1->mtimer, ktime_set(1, 0),  in xfrm_state_update()
2225 		if (READ_ONCE(x1->curlft.use_time))  in xfrm_state_update()
2228 		if (x->props.smark.m || x->props.smark.v || x->if_id) {  in xfrm_state_update()
2229 			spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update()
2231 			if (x->props.smark.m || x->props.smark.v)  in xfrm_state_update()
2232 				x1->props.smark = x->props.smark;  in xfrm_state_update()
2234 			if (x->if_id)  in xfrm_state_update()
2235 				x1->if_id = x->if_id;  in xfrm_state_update()
2238 			spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update()
2242 		x->km.state = XFRM_STATE_DEAD;  in xfrm_state_update()
2243 		__xfrm_state_put(x);  in xfrm_state_update()
2247 	spin_unlock_bh(&x1->lock);  in xfrm_state_update()
2255 int xfrm_state_check_expire(struct xfrm_state *x)  in xfrm_state_check_expire()  argument
2257 	xfrm_dev_state_update_stats(x);  in xfrm_state_check_expire()
2259 	if (!READ_ONCE(x->curlft.use_time))  in xfrm_state_check_expire()
2260 		WRITE_ONCE(x->curlft.use_time, ktime_get_real_seconds());  in xfrm_state_check_expire()
2262 	if (x->curlft.bytes >= x->lft.hard_byte_limit ||  in xfrm_state_check_expire()
2263 	    x->curlft.packets >= x->lft.hard_packet_limit) {  in xfrm_state_check_expire()
2264 		x->km.state = XFRM_STATE_EXPIRED;  in xfrm_state_check_expire()
2265 		hrtimer_start(&x->mtimer, 0, HRTIMER_MODE_REL_SOFT);  in xfrm_state_check_expire()
2266 		return -EINVAL;  in xfrm_state_check_expire()
2269 	if (!x->km.dying &&  in xfrm_state_check_expire()
2270 	    (x->curlft.bytes >= x->lft.soft_byte_limit ||  in xfrm_state_check_expire()
2271 	     x->curlft.packets >= x->lft.soft_packet_limit)) {  in xfrm_state_check_expire()
2272 		x->km.dying = 1;  in xfrm_state_check_expire()
2273 		km_state_expired(x, 0, 0);  in xfrm_state_check_expire()
2281 	struct xfrm_state *x;  in xfrm_state_update_stats()  local
2284 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update_stats()
2285 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_state_update_stats()
2286 		hlist_for_each_entry(x, net->xfrm.state_bydst + i, bydst)  in xfrm_state_update_stats()
2287 			xfrm_dev_state_update_stats(x);  in xfrm_state_update_stats()
2289 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update_stats()
2297 	struct xfrm_state *x;  in xfrm_state_lookup()  local
2302 	x = __xfrm_state_lookup(&state_ptrs, mark, daddr, spi, proto, family);  in xfrm_state_lookup()
2304 	return x;  in xfrm_state_lookup()
2314 	struct xfrm_state *x;  in xfrm_state_lookup_byaddr()  local
2316 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byaddr()
2320 	x = __xfrm_state_lookup_byaddr(&state_ptrs, mark, daddr, saddr, proto, family);  in xfrm_state_lookup_byaddr()
2321 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byaddr()
2322 	return x;  in xfrm_state_lookup_byaddr()
2331 	struct xfrm_state *x;  in xfrm_find_acq()  local
2333 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_find_acq()
2334 	x = __find_acq_core(net, mark, family, mode, reqid, if_id, pcpu_num,  in xfrm_find_acq()
2336 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_find_acq()
2338 	return x;  in xfrm_find_acq()
2361 		count[i] += count[i - 1];  in __xfrm6_sort()
2364 		dst[count[class[i] - 1]++] = src[i];  in __xfrm6_sort()
2381 	switch (v->props.mode) {  in __xfrm6_state_sort_cmp()
2383 		if (v->id.proto != IPPROTO_AH)  in __xfrm6_state_sort_cmp()
2411 	switch (v->mode) {  in __xfrm6_tmpl_sort_cmp()
2475 	struct xfrm_state *x;  in __xfrm_find_acq_byseq()  local
2477 	hlist_for_each_entry_rcu(x, net->xfrm.state_byseq + h, byseq) {  in __xfrm_find_acq_byseq()
2478 		if (x->km.seq == seq &&  in __xfrm_find_acq_byseq()
2479 		    (mark & x->mark.m) == x->mark.v &&  in __xfrm_find_acq_byseq()
2480 		    x->pcpu_num == pcpu_num &&  in __xfrm_find_acq_byseq()
2481 		    x->km.state == XFRM_STATE_ACQ) {  in __xfrm_find_acq_byseq()
2482 			xfrm_state_hold(x);  in __xfrm_find_acq_byseq()
2483 			return x;  in __xfrm_find_acq_byseq()
2492 	struct xfrm_state *x;  in xfrm_find_acq_byseq()  local
2494 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_find_acq_byseq()
2495 	x = __xfrm_find_acq_byseq(net, mark, seq, pcpu_num);  in xfrm_find_acq_byseq()
2496 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_find_acq_byseq()
2497 	return x;  in xfrm_find_acq_byseq()
2522 		/* IPCOMP spi is 16-bits. */  in verify_spi_info()
2525 			return -EINVAL;  in verify_spi_info()
2531 		return -EINVAL;  in verify_spi_info()
2536 		return -EINVAL;  in verify_spi_info()
2543 int xfrm_alloc_spi(struct xfrm_state *x, u32 low, u32 high,  in xfrm_alloc_spi()  argument
2546 	struct net *net = xs_net(x);  in xfrm_alloc_spi()
2549 	int err = -ENOENT;  in xfrm_alloc_spi()
2553 	u32 mark = x->mark.v & x->mark.m;  in xfrm_alloc_spi()
2555 	spin_lock_bh(&x->lock);  in xfrm_alloc_spi()
2556 	if (x->km.state == XFRM_STATE_DEAD) {  in xfrm_alloc_spi()
2562 	if (x->id.spi)  in xfrm_alloc_spi()
2565 	err = -ENOENT;  in xfrm_alloc_spi()
2568 		x0 = xfrm_state_lookup(net, mark, &x->id.daddr, minspi, x->id.proto, x->props.family);  in xfrm_alloc_spi()
2577 		for (h = 0; h < high-low+1; h++) {  in xfrm_alloc_spi()
2579 			x0 = xfrm_state_lookup(net, mark, &x->id.daddr, htonl(spi), x->id.proto, x->props.family);  in xfrm_alloc_spi()
2588 		spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_alloc_spi()
2589 		x->id.spi = newspi;  in xfrm_alloc_spi()
2590 		h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, x->props.family);  in xfrm_alloc_spi()
2591 		XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h,  in xfrm_alloc_spi()
2592 				  x->xso.type);  in xfrm_alloc_spi()
2593 		spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_alloc_spi()
2601 	spin_unlock_bh(&x->lock);  in xfrm_alloc_spi()
2607 static bool __xfrm_state_filter_match(struct xfrm_state *x,  in __xfrm_state_filter_match()  argument
2611 		if ((filter->family == AF_INET ||  in __xfrm_state_filter_match()
2612 		     filter->family == AF_INET6) &&  in __xfrm_state_filter_match()
2613 		    x->props.family != filter->family)  in __xfrm_state_filter_match()
2616 		return addr_match(&x->props.saddr, &filter->saddr,  in __xfrm_state_filter_match()
2617 				  filter->splen) &&  in __xfrm_state_filter_match()
2618 		       addr_match(&x->id.daddr, &filter->daddr,  in __xfrm_state_filter_match()
2619 				  filter->dplen);  in __xfrm_state_filter_match()
2629 	struct xfrm_state_walk *x;  in xfrm_state_walk()  local
2632 	if (walk->seq != 0 && list_empty(&walk->all))  in xfrm_state_walk()
2635 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_walk()
2636 	if (list_empty(&walk->all))  in xfrm_state_walk()
2637 		x = list_first_entry(&net->xfrm.state_all, struct xfrm_state_walk, all);  in xfrm_state_walk()
2639 		x = list_first_entry(&walk->all, struct xfrm_state_walk, all);  in xfrm_state_walk()
2640 	list_for_each_entry_from(x, &net->xfrm.state_all, all) {  in xfrm_state_walk()
2641 		if (x->state == XFRM_STATE_DEAD)  in xfrm_state_walk()
2643 		state = container_of(x, struct xfrm_state, km);  in xfrm_state_walk()
2644 		if (!xfrm_id_proto_match(state->id.proto, walk->proto))  in xfrm_state_walk()
2646 		if (!__xfrm_state_filter_match(state, walk->filter))  in xfrm_state_walk()
2648 		err = func(state, walk->seq, data);  in xfrm_state_walk()
2650 			list_move_tail(&walk->all, &x->all);  in xfrm_state_walk()
2653 		walk->seq++;  in xfrm_state_walk()
2655 	if (walk->seq == 0) {  in xfrm_state_walk()
2656 		err = -ENOENT;  in xfrm_state_walk()
2659 	list_del_init(&walk->all);  in xfrm_state_walk()
2661 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_walk()
2669 	INIT_LIST_HEAD(&walk->all);  in xfrm_state_walk_init()
2670 	walk->proto = proto;  in xfrm_state_walk_init()
2671 	walk->state = XFRM_STATE_DEAD;  in xfrm_state_walk_init()
2672 	walk->seq = 0;  in xfrm_state_walk_init()
2673 	walk->filter = filter;  in xfrm_state_walk_init()
2679 	kfree(walk->filter);  in xfrm_state_walk_done()
2681 	if (list_empty(&walk->all))  in xfrm_state_walk_done()
2684 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_walk_done()
2685 	list_del(&walk->all);  in xfrm_state_walk_done()
2686 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_walk_done()
2692 	struct xfrm_state *x = from_timer(x, t, rtimer);  in xfrm_replay_timer_handler()  local
2694 	spin_lock(&x->lock);  in xfrm_replay_timer_handler()
2696 	if (x->km.state == XFRM_STATE_VALID) {  in xfrm_replay_timer_handler()
2697 		if (xfrm_aevent_is_on(xs_net(x)))  in xfrm_replay_timer_handler()
2698 			xfrm_replay_notify(x, XFRM_REPLAY_TIMEOUT);  in xfrm_replay_timer_handler()
2700 			x->xflags |= XFRM_TIME_DEFER;  in xfrm_replay_timer_handler()
2703 	spin_unlock(&x->lock);  in xfrm_replay_timer_handler()
2714 		if (km->notify_policy)  in km_policy_notify()
2715 			km->notify_policy(xp, dir, c);  in km_policy_notify()
2719 void km_state_notify(struct xfrm_state *x, const struct km_event *c)  in km_state_notify()  argument
2724 		if (km->notify)  in km_state_notify()
2725 			km->notify(x, c);  in km_state_notify()
2732 void km_state_expired(struct xfrm_state *x, int hard, u32 portid)  in km_state_expired()  argument
2739 	km_state_notify(x, &c);  in km_state_expired()
2747 int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol)  in km_query()  argument
2749 	int err = -EINVAL, acqret;  in km_query()
2754 		acqret = km->acquire(x, t, pol);  in km_query()
2763 static int __km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport)  in __km_new_mapping()  argument
2765 	int err = -EINVAL;  in __km_new_mapping()
2770 		if (km->new_mapping)  in __km_new_mapping()
2771 			err = km->new_mapping(x, ipaddr, sport);  in __km_new_mapping()
2779 int km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport)  in km_new_mapping()  argument
2783 	if (x->mapping_maxage) {  in km_new_mapping()
2784 		if ((jiffies / HZ - x->new_mapping) > x->mapping_maxage ||  in km_new_mapping()
2785 		    x->new_mapping_sport != sport) {  in km_new_mapping()
2786 			x->new_mapping_sport = sport;  in km_new_mapping()
2787 			x->new_mapping = jiffies / HZ;  in km_new_mapping()
2788 			ret = __km_new_mapping(x, ipaddr, sport);  in km_new_mapping()
2791 		ret = __km_new_mapping(x, ipaddr, sport);  in km_new_mapping()
2815 	int err = -EINVAL;  in km_migrate()
2821 		if (km->migrate) {  in km_migrate()
2822 			ret = km->migrate(sel, dir, type, m, num_migrate, k,  in km_migrate()
2836 	int err = -EINVAL;  in km_report()
2842 		if (km->report) {  in km_report()
2843 			ret = km->report(net, proto, sel, addr);  in km_report()
2860 		if (km->is_alive && km->is_alive(c)) {  in km_is_alive()
2882 	if (!try_module_get(xtr->owner))  in xfrm_get_translator()
2892 	module_put(xtr->owner);  in xfrm_put_translator()
2902 		err = -EEXIST;  in xfrm_register_translator()
2918 			err = -EINVAL;  in xfrm_unregister_translator()
2945 		return -EMSGSIZE;  in xfrm_user_policy()
2956 			return -EOPNOTSUPP;  in xfrm_user_policy()
2959 		err = xtr->xlate_user_policy_sockptr(&data, optlen);  in xfrm_user_policy()
2967 	err = -EINVAL;  in xfrm_user_policy()
2970 		pol = km->compile_policy(sk, optname, data,  in xfrm_user_policy()
2994 	list_add_tail_rcu(&km->list, &xfrm_km_list);  in xfrm_register_km()
3002 	list_del_rcu(&km->list);  in xfrm_unregister_km()
3012 	if (WARN_ON(afinfo->family >= NPROTO))  in xfrm_state_register_afinfo()
3013 		return -EAFNOSUPPORT;  in xfrm_state_register_afinfo()
3016 	if (unlikely(xfrm_state_afinfo[afinfo->family] != NULL))  in xfrm_state_register_afinfo()
3017 		err = -EEXIST;  in xfrm_state_register_afinfo()
3019 		rcu_assign_pointer(xfrm_state_afinfo[afinfo->family], afinfo);  in xfrm_state_register_afinfo()
3027 	int err = 0, family = afinfo->family;  in xfrm_state_unregister_afinfo()
3030 		return -EAFNOSUPPORT;  in xfrm_state_unregister_afinfo()
3033 	if (likely(xfrm_state_afinfo[afinfo->family] != NULL)) {  in xfrm_state_unregister_afinfo()
3035 			err = -EINVAL;  in xfrm_state_unregister_afinfo()
3037 			RCU_INIT_POINTER(xfrm_state_afinfo[afinfo->family], NULL);  in xfrm_state_unregister_afinfo()
3073 void xfrm_state_delete_tunnel(struct xfrm_state *x)  in xfrm_state_delete_tunnel()  argument
3075 	if (x->tunnel) {  in xfrm_state_delete_tunnel()
3076 		struct xfrm_state *t = x->tunnel;  in xfrm_state_delete_tunnel()
3078 		if (atomic_read(&t->tunnel_users) == 2)  in xfrm_state_delete_tunnel()
3080 		atomic_dec(&t->tunnel_users);  in xfrm_state_delete_tunnel()
3082 		x->tunnel = NULL;  in xfrm_state_delete_tunnel()
3087 u32 xfrm_state_mtu(struct xfrm_state *x, int mtu)  in xfrm_state_mtu()  argument
3089 	const struct xfrm_type *type = READ_ONCE(x->type);  in xfrm_state_mtu()
3093 	if (x->km.state != XFRM_STATE_VALID ||  in xfrm_state_mtu()
3094 	    !type || type->proto != IPPROTO_ESP)  in xfrm_state_mtu()
3095 		return mtu - x->props.header_len;  in xfrm_state_mtu()
3097 	aead = x->data;  in xfrm_state_mtu()
3100 	switch (x->props.mode) {  in xfrm_state_mtu()
3103 		if (x->props.family == AF_INET)  in xfrm_state_mtu()
3105 		else if (x->props.family == AF_INET6)  in xfrm_state_mtu()
3111 		if (x->mode_cbs && x->mode_cbs->get_inner_mtu)  in xfrm_state_mtu()
3112 			return x->mode_cbs->get_inner_mtu(x, mtu);  in xfrm_state_mtu()
3118 	return ((mtu - x->props.header_len - crypto_aead_authsize(aead) -  in xfrm_state_mtu()
3119 		 net_adj) & ~(blksize - 1)) + net_adj - 2;  in xfrm_state_mtu()
3123 int __xfrm_init_state(struct xfrm_state *x, bool init_replay,  in __xfrm_init_state()  argument
3128 	int family = x->props.family;  in __xfrm_init_state()
3132 	    READ_ONCE(xs_net(x)->ipv4.sysctl_ip_no_pmtu_disc))  in __xfrm_init_state()
3133 		x->props.flags |= XFRM_STATE_NOPMTUDISC;  in __xfrm_init_state()
3135 	err = -EPROTONOSUPPORT;  in __xfrm_init_state()
3137 	if (x->sel.family != AF_UNSPEC) {  in __xfrm_init_state()
3138 		inner_mode = xfrm_get_mode(x->props.mode, x->sel.family);  in __xfrm_init_state()
3144 		if (!(inner_mode->flags & XFRM_MODE_FLAG_TUNNEL) &&  in __xfrm_init_state()
3145 		    family != x->sel.family) {  in __xfrm_init_state()
3150 		x->inner_mode = *inner_mode;  in __xfrm_init_state()
3155 		inner_mode = xfrm_get_mode(x->props.mode, x->props.family);  in __xfrm_init_state()
3161 		x->inner_mode = *inner_mode;  in __xfrm_init_state()
3163 		if (x->props.family == AF_INET)  in __xfrm_init_state()
3166 		inner_mode_iaf = xfrm_get_mode(x->props.mode, iafamily);  in __xfrm_init_state()
3168 			if (inner_mode_iaf->flags & XFRM_MODE_FLAG_TUNNEL)  in __xfrm_init_state()
3169 				x->inner_mode_iaf = *inner_mode_iaf;  in __xfrm_init_state()
3173 	x->type = xfrm_get_type(x->id.proto, family);  in __xfrm_init_state()
3174 	if (x->type == NULL) {  in __xfrm_init_state()
3179 	err = x->type->init_state(x, extack);  in __xfrm_init_state()
3183 	outer_mode = xfrm_get_mode(x->props.mode, family);  in __xfrm_init_state()
3186 		err = -EPROTONOSUPPORT;  in __xfrm_init_state()
3190 	x->outer_mode = *outer_mode;  in __xfrm_init_state()
3192 		err = xfrm_init_replay(x, extack);  in __xfrm_init_state()
3197 	if (x->nat_keepalive_interval) {  in __xfrm_init_state()
3198 		if (x->dir != XFRM_SA_DIR_OUT) {  in __xfrm_init_state()
3200 			err = -EINVAL;  in __xfrm_init_state()
3204 		if (!x->encap || x->encap->encap_type != UDP_ENCAP_ESPINUDP) {  in __xfrm_init_state()
3207 			err = -EINVAL;  in __xfrm_init_state()
3212 	x->mode_cbs = xfrm_get_mode_cbs(x->props.mode);  in __xfrm_init_state()
3213 	if (x->mode_cbs) {  in __xfrm_init_state()
3214 		if (x->mode_cbs->init_state)  in __xfrm_init_state()
3215 			err = x->mode_cbs->init_state(x);  in __xfrm_init_state()
3216 		module_put(x->mode_cbs->owner);  in __xfrm_init_state()
3224 int xfrm_init_state(struct xfrm_state *x)  in xfrm_init_state()  argument
3228 	err = __xfrm_init_state(x, true, NULL);  in xfrm_init_state()
3230 		x->km.state = XFRM_STATE_VALID;  in xfrm_init_state()
3245 	INIT_LIST_HEAD(&net->xfrm.state_all);  in xfrm_state_init()
3249 	net->xfrm.state_bydst = xfrm_hash_alloc(sz);  in xfrm_state_init()
3250 	if (!net->xfrm.state_bydst)  in xfrm_state_init()
3252 	net->xfrm.state_bysrc = xfrm_hash_alloc(sz);  in xfrm_state_init()
3253 	if (!net->xfrm.state_bysrc)  in xfrm_state_init()
3255 	net->xfrm.state_byspi = xfrm_hash_alloc(sz);  in xfrm_state_init()
3256 	if (!net->xfrm.state_byspi)  in xfrm_state_init()
3258 	net->xfrm.state_byseq = xfrm_hash_alloc(sz);  in xfrm_state_init()
3259 	if (!net->xfrm.state_byseq)  in xfrm_state_init()
3262 	net->xfrm.state_cache_input = alloc_percpu(struct hlist_head);  in xfrm_state_init()
3263 	if (!net->xfrm.state_cache_input)  in xfrm_state_init()
3266 	net->xfrm.state_hmask = ((sz / sizeof(struct hlist_head)) - 1);  in xfrm_state_init()
3268 	net->xfrm.state_num = 0;  in xfrm_state_init()
3269 	INIT_WORK(&net->xfrm.state_hash_work, xfrm_hash_resize);  in xfrm_state_init()
3270 	spin_lock_init(&net->xfrm.xfrm_state_lock);  in xfrm_state_init()
3271 	seqcount_spinlock_init(&net->xfrm.xfrm_state_hash_generation,  in xfrm_state_init()
3272 			       &net->xfrm.xfrm_state_lock);  in xfrm_state_init()
3276 	xfrm_hash_free(net->xfrm.state_byseq, sz);  in xfrm_state_init()
3278 	xfrm_hash_free(net->xfrm.state_byspi, sz);  in xfrm_state_init()
3280 	xfrm_hash_free(net->xfrm.state_bysrc, sz);  in xfrm_state_init()
3282 	xfrm_hash_free(net->xfrm.state_bydst, sz);  in xfrm_state_init()
3284 	return -ENOMEM;  in xfrm_state_init()
3291 	flush_work(&net->xfrm.state_hash_work);  in xfrm_state_fini()
3295 	WARN_ON(!list_empty(&net->xfrm.state_all));  in xfrm_state_fini()
3297 	sz = (net->xfrm.state_hmask + 1) * sizeof(struct hlist_head);  in xfrm_state_fini()
3298 	WARN_ON(!hlist_empty(net->xfrm.state_byseq));  in xfrm_state_fini()
3299 	xfrm_hash_free(net->xfrm.state_byseq, sz);  in xfrm_state_fini()
3300 	WARN_ON(!hlist_empty(net->xfrm.state_byspi));  in xfrm_state_fini()
3301 	xfrm_hash_free(net->xfrm.state_byspi, sz);  in xfrm_state_fini()
3302 	WARN_ON(!hlist_empty(net->xfrm.state_bysrc));  in xfrm_state_fini()
3303 	xfrm_hash_free(net->xfrm.state_bysrc, sz);  in xfrm_state_fini()
3304 	WARN_ON(!hlist_empty(net->xfrm.state_bydst));  in xfrm_state_fini()
3305 	xfrm_hash_free(net->xfrm.state_bydst, sz);  in xfrm_state_fini()
3306 	free_percpu(net->xfrm.state_cache_input);  in xfrm_state_fini()
3310 static void xfrm_audit_helper_sainfo(struct xfrm_state *x,  in xfrm_audit_helper_sainfo()  argument
3313 	struct xfrm_sec_ctx *ctx = x->security;  in xfrm_audit_helper_sainfo()
3314 	u32 spi = ntohl(x->id.spi);  in xfrm_audit_helper_sainfo()
3318 				 ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str);  in xfrm_audit_helper_sainfo()
3320 	switch (x->props.family) {  in xfrm_audit_helper_sainfo()
3323 				 &x->props.saddr.a4, &x->id.daddr.a4);  in xfrm_audit_helper_sainfo()
3327 				 x->props.saddr.a6, x->id.daddr.a6);  in xfrm_audit_helper_sainfo()
3331 	audit_log_format(audit_buf, " spi=%u(0x%x)", spi, spi);  in xfrm_audit_helper_sainfo()
3344 				 &iph4->saddr, &iph4->daddr);  in xfrm_audit_helper_pktinfo()
3349 				 " src=%pI6 dst=%pI6 flowlbl=0x%x%02x%02x",  in xfrm_audit_helper_pktinfo()
3350 				 &iph6->saddr, &iph6->daddr,  in xfrm_audit_helper_pktinfo()
3351 				 iph6->flow_lbl[0] & 0x0f,  in xfrm_audit_helper_pktinfo()
3352 				 iph6->flow_lbl[1],  in xfrm_audit_helper_pktinfo()
3353 				 iph6->flow_lbl[2]);  in xfrm_audit_helper_pktinfo()
3358 void xfrm_audit_state_add(struct xfrm_state *x, int result, bool task_valid)  in xfrm_audit_state_add()  argument
3362 	audit_buf = xfrm_audit_start("SAD-add");  in xfrm_audit_state_add()
3366 	xfrm_audit_helper_sainfo(x, audit_buf);  in xfrm_audit_state_add()
3372 void xfrm_audit_state_delete(struct xfrm_state *x, int result, bool task_valid)  in xfrm_audit_state_delete()  argument
3376 	audit_buf = xfrm_audit_start("SAD-delete");  in xfrm_audit_state_delete()
3380 	xfrm_audit_helper_sainfo(x, audit_buf);  in xfrm_audit_state_delete()
3386 void xfrm_audit_state_replay_overflow(struct xfrm_state *x,  in xfrm_audit_state_replay_overflow()  argument
3392 	audit_buf = xfrm_audit_start("SA-replay-overflow");  in xfrm_audit_state_replay_overflow()
3395 	xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf);  in xfrm_audit_state_replay_overflow()
3398 	spi = ntohl(x->id.spi);  in xfrm_audit_state_replay_overflow()
3399 	audit_log_format(audit_buf, " spi=%u(0x%x)", spi, spi);  in xfrm_audit_state_replay_overflow()
3404 void xfrm_audit_state_replay(struct xfrm_state *x,  in xfrm_audit_state_replay()  argument
3410 	audit_buf = xfrm_audit_start("SA-replayed-pkt");  in xfrm_audit_state_replay()
3413 	xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf);  in xfrm_audit_state_replay()
3414 	spi = ntohl(x->id.spi);  in xfrm_audit_state_replay()
3415 	audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u",  in xfrm_audit_state_replay()
3425 	audit_buf = xfrm_audit_start("SA-notfound");  in xfrm_audit_state_notfound_simple()
3439 	audit_buf = xfrm_audit_start("SA-notfound");  in xfrm_audit_state_notfound()
3444 	audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u",  in xfrm_audit_state_notfound()
3450 void xfrm_audit_state_icvfail(struct xfrm_state *x,  in xfrm_audit_state_icvfail()  argument
3457 	audit_buf = xfrm_audit_start("SA-icv-failure");  in xfrm_audit_state_icvfail()
3460 	xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf);  in xfrm_audit_state_icvfail()
3463 		audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u",  in xfrm_audit_state_icvfail()