Lines Matching +full:sp +full:- +full:disabled +full:- +full:ports
1 // SPDX-License-Identifier: GPL-2.0-only
12 * Split up af-specific portion
82 * +---- root_d: sorted by daddr:prefix
86 * | +- root: sorted by saddr/prefix
94 * | +- coarse policies and all any:daddr policies
96 * +---- root_s: sorted by saddr:prefix
104 * +---- coarse policies and all any:any policies
107 * 1. any:any list from top-level xfrm_pol_inexact_bin
116 * This replicates previous single-list-search algorithm which would
117 * return first matching policy in the (ordered-by-priority) list.
167 struct flow_dissector_key_ports ports; member
213 return refcount_inc_not_zero(&policy->refcnt); in xfrm_pol_hold_rcu()
219 const struct flowi4 *fl4 = &fl->u.ip4; in __xfrm4_selector_match()
221 return addr4_match(fl4->daddr, sel->daddr.a4, sel->prefixlen_d) && in __xfrm4_selector_match()
222 addr4_match(fl4->saddr, sel->saddr.a4, sel->prefixlen_s) && in __xfrm4_selector_match()
223 !((xfrm_flowi_dport(fl, &fl4->uli) ^ sel->dport) & sel->dport_mask) && in __xfrm4_selector_match()
224 !((xfrm_flowi_sport(fl, &fl4->uli) ^ sel->sport) & sel->sport_mask) && in __xfrm4_selector_match()
225 (fl4->flowi4_proto == sel->proto || !sel->proto) && in __xfrm4_selector_match()
226 (fl4->flowi4_oif == sel->ifindex || !sel->ifindex); in __xfrm4_selector_match()
232 const struct flowi6 *fl6 = &fl->u.ip6; in __xfrm6_selector_match()
234 return addr_match(&fl6->daddr, &sel->daddr, sel->prefixlen_d) && in __xfrm6_selector_match()
235 addr_match(&fl6->saddr, &sel->saddr, sel->prefixlen_s) && in __xfrm6_selector_match()
236 !((xfrm_flowi_dport(fl, &fl6->uli) ^ sel->dport) & sel->dport_mask) && in __xfrm6_selector_match()
237 !((xfrm_flowi_sport(fl, &fl6->uli) ^ sel->sport) & sel->sport_mask) && in __xfrm6_selector_match()
238 (fl6->flowi6_proto == sel->proto || !sel->proto) && in __xfrm6_selector_match()
239 (fl6->flowi6_oif == sel->ifindex || !sel->ifindex); in __xfrm6_selector_match()
281 return ERR_PTR(-EAFNOSUPPORT); in __xfrm_dst_lookup()
283 dst = afinfo->dst_lookup(params); in __xfrm_dst_lookup()
299 xfrm_address_t *saddr = &x->props.saddr; in xfrm_dst_lookup()
300 xfrm_address_t *daddr = &x->id.daddr; in xfrm_dst_lookup()
303 if (x->type->flags & XFRM_TYPE_LOCAL_COADDR) { in xfrm_dst_lookup()
304 saddr = x->coaddr; in xfrm_dst_lookup()
307 if (x->type->flags & XFRM_TYPE_REMOTE_COADDR) { in xfrm_dst_lookup()
309 daddr = x->coaddr; in xfrm_dst_lookup()
318 params.ipproto = x->id.proto; in xfrm_dst_lookup()
319 if (x->encap) { in xfrm_dst_lookup()
320 switch (x->encap->encap_type) { in xfrm_dst_lookup()
323 params.uli.ports.sport = x->encap->encap_sport; in xfrm_dst_lookup()
324 params.uli.ports.dport = x->encap->encap_dport; in xfrm_dst_lookup()
328 params.uli.ports.sport = x->encap->encap_sport; in xfrm_dst_lookup()
329 params.uli.ports.dport = x->encap->encap_dport; in xfrm_dst_lookup()
348 if (secs >= (MAX_SCHEDULE_TIMEOUT-1)/HZ) in make_jiffies()
349 return MAX_SCHEDULE_TIMEOUT-1; in make_jiffies()
362 read_lock(&xp->lock); in xfrm_policy_timer()
364 if (unlikely(xp->walk.dead)) in xfrm_policy_timer()
367 dir = xfrm_policy_id2dir(xp->index); in xfrm_policy_timer()
369 if (xp->lft.hard_add_expires_seconds) { in xfrm_policy_timer()
370 time64_t tmo = xp->lft.hard_add_expires_seconds + in xfrm_policy_timer()
371 xp->curlft.add_time - now; in xfrm_policy_timer()
377 if (xp->lft.hard_use_expires_seconds) { in xfrm_policy_timer()
378 time64_t tmo = xp->lft.hard_use_expires_seconds + in xfrm_policy_timer()
379 (READ_ONCE(xp->curlft.use_time) ? : xp->curlft.add_time) - now; in xfrm_policy_timer()
385 if (xp->lft.soft_add_expires_seconds) { in xfrm_policy_timer()
386 time64_t tmo = xp->lft.soft_add_expires_seconds + in xfrm_policy_timer()
387 xp->curlft.add_time - now; in xfrm_policy_timer()
395 if (xp->lft.soft_use_expires_seconds) { in xfrm_policy_timer()
396 time64_t tmo = xp->lft.soft_use_expires_seconds + in xfrm_policy_timer()
397 (READ_ONCE(xp->curlft.use_time) ? : xp->curlft.add_time) - now; in xfrm_policy_timer()
409 !mod_timer(&xp->timer, jiffies + make_jiffies(next))) in xfrm_policy_timer()
413 read_unlock(&xp->lock); in xfrm_policy_timer()
418 read_unlock(&xp->lock); in xfrm_policy_timer()
435 write_pnet(&policy->xp_net, net); in xfrm_policy_alloc()
436 INIT_LIST_HEAD(&policy->walk.all); in xfrm_policy_alloc()
437 INIT_HLIST_HEAD(&policy->state_cache_list); in xfrm_policy_alloc()
438 INIT_HLIST_NODE(&policy->bydst); in xfrm_policy_alloc()
439 INIT_HLIST_NODE(&policy->byidx); in xfrm_policy_alloc()
440 rwlock_init(&policy->lock); in xfrm_policy_alloc()
441 refcount_set(&policy->refcnt, 1); in xfrm_policy_alloc()
442 skb_queue_head_init(&policy->polq.hold_queue); in xfrm_policy_alloc()
443 timer_setup(&policy->timer, xfrm_policy_timer, 0); in xfrm_policy_alloc()
444 timer_setup(&policy->polq.hold_timer, in xfrm_policy_alloc()
455 security_xfrm_policy_free(policy->security); in xfrm_policy_destroy_rcu()
463 BUG_ON(!policy->walk.dead); in xfrm_policy_destroy()
465 if (del_timer(&policy->timer) || del_timer(&policy->polq.hold_timer)) in xfrm_policy_destroy()
469 call_rcu(&policy->rcu, xfrm_policy_destroy_rcu); in xfrm_policy_destroy()
484 write_lock_bh(&policy->lock); in xfrm_policy_kill()
485 policy->walk.dead = 1; in xfrm_policy_kill()
486 write_unlock_bh(&policy->lock); in xfrm_policy_kill()
488 atomic_inc(&policy->genid); in xfrm_policy_kill()
490 if (del_timer(&policy->polq.hold_timer)) in xfrm_policy_kill()
492 skb_queue_purge(&policy->polq.hold_queue); in xfrm_policy_kill()
494 if (del_timer(&policy->timer)) in xfrm_policy_kill()
498 spin_lock_bh(&net->xfrm.xfrm_state_lock); in xfrm_policy_kill()
499 hlist_for_each_entry_rcu(x, &policy->state_cache_list, state_cache) { in xfrm_policy_kill()
500 hlist_del_init_rcu(&x->state_cache); in xfrm_policy_kill()
502 spin_unlock_bh(&net->xfrm.xfrm_state_lock); in xfrm_policy_kill()
511 return __idx_hash(index, net->xfrm.policy_idx_hmask); in idx_hash()
521 *dbits = net->xfrm.policy_bydst[dir].dbits4; in __get_hash_thresh()
522 *sbits = net->xfrm.policy_bydst[dir].sbits4; in __get_hash_thresh()
526 *dbits = net->xfrm.policy_bydst[dir].dbits6; in __get_hash_thresh()
527 *sbits = net->xfrm.policy_bydst[dir].sbits6; in __get_hash_thresh()
540 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; in policy_hash_bysel()
551 return rcu_dereference_check(net->xfrm.policy_bydst[dir].table, in policy_hash_bysel()
552 lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash; in policy_hash_bysel()
560 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; in policy_hash_direct()
568 return rcu_dereference_check(net->xfrm.policy_bydst[dir].table, in policy_hash_direct()
569 lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash; in policy_hash_direct()
588 __get_hash_thresh(net, pol->family, dir, &dbits, &sbits); in xfrm_dst_hash_transfer()
589 h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr, in xfrm_dst_hash_transfer()
590 pol->family, nhashmask, dbits, sbits); in xfrm_dst_hash_transfer()
591 if (!entry0 || pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) { in xfrm_dst_hash_transfer()
592 hlist_del_rcu(&pol->bydst); in xfrm_dst_hash_transfer()
593 hlist_add_head_rcu(&pol->bydst, ndsttable + h); in xfrm_dst_hash_transfer()
598 hlist_del_rcu(&pol->bydst); in xfrm_dst_hash_transfer()
599 hlist_add_behind_rcu(&pol->bydst, entry0); in xfrm_dst_hash_transfer()
601 entry0 = &pol->bydst; in xfrm_dst_hash_transfer()
619 h = __idx_hash(pol->index, nhashmask); in xfrm_idx_hash_transfer()
620 hlist_add_head(&pol->byidx, nidxtable+h); in xfrm_idx_hash_transfer()
626 return ((old_hmask + 1) << 1) - 1; in xfrm_new_hash_mask()
631 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; in xfrm_bydst_resize()
641 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_bydst_resize()
642 write_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation); in xfrm_bydst_resize()
644 odst = rcu_dereference_protected(net->xfrm.policy_bydst[dir].table, in xfrm_bydst_resize()
645 lockdep_is_held(&net->xfrm.xfrm_policy_lock)); in xfrm_bydst_resize()
647 for (i = hmask; i >= 0; i--) in xfrm_bydst_resize()
650 rcu_assign_pointer(net->xfrm.policy_bydst[dir].table, ndst); in xfrm_bydst_resize()
651 net->xfrm.policy_bydst[dir].hmask = nhashmask; in xfrm_bydst_resize()
653 write_seqcount_end(&net->xfrm.xfrm_policy_hash_generation); in xfrm_bydst_resize()
654 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_bydst_resize()
663 unsigned int hmask = net->xfrm.policy_idx_hmask; in xfrm_byidx_resize()
666 struct hlist_head *oidx = net->xfrm.policy_byidx; in xfrm_byidx_resize()
673 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_byidx_resize()
675 for (i = hmask; i >= 0; i--) in xfrm_byidx_resize()
678 net->xfrm.policy_byidx = nidx; in xfrm_byidx_resize()
679 net->xfrm.policy_idx_hmask = nhashmask; in xfrm_byidx_resize()
681 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_byidx_resize()
688 unsigned int cnt = net->xfrm.policy_count[dir]; in xfrm_bydst_should_resize()
689 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; in xfrm_bydst_should_resize()
703 unsigned int hmask = net->xfrm.policy_idx_hmask; in xfrm_byidx_should_resize()
714 si->incnt = net->xfrm.policy_count[XFRM_POLICY_IN]; in xfrm_spd_getinfo()
715 si->outcnt = net->xfrm.policy_count[XFRM_POLICY_OUT]; in xfrm_spd_getinfo()
716 si->fwdcnt = net->xfrm.policy_count[XFRM_POLICY_FWD]; in xfrm_spd_getinfo()
717 si->inscnt = net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX]; in xfrm_spd_getinfo()
718 si->outscnt = net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX]; in xfrm_spd_getinfo()
719 si->fwdscnt = net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX]; in xfrm_spd_getinfo()
720 si->spdhcnt = net->xfrm.policy_idx_hmask; in xfrm_spd_getinfo()
721 si->spdhmcnt = xfrm_policy_hashmax; in xfrm_spd_getinfo()
753 .family = pol->family, in xfrm_policy_inexact_alloc_bin()
754 .type = pol->type, in xfrm_policy_inexact_alloc_bin()
756 .if_id = pol->if_id, in xfrm_policy_inexact_alloc_bin()
760 lockdep_assert_held(&net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_alloc_bin()
772 bin->k = k; in xfrm_policy_inexact_alloc_bin()
773 INIT_HLIST_HEAD(&bin->hhead); in xfrm_policy_inexact_alloc_bin()
774 bin->root_d = RB_ROOT; in xfrm_policy_inexact_alloc_bin()
775 bin->root_s = RB_ROOT; in xfrm_policy_inexact_alloc_bin()
776 seqcount_spinlock_init(&bin->count, &net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_alloc_bin()
779 &bin->k, &bin->head, in xfrm_policy_inexact_alloc_bin()
782 list_add(&bin->inexact_bins, &net->xfrm.inexact_bins); in xfrm_policy_inexact_alloc_bin()
813 addr = &policy->selector.saddr; in xfrm_policy_inexact_insert_use_any_list()
814 prefixlen = policy->selector.prefixlen_s; in xfrm_policy_inexact_insert_use_any_list()
817 policy->family, in xfrm_policy_inexact_insert_use_any_list()
819 addr = &policy->selector.daddr; in xfrm_policy_inexact_insert_use_any_list()
820 prefixlen = policy->selector.prefixlen_d; in xfrm_policy_inexact_insert_use_any_list()
822 policy->family, in xfrm_policy_inexact_insert_use_any_list()
830 node->addr = *addr; in xfrm_pol_inexact_node_init()
831 node->prefixlen = prefixlen; in xfrm_pol_inexact_node_init()
858 mask = ~0U << (32 - prefixlen); in xfrm_policy_addr_delta()
859 ma = ntohl(a->a4) & mask; in xfrm_policy_addr_delta()
860 mb = ntohl(b->a4) & mask; in xfrm_policy_addr_delta()
862 delta = -1; in xfrm_policy_addr_delta()
871 delta = memcmp(a->a6, b->a6, pdw << 2); in xfrm_policy_addr_delta()
876 mask = ~0U << (32 - pbi); in xfrm_policy_addr_delta()
877 ma = ntohl(a->a6[pdw]) & mask; in xfrm_policy_addr_delta()
878 mb = ntohl(b->a6[pdw]) & mask; in xfrm_policy_addr_delta()
880 delta = -1; in xfrm_policy_addr_delta()
902 list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) { in xfrm_policy_inexact_list_reinsert()
906 if (policy->walk.dead || !policy->bydst_reinsert) in xfrm_policy_inexact_list_reinsert()
909 WARN_ON_ONCE(policy->family != family); in xfrm_policy_inexact_list_reinsert()
911 policy->bydst_reinsert = false; in xfrm_policy_inexact_list_reinsert()
912 hlist_for_each_entry(p, &n->hhead, bydst) { in xfrm_policy_inexact_list_reinsert()
913 if (policy->priority > p->priority) in xfrm_policy_inexact_list_reinsert()
914 newpos = &p->bydst; in xfrm_policy_inexact_list_reinsert()
915 else if (policy->priority == p->priority && in xfrm_policy_inexact_list_reinsert()
916 policy->pos > p->pos) in xfrm_policy_inexact_list_reinsert()
917 newpos = &p->bydst; in xfrm_policy_inexact_list_reinsert()
922 if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) in xfrm_policy_inexact_list_reinsert()
923 hlist_add_behind_rcu(&policy->bydst, newpos); in xfrm_policy_inexact_list_reinsert()
925 hlist_add_head_rcu(&policy->bydst, &n->hhead); in xfrm_policy_inexact_list_reinsert()
935 matches_s = xfrm_policy_addr_delta(&policy->selector.saddr, in xfrm_policy_inexact_list_reinsert()
936 &n->addr, in xfrm_policy_inexact_list_reinsert()
937 n->prefixlen, in xfrm_policy_inexact_list_reinsert()
939 matches_d = xfrm_policy_addr_delta(&policy->selector.daddr, in xfrm_policy_inexact_list_reinsert()
940 &n->addr, in xfrm_policy_inexact_list_reinsert()
941 n->prefixlen, in xfrm_policy_inexact_list_reinsert()
964 WARN_ON_ONCE(!RB_EMPTY_ROOT(&n->root)); in xfrm_policy_inexact_node_reinsert()
967 p = &new->rb_node; in xfrm_policy_inexact_node_reinsert()
975 prefixlen = min(node->prefixlen, n->prefixlen); in xfrm_policy_inexact_node_reinsert()
977 delta = xfrm_policy_addr_delta(&n->addr, &node->addr, in xfrm_policy_inexact_node_reinsert()
980 p = &parent->rb_left; in xfrm_policy_inexact_node_reinsert()
982 p = &parent->rb_right; in xfrm_policy_inexact_node_reinsert()
984 bool same_prefixlen = node->prefixlen == n->prefixlen; in xfrm_policy_inexact_node_reinsert()
987 hlist_for_each_entry(tmp, &n->hhead, bydst) { in xfrm_policy_inexact_node_reinsert()
988 tmp->bydst_reinsert = true; in xfrm_policy_inexact_node_reinsert()
989 hlist_del_rcu(&tmp->bydst); in xfrm_policy_inexact_node_reinsert()
992 node->prefixlen = prefixlen; in xfrm_policy_inexact_node_reinsert()
1008 rb_link_node_rcu(&n->node, parent, p); in xfrm_policy_inexact_node_reinsert()
1009 rb_insert_color(&n->node, new); in xfrm_policy_inexact_node_reinsert()
1022 /* To-be-merged node v has a subtree. in xfrm_policy_inexact_node_merge()
1024 * Dismantle it and insert its nodes to n->root. in xfrm_policy_inexact_node_merge()
1026 while ((rnode = rb_first(&v->root)) != NULL) { in xfrm_policy_inexact_node_merge()
1028 rb_erase(&node->node, &v->root); in xfrm_policy_inexact_node_merge()
1029 xfrm_policy_inexact_node_reinsert(net, node, &n->root, in xfrm_policy_inexact_node_merge()
1033 hlist_for_each_entry(tmp, &v->hhead, bydst) { in xfrm_policy_inexact_node_merge()
1034 tmp->bydst_reinsert = true; in xfrm_policy_inexact_node_merge()
1035 hlist_del_rcu(&tmp->bydst); in xfrm_policy_inexact_node_merge()
1051 p = &root->rb_node; in xfrm_policy_inexact_insert_node()
1058 delta = xfrm_policy_addr_delta(addr, &node->addr, in xfrm_policy_inexact_insert_node()
1059 node->prefixlen, in xfrm_policy_inexact_insert_node()
1061 if (delta == 0 && prefixlen >= node->prefixlen) { in xfrm_policy_inexact_insert_node()
1067 p = &parent->rb_left; in xfrm_policy_inexact_insert_node()
1069 p = &parent->rb_right; in xfrm_policy_inexact_insert_node()
1071 if (prefixlen < node->prefixlen) { in xfrm_policy_inexact_insert_node()
1072 delta = xfrm_policy_addr_delta(addr, &node->addr, in xfrm_policy_inexact_insert_node()
1079 * to be removed and re-inserted with the smaller in xfrm_policy_inexact_insert_node()
1083 rb_erase(&node->node, root); in xfrm_policy_inexact_insert_node()
1091 * prefixlen. Merge the to-be-reinserted in xfrm_policy_inexact_insert_node()
1100 p = &root->rb_node; in xfrm_policy_inexact_insert_node()
1112 rb_link_node_rcu(&node->node, parent, p); in xfrm_policy_inexact_insert_node()
1113 rb_insert_color(&node->node, root); in xfrm_policy_inexact_insert_node()
1126 xfrm_policy_inexact_gc_tree(&node->root, rm); in xfrm_policy_inexact_gc_tree()
1129 if (!hlist_empty(&node->hhead) || !RB_EMPTY_ROOT(&node->root)) { in xfrm_policy_inexact_gc_tree()
1134 rb_erase(&node->node, r); in xfrm_policy_inexact_gc_tree()
1141 write_seqcount_begin(&b->count); in __xfrm_policy_inexact_prune_bin()
1142 xfrm_policy_inexact_gc_tree(&b->root_d, net_exit); in __xfrm_policy_inexact_prune_bin()
1143 xfrm_policy_inexact_gc_tree(&b->root_s, net_exit); in __xfrm_policy_inexact_prune_bin()
1144 write_seqcount_end(&b->count); in __xfrm_policy_inexact_prune_bin()
1146 if (!RB_EMPTY_ROOT(&b->root_d) || !RB_EMPTY_ROOT(&b->root_s) || in __xfrm_policy_inexact_prune_bin()
1147 !hlist_empty(&b->hhead)) { in __xfrm_policy_inexact_prune_bin()
1152 if (rhashtable_remove_fast(&xfrm_policy_inexact_table, &b->head, in __xfrm_policy_inexact_prune_bin()
1154 list_del(&b->inexact_bins); in __xfrm_policy_inexact_prune_bin()
1161 struct net *net = read_pnet(&b->k.net); in xfrm_policy_inexact_prune_bin()
1163 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_prune_bin()
1165 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_prune_bin()
1172 lockdep_assert_held(&net->xfrm.xfrm_policy_lock); in __xfrm_policy_inexact_flush()
1174 list_for_each_entry_safe(bin, t, &net->xfrm.inexact_bins, inexact_bins) in __xfrm_policy_inexact_flush()
1186 lockdep_assert_held(&net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_alloc_chain()
1189 return &bin->hhead; in xfrm_policy_inexact_alloc_chain()
1191 if (xfrm_pol_inexact_addr_use_any_list(&policy->selector.daddr, in xfrm_policy_inexact_alloc_chain()
1192 policy->family, in xfrm_policy_inexact_alloc_chain()
1193 policy->selector.prefixlen_d)) { in xfrm_policy_inexact_alloc_chain()
1194 write_seqcount_begin(&bin->count); in xfrm_policy_inexact_alloc_chain()
1196 &bin->root_s, in xfrm_policy_inexact_alloc_chain()
1197 &policy->selector.saddr, in xfrm_policy_inexact_alloc_chain()
1198 policy->family, in xfrm_policy_inexact_alloc_chain()
1199 policy->selector.prefixlen_s, in xfrm_policy_inexact_alloc_chain()
1201 write_seqcount_end(&bin->count); in xfrm_policy_inexact_alloc_chain()
1205 return &n->hhead; in xfrm_policy_inexact_alloc_chain()
1209 write_seqcount_begin(&bin->count); in xfrm_policy_inexact_alloc_chain()
1211 &bin->root_d, in xfrm_policy_inexact_alloc_chain()
1212 &policy->selector.daddr, in xfrm_policy_inexact_alloc_chain()
1213 policy->family, in xfrm_policy_inexact_alloc_chain()
1214 policy->selector.prefixlen_d, dir); in xfrm_policy_inexact_alloc_chain()
1215 write_seqcount_end(&bin->count); in xfrm_policy_inexact_alloc_chain()
1220 if (xfrm_pol_inexact_addr_use_any_list(&policy->selector.saddr, in xfrm_policy_inexact_alloc_chain()
1221 policy->family, in xfrm_policy_inexact_alloc_chain()
1222 policy->selector.prefixlen_s)) in xfrm_policy_inexact_alloc_chain()
1223 return &n->hhead; in xfrm_policy_inexact_alloc_chain()
1225 write_seqcount_begin(&bin->count); in xfrm_policy_inexact_alloc_chain()
1227 &n->root, in xfrm_policy_inexact_alloc_chain()
1228 &policy->selector.saddr, in xfrm_policy_inexact_alloc_chain()
1229 policy->family, in xfrm_policy_inexact_alloc_chain()
1230 policy->selector.prefixlen_s, dir); in xfrm_policy_inexact_alloc_chain()
1231 write_seqcount_end(&bin->count); in xfrm_policy_inexact_alloc_chain()
1235 return &n->hhead; in xfrm_policy_inexact_alloc_chain()
1248 return ERR_PTR(-ENOMEM); in xfrm_policy_inexact_insert()
1251 lockdep_assert_held(&net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_insert()
1256 return ERR_PTR(-ENOMEM); in xfrm_policy_inexact_insert()
1262 return ERR_PTR(-EEXIST); in xfrm_policy_inexact_insert()
1275 if (policy->walk.dead) in xfrm_policy_is_dead_or_sk()
1278 dir = xfrm_policy_id2dir(policy->index); in xfrm_policy_is_dead_or_sk()
1298 seq = read_seqbegin(&net->xfrm.policy_hthresh.lock); in xfrm_hash_rebuild()
1300 lbits4 = net->xfrm.policy_hthresh.lbits4; in xfrm_hash_rebuild()
1301 rbits4 = net->xfrm.policy_hthresh.rbits4; in xfrm_hash_rebuild()
1302 lbits6 = net->xfrm.policy_hthresh.lbits6; in xfrm_hash_rebuild()
1303 rbits6 = net->xfrm.policy_hthresh.rbits6; in xfrm_hash_rebuild()
1304 } while (read_seqretry(&net->xfrm.policy_hthresh.lock, seq)); in xfrm_hash_rebuild()
1306 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_hash_rebuild()
1307 write_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation); in xfrm_hash_rebuild()
1312 list_for_each_entry(policy, &net->xfrm.policy_all, walk.all) { in xfrm_hash_rebuild()
1319 dir = xfrm_policy_id2dir(policy->index); in xfrm_hash_rebuild()
1321 if (policy->family == AF_INET) { in xfrm_hash_rebuild()
1329 if (policy->family == AF_INET) { in xfrm_hash_rebuild()
1338 if (policy->selector.prefixlen_d < dbits || in xfrm_hash_rebuild()
1339 policy->selector.prefixlen_s < sbits) in xfrm_hash_rebuild()
1353 net->xfrm.policy_bydst[dir].dbits4 = rbits4; in xfrm_hash_rebuild()
1354 net->xfrm.policy_bydst[dir].sbits4 = lbits4; in xfrm_hash_rebuild()
1355 net->xfrm.policy_bydst[dir].dbits6 = rbits6; in xfrm_hash_rebuild()
1356 net->xfrm.policy_bydst[dir].sbits6 = lbits6; in xfrm_hash_rebuild()
1359 net->xfrm.policy_bydst[dir].dbits4 = lbits4; in xfrm_hash_rebuild()
1360 net->xfrm.policy_bydst[dir].sbits4 = rbits4; in xfrm_hash_rebuild()
1361 net->xfrm.policy_bydst[dir].dbits6 = lbits6; in xfrm_hash_rebuild()
1362 net->xfrm.policy_bydst[dir].sbits6 = rbits6; in xfrm_hash_rebuild()
1366 /* re-insert all policies by order of creation */ in xfrm_hash_rebuild()
1367 list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) { in xfrm_hash_rebuild()
1371 hlist_del_rcu(&policy->bydst); in xfrm_hash_rebuild()
1374 dir = xfrm_policy_id2dir(policy->index); in xfrm_hash_rebuild()
1375 chain = policy_hash_bysel(net, &policy->selector, in xfrm_hash_rebuild()
1376 policy->family, dir); in xfrm_hash_rebuild()
1386 if (policy->priority >= pol->priority) in xfrm_hash_rebuild()
1387 newpos = &pol->bydst; in xfrm_hash_rebuild()
1391 if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) in xfrm_hash_rebuild()
1392 hlist_add_behind_rcu(&policy->bydst, newpos); in xfrm_hash_rebuild()
1394 hlist_add_head_rcu(&policy->bydst, chain); in xfrm_hash_rebuild()
1399 write_seqcount_end(&net->xfrm.xfrm_policy_hash_generation); in xfrm_hash_rebuild()
1400 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_hash_rebuild()
1407 schedule_work(&net->xfrm.policy_hthresh.work); in xfrm_policy_hash_rebuild()
1422 idx = (net->xfrm.idx_generator | dir); in xfrm_gen_index()
1423 net->xfrm.idx_generator += 8; in xfrm_gen_index()
1431 list = net->xfrm.policy_byidx + idx_hash(net, idx); in xfrm_gen_index()
1434 if (p->index == idx) { in xfrm_gen_index()
1462 struct xfrm_policy_queue *pq = &old->polq; in xfrm_policy_requeue()
1465 if (skb_queue_empty(&pq->hold_queue)) in xfrm_policy_requeue()
1470 spin_lock_bh(&pq->hold_queue.lock); in xfrm_policy_requeue()
1471 skb_queue_splice_init(&pq->hold_queue, &list); in xfrm_policy_requeue()
1472 if (del_timer(&pq->hold_timer)) in xfrm_policy_requeue()
1474 spin_unlock_bh(&pq->hold_queue.lock); in xfrm_policy_requeue()
1476 pq = &new->polq; in xfrm_policy_requeue()
1478 spin_lock_bh(&pq->hold_queue.lock); in xfrm_policy_requeue()
1479 skb_queue_splice(&list, &pq->hold_queue); in xfrm_policy_requeue()
1480 pq->timeout = XFRM_QUEUE_TMO_MIN; in xfrm_policy_requeue()
1481 if (!mod_timer(&pq->hold_timer, jiffies)) in xfrm_policy_requeue()
1483 spin_unlock_bh(&pq->hold_queue.lock); in xfrm_policy_requeue()
1489 return mark->v == pol->mark.v && mark->m == pol->mark.m; in xfrm_policy_mark_match()
1495 u32 a = k->type << 24 | k->dir << 16 | k->family; in xfrm_pol_bin_key()
1497 return jhash_3words(a, k->if_id, net_hash_mix(read_pnet(&k->net)), in xfrm_pol_bin_key()
1505 return xfrm_pol_bin_key(&b->k, 0, seed); in xfrm_pol_bin_obj()
1511 const struct xfrm_pol_inexact_key *key = arg->key; in xfrm_pol_bin_cmp()
1515 if (!net_eq(read_pnet(&b->k.net), read_pnet(&key->net))) in xfrm_pol_bin_cmp()
1516 return -1; in xfrm_pol_bin_cmp()
1518 ret = b->k.dir ^ key->dir; in xfrm_pol_bin_cmp()
1522 ret = b->k.type ^ key->type; in xfrm_pol_bin_cmp()
1526 ret = b->k.family ^ key->family; in xfrm_pol_bin_cmp()
1530 return b->k.if_id ^ key->if_id; in xfrm_pol_bin_cmp()
1548 if (pol->type == policy->type && in xfrm_policy_insert_list()
1549 pol->if_id == policy->if_id && in xfrm_policy_insert_list()
1550 !selector_cmp(&pol->selector, &policy->selector) && in xfrm_policy_insert_list()
1551 xfrm_policy_mark_match(&policy->mark, pol) && in xfrm_policy_insert_list()
1552 xfrm_sec_ctx_match(pol->security, policy->security) && in xfrm_policy_insert_list()
1555 return ERR_PTR(-EEXIST); in xfrm_policy_insert_list()
1557 if (policy->priority > pol->priority) in xfrm_policy_insert_list()
1559 } else if (policy->priority >= pol->priority) { in xfrm_policy_insert_list()
1567 if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) in xfrm_policy_insert_list()
1568 hlist_add_behind_rcu(&policy->bydst, &newpos->bydst); in xfrm_policy_insert_list()
1571 * to speed-up lookups. in xfrm_policy_insert_list()
1573 hlist_add_head_rcu(&policy->bydst, chain); in xfrm_policy_insert_list()
1584 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_insert()
1585 chain = policy_hash_bysel(net, &policy->selector, policy->family, dir); in xfrm_policy_insert()
1592 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_insert()
1599 if (policy->family == AF_INET) in xfrm_policy_insert()
1608 policy->index = delpol ? delpol->index : xfrm_gen_index(net, dir, policy->index); in xfrm_policy_insert()
1609 hlist_add_head(&policy->byidx, net->xfrm.policy_byidx+idx_hash(net, policy->index)); in xfrm_policy_insert()
1610 policy->curlft.add_time = ktime_get_real_seconds(); in xfrm_policy_insert()
1611 policy->curlft.use_time = 0; in xfrm_policy_insert()
1612 if (!mod_timer(&policy->timer, jiffies + HZ)) in xfrm_policy_insert()
1614 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_insert()
1619 schedule_work(&net->xfrm.policy_hash_work); in xfrm_policy_insert()
1636 if (pol->type == type && in __xfrm_policy_bysel_ctx()
1637 pol->if_id == if_id && in __xfrm_policy_bysel_ctx()
1639 !selector_cmp(sel, &pol->selector) && in __xfrm_policy_bysel_ctx()
1640 xfrm_sec_ctx_match(ctx, pol->security)) in __xfrm_policy_bysel_ctx()
1657 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_bysel_ctx()
1658 chain = policy_hash_bysel(net, sel, sel->family, dir); in xfrm_policy_bysel_ctx()
1664 sel->family, dir, if_id); in xfrm_policy_bysel_ctx()
1666 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_bysel_ctx()
1671 &sel->saddr, in xfrm_policy_bysel_ctx()
1672 &sel->daddr)) { in xfrm_policy_bysel_ctx()
1673 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_bysel_ctx()
1687 if (!pol || tmp->pos < pol->pos) in xfrm_policy_bysel_ctx()
1698 *err = security_xfrm_policy_delete(pol->security); in xfrm_policy_bysel_ctx()
1700 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_bysel_ctx()
1707 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_bysel_ctx()
1724 *err = -ENOENT; in xfrm_policy_byid()
1729 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_byid()
1730 chain = net->xfrm.policy_byidx + idx_hash(net, id); in xfrm_policy_byid()
1733 if (pol->type == type && pol->index == id && in xfrm_policy_byid()
1734 pol->if_id == if_id && xfrm_policy_mark_match(mark, pol)) { in xfrm_policy_byid()
1738 pol->security); in xfrm_policy_byid()
1740 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_byid()
1749 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_byid()
1764 list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { in xfrm_policy_flush_secctx_check()
1765 if (pol->walk.dead || in xfrm_policy_flush_secctx_check()
1766 xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX || in xfrm_policy_flush_secctx_check()
1767 pol->type != type) in xfrm_policy_flush_secctx_check()
1770 err = security_xfrm_policy_delete(pol->security); in xfrm_policy_flush_secctx_check()
1786 list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { in xfrm_dev_policy_flush_secctx_check()
1787 if (pol->walk.dead || in xfrm_dev_policy_flush_secctx_check()
1788 xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX || in xfrm_dev_policy_flush_secctx_check()
1789 pol->xdo.dev != dev) in xfrm_dev_policy_flush_secctx_check()
1792 err = security_xfrm_policy_delete(pol->security); in xfrm_dev_policy_flush_secctx_check()
1820 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_flush()
1827 list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { in xfrm_policy_flush()
1828 if (pol->walk.dead) in xfrm_policy_flush()
1831 dir = xfrm_policy_id2dir(pol->index); in xfrm_policy_flush()
1833 pol->type != type) in xfrm_policy_flush()
1837 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_flush()
1841 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_flush()
1847 err = -ESRCH; in xfrm_policy_flush()
1849 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_flush()
1860 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_dev_policy_flush()
1867 list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { in xfrm_dev_policy_flush()
1868 if (pol->walk.dead) in xfrm_dev_policy_flush()
1871 dir = xfrm_policy_id2dir(pol->index); in xfrm_dev_policy_flush()
1873 pol->xdo.dev != dev) in xfrm_dev_policy_flush()
1877 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_dev_policy_flush()
1881 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_dev_policy_flush()
1887 err = -ESRCH; in xfrm_dev_policy_flush()
1889 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_dev_policy_flush()
1902 if (walk->type >= XFRM_POLICY_TYPE_MAX && in xfrm_policy_walk()
1903 walk->type != XFRM_POLICY_TYPE_ANY) in xfrm_policy_walk()
1904 return -EINVAL; in xfrm_policy_walk()
1906 if (list_empty(&walk->walk.all) && walk->seq != 0) in xfrm_policy_walk()
1909 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_walk()
1910 if (list_empty(&walk->walk.all)) in xfrm_policy_walk()
1911 x = list_first_entry(&net->xfrm.policy_all, struct xfrm_policy_walk_entry, all); in xfrm_policy_walk()
1913 x = list_first_entry(&walk->walk.all, in xfrm_policy_walk()
1916 list_for_each_entry_from(x, &net->xfrm.policy_all, all) { in xfrm_policy_walk()
1917 if (x->dead) in xfrm_policy_walk()
1920 if (walk->type != XFRM_POLICY_TYPE_ANY && in xfrm_policy_walk()
1921 walk->type != pol->type) in xfrm_policy_walk()
1923 error = func(pol, xfrm_policy_id2dir(pol->index), in xfrm_policy_walk()
1924 walk->seq, data); in xfrm_policy_walk()
1926 list_move_tail(&walk->walk.all, &x->all); in xfrm_policy_walk()
1929 walk->seq++; in xfrm_policy_walk()
1931 if (walk->seq == 0) { in xfrm_policy_walk()
1932 error = -ENOENT; in xfrm_policy_walk()
1935 list_del_init(&walk->walk.all); in xfrm_policy_walk()
1937 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_walk()
1944 INIT_LIST_HEAD(&walk->walk.all); in xfrm_policy_walk_init()
1945 walk->walk.dead = 1; in xfrm_policy_walk_init()
1946 walk->type = type; in xfrm_policy_walk_init()
1947 walk->seq = 0; in xfrm_policy_walk_init()
1953 if (list_empty(&walk->walk.all)) in xfrm_policy_walk_done()
1956 spin_lock_bh(&net->xfrm.xfrm_policy_lock); /*FIXME where is net? */ in xfrm_policy_walk_done()
1957 list_del(&walk->walk.all); in xfrm_policy_walk_done()
1958 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_walk_done()
1965 * Returns 0 if policy found, else an -errno.
1971 const struct xfrm_selector *sel = &pol->selector; in xfrm_policy_match()
1972 int ret = -ESRCH; in xfrm_policy_match()
1975 if (pol->family != family || in xfrm_policy_match()
1976 pol->if_id != if_id || in xfrm_policy_match()
1977 (fl->flowi_mark & pol->mark.m) != pol->mark.v || in xfrm_policy_match()
1978 pol->type != type) in xfrm_policy_match()
1983 ret = security_xfrm_policy_lookup(pol->security, fl->flowi_secid); in xfrm_policy_match()
1998 parent = rcu_dereference_raw(r->rb_node); in xfrm_policy_lookup_inexact_addr()
2005 delta = xfrm_policy_addr_delta(addr, &node->addr, in xfrm_policy_lookup_inexact_addr()
2006 node->prefixlen, family); in xfrm_policy_lookup_inexact_addr()
2008 parent = rcu_dereference_raw(parent->rb_left); in xfrm_policy_lookup_inexact_addr()
2011 parent = rcu_dereference_raw(parent->rb_right); in xfrm_policy_lookup_inexact_addr()
2036 family = b->k.family; in xfrm_policy_find_inexact_candidates()
2038 cand->res[XFRM_POL_CAND_ANY] = &b->hhead; in xfrm_policy_find_inexact_candidates()
2040 n = xfrm_policy_lookup_inexact_addr(&b->root_d, &b->count, daddr, in xfrm_policy_find_inexact_candidates()
2043 cand->res[XFRM_POL_CAND_DADDR] = &n->hhead; in xfrm_policy_find_inexact_candidates()
2044 n = xfrm_policy_lookup_inexact_addr(&n->root, &b->count, saddr, in xfrm_policy_find_inexact_candidates()
2047 cand->res[XFRM_POL_CAND_BOTH] = &n->hhead; in xfrm_policy_find_inexact_candidates()
2050 n = xfrm_policy_lookup_inexact_addr(&b->root_s, &b->count, saddr, in xfrm_policy_find_inexact_candidates()
2053 cand->res[XFRM_POL_CAND_SADDR] = &n->hhead; in xfrm_policy_find_inexact_candidates()
2081 lockdep_assert_held(&net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_lookup()
2096 u32 priority = prefer ? prefer->priority : ~0u; in __xfrm_policy_eval_candidates()
2105 if (pol->priority > priority) in __xfrm_policy_eval_candidates()
2110 if (err != -ESRCH) in __xfrm_policy_eval_candidates()
2118 if (pol->priority == priority && in __xfrm_policy_eval_candidates()
2119 prefer->pos < pol->pos) in __xfrm_policy_eval_candidates()
2138 for (i = 0; i < ARRAY_SIZE(cand->res); i++) { in xfrm_policy_eval_candidates()
2139 tmp = __xfrm_policy_eval_candidates(cand->res[i], in xfrm_policy_eval_candidates()
2174 sequence = read_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation); in xfrm_policy_lookup_bytype()
2176 } while (read_seqcount_retry(&net->xfrm.xfrm_policy_hash_generation, sequence)); in xfrm_policy_lookup_bytype()
2182 if (err == -ESRCH) in xfrm_policy_lookup_bytype()
2193 if (ret && ret->xdo.type == XFRM_DEV_OFFLOAD_PACKET) in xfrm_policy_lookup_bytype()
2210 if (read_seqcount_retry(&net->xfrm.xfrm_policy_hash_generation, sequence)) in xfrm_policy_lookup_bytype()
2245 pol = rcu_dereference(sk->sk_policy[dir]); in xfrm_sk_policy_lookup()
2250 if (pol->family != family) { in xfrm_sk_policy_lookup()
2255 match = xfrm_selector_match(&pol->selector, fl, family); in xfrm_sk_policy_lookup()
2257 if ((READ_ONCE(sk->sk_mark) & pol->mark.m) != pol->mark.v || in xfrm_sk_policy_lookup()
2258 pol->if_id != if_id) { in xfrm_sk_policy_lookup()
2262 err = security_xfrm_policy_lookup(pol->security, in xfrm_sk_policy_lookup()
2263 fl->flowi_secid); in xfrm_sk_policy_lookup()
2267 } else if (err == -ESRCH) { in xfrm_sk_policy_lookup()
2286 list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) { in xfrm_gen_pos_slow()
2288 policy->pos = ++i; in xfrm_gen_pos_slow()
2300 list_for_each_entry(policy, &net->xfrm.policy_all, walk.all) { in xfrm_gen_pos()
2304 if (policy->pos == UINT_MAX) in xfrm_gen_pos()
2307 i = policy->pos + 1; in xfrm_gen_pos()
2322 pol->pos = xfrm_gen_pos(net); in __xfrm_policy_link()
2326 list_add(&pol->walk.all, &net->xfrm.policy_all); in __xfrm_policy_link()
2327 net->xfrm.policy_count[dir]++; in __xfrm_policy_link()
2336 if (list_empty(&pol->walk.all)) in __xfrm_policy_unlink()
2340 if (!hlist_unhashed(&pol->bydst)) { in __xfrm_policy_unlink()
2341 hlist_del_rcu(&pol->bydst); in __xfrm_policy_unlink()
2342 hlist_del(&pol->byidx); in __xfrm_policy_unlink()
2345 list_del_init(&pol->walk.all); in __xfrm_policy_unlink()
2346 net->xfrm.policy_count[dir]--; in __xfrm_policy_unlink()
2365 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_delete()
2367 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_delete()
2372 return -ENOENT; in xfrm_policy_delete()
2382 if (pol && pol->type != XFRM_POLICY_TYPE_MAIN) in xfrm_sk_policy_insert()
2383 return -EINVAL; in xfrm_sk_policy_insert()
2386 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_sk_policy_insert()
2387 old_pol = rcu_dereference_protected(sk->sk_policy[dir], in xfrm_sk_policy_insert()
2388 lockdep_is_held(&net->xfrm.xfrm_policy_lock)); in xfrm_sk_policy_insert()
2390 pol->curlft.add_time = ktime_get_real_seconds(); in xfrm_sk_policy_insert()
2391 pol->index = xfrm_gen_index(net, XFRM_POLICY_MAX+dir, 0); in xfrm_sk_policy_insert()
2394 rcu_assign_pointer(sk->sk_policy[dir], pol); in xfrm_sk_policy_insert()
2404 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_sk_policy_insert()
2418 newp->selector = old->selector; in clone_policy()
2419 if (security_xfrm_policy_clone(old->security, in clone_policy()
2420 &newp->security)) { in clone_policy()
2424 newp->lft = old->lft; in clone_policy()
2425 newp->curlft = old->curlft; in clone_policy()
2426 newp->mark = old->mark; in clone_policy()
2427 newp->if_id = old->if_id; in clone_policy()
2428 newp->action = old->action; in clone_policy()
2429 newp->flags = old->flags; in clone_policy()
2430 newp->xfrm_nr = old->xfrm_nr; in clone_policy()
2431 newp->index = old->index; in clone_policy()
2432 newp->type = old->type; in clone_policy()
2433 newp->family = old->family; in clone_policy()
2434 memcpy(newp->xfrm_vec, old->xfrm_vec, in clone_policy()
2435 newp->xfrm_nr*sizeof(struct xfrm_tmpl)); in clone_policy()
2436 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in clone_policy()
2438 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in clone_policy()
2452 p = rcu_dereference(osk->sk_policy[i]); in __xfrm_sk_clone_policy()
2456 ret = -ENOMEM; in __xfrm_sk_clone_policy()
2459 rcu_assign_pointer(sk->sk_policy[i], np); in __xfrm_sk_clone_policy()
2474 return -EINVAL; in xfrm_get_saddr()
2475 err = afinfo->get_saddr(saddr, params); in xfrm_get_saddr()
2493 for (nx = 0, i = 0; i < policy->xfrm_nr; i++) { in xfrm_tmpl_resolve_one()
2497 struct xfrm_tmpl *tmpl = &policy->xfrm_vec[i]; in xfrm_tmpl_resolve_one()
2499 if (tmpl->mode == XFRM_MODE_TUNNEL || in xfrm_tmpl_resolve_one()
2500 tmpl->mode == XFRM_MODE_IPTFS || in xfrm_tmpl_resolve_one()
2501 tmpl->mode == XFRM_MODE_BEET) { in xfrm_tmpl_resolve_one()
2502 remote = &tmpl->id.daddr; in xfrm_tmpl_resolve_one()
2503 local = &tmpl->saddr; in xfrm_tmpl_resolve_one()
2504 if (xfrm_addr_any(local, tmpl->encap_family)) { in xfrm_tmpl_resolve_one()
2509 params.oif = fl->flowi_oif; in xfrm_tmpl_resolve_one()
2511 error = xfrm_get_saddr(tmpl->encap_family, &tmp, in xfrm_tmpl_resolve_one()
2520 family, policy->if_id); in xfrm_tmpl_resolve_one()
2521 if (x && x->dir && x->dir != XFRM_SA_DIR_OUT) { in xfrm_tmpl_resolve_one()
2524 error = -EINVAL; in xfrm_tmpl_resolve_one()
2528 if (x && x->km.state == XFRM_STATE_VALID) { in xfrm_tmpl_resolve_one()
2535 error = (x->km.state == XFRM_STATE_ERROR ? in xfrm_tmpl_resolve_one()
2536 -EINVAL : -EAGAIN); in xfrm_tmpl_resolve_one()
2538 } else if (error == -ESRCH) { in xfrm_tmpl_resolve_one()
2539 error = -EAGAIN; in xfrm_tmpl_resolve_one()
2542 if (!tmpl->optional) in xfrm_tmpl_resolve_one()
2548 for (nx--; nx >= 0; nx--) in xfrm_tmpl_resolve_one()
2565 if (cnx + pols[i]->xfrm_nr >= XFRM_MAX_DEPTH) { in xfrm_tmpl_resolve()
2566 error = -ENOBUFS; in xfrm_tmpl_resolve()
2585 for (cnx--; cnx >= 0; cnx--) in xfrm_tmpl_resolve()
2594 return inet_dsfield_to_dscp(fl->u.ip4.flowi4_tos); in xfrm_get_dscp()
2606 return ERR_PTR(-EINVAL); in xfrm_alloc_dst()
2610 dst_ops = &net->xfrm.xfrm4_dst_ops; in xfrm_alloc_dst()
2614 dst_ops = &net->xfrm.xfrm6_dst_ops; in xfrm_alloc_dst()
2625 xdst = ERR_PTR(-ENOBUFS); in xfrm_alloc_dst()
2635 if (dst->ops->family == AF_INET6) { in xfrm_init_path()
2636 path->path_cookie = rt6_get_cookie(dst_rt6_info(dst)); in xfrm_init_path()
2637 path->u.rt6.rt6i_nfheader_len = nfheader_len; in xfrm_init_path()
2645 xfrm_policy_get_afinfo(xdst->u.dst.ops->family); in xfrm_fill_dst()
2649 return -EINVAL; in xfrm_fill_dst()
2651 err = afinfo->fill_dst(xdst, dev, fl); in xfrm_fill_dst()
2682 int family = policy->selector.family; in xfrm_bundle_create()
2694 struct dst_entry *dst1 = &xdst->u.dst; in xfrm_bundle_create()
2709 xfrm_dst_set_child(xdst_prev, &xdst->u.dst); in xfrm_bundle_create()
2711 if (xfrm[i]->sel.family == AF_UNSPEC) { in xfrm_bundle_create()
2715 err = -EAFNOSUPPORT; in xfrm_bundle_create()
2720 inner_mode = &xfrm[i]->inner_mode; in xfrm_bundle_create()
2722 xdst->route = dst; in xfrm_bundle_create()
2725 if (xfrm[i]->props.mode != XFRM_MODE_TRANSPORT) { in xfrm_bundle_create()
2729 if (xfrm[i]->props.smark.v || xfrm[i]->props.smark.m) in xfrm_bundle_create()
2730 mark = xfrm_smark_get(fl->flowi_mark, xfrm[i]); in xfrm_bundle_create()
2732 if (xfrm[i]->xso.type != XFRM_DEV_OFFLOAD_PACKET) in xfrm_bundle_create()
2733 family = xfrm[i]->props.family; in xfrm_bundle_create()
2735 oif = fl->flowi_oif ? : fl->flowi_l3mdev; in xfrm_bundle_create()
2744 dst1->xfrm = xfrm[i]; in xfrm_bundle_create()
2745 xdst->xfrm_genid = xfrm[i]->genid; in xfrm_bundle_create()
2747 dst1->obsolete = DST_OBSOLETE_FORCE_CHK; in xfrm_bundle_create()
2748 dst1->lastuse = now; in xfrm_bundle_create()
2750 dst1->input = dst_discard; in xfrm_bundle_create()
2752 if (xfrm[i]->mode_cbs && xfrm[i]->mode_cbs->output) { in xfrm_bundle_create()
2753 dst1->output = xfrm[i]->mode_cbs->output; in xfrm_bundle_create()
2756 afinfo = xfrm_state_afinfo_get_rcu(inner_mode->family); in xfrm_bundle_create()
2758 dst1->output = afinfo->output; in xfrm_bundle_create()
2760 dst1->output = dst_discard_out; in xfrm_bundle_create()
2766 header_len += xfrm[i]->props.header_len; in xfrm_bundle_create()
2767 if (xfrm[i]->type->flags & XFRM_TYPE_NON_FRAGMENT) in xfrm_bundle_create()
2768 nfheader_len += xfrm[i]->props.header_len; in xfrm_bundle_create()
2769 trailer_len += xfrm[i]->props.trailer_len; in xfrm_bundle_create()
2773 xdst0->path = dst; in xfrm_bundle_create()
2775 err = -ENODEV; in xfrm_bundle_create()
2776 dev = dst->dev; in xfrm_bundle_create()
2784 xdst_prev = (struct xfrm_dst *) xfrm_dst_child(&xdst_prev->u.dst)) { in xfrm_bundle_create()
2789 xdst_prev->u.dst.header_len = header_len; in xfrm_bundle_create()
2790 xdst_prev->u.dst.trailer_len = trailer_len; in xfrm_bundle_create()
2791 header_len -= xdst_prev->u.dst.xfrm->props.header_len; in xfrm_bundle_create()
2792 trailer_len -= xdst_prev->u.dst.xfrm->props.trailer_len; in xfrm_bundle_create()
2795 return &xdst0->u.dst; in xfrm_bundle_create()
2802 dst_release_immediate(&xdst0->u.dst); in xfrm_bundle_create()
2823 *num_xfrms = pols[0]->xfrm_nr; in xfrm_expand_policies()
2826 if (pols[0]->action == XFRM_POLICY_ALLOW && in xfrm_expand_policies()
2827 pols[0]->type != XFRM_POLICY_TYPE_MAIN) { in xfrm_expand_policies()
2832 pols[0]->if_id); in xfrm_expand_policies()
2840 (*num_xfrms) += pols[1]->xfrm_nr; in xfrm_expand_policies()
2845 if (pols[i]->action != XFRM_POLICY_ALLOW) { in xfrm_expand_policies()
2846 *num_xfrms = -1; in xfrm_expand_policies()
2873 if (err != -EAGAIN) in xfrm_resolve_and_create_bundle()
2885 xdst->num_xfrms = err; in xfrm_resolve_and_create_bundle()
2886 xdst->num_pols = num_pols; in xfrm_resolve_and_create_bundle()
2887 memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols); in xfrm_resolve_and_create_bundle()
2888 xdst->policy_genid = atomic_read(&pols[0]->genid); in xfrm_resolve_and_create_bundle()
2900 struct xfrm_policy_queue *pq = &pol->polq; in xfrm_policy_queue_process()
2905 spin_lock(&pq->hold_queue.lock); in xfrm_policy_queue_process()
2906 skb = skb_peek(&pq->hold_queue); in xfrm_policy_queue_process()
2908 spin_unlock(&pq->hold_queue.lock); in xfrm_policy_queue_process()
2912 sk = skb->sk; in xfrm_policy_queue_process()
2915 skb_mark = skb->mark; in xfrm_policy_queue_process()
2916 skb->mark = pol->mark.v; in xfrm_policy_queue_process()
2917 xfrm_decode_session(net, skb, &fl, dst->ops->family); in xfrm_policy_queue_process()
2918 skb->mark = skb_mark; in xfrm_policy_queue_process()
2919 spin_unlock(&pq->hold_queue.lock); in xfrm_policy_queue_process()
2926 if (dst->flags & DST_XFRM_QUEUE) { in xfrm_policy_queue_process()
2929 if (pq->timeout >= XFRM_QUEUE_TMO_MAX) in xfrm_policy_queue_process()
2932 pq->timeout = pq->timeout << 1; in xfrm_policy_queue_process()
2933 if (!mod_timer(&pq->hold_timer, jiffies + pq->timeout)) in xfrm_policy_queue_process()
2942 spin_lock(&pq->hold_queue.lock); in xfrm_policy_queue_process()
2943 pq->timeout = 0; in xfrm_policy_queue_process()
2944 skb_queue_splice_init(&pq->hold_queue, &list); in xfrm_policy_queue_process()
2945 spin_unlock(&pq->hold_queue.lock); in xfrm_policy_queue_process()
2951 skb_mark = skb->mark; in xfrm_policy_queue_process()
2952 skb->mark = pol->mark.v; in xfrm_policy_queue_process()
2953 xfrm_decode_session(net, skb, &fl, skb_dst(skb)->ops->family); in xfrm_policy_queue_process()
2954 skb->mark = skb_mark; in xfrm_policy_queue_process()
2957 dst = xfrm_lookup(net, xfrm_dst_path(skb_dst(skb)), &fl, skb->sk, 0); in xfrm_policy_queue_process()
2975 pq->timeout = 0; in xfrm_policy_queue_process()
2976 skb_queue_purge(&pq->hold_queue); in xfrm_policy_queue_process()
2985 struct xfrm_policy *pol = xdst->pols[0]; in xdst_queue_output()
2986 struct xfrm_policy_queue *pq = &pol->polq; in xdst_queue_output()
2993 if (pq->hold_queue.qlen > XFRM_MAX_QUEUE_LEN) { in xdst_queue_output()
2995 return -EAGAIN; in xdst_queue_output()
3000 spin_lock_bh(&pq->hold_queue.lock); in xdst_queue_output()
3002 if (!pq->timeout) in xdst_queue_output()
3003 pq->timeout = XFRM_QUEUE_TMO_MIN; in xdst_queue_output()
3005 sched_next = jiffies + pq->timeout; in xdst_queue_output()
3007 if (del_timer(&pq->hold_timer)) { in xdst_queue_output()
3008 if (time_before(pq->hold_timer.expires, sched_next)) in xdst_queue_output()
3009 sched_next = pq->hold_timer.expires; in xdst_queue_output()
3013 __skb_queue_tail(&pq->hold_queue, skb); in xdst_queue_output()
3014 if (!mod_timer(&pq->hold_timer, sched_next)) in xdst_queue_output()
3017 spin_unlock_bh(&pq->hold_queue.lock); in xdst_queue_output()
3038 if (!(xflo->flags & XFRM_LOOKUP_QUEUE) || in xfrm_create_dummy_bundle()
3039 net->xfrm.sysctl_larval_drop || in xfrm_create_dummy_bundle()
3043 dst = xflo->dst_orig; in xfrm_create_dummy_bundle()
3044 dst1 = &xdst->u.dst; in xfrm_create_dummy_bundle()
3046 xdst->route = dst; in xfrm_create_dummy_bundle()
3050 dst1->obsolete = DST_OBSOLETE_FORCE_CHK; in xfrm_create_dummy_bundle()
3051 dst1->flags |= DST_XFRM_QUEUE; in xfrm_create_dummy_bundle()
3052 dst1->lastuse = jiffies; in xfrm_create_dummy_bundle()
3054 dst1->input = dst_discard; in xfrm_create_dummy_bundle()
3055 dst1->output = xdst_queue_output; in xfrm_create_dummy_bundle()
3059 xdst->path = dst; in xfrm_create_dummy_bundle()
3063 err = -ENODEV; in xfrm_create_dummy_bundle()
3064 dev = dst->dev; in xfrm_create_dummy_bundle()
3104 xflo->dst_orig); in xfrm_bundle_lookup()
3107 if (err == -EREMOTE) { in xfrm_bundle_lookup()
3112 if (err != -EAGAIN) in xfrm_bundle_lookup()
3131 xdst->num_pols = num_pols; in xfrm_bundle_lookup()
3132 xdst->num_xfrms = num_xfrms; in xfrm_bundle_lookup()
3133 memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols); in xfrm_bundle_lookup()
3152 return ERR_PTR(-EINVAL); in make_blackhole()
3154 ret = afinfo->blackhole_route(net, dst_orig); in make_blackhole()
3164 * on interfaces with disabled IPsec.
3178 u16 family = dst_orig->ops->family; in xfrm_lookup_with_ifid()
3187 if (sk && sk->sk_policy[XFRM_POLICY_OUT]) { in xfrm_lookup_with_ifid()
3209 if (err == -EREMOTE) in xfrm_lookup_with_ifid()
3219 route = xdst->route; in xfrm_lookup_with_ifid()
3230 if (!if_id && ((dst_orig->flags & DST_NOXFRM) || in xfrm_lookup_with_ifid()
3231 !net->xfrm.policy_count[XFRM_POLICY_OUT])) in xfrm_lookup_with_ifid()
3242 num_pols = xdst->num_pols; in xfrm_lookup_with_ifid()
3243 num_xfrms = xdst->num_xfrms; in xfrm_lookup_with_ifid()
3244 memcpy(pols, xdst->pols, sizeof(struct xfrm_policy *) * num_pols); in xfrm_lookup_with_ifid()
3245 route = xdst->route; in xfrm_lookup_with_ifid()
3248 dst = &xdst->u.dst; in xfrm_lookup_with_ifid()
3256 if (net->xfrm.sysctl_larval_drop) { in xfrm_lookup_with_ifid()
3258 err = -EREMOTE; in xfrm_lookup_with_ifid()
3262 err = -EAGAIN; in xfrm_lookup_with_ifid()
3273 !(pols[0]->flags & XFRM_POLICY_ICMP)) { in xfrm_lookup_with_ifid()
3274 err = -ENOENT; in xfrm_lookup_with_ifid()
3279 WRITE_ONCE(pols[i]->curlft.use_time, ktime_get_real_seconds()); in xfrm_lookup_with_ifid()
3284 err = -EPERM; in xfrm_lookup_with_ifid()
3297 if (dst && dst->xfrm && in xfrm_lookup_with_ifid()
3298 (dst->xfrm->props.mode == XFRM_MODE_TUNNEL || in xfrm_lookup_with_ifid()
3299 dst->xfrm->props.mode == XFRM_MODE_IPTFS)) in xfrm_lookup_with_ifid()
3300 dst->flags |= DST_XFRM_TUNNEL; in xfrm_lookup_with_ifid()
3304 if ((!dst_orig->dev || !(dst_orig->dev->flags & IFF_LOOPBACK)) && in xfrm_lookup_with_ifid()
3305 net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) { in xfrm_lookup_with_ifid()
3306 err = -EPERM; in xfrm_lookup_with_ifid()
3313 err = -ENOENT; in xfrm_lookup_with_ifid()
3327 * on interfaces with disabled IPsec.
3348 if (PTR_ERR(dst) == -EREMOTE) in xfrm_lookup_route()
3349 return make_blackhole(net, dst_orig->ops->family, dst_orig); in xfrm_lookup_route()
3361 struct sec_path *sp = skb_sec_path(skb); in xfrm_secpath_reject() local
3364 if (!sp || idx < 0 || idx >= sp->len) in xfrm_secpath_reject()
3366 x = sp->xvec[idx]; in xfrm_secpath_reject()
3367 if (!x->type->reject) in xfrm_secpath_reject()
3369 return x->type->reject(x, skb, fl); in xfrm_secpath_reject()
3374 * stupid way. Shame on me. :-) Of course, connected sockets must
3383 return tmpl->optional && !xfrm_state_addr_cmp(tmpl, x, tmpl->encap_family); in xfrm_state_ok()
3384 return x->id.proto == tmpl->id.proto && in xfrm_state_ok()
3385 (x->id.spi == tmpl->id.spi || !tmpl->id.spi) && in xfrm_state_ok()
3386 (x->props.reqid == tmpl->reqid || !tmpl->reqid) && in xfrm_state_ok()
3387 x->props.mode == tmpl->mode && in xfrm_state_ok()
3388 (tmpl->allalgs || (tmpl->aalgos & (1<<x->props.aalgo)) || in xfrm_state_ok()
3389 !(xfrm_id_proto_match(tmpl->id.proto, IPSEC_PROTO_ANY))) && in xfrm_state_ok()
3390 !(x->props.mode != XFRM_MODE_TRANSPORT && in xfrm_state_ok()
3392 (if_id == 0 || if_id == x->if_id); in xfrm_state_ok()
3399 * -1 is returned when no matching template is found.
3400 * Otherwise "-2 - errored_index" is returned.
3403 xfrm_policy_ok(const struct xfrm_tmpl *tmpl, const struct sec_path *sp, int start, in xfrm_policy_ok() argument
3408 if (tmpl->optional) { in xfrm_policy_ok()
3409 if (tmpl->mode == XFRM_MODE_TRANSPORT) in xfrm_policy_ok()
3412 start = -1; in xfrm_policy_ok()
3413 for (; idx < sp->len; idx++) { in xfrm_policy_ok()
3414 if (xfrm_state_ok(tmpl, sp->xvec[idx], family, if_id)) in xfrm_policy_ok()
3416 if (sp->xvec[idx]->props.mode != XFRM_MODE_TRANSPORT) { in xfrm_policy_ok()
3417 if (idx < sp->verified_cnt) { in xfrm_policy_ok()
3424 if (start == -1) in xfrm_policy_ok()
3425 start = -2-idx; in xfrm_policy_ok()
3435 struct flowi4 *fl4 = &fl->u.ip4; in decode_session4()
3440 fl4->saddr = flkeys->addrs.ipv4.dst; in decode_session4()
3441 fl4->daddr = flkeys->addrs.ipv4.src; in decode_session4()
3442 fl4->fl4_sport = flkeys->ports.dst; in decode_session4()
3443 fl4->fl4_dport = flkeys->ports.src; in decode_session4()
3445 fl4->saddr = flkeys->addrs.ipv4.src; in decode_session4()
3446 fl4->daddr = flkeys->addrs.ipv4.dst; in decode_session4()
3447 fl4->fl4_sport = flkeys->ports.src; in decode_session4()
3448 fl4->fl4_dport = flkeys->ports.dst; in decode_session4()
3451 switch (flkeys->basic.ip_proto) { in decode_session4()
3453 fl4->fl4_gre_key = flkeys->gre.keyid; in decode_session4()
3456 fl4->fl4_icmp_type = flkeys->icmp.type; in decode_session4()
3457 fl4->fl4_icmp_code = flkeys->icmp.code; in decode_session4()
3461 fl4->flowi4_proto = flkeys->basic.ip_proto; in decode_session4()
3462 fl4->flowi4_tos = flkeys->ip.tos & ~INET_ECN_MASK; in decode_session4()
3469 struct flowi6 *fl6 = &fl->u.ip6; in decode_session6()
3474 fl6->saddr = flkeys->addrs.ipv6.dst; in decode_session6()
3475 fl6->daddr = flkeys->addrs.ipv6.src; in decode_session6()
3476 fl6->fl6_sport = flkeys->ports.dst; in decode_session6()
3477 fl6->fl6_dport = flkeys->ports.src; in decode_session6()
3479 fl6->saddr = flkeys->addrs.ipv6.src; in decode_session6()
3480 fl6->daddr = flkeys->addrs.ipv6.dst; in decode_session6()
3481 fl6->fl6_sport = flkeys->ports.src; in decode_session6()
3482 fl6->fl6_dport = flkeys->ports.dst; in decode_session6()
3485 switch (flkeys->basic.ip_proto) { in decode_session6()
3487 fl6->fl6_gre_key = flkeys->gre.keyid; in decode_session6()
3490 fl6->fl6_icmp_type = flkeys->icmp.type; in decode_session6()
3491 fl6->fl6_icmp_code = flkeys->icmp.code; in decode_session6()
3495 fl6->flowi6_proto = flkeys->basic.ip_proto; in decode_session6()
3518 return -EAFNOSUPPORT; in __xfrm_decode_session()
3521 fl->flowi_mark = skb->mark; in __xfrm_decode_session()
3523 fl->flowi_oif = skb->skb_iif; in __xfrm_decode_session()
3527 if (skb_dst(skb) && skb_dst(skb)->dev) in __xfrm_decode_session()
3528 oif = skb_dst(skb)->dev->ifindex; in __xfrm_decode_session()
3530 fl->flowi_oif = oif; in __xfrm_decode_session()
3533 return security_xfrm_decode_session(skb, &fl->flowi_secid); in __xfrm_decode_session()
3537 static inline int secpath_has_nontransport(const struct sec_path *sp, int k, int *idxp) in secpath_has_nontransport() argument
3539 for (; k < sp->len; k++) { in secpath_has_nontransport()
3540 if (sp->xvec[k]->props.mode != XFRM_MODE_TRANSPORT) { in secpath_has_nontransport()
3551 const struct flowi4 *fl4 = &fl->u.ip4; in icmp_err_packet()
3554 fl4->flowi4_proto == IPPROTO_ICMP && in icmp_err_packet()
3555 (fl4->fl4_icmp_type == ICMP_DEST_UNREACH || in icmp_err_packet()
3556 fl4->fl4_icmp_type == ICMP_TIME_EXCEEDED)) in icmp_err_packet()
3561 const struct flowi6 *fl6 = &fl->u.ip6; in icmp_err_packet()
3563 if (fl6->flowi6_proto == IPPROTO_ICMPV6 && in icmp_err_packet()
3564 (fl6->fl6_icmp_type == ICMPV6_DEST_UNREACH || in icmp_err_packet()
3565 fl6->fl6_icmp_type == ICMPV6_PKT_TOOBIG || in icmp_err_packet()
3566 fl6->fl6_icmp_type == ICMPV6_TIME_EXCEED)) in icmp_err_packet()
3589 if (xfrm_decode_session_reverse(dev_net(skb->dev), newskb, fl1, family) < 0) in xfrm_icmp_flow_decode()
3592 fl1->flowi_oif = fl->flowi_oif; in xfrm_icmp_flow_decode()
3593 fl1->flowi_mark = fl->flowi_mark; in xfrm_icmp_flow_decode()
3594 fl1->flowi_tos = fl->flowi_tos; in xfrm_icmp_flow_decode()
3630 struct net *net = dev_net(skb->dev); in xfrm_in_fwd_icmp()
3648 struct net *net = dev_net(skb->dev); in xfrm_out_fwd_icmp()
3662 if (dst2->xfrm) { in xfrm_out_fwd_icmp()
3676 struct net *net = dev_net(skb->dev); in __xfrm_policy_check()
3684 int xerr_idx = -1; in __xfrm_policy_check()
3686 struct sec_path *sp; in __xfrm_policy_check() local
3695 if (ifcb->decode_session(skb, family, &r)) { in __xfrm_policy_check()
3713 sp = skb_sec_path(skb); in __xfrm_policy_check()
3714 if (sp) { in __xfrm_policy_check()
3717 for (i = sp->len - 1; i >= 0; i--) { in __xfrm_policy_check()
3718 struct xfrm_state *x = sp->xvec[i]; in __xfrm_policy_check()
3721 if (!xfrm_selector_match(&x->sel, &fl, family)) { in __xfrm_policy_check()
3723 if (x->props.flags & XFRM_STATE_ICMP && in __xfrm_policy_check()
3724 xfrm_selector_inner_icmp_match(skb, family, &x->sel, &fl)) in __xfrm_policy_check()
3736 if (sk && sk->sk_policy[dir]) { in __xfrm_policy_check()
3756 const bool is_crypto_offload = sp && in __xfrm_policy_check()
3757 (xfrm_input_state(skb)->xso.type == XFRM_DEV_OFFLOAD_CRYPTO); in __xfrm_policy_check()
3759 if (net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) { in __xfrm_policy_check()
3764 if (sp && secpath_has_nontransport(sp, 0, &xerr_idx) && !is_crypto_offload) { in __xfrm_policy_check()
3773 WRITE_ONCE(pol->curlft.use_time, ktime_get_real_seconds()); in __xfrm_policy_check()
3778 if (pols[0]->type != XFRM_POLICY_TYPE_MAIN) { in __xfrm_policy_check()
3789 WRITE_ONCE(pols[1]->curlft.use_time, in __xfrm_policy_check()
3796 if (pol->action == XFRM_POLICY_ALLOW) { in __xfrm_policy_check()
3804 sp = skb_sec_path(skb); in __xfrm_policy_check()
3805 if (!sp) in __xfrm_policy_check()
3806 sp = &dummy; in __xfrm_policy_check()
3810 pols[pi]->action != XFRM_POLICY_ALLOW) { in __xfrm_policy_check()
3814 if (ti + pols[pi]->xfrm_nr >= XFRM_MAX_DEPTH) { in __xfrm_policy_check()
3818 for (i = 0; i < pols[pi]->xfrm_nr; i++) in __xfrm_policy_check()
3819 tpp[ti++] = &pols[pi]->xfrm_vec[i]; in __xfrm_policy_check()
3837 for (i = xfrm_nr-1, k = 0; i >= 0; i--) { in __xfrm_policy_check()
3838 k = xfrm_policy_ok(tpp[i], sp, k, family, if_id); in __xfrm_policy_check()
3840 if (k < -1) in __xfrm_policy_check()
3841 /* "-2 - errored_index" returned */ in __xfrm_policy_check()
3842 xerr_idx = -(2+k); in __xfrm_policy_check()
3848 if (secpath_has_nontransport(sp, k, &xerr_idx)) { in __xfrm_policy_check()
3854 sp->verified_cnt = k; in __xfrm_policy_check()
3870 struct net *net = dev_net(skb->dev); in __xfrm_route_forward()
3892 if (dst && !dst->xfrm) in __xfrm_route_forward()
3904 /* Code (such as __xfrm4_bundle_create()) sets dst->obsolete in xfrm_dst_check()
3906 * get validated by dst_ops->check on every use. We do this in xfrm_dst_check()
3913 * XFRM dst A --> IPv4 dst X in xfrm_dst_check()
3915 * X is the "xdst->route" of A (X is also the "dst->path" of A in xfrm_dst_check()
3925 if (dst->obsolete < 0 && !stale_bundle(dst)) in xfrm_dst_check()
3938 while ((dst = xfrm_dst_child(dst)) && dst->xfrm && dst->dev == dev) { in xfrm_dst_ifdown()
3939 dst->dev = blackhole_netdev; in xfrm_dst_ifdown()
3940 dev_hold(dst->dev); in xfrm_dst_ifdown()
3953 if (dst->obsolete) in xfrm_negative_advice()
3959 while (nr--) { in xfrm_init_pmtu()
3964 dst = &xdst->u.dst; in xfrm_init_pmtu()
3966 xdst->child_mtu_cached = pmtu; in xfrm_init_pmtu()
3968 pmtu = xfrm_state_mtu(dst->xfrm, pmtu); in xfrm_init_pmtu()
3970 route_mtu_cached = dst_mtu(xdst->route); in xfrm_init_pmtu()
3971 xdst->route_mtu_cached = route_mtu_cached; in xfrm_init_pmtu()
3987 struct dst_entry *dst = &first->u.dst; in xfrm_bundle_ok()
3992 if (!dst_check(xfrm_dst_path(dst), ((struct xfrm_dst *)dst)->path_cookie) || in xfrm_bundle_ok()
3993 (dst->dev && !netif_running(dst->dev))) in xfrm_bundle_ok()
3996 if (dst->flags & DST_XFRM_QUEUE) in xfrm_bundle_ok()
4003 if (dst->xfrm->km.state != XFRM_STATE_VALID) in xfrm_bundle_ok()
4005 if (xdst->xfrm_genid != dst->xfrm->genid) in xfrm_bundle_ok()
4007 if (xdst->num_pols > 0 && in xfrm_bundle_ok()
4008 xdst->policy_genid != atomic_read(&xdst->pols[0]->genid)) in xfrm_bundle_ok()
4014 if (xdst->child_mtu_cached != mtu) { in xfrm_bundle_ok()
4016 xdst->child_mtu_cached = mtu; in xfrm_bundle_ok()
4019 if (!dst_check(xdst->route, xdst->route_cookie)) in xfrm_bundle_ok()
4021 mtu = dst_mtu(xdst->route); in xfrm_bundle_ok()
4022 if (xdst->route_mtu_cached != mtu) { in xfrm_bundle_ok()
4024 xdst->route_mtu_cached = mtu; in xfrm_bundle_ok()
4028 } while (dst->xfrm); in xfrm_bundle_ok()
4033 xdst = bundle[start_from - 1]; in xfrm_bundle_ok()
4034 mtu = xdst->child_mtu_cached; in xfrm_bundle_ok()
4035 while (start_from--) { in xfrm_bundle_ok()
4036 dst = &xdst->u.dst; in xfrm_bundle_ok()
4038 mtu = xfrm_state_mtu(dst->xfrm, mtu); in xfrm_bundle_ok()
4039 if (mtu > xdst->route_mtu_cached) in xfrm_bundle_ok()
4040 mtu = xdst->route_mtu_cached; in xfrm_bundle_ok()
4045 xdst = bundle[start_from - 1]; in xfrm_bundle_ok()
4046 xdst->child_mtu_cached = mtu; in xfrm_bundle_ok()
4067 while (dst->xfrm) { in xfrm_get_dst_nexthop()
4068 const struct xfrm_state *xfrm = dst->xfrm; in xfrm_get_dst_nexthop()
4072 if (xfrm->props.mode == XFRM_MODE_TRANSPORT) in xfrm_get_dst_nexthop()
4074 if (xfrm->type->flags & XFRM_TYPE_REMOTE_COADDR) in xfrm_get_dst_nexthop()
4075 daddr = xfrm->coaddr; in xfrm_get_dst_nexthop()
4076 else if (!(xfrm->type->flags & XFRM_TYPE_LOCAL_COADDR)) in xfrm_get_dst_nexthop()
4077 daddr = &xfrm->id.daddr; in xfrm_get_dst_nexthop()
4090 return path->ops->neigh_lookup(path, skb, daddr); in xfrm_neigh_lookup()
4098 path->ops->confirm_neigh(path, daddr); in xfrm_confirm_neigh()
4106 return -EAFNOSUPPORT; in xfrm_policy_register_afinfo()
4110 err = -EEXIST; in xfrm_policy_register_afinfo()
4112 struct dst_ops *dst_ops = afinfo->dst_ops; in xfrm_policy_register_afinfo()
4113 if (likely(dst_ops->kmem_cachep == NULL)) in xfrm_policy_register_afinfo()
4114 dst_ops->kmem_cachep = xfrm_dst_cache; in xfrm_policy_register_afinfo()
4115 if (likely(dst_ops->check == NULL)) in xfrm_policy_register_afinfo()
4116 dst_ops->check = xfrm_dst_check; in xfrm_policy_register_afinfo()
4117 if (likely(dst_ops->default_advmss == NULL)) in xfrm_policy_register_afinfo()
4118 dst_ops->default_advmss = xfrm_default_advmss; in xfrm_policy_register_afinfo()
4119 if (likely(dst_ops->mtu == NULL)) in xfrm_policy_register_afinfo()
4120 dst_ops->mtu = xfrm_mtu; in xfrm_policy_register_afinfo()
4121 if (likely(dst_ops->negative_advice == NULL)) in xfrm_policy_register_afinfo()
4122 dst_ops->negative_advice = xfrm_negative_advice; in xfrm_policy_register_afinfo()
4123 if (likely(dst_ops->link_failure == NULL)) in xfrm_policy_register_afinfo()
4124 dst_ops->link_failure = xfrm_link_failure; in xfrm_policy_register_afinfo()
4125 if (likely(dst_ops->neigh_lookup == NULL)) in xfrm_policy_register_afinfo()
4126 dst_ops->neigh_lookup = xfrm_neigh_lookup; in xfrm_policy_register_afinfo()
4127 if (likely(!dst_ops->confirm_neigh)) in xfrm_policy_register_afinfo()
4128 dst_ops->confirm_neigh = xfrm_confirm_neigh; in xfrm_policy_register_afinfo()
4139 struct dst_ops *dst_ops = afinfo->dst_ops; in xfrm_policy_unregister_afinfo()
4151 dst_ops->kmem_cachep = NULL; in xfrm_policy_unregister_afinfo()
4152 dst_ops->check = NULL; in xfrm_policy_unregister_afinfo()
4153 dst_ops->negative_advice = NULL; in xfrm_policy_unregister_afinfo()
4154 dst_ops->link_failure = NULL; in xfrm_policy_unregister_afinfo()
4177 net->mib.xfrm_statistics = alloc_percpu(struct linux_xfrm_mib); in xfrm_statistics_init()
4178 if (!net->mib.xfrm_statistics) in xfrm_statistics_init()
4179 return -ENOMEM; in xfrm_statistics_init()
4182 free_percpu(net->mib.xfrm_statistics); in xfrm_statistics_init()
4189 free_percpu(net->mib.xfrm_statistics); in xfrm_statistics_fini()
4214 hmask = 8 - 1; in xfrm_policy_init()
4217 net->xfrm.policy_byidx = xfrm_hash_alloc(sz); in xfrm_policy_init()
4218 if (!net->xfrm.policy_byidx) in xfrm_policy_init()
4220 net->xfrm.policy_idx_hmask = hmask; in xfrm_policy_init()
4225 net->xfrm.policy_count[dir] = 0; in xfrm_policy_init()
4226 net->xfrm.policy_count[XFRM_POLICY_MAX + dir] = 0; in xfrm_policy_init()
4228 htab = &net->xfrm.policy_bydst[dir]; in xfrm_policy_init()
4229 htab->table = xfrm_hash_alloc(sz); in xfrm_policy_init()
4230 if (!htab->table) in xfrm_policy_init()
4232 htab->hmask = hmask; in xfrm_policy_init()
4233 htab->dbits4 = 32; in xfrm_policy_init()
4234 htab->sbits4 = 32; in xfrm_policy_init()
4235 htab->dbits6 = 128; in xfrm_policy_init()
4236 htab->sbits6 = 128; in xfrm_policy_init()
4238 net->xfrm.policy_hthresh.lbits4 = 32; in xfrm_policy_init()
4239 net->xfrm.policy_hthresh.rbits4 = 32; in xfrm_policy_init()
4240 net->xfrm.policy_hthresh.lbits6 = 128; in xfrm_policy_init()
4241 net->xfrm.policy_hthresh.rbits6 = 128; in xfrm_policy_init()
4243 seqlock_init(&net->xfrm.policy_hthresh.lock); in xfrm_policy_init()
4245 INIT_LIST_HEAD(&net->xfrm.policy_all); in xfrm_policy_init()
4246 INIT_LIST_HEAD(&net->xfrm.inexact_bins); in xfrm_policy_init()
4247 INIT_WORK(&net->xfrm.policy_hash_work, xfrm_hash_resize); in xfrm_policy_init()
4248 INIT_WORK(&net->xfrm.policy_hthresh.work, xfrm_hash_rebuild); in xfrm_policy_init()
4252 for (dir--; dir >= 0; dir--) { in xfrm_policy_init()
4255 htab = &net->xfrm.policy_bydst[dir]; in xfrm_policy_init()
4256 xfrm_hash_free(htab->table, sz); in xfrm_policy_init()
4258 xfrm_hash_free(net->xfrm.policy_byidx, sz); in xfrm_policy_init()
4260 return -ENOMEM; in xfrm_policy_init()
4269 flush_work(&net->xfrm.policy_hash_work); in xfrm_policy_fini()
4275 WARN_ON(!list_empty(&net->xfrm.policy_all)); in xfrm_policy_fini()
4280 htab = &net->xfrm.policy_bydst[dir]; in xfrm_policy_fini()
4281 sz = (htab->hmask + 1) * sizeof(struct hlist_head); in xfrm_policy_fini()
4282 WARN_ON(!hlist_empty(htab->table)); in xfrm_policy_fini()
4283 xfrm_hash_free(htab->table, sz); in xfrm_policy_fini()
4286 sz = (net->xfrm.policy_idx_hmask + 1) * sizeof(struct hlist_head); in xfrm_policy_fini()
4287 WARN_ON(!hlist_empty(net->xfrm.policy_byidx)); in xfrm_policy_fini()
4288 xfrm_hash_free(net->xfrm.policy_byidx, sz); in xfrm_policy_fini()
4290 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_fini()
4291 list_for_each_entry_safe(b, t, &net->xfrm.inexact_bins, inexact_bins) in xfrm_policy_fini()
4293 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_fini()
4300 /* Initialize the per-net locks here */ in xfrm_net_init()
4301 spin_lock_init(&net->xfrm.xfrm_state_lock); in xfrm_net_init()
4302 spin_lock_init(&net->xfrm.xfrm_policy_lock); in xfrm_net_init()
4303 seqcount_spinlock_init(&net->xfrm.xfrm_policy_hash_generation, &net->xfrm.xfrm_policy_lock); in xfrm_net_init()
4304 mutex_init(&net->xfrm.xfrm_cfg_mutex); in xfrm_net_init()
4305 net->xfrm.policy_default[XFRM_POLICY_IN] = XFRM_USERPOLICY_ACCEPT; in xfrm_net_init()
4306 net->xfrm.policy_default[XFRM_POLICY_FWD] = XFRM_USERPOLICY_ACCEPT; in xfrm_net_init()
4307 net->xfrm.policy_default[XFRM_POLICY_OUT] = XFRM_USERPOLICY_ACCEPT; in xfrm_net_init()
4373 .offset = offsetof(struct xfrm_flow_keys, ports),
4411 struct xfrm_sec_ctx *ctx = xp->security; in xfrm_audit_common_policyinfo()
4412 struct xfrm_selector *sel = &xp->selector; in xfrm_audit_common_policyinfo()
4416 ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str); in xfrm_audit_common_policyinfo()
4418 switch (sel->family) { in xfrm_audit_common_policyinfo()
4420 audit_log_format(audit_buf, " src=%pI4", &sel->saddr.a4); in xfrm_audit_common_policyinfo()
4421 if (sel->prefixlen_s != 32) in xfrm_audit_common_policyinfo()
4423 sel->prefixlen_s); in xfrm_audit_common_policyinfo()
4424 audit_log_format(audit_buf, " dst=%pI4", &sel->daddr.a4); in xfrm_audit_common_policyinfo()
4425 if (sel->prefixlen_d != 32) in xfrm_audit_common_policyinfo()
4427 sel->prefixlen_d); in xfrm_audit_common_policyinfo()
4430 audit_log_format(audit_buf, " src=%pI6", sel->saddr.a6); in xfrm_audit_common_policyinfo()
4431 if (sel->prefixlen_s != 128) in xfrm_audit_common_policyinfo()
4433 sel->prefixlen_s); in xfrm_audit_common_policyinfo()
4434 audit_log_format(audit_buf, " dst=%pI6", sel->daddr.a6); in xfrm_audit_common_policyinfo()
4435 if (sel->prefixlen_d != 128) in xfrm_audit_common_policyinfo()
4437 sel->prefixlen_d); in xfrm_audit_common_policyinfo()
4446 audit_buf = xfrm_audit_start("SPD-add"); in xfrm_audit_policy_add()
4461 audit_buf = xfrm_audit_start("SPD-delete"); in xfrm_audit_policy_delete()
4481 fl.flowi_proto = sel->proto; in xfrm_migrate_policy_find()
4483 switch (sel->family) { in xfrm_migrate_policy_find()
4485 fl.u.ip4.saddr = sel->saddr.a4; in xfrm_migrate_policy_find()
4486 fl.u.ip4.daddr = sel->daddr.a4; in xfrm_migrate_policy_find()
4487 if (sel->proto == IPSEC_ULPROTO_ANY) in xfrm_migrate_policy_find()
4489 fl.u.flowi4_oif = sel->ifindex; in xfrm_migrate_policy_find()
4490 fl.u.ip4.fl4_sport = sel->sport; in xfrm_migrate_policy_find()
4491 fl.u.ip4.fl4_dport = sel->dport; in xfrm_migrate_policy_find()
4494 fl.u.ip6.saddr = sel->saddr.in6; in xfrm_migrate_policy_find()
4495 fl.u.ip6.daddr = sel->daddr.in6; in xfrm_migrate_policy_find()
4496 if (sel->proto == IPSEC_ULPROTO_ANY) in xfrm_migrate_policy_find()
4498 fl.u.flowi6_oif = sel->ifindex; in xfrm_migrate_policy_find()
4499 fl.u.ip6.fl4_sport = sel->sport; in xfrm_migrate_policy_find()
4500 fl.u.ip6.fl4_dport = sel->dport; in xfrm_migrate_policy_find()
4503 return ERR_PTR(-EAFNOSUPPORT); in xfrm_migrate_policy_find()
4508 pol = xfrm_policy_lookup_bytype(net, type, &fl, sel->family, dir, if_id); in xfrm_migrate_policy_find()
4523 if (t->mode == m->mode && t->id.proto == m->proto && in migrate_tmpl_match()
4524 (m->reqid == 0 || t->reqid == m->reqid)) { in migrate_tmpl_match()
4525 switch (t->mode) { in migrate_tmpl_match()
4529 if (xfrm_addr_equal(&t->id.daddr, &m->old_daddr, in migrate_tmpl_match()
4530 m->old_family) && in migrate_tmpl_match()
4531 xfrm_addr_equal(&t->saddr, &m->old_saddr, in migrate_tmpl_match()
4532 m->old_family)) { in migrate_tmpl_match()
4557 write_lock_bh(&pol->lock); in xfrm_policy_migrate()
4558 if (unlikely(pol->walk.dead)) { in xfrm_policy_migrate()
4561 write_unlock_bh(&pol->lock); in xfrm_policy_migrate()
4562 return -ENOENT; in xfrm_policy_migrate()
4565 for (i = 0; i < pol->xfrm_nr; i++) { in xfrm_policy_migrate()
4567 if (!migrate_tmpl_match(mp, &pol->xfrm_vec[i])) in xfrm_policy_migrate()
4570 if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL && in xfrm_policy_migrate()
4571 pol->xfrm_vec[i].mode != XFRM_MODE_BEET && in xfrm_policy_migrate()
4572 pol->xfrm_vec[i].mode != XFRM_MODE_IPTFS) in xfrm_policy_migrate()
4575 memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr, in xfrm_policy_migrate()
4576 sizeof(pol->xfrm_vec[i].id.daddr)); in xfrm_policy_migrate()
4577 memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr, in xfrm_policy_migrate()
4578 sizeof(pol->xfrm_vec[i].saddr)); in xfrm_policy_migrate()
4579 pol->xfrm_vec[i].encap_family = mp->new_family; in xfrm_policy_migrate()
4581 atomic_inc(&pol->genid); in xfrm_policy_migrate()
4585 write_unlock_bh(&pol->lock); in xfrm_policy_migrate()
4588 return -ENODATA; in xfrm_policy_migrate()
4600 return -EINVAL; in xfrm_migrate_check()
4607 return -EINVAL; in xfrm_migrate_check()
4621 return -EINVAL; in xfrm_migrate_check()
4642 /* Stage 0 - sanity checks */ in xfrm_migrate()
4649 err = -EINVAL; in xfrm_migrate()
4653 /* Stage 1 - find policy */ in xfrm_migrate()
4657 err = IS_ERR(pol) ? PTR_ERR(pol) : -ENOENT; in xfrm_migrate()
4661 /* Stage 2 - find and update state(s) */ in xfrm_migrate()
4671 err = -ENODATA; in xfrm_migrate()
4677 /* Stage 3 - update policy */ in xfrm_migrate()
4682 /* Stage 4 - delete old state(s) */ in xfrm_migrate()
4688 /* Stage 5 - announce */ in xfrm_migrate()