Lines Matching +full:common +full:- +full:rules
1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
30 * get_hashentry() - Wrapper around hashtable lookup.
51 return ERR_PTR(-ENOMEM); in get_hashentry()
59 struct flow_dissector *dissector = rule->match.dissector; in is_pre_ct_flow()
64 if (dissector->used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CT)) { in is_pre_ct_flow()
66 if (ct.key->ct_state) in is_pre_ct_flow()
70 if (flow->common.chain_index) in is_pre_ct_flow()
73 flow_action_for_each(i, act, &flow->rule->action) { in is_pre_ct_flow()
74 if (act->id == FLOW_ACTION_CT) { in is_pre_ct_flow()
78 if ((!act->ct.action || act->ct.action == TCA_CT_ACT_NAT)) in is_pre_ct_flow()
91 struct flow_dissector *dissector = rule->match.dissector; in is_post_ct_flow()
97 if (dissector->used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CT)) { in is_post_ct_flow()
99 if (ct.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED) in is_post_ct_flow()
103 flow_action_for_each(i, act, &flow->rule->action) { in is_post_ct_flow()
104 if (act->id == FLOW_ACTION_CT) { in is_post_ct_flow()
106 if (act->ct.action == TCA_CT_ACT_CLEAR) { in is_post_ct_flow()
118 if (flow->common.chain_index && exist_ct_clear) in is_post_ct_flow()
126 * get_mangled_key() - Mangle the key if mangle act exists
144 flow_action_for_each(i, act, &rule->action) { in get_mangled_key()
145 if (act->id == FLOW_ACTION_MANGLE && in get_mangled_key()
146 act->mangle.htype == htype) { in get_mangled_key()
147 off = act->mangle.offset - offset; in get_mangled_key()
148 msk = act->mangle.mask; in get_mangled_key()
149 key = act->mangle.val; in get_mangled_key()
183 tmp = FIELD_PREP(NFP_IPV6_TCLASS_MASK, match.key->tos); in get_mangled_tos_ttl()
185 tmp = FIELD_PREP(NFP_IPV6_HLIMIT_MASK, match.key->ttl); in get_mangled_tos_ttl()
189 tmp = FIELD_PREP(NFP_IPV4_TOS_MASK, match.key->tos); in get_mangled_tos_ttl()
191 tmp = FIELD_PREP(NFP_IPV4_TTL_MASK, match.key->ttl); in get_mangled_tos_ttl()
204 match.key->tos = FIELD_GET(NFP_IPV6_TCLASS_MASK, tmp); in get_mangled_tos_ttl()
206 match.key->ttl = FIELD_GET(NFP_IPV6_HLIMIT_MASK, tmp); in get_mangled_tos_ttl()
209 match.key->tos = FIELD_GET(NFP_IPV4_TOS_MASK, tmp); in get_mangled_tos_ttl()
211 match.key->ttl = FIELD_GET(NFP_IPV4_TTL_MASK, tmp); in get_mangled_tos_ttl()
224 if ((entry1->flags & NFP_FL_ACTION_DO_NAT) && in nfp_ct_merge_check_cannot_skip()
225 entry2->type == CT_TYPE_POST_CT) in nfp_ct_merge_check_cannot_skip()
242 ovlp_keys = entry1->rule->match.dissector->used_keys & in nfp_ct_merge_check()
243 entry2->rule->match.dissector->used_keys; in nfp_ct_merge_check()
254 if (entry1->netdev && entry2->netdev && in nfp_ct_merge_check()
255 entry1->netdev != entry2->netdev) in nfp_ct_merge_check()
256 return -EINVAL; in nfp_ct_merge_check()
264 flow_rule_match_control(entry1->rule, &match1); in nfp_ct_merge_check()
265 flow_rule_match_control(entry2->rule, &match2); in nfp_ct_merge_check()
274 flow_rule_match_basic(entry1->rule, &match1); in nfp_ct_merge_check()
275 flow_rule_match_basic(entry2->rule, &match2); in nfp_ct_merge_check()
277 /* n_proto field is a must in ct-related flows, in nfp_ct_merge_check()
280 is_v6 = match1.key->n_proto == htons(ETH_P_IPV6); in nfp_ct_merge_check()
282 ip_proto = match1.key->ip_proto; in nfp_ct_merge_check()
297 flow_rule_match_ipv4_addrs(entry1->rule, &match1); in nfp_ct_merge_check()
298 flow_rule_match_ipv4_addrs(entry2->rule, &match2); in nfp_ct_merge_check()
301 match1.key = get_mangled_key(entry1->rule, buf, in nfp_ct_merge_check()
319 flow_rule_match_ipv6_addrs(entry1->rule, &match1); in nfp_ct_merge_check()
320 flow_rule_match_ipv6_addrs(entry2->rule, &match2); in nfp_ct_merge_check()
323 match1.key = get_mangled_key(entry1->rule, buf, in nfp_ct_merge_check()
342 flow_rule_match_ports(entry1->rule, &match1); in nfp_ct_merge_check()
343 flow_rule_match_ports(entry2->rule, &match2); in nfp_ct_merge_check()
351 match1.key = get_mangled_key(entry1->rule, buf, 0, in nfp_ct_merge_check()
362 flow_rule_match_eth_addrs(entry1->rule, &match1); in nfp_ct_merge_check()
363 flow_rule_match_eth_addrs(entry2->rule, &match2); in nfp_ct_merge_check()
366 match1.key = get_mangled_key(entry1->rule, buf, 0, in nfp_ct_merge_check()
378 flow_rule_match_vlan(entry1->rule, &match1); in nfp_ct_merge_check()
379 flow_rule_match_vlan(entry2->rule, &match2); in nfp_ct_merge_check()
388 flow_rule_match_mpls(entry1->rule, &match1); in nfp_ct_merge_check()
389 flow_rule_match_mpls(entry2->rule, &match2); in nfp_ct_merge_check()
398 flow_rule_match_tcp(entry1->rule, &match1); in nfp_ct_merge_check()
399 flow_rule_match_tcp(entry2->rule, &match2); in nfp_ct_merge_check()
408 flow_rule_match_ip(entry1->rule, &match1); in nfp_ct_merge_check()
409 flow_rule_match_ip(entry2->rule, &match2); in nfp_ct_merge_check()
411 match1.key = get_mangled_tos_ttl(entry1->rule, buf, is_v6); in nfp_ct_merge_check()
420 flow_rule_match_enc_keyid(entry1->rule, &match1); in nfp_ct_merge_check()
421 flow_rule_match_enc_keyid(entry2->rule, &match2); in nfp_ct_merge_check()
430 flow_rule_match_enc_ipv4_addrs(entry1->rule, &match1); in nfp_ct_merge_check()
431 flow_rule_match_enc_ipv4_addrs(entry2->rule, &match2); in nfp_ct_merge_check()
440 flow_rule_match_enc_ipv6_addrs(entry1->rule, &match1); in nfp_ct_merge_check()
441 flow_rule_match_enc_ipv6_addrs(entry2->rule, &match2); in nfp_ct_merge_check()
450 flow_rule_match_enc_control(entry1->rule, &match1); in nfp_ct_merge_check()
451 flow_rule_match_enc_control(entry2->rule, &match2); in nfp_ct_merge_check()
460 flow_rule_match_enc_ip(entry1->rule, &match1); in nfp_ct_merge_check()
461 flow_rule_match_enc_ip(entry2->rule, &match2); in nfp_ct_merge_check()
470 flow_rule_match_enc_opts(entry1->rule, &match1); in nfp_ct_merge_check()
471 flow_rule_match_enc_opts(entry2->rule, &match2); in nfp_ct_merge_check()
480 return -EINVAL; in nfp_ct_merge_check()
489 return -EOPNOTSUPP; in nfp_ct_check_vlan_merge()
495 switch (a_in->id) { in nfp_ct_check_vlan_merge()
498 return -EOPNOTSUPP; in nfp_ct_check_vlan_merge()
504 if ((match.key->vlan_id & match.mask->vlan_id) ^ in nfp_ct_check_vlan_merge()
505 (a_in->vlan.vid & match.mask->vlan_id)) in nfp_ct_check_vlan_merge()
506 return -EOPNOTSUPP; in nfp_ct_check_vlan_merge()
509 if ((match.key->vlan_tpid & match.mask->vlan_tpid) ^ in nfp_ct_check_vlan_merge()
510 (a_in->vlan.proto & match.mask->vlan_tpid)) in nfp_ct_check_vlan_merge()
511 return -EOPNOTSUPP; in nfp_ct_check_vlan_merge()
514 if ((match.key->vlan_priority & match.mask->vlan_priority) ^ in nfp_ct_check_vlan_merge()
515 (a_in->vlan.prio & match.mask->vlan_priority)) in nfp_ct_check_vlan_merge()
516 return -EOPNOTSUPP; in nfp_ct_check_vlan_merge()
520 return -EOPNOTSUPP; in nfp_ct_check_vlan_merge()
526 /* Extra check for multiple ct-zones merge
535 pre_ct_entry = tc_m_entry->pre_ct_parent; in nfp_ct_merge_extra_check()
536 prev_nft_m_entry = pre_ct_entry->prev_m_entries[pre_ct_entry->num_prev_m_entries - 1]; in nfp_ct_merge_extra_check()
538 return nfp_ct_merge_check(prev_nft_m_entry->nft_parent, nft_entry); in nfp_ct_merge_extra_check()
548 /* Check for pre_ct->action conflicts */ in nfp_ct_merge_act_check()
549 flow_action_for_each(i, act, &pre_ct_entry->rule->action) { in nfp_ct_merge_act_check()
550 switch (act->id) { in nfp_ct_merge_act_check()
554 err = nfp_ct_check_vlan_merge(act, post_ct_entry->rule); in nfp_ct_merge_act_check()
561 return -EOPNOTSUPP; in nfp_ct_merge_act_check()
567 /* Check for nft->action conflicts */ in nfp_ct_merge_act_check()
568 flow_action_for_each(i, act, &nft_entry->rule->action) { in nfp_ct_merge_act_check()
569 switch (act->id) { in nfp_ct_merge_act_check()
576 return -EOPNOTSUPP; in nfp_ct_merge_act_check()
587 struct flow_dissector *dissector = post_ct_entry->rule->match.dissector; in nfp_ct_check_meta()
592 ct_met = get_flow_act(nft_entry->rule, FLOW_ACTION_CT_METADATA); in nfp_ct_check_meta()
593 if (ct_met && (dissector->used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CT))) { in nfp_ct_check_meta()
596 act_lbl = ct_met->ct_metadata.labels; in nfp_ct_check_meta()
597 flow_rule_match_ct(post_ct_entry->rule, &ct); in nfp_ct_check_meta()
599 if ((ct.key->ct_labels[i] & ct.mask->ct_labels[i]) ^ in nfp_ct_check_meta()
600 (act_lbl[i] & ct.mask->ct_labels[i])) in nfp_ct_check_meta()
601 return -EINVAL; in nfp_ct_check_meta()
604 if ((ct.key->ct_mark & ct.mask->ct_mark) ^ in nfp_ct_check_meta()
605 (ct_met->ct_metadata.mark & ct.mask->ct_mark)) in nfp_ct_check_meta()
606 return -EINVAL; in nfp_ct_check_meta()
613 if (nft_entry->flags & NFP_FL_ACTION_DO_MANGLE) in nfp_ct_check_meta()
617 return -EINVAL; in nfp_ct_check_meta()
687 if (a_in->id != FLOW_ACTION_MANGLE) in nfp_fl_get_csum_flag()
690 switch (a_in->mangle.htype) { in nfp_fl_get_csum_flag()
709 static int nfp_fl_merge_actions_offload(struct flow_rule **rules, in nfp_fl_merge_actions_offload() argument
722 num_actions += rules[i]->action.num_entries; in nfp_fl_merge_actions_offload()
729 return -ENOMEM; in nfp_fl_merge_actions_offload()
732 if (rules[num_rules - 1]->action.num_entries != 0) in nfp_fl_merge_actions_offload()
733 tmp_stats = rules[num_rules - 1]->action.entries[0].hw_stats; in nfp_fl_merge_actions_offload()
736 a_rule->match = rules[0]->match; in nfp_fl_merge_actions_offload()
743 if (flow_rule_match_key(rules[j], FLOW_DISSECTOR_KEY_BASIC)) { in nfp_fl_merge_actions_offload()
753 flow_rule_match_basic(rules[j], &match); in nfp_fl_merge_actions_offload()
754 if (match.mask->ip_proto) { in nfp_fl_merge_actions_offload()
755 a_rule->match = rules[j]->match; in nfp_fl_merge_actions_offload()
756 ip_proto = match.key->ip_proto; in nfp_fl_merge_actions_offload()
760 for (i = 0; i < rules[j]->action.num_entries; i++) { in nfp_fl_merge_actions_offload()
761 a_in = &rules[j]->action.entries[i]; in nfp_fl_merge_actions_offload()
762 id = a_in->id; in nfp_fl_merge_actions_offload()
777 * nft entry flow rules are at odd array index. in nfp_fl_merge_actions_offload()
780 if (a_in->hw_stats == FLOW_ACTION_HW_STATS_DONT_CARE) in nfp_fl_merge_actions_offload()
781 a_in->hw_stats = tmp_stats; in nfp_fl_merge_actions_offload()
784 memcpy(&a_rule->action.entries[offset++], in nfp_fl_merge_actions_offload()
796 csum_action = &a_rule->action.entries[offset++]; in nfp_fl_merge_actions_offload()
797 csum_action->id = FLOW_ACTION_CSUM; in nfp_fl_merge_actions_offload()
798 csum_action->csum_flags = csum_updated; in nfp_fl_merge_actions_offload()
799 csum_action->hw_stats = tmp_stats; in nfp_fl_merge_actions_offload()
804 a_rule->action.num_entries = offset; in nfp_fl_merge_actions_offload()
805 err = nfp_flower_compile_action(priv->app, a_rule, netdev, flow_pay, NULL); in nfp_fl_merge_actions_offload()
814 struct nfp_fl_ct_zone_entry *zt = m_entry->zt; in nfp_fl_ct_add_offload()
815 struct flow_rule *rules[NFP_MAX_ENTRY_RULES]; in nfp_fl_ct_add_offload() local
818 struct nfp_flower_priv *priv = zt->priv; in nfp_fl_ct_add_offload()
829 netdev = m_entry->netdev; in nfp_fl_ct_add_offload()
830 qinq_sup = !!(priv->flower_ext_feats & NFP_FL_FEATS_VLAN_QINQ); in nfp_fl_ct_add_offload()
832 pre_ct_entry = m_entry->tc_m_parent->pre_ct_parent; in nfp_fl_ct_add_offload()
833 num_rules = pre_ct_entry->num_prev_m_entries * 2 + _CT_TYPE_MAX; in nfp_fl_ct_add_offload()
835 for (i = 0; i < pre_ct_entry->num_prev_m_entries; i++) { in nfp_fl_ct_add_offload()
836 rules[j++] = pre_ct_entry->prev_m_entries[i]->tc_m_parent->pre_ct_parent->rule; in nfp_fl_ct_add_offload()
837 rules[j++] = pre_ct_entry->prev_m_entries[i]->nft_parent->rule; in nfp_fl_ct_add_offload()
840 rules[j++] = m_entry->tc_m_parent->pre_ct_parent->rule; in nfp_fl_ct_add_offload()
841 rules[j++] = m_entry->nft_parent->rule; in nfp_fl_ct_add_offload()
842 rules[j++] = m_entry->tc_m_parent->post_ct_parent->rule; in nfp_fl_ct_add_offload()
849 err = nfp_flower_calculate_key_layers(priv->app, in nfp_fl_ct_add_offload()
850 m_entry->netdev, in nfp_fl_ct_add_offload()
851 &tmp_layer, rules[i], in nfp_fl_ct_add_offload()
863 return -ENOMEM; in nfp_fl_ct_add_offload()
865 memset(flow_pay->unmasked_data, 0, key_layer.key_size); in nfp_fl_ct_add_offload()
866 memset(flow_pay->mask_data, 0, key_layer.key_size); in nfp_fl_ct_add_offload()
868 kdata = flow_pay->unmasked_data; in nfp_fl_ct_add_offload()
869 mdata = flow_pay->mask_data; in nfp_fl_ct_add_offload()
888 /* Using in_port from the -trk rule. The tc merge checks should already in nfp_fl_ct_add_offload()
891 port_id = nfp_flower_get_port_id_from_netdev(priv->app, netdev); in nfp_fl_ct_add_offload()
906 * layers. Here we iterate through all three rules and merge their respective in nfp_fl_ct_add_offload()
910 * If none of the rules contains a match that is also fine, that simply means in nfp_fl_ct_add_offload()
920 rules[i]); in nfp_fl_ct_add_offload()
931 rules[i]); in nfp_fl_ct_add_offload()
934 rules[i], NULL); in nfp_fl_ct_add_offload()
947 rules[i]); in nfp_fl_ct_add_offload()
958 rules[i]); in nfp_fl_ct_add_offload()
969 rules[i]); in nfp_fl_ct_add_offload()
980 rules[i]); in nfp_fl_ct_add_offload()
995 (void *)msk, rules[i]); in nfp_fl_ct_add_offload()
998 dst = &gre_match->ipv6.dst; in nfp_fl_ct_add_offload()
1000 entry = nfp_tunnel_add_ipv6_off(priv->app, dst); in nfp_fl_ct_add_offload()
1002 err = -ENOMEM; in nfp_fl_ct_add_offload()
1006 flow_pay->nfp_tun_ipv6 = entry; in nfp_fl_ct_add_offload()
1012 (void *)msk, rules[i]); in nfp_fl_ct_add_offload()
1014 dst = ((struct nfp_flower_ipv4_gre_tun *)key)->ipv4.dst; in nfp_fl_ct_add_offload()
1019 flow_pay->nfp_tun_ipv4_addr = dst; in nfp_fl_ct_add_offload()
1020 nfp_tunnel_add_ipv4_off(priv->app, dst); in nfp_fl_ct_add_offload()
1036 (void *)msk, rules[i]); in nfp_fl_ct_add_offload()
1039 dst = &udp_match->ipv6.dst; in nfp_fl_ct_add_offload()
1041 entry = nfp_tunnel_add_ipv6_off(priv->app, dst); in nfp_fl_ct_add_offload()
1043 err = -ENOMEM; in nfp_fl_ct_add_offload()
1047 flow_pay->nfp_tun_ipv6 = entry; in nfp_fl_ct_add_offload()
1053 (void *)msk, rules[i]); in nfp_fl_ct_add_offload()
1055 dst = ((struct nfp_flower_ipv4_udp_tun *)key)->ipv4.dst; in nfp_fl_ct_add_offload()
1060 flow_pay->nfp_tun_ipv4_addr = dst; in nfp_fl_ct_add_offload()
1061 nfp_tunnel_add_ipv4_off(priv->app, dst); in nfp_fl_ct_add_offload()
1069 nfp_flower_compile_geneve_opt(key, msk, rules[i]); in nfp_fl_ct_add_offload()
1074 err = nfp_fl_merge_actions_offload(rules, priv, netdev, flow_pay, num_rules); in nfp_fl_ct_add_offload()
1083 flow_pay->tc_flower_cookie = ((unsigned long)flow_pay) | 0x1; in nfp_fl_ct_add_offload()
1084 err = nfp_compile_flow_metadata(priv->app, flow_pay->tc_flower_cookie, in nfp_fl_ct_add_offload()
1092 err = rhashtable_insert_fast(&priv->flow_table, &flow_pay->fl_node, in nfp_fl_ct_add_offload()
1097 err = nfp_flower_xmit_flow(priv->app, flow_pay, in nfp_fl_ct_add_offload()
1102 m_entry->tc_flower_cookie = flow_pay->tc_flower_cookie; in nfp_fl_ct_add_offload()
1103 m_entry->flow_pay = flow_pay; in nfp_fl_ct_add_offload()
1106 port->tc_offload_cnt++; in nfp_fl_ct_add_offload()
1111 WARN_ON_ONCE(rhashtable_remove_fast(&priv->flow_table, in nfp_fl_ct_add_offload()
1112 &flow_pay->fl_node, in nfp_fl_ct_add_offload()
1115 nfp_modify_flow_metadata(priv->app, flow_pay); in nfp_fl_ct_add_offload()
1117 if (flow_pay->nfp_tun_ipv4_addr) in nfp_fl_ct_add_offload()
1118 nfp_tunnel_del_ipv4_off(priv->app, flow_pay->nfp_tun_ipv4_addr); in nfp_fl_ct_add_offload()
1119 if (flow_pay->nfp_tun_ipv6) in nfp_fl_ct_add_offload()
1120 nfp_tunnel_put_ipv6_off(priv->app, flow_pay->nfp_tun_ipv6); in nfp_fl_ct_add_offload()
1121 kfree(flow_pay->action_data); in nfp_fl_ct_add_offload()
1122 kfree(flow_pay->mask_data); in nfp_fl_ct_add_offload()
1123 kfree(flow_pay->unmasked_data); in nfp_fl_ct_add_offload()
1131 struct nfp_flower_priv *priv = app->priv; in nfp_fl_ct_del_offload()
1141 return -ENOENT; in nfp_fl_ct_del_offload()
1147 if (flow_pay->nfp_tun_ipv4_addr) in nfp_fl_ct_del_offload()
1148 nfp_tunnel_del_ipv4_off(app, flow_pay->nfp_tun_ipv4_addr); in nfp_fl_ct_del_offload()
1150 if (flow_pay->nfp_tun_ipv6) in nfp_fl_ct_del_offload()
1151 nfp_tunnel_put_ipv6_off(app, flow_pay->nfp_tun_ipv6); in nfp_fl_ct_del_offload()
1153 if (!flow_pay->in_hw) { in nfp_fl_ct_del_offload()
1164 port->tc_offload_cnt--; in nfp_fl_ct_del_offload()
1165 kfree(flow_pay->action_data); in nfp_fl_ct_del_offload()
1166 kfree(flow_pay->mask_data); in nfp_fl_ct_del_offload()
1167 kfree(flow_pay->unmasked_data); in nfp_fl_ct_del_offload()
1168 WARN_ON_ONCE(rhashtable_remove_fast(&priv->flow_table, in nfp_fl_ct_del_offload()
1169 &flow_pay->fl_node, in nfp_fl_ct_del_offload()
1184 pre_ct_entry = tc_m_entry->pre_ct_parent; in nfp_ct_do_nft_merge()
1185 post_ct_entry = tc_m_entry->post_ct_parent; in nfp_ct_do_nft_merge()
1207 if (pre_ct_entry->num_prev_m_entries > 0) { in nfp_ct_do_nft_merge()
1214 new_cookie[0] = tc_m_entry->cookie[0]; in nfp_ct_do_nft_merge()
1215 new_cookie[1] = tc_m_entry->cookie[1]; in nfp_ct_do_nft_merge()
1216 new_cookie[2] = nft_entry->cookie; in nfp_ct_do_nft_merge()
1217 nft_m_entry = get_hashentry(&zt->nft_merge_tb, in nfp_ct_do_nft_merge()
1226 if (!memcmp(&new_cookie, nft_m_entry->cookie, sizeof(new_cookie))) in nfp_ct_do_nft_merge()
1229 memcpy(&nft_m_entry->cookie, &new_cookie, sizeof(new_cookie)); in nfp_ct_do_nft_merge()
1230 nft_m_entry->zt = zt; in nfp_ct_do_nft_merge()
1231 nft_m_entry->tc_m_parent = tc_m_entry; in nfp_ct_do_nft_merge()
1232 nft_m_entry->nft_parent = nft_entry; in nfp_ct_do_nft_merge()
1233 nft_m_entry->tc_flower_cookie = 0; in nfp_ct_do_nft_merge()
1237 nft_m_entry->netdev = pre_ct_entry->netdev; in nfp_ct_do_nft_merge()
1240 list_add(&nft_m_entry->tc_merge_list, &tc_m_entry->children); in nfp_ct_do_nft_merge()
1241 list_add(&nft_m_entry->nft_flow_list, &nft_entry->children); in nfp_ct_do_nft_merge()
1243 err = rhashtable_insert_fast(&zt->nft_merge_tb, &nft_m_entry->hash_node, in nfp_ct_do_nft_merge()
1248 zt->nft_merge_count++; in nfp_ct_do_nft_merge()
1250 if (post_ct_entry->goto_chain_index > 0) in nfp_ct_do_nft_merge()
1261 nfp_fl_ct_del_offload(zt->priv->app, nft_m_entry->tc_flower_cookie, in nfp_ct_do_nft_merge()
1262 nft_m_entry->netdev); in nfp_ct_do_nft_merge()
1264 list_del(&nft_m_entry->tc_merge_list); in nfp_ct_do_nft_merge()
1265 list_del(&nft_m_entry->nft_flow_list); in nfp_ct_do_nft_merge()
1280 if (ct_entry1->type == CT_TYPE_PRE_CT) { in nfp_ct_do_tc_merge()
1291 if (post_ct_entry->chain_index != pre_ct_entry->goto_chain_index) in nfp_ct_do_tc_merge()
1292 return -EINVAL; in nfp_ct_do_tc_merge()
1298 new_cookie[0] = pre_ct_entry->cookie; in nfp_ct_do_tc_merge()
1299 new_cookie[1] = post_ct_entry->cookie; in nfp_ct_do_tc_merge()
1300 m_entry = get_hashentry(&zt->tc_merge_tb, &new_cookie, in nfp_ct_do_tc_merge()
1306 if (!memcmp(&new_cookie, m_entry->cookie, sizeof(new_cookie))) in nfp_ct_do_tc_merge()
1309 memcpy(&m_entry->cookie, &new_cookie, sizeof(new_cookie)); in nfp_ct_do_tc_merge()
1310 m_entry->zt = zt; in nfp_ct_do_tc_merge()
1311 m_entry->post_ct_parent = post_ct_entry; in nfp_ct_do_tc_merge()
1312 m_entry->pre_ct_parent = pre_ct_entry; in nfp_ct_do_tc_merge()
1315 list_add(&m_entry->post_ct_list, &post_ct_entry->children); in nfp_ct_do_tc_merge()
1316 list_add(&m_entry->pre_ct_list, &pre_ct_entry->children); in nfp_ct_do_tc_merge()
1317 INIT_LIST_HEAD(&m_entry->children); in nfp_ct_do_tc_merge()
1319 err = rhashtable_insert_fast(&zt->tc_merge_tb, &m_entry->hash_node, in nfp_ct_do_tc_merge()
1323 zt->tc_merge_count++; in nfp_ct_do_tc_merge()
1326 list_for_each_entry_safe(nft_entry, nft_tmp, &zt->nft_flows_list, in nfp_ct_do_tc_merge()
1334 list_del(&m_entry->post_ct_list); in nfp_ct_do_tc_merge()
1335 list_del(&m_entry->pre_ct_list); in nfp_ct_do_tc_merge()
1347 if (wildcarded && priv->ct_zone_wc) in get_nfp_zone_entry()
1348 return priv->ct_zone_wc; in get_nfp_zone_entry()
1351 zt = get_hashentry(&priv->ct_zone_table, &zone, in get_nfp_zone_entry()
1355 if (IS_ERR(zt) || zt->priv) in get_nfp_zone_entry()
1360 return ERR_PTR(-ENOMEM); in get_nfp_zone_entry()
1363 zt->zone = zone; in get_nfp_zone_entry()
1364 zt->priv = priv; in get_nfp_zone_entry()
1365 zt->nft = NULL; in get_nfp_zone_entry()
1368 INIT_LIST_HEAD(&zt->pre_ct_list); in get_nfp_zone_entry()
1369 INIT_LIST_HEAD(&zt->post_ct_list); in get_nfp_zone_entry()
1370 INIT_LIST_HEAD(&zt->nft_flows_list); in get_nfp_zone_entry()
1372 err = rhashtable_init(&zt->tc_merge_tb, &nfp_tc_ct_merge_params); in get_nfp_zone_entry()
1376 err = rhashtable_init(&zt->nft_merge_tb, &nfp_nft_ct_merge_params); in get_nfp_zone_entry()
1381 priv->ct_zone_wc = zt; in get_nfp_zone_entry()
1383 err = rhashtable_insert_fast(&priv->ct_zone_table, in get_nfp_zone_entry()
1384 &zt->hash_node, in get_nfp_zone_entry()
1393 rhashtable_destroy(&zt->nft_merge_tb); in get_nfp_zone_entry()
1395 rhashtable_destroy(&zt->tc_merge_tb); in get_nfp_zone_entry()
1407 if (match.key->ingress_ifindex & match.mask->ingress_ifindex) in get_netdev_from_rule()
1409 match.key->ingress_ifindex); in get_netdev_from_rule()
1417 if (mangle_action->id != FLOW_ACTION_MANGLE) in nfp_nft_ct_translate_mangle_action()
1420 switch (mangle_action->mangle.htype) { in nfp_nft_ct_translate_mangle_action()
1423 mangle_action->mangle.val = (__force u32)cpu_to_be32(mangle_action->mangle.val); in nfp_nft_ct_translate_mangle_action()
1424 mangle_action->mangle.mask = (__force u32)cpu_to_be32(mangle_action->mangle.mask); in nfp_nft_ct_translate_mangle_action()
1434 if (mangle_action->mangle.offset == offsetof(struct tcphdr, source)) { in nfp_nft_ct_translate_mangle_action()
1435 mangle_action->mangle.val = in nfp_nft_ct_translate_mangle_action()
1436 (__force u32)cpu_to_be32(mangle_action->mangle.val << 16); in nfp_nft_ct_translate_mangle_action()
1439 * instead of rotate-left operation. in nfp_nft_ct_translate_mangle_action()
1441 mangle_action->mangle.mask = in nfp_nft_ct_translate_mangle_action()
1442 (__force u32)cpu_to_be32(mangle_action->mangle.mask << 16 | 0xFFFF); in nfp_nft_ct_translate_mangle_action()
1444 if (mangle_action->mangle.offset == offsetof(struct tcphdr, dest)) { in nfp_nft_ct_translate_mangle_action()
1445 mangle_action->mangle.offset = 0; in nfp_nft_ct_translate_mangle_action()
1446 mangle_action->mangle.val = in nfp_nft_ct_translate_mangle_action()
1447 (__force u32)cpu_to_be32(mangle_action->mangle.val); in nfp_nft_ct_translate_mangle_action()
1448 mangle_action->mangle.mask = in nfp_nft_ct_translate_mangle_action()
1449 (__force u32)cpu_to_be32(mangle_action->mangle.mask); in nfp_nft_ct_translate_mangle_action()
1461 switch (act->id) { in nfp_nft_ct_set_flow_flag()
1463 if (act->ct.action == TCA_CT_ACT_NAT) in nfp_nft_ct_set_flow_flag()
1464 entry->flags |= NFP_FL_ACTION_DO_NAT; in nfp_nft_ct_set_flow_flag()
1468 entry->flags |= NFP_FL_ACTION_DO_MANGLE; in nfp_nft_ct_set_flow_flag()
1492 return ERR_PTR(-ENOMEM); in nfp_fl_ct_add_flow()
1494 entry->rule = flow_rule_alloc(flow->rule->action.num_entries); in nfp_fl_ct_add_flow()
1495 if (!entry->rule) { in nfp_fl_ct_add_flow()
1496 err = -ENOMEM; in nfp_fl_ct_add_flow()
1506 err = -ENOMEM; in nfp_fl_ct_add_flow()
1509 memcpy(&nft_match->dissector, flow->rule->match.dissector, in nfp_fl_ct_add_flow()
1510 sizeof(nft_match->dissector)); in nfp_fl_ct_add_flow()
1511 memcpy(&nft_match->mask, flow->rule->match.mask, in nfp_fl_ct_add_flow()
1512 sizeof(nft_match->mask)); in nfp_fl_ct_add_flow()
1513 memcpy(&nft_match->key, flow->rule->match.key, in nfp_fl_ct_add_flow()
1514 sizeof(nft_match->key)); in nfp_fl_ct_add_flow()
1515 entry->rule->match.dissector = &nft_match->dissector; in nfp_fl_ct_add_flow()
1516 entry->rule->match.mask = &nft_match->mask; in nfp_fl_ct_add_flow()
1517 entry->rule->match.key = &nft_match->key; in nfp_fl_ct_add_flow()
1520 netdev = get_netdev_from_rule(entry->rule); in nfp_fl_ct_add_flow()
1522 entry->rule->match.dissector = flow->rule->match.dissector; in nfp_fl_ct_add_flow()
1523 entry->rule->match.mask = flow->rule->match.mask; in nfp_fl_ct_add_flow()
1524 entry->rule->match.key = flow->rule->match.key; in nfp_fl_ct_add_flow()
1527 entry->zt = zt; in nfp_fl_ct_add_flow()
1528 entry->netdev = netdev; in nfp_fl_ct_add_flow()
1529 entry->cookie = flow->cookie > 0 ? flow->cookie : (unsigned long)entry; in nfp_fl_ct_add_flow()
1530 entry->chain_index = flow->common.chain_index; in nfp_fl_ct_add_flow()
1531 entry->tun_offset = NFP_FL_CT_NO_TUN; in nfp_fl_ct_add_flow()
1538 entry->rule->action.num_entries = flow->rule->action.num_entries; in nfp_fl_ct_add_flow()
1539 flow_action_for_each(i, act, &flow->rule->action) { in nfp_fl_ct_add_flow()
1542 new_act = &entry->rule->action.entries[i]; in nfp_fl_ct_add_flow()
1554 if (act->id == FLOW_ACTION_TUNNEL_ENCAP) { in nfp_fl_ct_add_flow()
1555 struct ip_tunnel_info *tun = act->tunnel; in nfp_fl_ct_add_flow()
1556 size_t tun_size = sizeof(*tun) + tun->options_len; in nfp_fl_ct_add_flow()
1558 new_act->tunnel = kmemdup(tun, tun_size, GFP_ATOMIC); in nfp_fl_ct_add_flow()
1559 if (!new_act->tunnel) { in nfp_fl_ct_add_flow()
1560 err = -ENOMEM; in nfp_fl_ct_add_flow()
1563 entry->tun_offset = i; in nfp_fl_ct_add_flow()
1567 INIT_LIST_HEAD(&entry->children); in nfp_fl_ct_add_flow()
1569 if (flow->cookie == 0) in nfp_fl_ct_add_flow()
1572 /* Now add a ct map entry to flower-priv */ in nfp_fl_ct_add_flow()
1573 map = get_hashentry(&zt->priv->ct_map_table, &flow->cookie, in nfp_fl_ct_add_flow()
1578 err = -ENOMEM; in nfp_fl_ct_add_flow()
1581 map->cookie = flow->cookie; in nfp_fl_ct_add_flow()
1582 map->ct_entry = entry; in nfp_fl_ct_add_flow()
1583 err = rhashtable_insert_fast(&zt->priv->ct_map_table, in nfp_fl_ct_add_flow()
1584 &map->hash_node, in nfp_fl_ct_add_flow()
1597 if (entry->tun_offset != NFP_FL_CT_NO_TUN) in nfp_fl_ct_add_flow()
1598 kfree(entry->rule->action.entries[entry->tun_offset].tunnel); in nfp_fl_ct_add_flow()
1602 kfree(entry->rule); in nfp_fl_ct_add_flow()
1613 zt = m_entry->zt; in cleanup_nft_merge_entry()
1616 if (m_entry->tc_flower_cookie) { in cleanup_nft_merge_entry()
1617 err = nfp_fl_ct_del_offload(zt->priv->app, m_entry->tc_flower_cookie, in cleanup_nft_merge_entry()
1618 m_entry->netdev); in cleanup_nft_merge_entry()
1623 WARN_ON_ONCE(rhashtable_remove_fast(&zt->nft_merge_tb, in cleanup_nft_merge_entry()
1624 &m_entry->hash_node, in cleanup_nft_merge_entry()
1626 zt->nft_merge_count--; in cleanup_nft_merge_entry()
1627 list_del(&m_entry->tc_merge_list); in cleanup_nft_merge_entry()
1628 list_del(&m_entry->nft_flow_list); in cleanup_nft_merge_entry()
1630 if (m_entry->next_pre_ct_entry) { in cleanup_nft_merge_entry()
1633 pre_ct_map_ent.ct_entry = m_entry->next_pre_ct_entry; in cleanup_nft_merge_entry()
1654 list_for_each_entry_safe(m_entry, tmp, &ct_entry->children, in nfp_free_nft_merge_children()
1662 list_for_each_entry_safe(m_entry, tmp, &ct_entry->children, in nfp_free_nft_merge_children()
1674 zt = m_ent->zt; in nfp_del_tc_merge_entry()
1675 err = rhashtable_remove_fast(&zt->tc_merge_tb, in nfp_del_tc_merge_entry()
1676 &m_ent->hash_node, in nfp_del_tc_merge_entry()
1680 zt->tc_merge_count--; in nfp_del_tc_merge_entry()
1681 list_del(&m_ent->post_ct_list); in nfp_del_tc_merge_entry()
1682 list_del(&m_ent->pre_ct_list); in nfp_del_tc_merge_entry()
1684 if (!list_empty(&m_ent->children)) in nfp_del_tc_merge_entry()
1693 switch (entry->type) { in nfp_free_tc_merge_children()
1695 list_for_each_entry_safe(m_ent, tmp, &entry->children, pre_ct_list) { in nfp_free_tc_merge_children()
1700 list_for_each_entry_safe(m_ent, tmp, &entry->children, post_ct_list) { in nfp_free_tc_merge_children()
1711 list_del(&entry->list_node); in nfp_fl_ct_clean_flow_entry()
1713 if (!list_empty(&entry->children)) { in nfp_fl_ct_clean_flow_entry()
1714 if (entry->type == CT_TYPE_NFT) in nfp_fl_ct_clean_flow_entry()
1720 if (entry->tun_offset != NFP_FL_CT_NO_TUN) in nfp_fl_ct_clean_flow_entry()
1721 kfree(entry->rule->action.entries[entry->tun_offset].tunnel); in nfp_fl_ct_clean_flow_entry()
1723 if (entry->type == CT_TYPE_NFT) { in nfp_fl_ct_clean_flow_entry()
1726 nft_match = container_of(entry->rule->match.dissector, in nfp_fl_ct_clean_flow_entry()
1731 kfree(entry->rule); in nfp_fl_ct_clean_flow_entry()
1743 flow_action_for_each(i, act, &rule->action) { in get_flow_act_ct()
1744 if (act->id == FLOW_ACTION_CT && act->ct.action != TCA_CT_ACT_CLEAR) in get_flow_act_ct()
1757 flow_action_for_each(i, act, &rule->action) { in get_flow_act()
1758 if (act->id == act_id) in get_flow_act()
1772 if (ct_entry1->type == CT_TYPE_PRE_CT) in nfp_ct_merge_tc_entries()
1773 ct_list = &zt_src->post_ct_list; in nfp_ct_merge_tc_entries()
1774 else if (ct_entry1->type == CT_TYPE_POST_CT) in nfp_ct_merge_tc_entries()
1775 ct_list = &zt_src->pre_ct_list; in nfp_ct_merge_tc_entries()
1792 rhashtable_walk_enter(&zt->tc_merge_tb, &iter); in nfp_ct_merge_nft_with_tc()
1816 ct_act = get_flow_act_ct(flow->rule); in nfp_fl_ct_handle_pre_ct()
1820 return -EOPNOTSUPP; in nfp_fl_ct_handle_pre_ct()
1823 ct_goto = get_flow_act(flow->rule, FLOW_ACTION_GOTO); in nfp_fl_ct_handle_pre_ct()
1827 return -EOPNOTSUPP; in nfp_fl_ct_handle_pre_ct()
1830 zt = get_nfp_zone_entry(priv, ct_act->ct.zone, false); in nfp_fl_ct_handle_pre_ct()
1837 if (!zt->nft) { in nfp_fl_ct_handle_pre_ct()
1838 zt->nft = ct_act->ct.flow_table; in nfp_fl_ct_handle_pre_ct()
1839 err = nf_flow_table_offload_add_cb(zt->nft, nfp_fl_ct_handle_nft_flow, zt); in nfp_fl_ct_handle_pre_ct()
1851 ct_entry->type = CT_TYPE_PRE_CT; in nfp_fl_ct_handle_pre_ct()
1852 ct_entry->chain_index = flow->common.chain_index; in nfp_fl_ct_handle_pre_ct()
1853 ct_entry->goto_chain_index = ct_goto->chain_index; in nfp_fl_ct_handle_pre_ct()
1859 pre_ct_entry = m_entry->tc_m_parent->pre_ct_parent; in nfp_fl_ct_handle_pre_ct()
1860 for (i = 0; i < pre_ct_entry->num_prev_m_entries; i++) in nfp_fl_ct_handle_pre_ct()
1861 ct_entry->prev_m_entries[i] = pre_ct_entry->prev_m_entries[i]; in nfp_fl_ct_handle_pre_ct()
1862 ct_entry->prev_m_entries[i++] = m_entry; in nfp_fl_ct_handle_pre_ct()
1863 ct_entry->num_prev_m_entries = i; in nfp_fl_ct_handle_pre_ct()
1865 m_entry->next_pre_ct_entry = ct_entry; in nfp_fl_ct_handle_pre_ct()
1868 list_add(&ct_entry->list_node, &zt->pre_ct_list); in nfp_fl_ct_handle_pre_ct()
1869 zt->pre_ct_count++; in nfp_fl_ct_handle_pre_ct()
1874 if (priv->ct_zone_wc) in nfp_fl_ct_handle_pre_ct()
1875 nfp_ct_merge_tc_entries(ct_entry, priv->ct_zone_wc, zt); in nfp_fl_ct_handle_pre_ct()
1894 flow_action_for_each(i, act, &rule->action) { in nfp_fl_ct_handle_post_ct()
1895 switch (act->id) { in nfp_fl_ct_handle_post_ct()
1900 if (act->dev->rtnl_link_ops && in nfp_fl_ct_handle_post_ct()
1901 !strcmp(act->dev->rtnl_link_ops->kind, "openvswitch")) { in nfp_fl_ct_handle_post_ct()
1904 return -EOPNOTSUPP; in nfp_fl_ct_handle_post_ct()
1913 if (!ct.mask->ct_zone) { in nfp_fl_ct_handle_post_ct()
1915 } else if (ct.mask->ct_zone != U16_MAX) { in nfp_fl_ct_handle_post_ct()
1918 return -EOPNOTSUPP; in nfp_fl_ct_handle_post_ct()
1921 zt = get_nfp_zone_entry(priv, ct.key->ct_zone, wildcarded); in nfp_fl_ct_handle_post_ct()
1933 ct_entry->type = CT_TYPE_POST_CT; in nfp_fl_ct_handle_post_ct()
1934 ct_entry->chain_index = flow->common.chain_index; in nfp_fl_ct_handle_post_ct()
1935 ct_goto = get_flow_act(flow->rule, FLOW_ACTION_GOTO); in nfp_fl_ct_handle_post_ct()
1936 ct_entry->goto_chain_index = ct_goto ? ct_goto->chain_index : 0; in nfp_fl_ct_handle_post_ct()
1937 list_add(&ct_entry->list_node, &zt->post_ct_list); in nfp_fl_ct_handle_post_ct()
1938 zt->post_ct_count++; in nfp_fl_ct_handle_post_ct()
1947 rhashtable_walk_enter(&priv->ct_zone_table, &iter); in nfp_fl_ct_handle_post_ct()
1971 pre_ct_entry = m_entry->tc_m_parent->pre_ct_parent; in nfp_fl_create_new_pre_ct()
1972 if (pre_ct_entry->num_prev_m_entries >= NFP_MAX_RECIRC_CT_ZONES - 1) in nfp_fl_create_new_pre_ct()
1973 return -1; in nfp_fl_create_new_pre_ct()
1975 post_ct_entry = m_entry->tc_m_parent->post_ct_parent; in nfp_fl_create_new_pre_ct()
1977 new_pre_ct_flow.rule = post_ct_entry->rule; in nfp_fl_create_new_pre_ct()
1978 new_pre_ct_flow.common.chain_index = post_ct_entry->chain_index; in nfp_fl_create_new_pre_ct()
1980 err = nfp_fl_ct_handle_pre_ct(pre_ct_entry->zt->priv, in nfp_fl_create_new_pre_ct()
1981 pre_ct_entry->netdev, in nfp_fl_create_new_pre_ct()
1992 struct nfp_flower_priv *priv = nft_merge->zt->priv; in nfp_fl_ct_sub_stats()
1996 nfp_flow = nft_merge->flow_pay; in nfp_fl_ct_sub_stats()
2000 ctx_id = be32_to_cpu(nfp_flow->meta.host_ctx_id); in nfp_fl_ct_sub_stats()
2001 *m_pkts += priv->stats[ctx_id].pkts; in nfp_fl_ct_sub_stats()
2002 *m_bytes += priv->stats[ctx_id].bytes; in nfp_fl_ct_sub_stats()
2003 *m_used = max_t(u64, *m_used, priv->stats[ctx_id].used); in nfp_fl_ct_sub_stats()
2008 if (!list_empty(&nfp_flow->linked_flows)) in nfp_fl_ct_sub_stats()
2009 nfp_flower_update_merge_stats(priv->app, nfp_flow); in nfp_fl_ct_sub_stats()
2013 flow_stats_update(&nft_merge->nft_parent->stats, in nfp_fl_ct_sub_stats()
2014 priv->stats[ctx_id].bytes, in nfp_fl_ct_sub_stats()
2015 priv->stats[ctx_id].pkts, in nfp_fl_ct_sub_stats()
2016 0, priv->stats[ctx_id].used, in nfp_fl_ct_sub_stats()
2020 flow_stats_update(&nft_merge->tc_m_parent->pre_ct_parent->stats, in nfp_fl_ct_sub_stats()
2021 priv->stats[ctx_id].bytes, in nfp_fl_ct_sub_stats()
2022 priv->stats[ctx_id].pkts, in nfp_fl_ct_sub_stats()
2023 0, priv->stats[ctx_id].used, in nfp_fl_ct_sub_stats()
2026 flow_stats_update(&nft_merge->tc_m_parent->post_ct_parent->stats, in nfp_fl_ct_sub_stats()
2027 priv->stats[ctx_id].bytes, in nfp_fl_ct_sub_stats()
2028 priv->stats[ctx_id].pkts, in nfp_fl_ct_sub_stats()
2029 0, priv->stats[ctx_id].used, in nfp_fl_ct_sub_stats()
2034 if (nft_merge->tc_m_parent->pre_ct_parent->num_prev_m_entries > 0) { in nfp_fl_ct_sub_stats()
2038 for (i = 0; i < nft_merge->tc_m_parent->pre_ct_parent->num_prev_m_entries; i++) { in nfp_fl_ct_sub_stats()
2039 tmp_nft_merge = nft_merge->tc_m_parent->pre_ct_parent->prev_m_entries[i]; in nfp_fl_ct_sub_stats()
2040 flow_stats_update(&tmp_nft_merge->tc_m_parent->pre_ct_parent->stats, in nfp_fl_ct_sub_stats()
2041 priv->stats[ctx_id].bytes, in nfp_fl_ct_sub_stats()
2042 priv->stats[ctx_id].pkts, in nfp_fl_ct_sub_stats()
2043 0, priv->stats[ctx_id].used, in nfp_fl_ct_sub_stats()
2045 flow_stats_update(&tmp_nft_merge->tc_m_parent->post_ct_parent->stats, in nfp_fl_ct_sub_stats()
2046 priv->stats[ctx_id].bytes, in nfp_fl_ct_sub_stats()
2047 priv->stats[ctx_id].pkts, in nfp_fl_ct_sub_stats()
2048 0, priv->stats[ctx_id].used, in nfp_fl_ct_sub_stats()
2050 flow_stats_update(&tmp_nft_merge->nft_parent->stats, in nfp_fl_ct_sub_stats()
2051 priv->stats[ctx_id].bytes, in nfp_fl_ct_sub_stats()
2052 priv->stats[ctx_id].pkts, in nfp_fl_ct_sub_stats()
2053 0, priv->stats[ctx_id].used, in nfp_fl_ct_sub_stats()
2059 priv->stats[ctx_id].pkts = 0; in nfp_fl_ct_sub_stats()
2060 priv->stats[ctx_id].bytes = 0; in nfp_fl_ct_sub_stats()
2066 struct nfp_fl_ct_flow_entry *ct_entry = ct_map_ent->ct_entry; in nfp_fl_ct_stats()
2073 spin_lock_bh(&ct_entry->zt->priv->stats_lock); in nfp_fl_ct_stats()
2075 if (ct_entry->type == CT_TYPE_PRE_CT) { in nfp_fl_ct_stats()
2077 list_for_each_entry_safe(tc_merge, tc_m_tmp, &ct_entry->children, in nfp_fl_ct_stats()
2083 list_for_each_entry_safe(nft_merge, nft_m_tmp, &tc_merge->children, in nfp_fl_ct_stats()
2092 flow_stats_update(&tc_merge->post_ct_parent->stats, in nfp_fl_ct_stats()
2096 } else if (ct_entry->type == CT_TYPE_POST_CT) { in nfp_fl_ct_stats()
2098 list_for_each_entry_safe(tc_merge, tc_m_tmp, &ct_entry->children, in nfp_fl_ct_stats()
2104 list_for_each_entry_safe(nft_merge, nft_m_tmp, &tc_merge->children, in nfp_fl_ct_stats()
2113 flow_stats_update(&tc_merge->pre_ct_parent->stats, in nfp_fl_ct_stats()
2119 list_for_each_entry_safe(nft_merge, nft_m_tmp, &ct_entry->children, in nfp_fl_ct_stats()
2129 flow_stats_update(&ct_entry->stats, bytes, pkts, 0, used, in nfp_fl_ct_stats()
2132 flow_stats_update(&flow->stats, ct_entry->stats.bytes, in nfp_fl_ct_stats()
2133 ct_entry->stats.pkts, 0, in nfp_fl_ct_stats()
2134 ct_entry->stats.lastused, in nfp_fl_ct_stats()
2139 ct_entry->stats.pkts = 0; in nfp_fl_ct_stats()
2140 ct_entry->stats.bytes = 0; in nfp_fl_ct_stats()
2141 spin_unlock_bh(&ct_entry->zt->priv->stats_lock); in nfp_fl_ct_stats()
2149 struct flow_rule *flow_rule = flow->rule; in nfp_fl_ct_offload_nft_supported()
2151 &flow_rule->action; in nfp_fl_ct_offload_nft_supported()
2156 if (act->id == FLOW_ACTION_CT_METADATA) { in nfp_fl_ct_offload_nft_supported()
2158 act->ct_metadata.cookie & NFCT_INFOMASK; in nfp_fl_ct_offload_nft_supported()
2174 extack = flow->common.extack; in nfp_fl_ct_offload_nft_flow()
2175 switch (flow->command) { in nfp_fl_ct_offload_nft_flow()
2178 return -EOPNOTSUPP; in nfp_fl_ct_offload_nft_flow()
2181 * flow - protect against adding duplicates. in nfp_fl_ct_offload_nft_flow()
2183 ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie, in nfp_fl_ct_offload_nft_flow()
2189 ct_entry->type = CT_TYPE_NFT; in nfp_fl_ct_offload_nft_flow()
2190 list_add(&ct_entry->list_node, &zt->nft_flows_list); in nfp_fl_ct_offload_nft_flow()
2191 zt->nft_flows_count++; in nfp_fl_ct_offload_nft_flow()
2196 ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie, in nfp_fl_ct_offload_nft_flow()
2200 ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie, in nfp_fl_ct_offload_nft_flow()
2208 return -EINVAL; in nfp_fl_ct_offload_nft_flow()
2215 int err = -EOPNOTSUPP; in nfp_fl_ct_handle_nft_flow()
2219 while (!mutex_trylock(&zt->priv->nfp_fl_lock)) { in nfp_fl_ct_handle_nft_flow()
2220 if (!zt->nft) /* avoid deadlock */ in nfp_fl_ct_handle_nft_flow()
2225 mutex_unlock(&zt->priv->nfp_fl_lock); in nfp_fl_ct_handle_nft_flow()
2228 return -EOPNOTSUPP; in nfp_fl_ct_handle_nft_flow()
2239 list_for_each_entry_safe(nft_entry, ct_tmp, &zt->nft_flows_list, in nfp_fl_ct_clean_nft_entries()
2241 ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, in nfp_fl_ct_clean_nft_entries()
2242 &nft_entry->cookie, in nfp_fl_ct_clean_nft_entries()
2256 return -ENOENT; in nfp_fl_ct_del_flow()
2258 zt = ct_map_ent->ct_entry->zt; in nfp_fl_ct_del_flow()
2259 ct_entry = ct_map_ent->ct_entry; in nfp_fl_ct_del_flow()
2260 m_table = &zt->priv->ct_map_table; in nfp_fl_ct_del_flow()
2262 switch (ct_entry->type) { in nfp_fl_ct_del_flow()
2264 zt->pre_ct_count--; in nfp_fl_ct_del_flow()
2265 if (ct_map_ent->cookie > 0) in nfp_fl_ct_del_flow()
2266 rhashtable_remove_fast(m_table, &ct_map_ent->hash_node, in nfp_fl_ct_del_flow()
2269 if (ct_map_ent->cookie > 0) in nfp_fl_ct_del_flow()
2272 if (!zt->pre_ct_count && zt->nft) { in nfp_fl_ct_del_flow()
2273 nft = zt->nft; in nfp_fl_ct_del_flow()
2274 zt->nft = NULL; /* avoid deadlock */ in nfp_fl_ct_del_flow()
2282 zt->post_ct_count--; in nfp_fl_ct_del_flow()
2283 rhashtable_remove_fast(m_table, &ct_map_ent->hash_node, in nfp_fl_ct_del_flow()
2289 zt->nft_flows_count--; in nfp_fl_ct_del_flow()
2290 rhashtable_remove_fast(m_table, &ct_map_ent->hash_node, in nfp_fl_ct_del_flow()
2292 nfp_fl_ct_clean_flow_entry(ct_map_ent->ct_entry); in nfp_fl_ct_del_flow()