From: Yashaswini Kanaujia <[email protected]> added support for asynchronous rte_flow APIs for cn10k device
Signed-off-by: Yashaswini Kanaujia <[email protected]> --- drivers/common/cnxk/meson.build | 1 + drivers/common/cnxk/roc_npc.h | 97 ++ drivers/common/cnxk/roc_npc_flow_template.c | 1326 ++++++++++++++++ drivers/common/cnxk/roc_npc_mcam.c | 45 +- drivers/common/cnxk/roc_npc_priv.h | 98 ++ drivers/common/cnxk/roc_npc_utils.c | 4 +- .../common/cnxk/roc_platform_base_symbols.c | 16 + drivers/net/cnxk/cn10k_ethdev.c | 23 + drivers/net/cnxk/cn10k_flow.c | 43 + drivers/net/cnxk/cn10k_flow.h | 2 + drivers/net/cnxk/cn20k_ethdev.c | 24 +- drivers/net/cnxk/cn20k_flow.c | 43 + drivers/net/cnxk/cn20k_flow.h | 2 + drivers/net/cnxk/cnxk_ethdev.h | 44 + drivers/net/cnxk/cnxk_flow.c | 1338 ++++++++++++++++- drivers/net/cnxk/cnxk_flow.h | 77 + 16 files changed, 3161 insertions(+), 22 deletions(-) create mode 100644 drivers/common/cnxk/roc_npc_flow_template.c diff --git a/drivers/common/cnxk/meson.build b/drivers/common/cnxk/meson.build index 3303ad9354..68374ee135 100644 --- a/drivers/common/cnxk/meson.build +++ b/drivers/common/cnxk/meson.build @@ -59,6 +59,7 @@ sources = files( 'roc_npa_type.c', 'roc_npc.c', 'roc_npc_aging.c', + 'roc_npc_flow_template.c', 'roc_npc_mcam.c', 'roc_npc_mcam_dump.c', 'roc_npc_parse.c', diff --git a/drivers/common/cnxk/roc_npc.h b/drivers/common/cnxk/roc_npc.h index a7254f35ca..21fa493c74 100644 --- a/drivers/common/cnxk/roc_npc.h +++ b/drivers/common/cnxk/roc_npc.h @@ -473,6 +473,9 @@ int __roc_api roc_npc_mcam_alloc_entries(struct roc_npc *roc_npc, int ref_entry, int __roc_api roc_npc_mcam_ena_dis_entry(struct roc_npc *roc_npc, struct roc_npc_flow *mcam, bool enable); int __roc_api roc_npc_mcam_write_entry(struct roc_npc *roc_npc, struct roc_npc_flow *mcam); + +void __roc_api roc_npc_mcam_write_rx_finalize(struct roc_npc *roc_npc, struct roc_npc_flow *flow); + void __roc_api roc_npc_defrag_mcam_banks(struct roc_npc *roc_npc); uint8_t __roc_api roc_npc_get_key_type(struct roc_npc *roc_npc, struct roc_npc_flow *flow); uint8_t __roc_api roc_npc_kex_key_type_config_get(struct roc_npc *roc_npc); @@ -508,4 +511,98 @@ void __roc_api roc_npc_sdp_channel_get(struct roc_npc *roc_npc, uint16_t *chan_b int __roc_api roc_npc_mcam_get_stats(struct roc_npc *roc_npc, struct roc_npc_flow *flow, uint64_t *count); int __roc_api roc_npc_skip_size_pkind_get(struct roc_npc *roc_npc); + +/* + * Queue-based (async) template flow API. + */ +enum roc_npc_template_table_insertion_type { + /* HW MCAM entry allocated per rule at create time (match based). */ + ROC_NPC_TEMPLATE_INSERTION_PATTERN, + /* MCAM range reserved up front; rules placed at an app-chosen index. */ + ROC_NPC_TEMPLATE_INSERTION_INDEX_WITH_PATTERN, +}; + +struct roc_npc_flow_queue_attr { + uint32_t size; /**< Number of in-flight async ops the queue can hold. */ +}; + +struct roc_npc_pattern_template_attr { + uint32_t relaxed_matching : 1; + uint32_t ingress : 1; + uint32_t egress : 1; + uint32_t transfer : 1; + uint32_t reserved : 28; +}; + +struct roc_npc_actions_template_attr { + uint32_t ingress : 1; + uint32_t egress : 1; + uint32_t transfer : 1; + uint32_t reserved : 29; +}; + +struct roc_npc_template_table_attr { + struct roc_npc_attr flow_attr; /**< Priority / ingress / egress. */ + uint32_t transfer : 1; /**< roc_npc_attr has no transfer bit. */ + uint32_t reserved : 31; + enum roc_npc_template_table_insertion_type insertion_type; + uint32_t nb_flows; /**< Max rules the table can hold. */ + void *cookie; /**< Opaque consumer context, retrievable per rule. */ +}; + +/** Result of one completed async operation, returned by roc_npc_flow_pull. */ +struct roc_npc_flow_op_result { + void *user_data; /**< user_data passed at async_create/destroy time. */ + int rc; /**< 0 on success, negative errno on failure. */ +}; + +/* Opaque template/table/flow handles owned by the roc layer. */ +struct roc_npc_pattern_template; +struct roc_npc_actions_template; +struct roc_npc_template_table; +struct roc_npc_template_flow; + +int __roc_api roc_npc_flow_configure(struct roc_npc *roc_npc, uint16_t nb_queues, + const struct roc_npc_flow_queue_attr *queue_attr[]); + +struct roc_npc_pattern_template *__roc_api roc_npc_pattern_template_create( + struct roc_npc *roc_npc, const struct roc_npc_pattern_template_attr *attr, + const struct roc_npc_item_info pattern[], int *errcode); +int __roc_api roc_npc_pattern_template_destroy(struct roc_npc *roc_npc, + struct roc_npc_pattern_template *tmpl); + +struct roc_npc_actions_template *__roc_api roc_npc_actions_template_create( + struct roc_npc *roc_npc, const struct roc_npc_actions_template_attr *attr, + const struct roc_npc_action actions[], const struct roc_npc_action masks[], int *errcode); +int __roc_api roc_npc_actions_template_destroy(struct roc_npc *roc_npc, + struct roc_npc_actions_template *tmpl); + +struct roc_npc_template_table *__roc_api roc_npc_template_table_create( + struct roc_npc *roc_npc, const struct roc_npc_template_table_attr *attr, + struct roc_npc_pattern_template *pattern_templates[], uint8_t nb_pattern_templates, + struct roc_npc_actions_template *actions_templates[], uint8_t nb_actions_templates, + int *errcode); +int __roc_api roc_npc_template_table_destroy(struct roc_npc *roc_npc, + struct roc_npc_template_table *table); + +struct roc_npc_template_flow *__roc_api roc_npc_async_flow_create( + struct roc_npc *roc_npc, uint32_t queue_id, struct roc_npc_template_table *table, + const struct roc_npc_item_info pattern[], const struct roc_npc_action actions[], + uint16_t dst_pf_func, uint64_t npc_default_action, void *user_data, int *errcode); +int __roc_api roc_npc_async_flow_destroy(struct roc_npc *roc_npc, uint32_t queue_id, + struct roc_npc_template_flow *flow, void *user_data); +struct roc_npc_template_flow *__roc_api roc_npc_async_flow_create_by_index_with_pattern( + struct roc_npc *roc_npc, uint32_t queue_id, struct roc_npc_template_table *table, + uint32_t rule_index, const struct roc_npc_item_info pattern[], + const struct roc_npc_action actions[], void *user_data, int *errcode); +int __roc_api roc_npc_async_flow_actions_update(struct roc_npc *roc_npc, uint32_t queue_id, + struct roc_npc_template_flow *flow, + const struct roc_npc_action actions[], + void *user_data); +int __roc_api roc_npc_flow_push(struct roc_npc *roc_npc, uint32_t queue_id); +int __roc_api roc_npc_flow_pull(struct roc_npc *roc_npc, uint32_t queue_id, + struct roc_npc_flow_op_result res[], uint16_t n_res); +bool __roc_api roc_npc_async_flow_resolve(struct roc_npc *roc_npc, void *handle, + struct roc_npc_flow **flow); +void *__roc_api roc_npc_async_flow_table_cookie(struct roc_npc_template_flow *flow); #endif /* _ROC_NPC_H_ */ diff --git a/drivers/common/cnxk/roc_npc_flow_template.c b/drivers/common/cnxk/roc_npc_flow_template.c new file mode 100644 index 0000000000..5c10cfe39a --- /dev/null +++ b/drivers/common/cnxk/roc_npc_flow_template.c @@ -0,0 +1,1326 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(C) 2024 Marvell. + */ + +#include "roc_api.h" +#include "roc_priv.h" + +/* Size of an action's conf for the supported set; -1 = nested/unsupported. */ +static int +roc_npc_action_conf_size(enum roc_npc_action_type type) +{ + switch (type) { + case ROC_NPC_ACTION_TYPE_END: + case ROC_NPC_ACTION_TYPE_VOID: + case ROC_NPC_ACTION_TYPE_FLAG: + case ROC_NPC_ACTION_TYPE_DROP: + case ROC_NPC_ACTION_TYPE_COUNT: + case ROC_NPC_ACTION_TYPE_PF: + case ROC_NPC_ACTION_TYPE_VLAN_STRIP: + return 0; /* no conf */ + case ROC_NPC_ACTION_TYPE_MARK: + return sizeof(struct roc_npc_action_mark); + case ROC_NPC_ACTION_TYPE_VF: + return sizeof(struct roc_npc_action_vf); + case ROC_NPC_ACTION_TYPE_PORT_ID: + return sizeof(struct roc_npc_action_port_id); + case ROC_NPC_ACTION_TYPE_QUEUE: + return sizeof(struct roc_npc_action_queue); + case ROC_NPC_ACTION_TYPE_VLAN_INSERT: + return sizeof(struct roc_npc_action_of_set_vlan_vid); + case ROC_NPC_ACTION_TYPE_VLAN_ETHTYPE_INSERT: + return sizeof(struct roc_npc_action_of_push_vlan); + case ROC_NPC_ACTION_TYPE_VLAN_PCP_INSERT: + return sizeof(struct roc_npc_action_of_set_vlan_pcp); + case ROC_NPC_ACTION_TYPE_AGE: + return sizeof(struct roc_npc_action_age); + default: + return -1; /* RSS handled separately; others unsupported */ + } +} + +/* Deep-copy an item spec/last/mask payload; *dst = NULL when nothing to copy. */ +static int +roc_npc_dup_item_field(const void *src, uint32_t sz, const void **dst) +{ + void *copy; + + *dst = NULL; + if (src == NULL || sz == 0) + return 0; + copy = plt_zmalloc(sz, 0); + if (copy == NULL) + return -ENOMEM; + memcpy(copy, src, sz); + *dst = copy; + return 0; +} + +/* Deep-copy an action conf; RSS packs queue[]/key[] into one blob. */ +static void * +roc_npc_dup_action_conf(const struct roc_npc_action *act) +{ + if (act->conf == NULL) + return NULL; + + if (act->type == ROC_NPC_ACTION_TYPE_RSS) { + const struct roc_npc_action_rss *rss = act->conf; + size_t rss_queue_bytes = (size_t)rss->queue_num * sizeof(uint16_t); + size_t rss_key_bytes = rss->key_len; + size_t rss_blob_bytes = sizeof(*rss); + struct roc_npc_action_rss *rss_copy; + uint8_t *rss_blob; + + if ((rss_queue_bytes && rss->queue == NULL) || (rss_key_bytes && rss->key == NULL)) + return NULL; + + rss_blob = plt_zmalloc(rss_blob_bytes + rss_queue_bytes + rss_key_bytes, 0); + if (rss_blob == NULL) + return NULL; + rss_copy = (struct roc_npc_action_rss *)rss_blob; + *rss_copy = *rss; + if (rss_queue_bytes) { + memcpy(rss_blob + rss_blob_bytes, rss->queue, rss_queue_bytes); + rss_copy->queue = (const uint16_t *)(rss_blob + rss_blob_bytes); + } else { + rss_copy->queue = NULL; + } + if (rss_key_bytes) { + memcpy(rss_blob + rss_blob_bytes + rss_queue_bytes, rss->key, + rss_key_bytes); + rss_copy->key = + (const uint8_t *)(rss_blob + rss_blob_bytes + rss_queue_bytes); + } else { + rss_copy->key = NULL; + } + return rss_copy; + } + + { + int conf_size = roc_npc_action_conf_size(act->type); + void *copy; + + if (conf_size <= 0) + return NULL; + copy = plt_zmalloc(conf_size, 0); + if (copy != NULL) + memcpy(copy, act->conf, conf_size); + return copy; + } +} + +/* Free the deep-copied items/actions arrays retained for a rule. */ +static void +roc_npc_template_flow_free_rule(struct roc_npc_template_flow *flow) +{ + uint16_t k; + + if (flow->items != NULL) { + for (k = 0; k < flow->nb_items; k++) { + plt_free((void *)(uintptr_t)flow->items[k].spec); + plt_free((void *)(uintptr_t)flow->items[k].last); + plt_free((void *)(uintptr_t)flow->items[k].mask); + } + + plt_free(flow->items); + flow->items = NULL; + } + if (flow->actions != NULL) { + for (k = 0; k < flow->nb_actions; k++) + plt_free((void *)(uintptr_t)flow->actions[k].conf); + + plt_free(flow->actions); + flow->actions = NULL; + } +} + +int +roc_npc_flow_configure(struct roc_npc *roc_npc, uint16_t nb_queues, + const struct roc_npc_flow_queue_attr *queue_attr[]) +{ + struct npc *npc = roc_npc_to_npc_priv(roc_npc); + struct roc_npc_flow_queue *queues; + int rc = -ENOMEM; + uint16_t i; + + if (nb_queues == 0) + return 0; + + if (queue_attr == NULL) + return -EINVAL; + + if (npc->flow_queues != NULL) { + /* Don't tear down live queues while async ops are in flight. */ + for (i = 0; i < npc->nb_flow_queues; i++) { + if (npc->flow_queues[i].head != npc->flow_queues[i].tail) + return -EBUSY; + } + } + + if (npc->flow_queues != NULL) { + for (i = 0; i < npc->nb_flow_queues; i++) + plt_free(npc->flow_queues[i].ops); + + plt_free(npc->flow_queues); + npc->flow_queues = NULL; + npc->nb_flow_queues = 0; + } + + queues = plt_zmalloc(nb_queues * sizeof(struct roc_npc_flow_queue), 0); + if (queues == NULL) + return -ENOMEM; + + for (i = 0; i < nb_queues; i++) { + if (queue_attr[i] == NULL || queue_attr[i]->size == 0) { + rc = -EINVAL; + goto err_free; + } + queues[i].size = plt_align32pow2(queue_attr[i]->size); + queues[i].ops = plt_zmalloc(queues[i].size * sizeof(struct roc_npc_async_op), 0); + if (queues[i].ops == NULL) { + rc = -ENOMEM; + goto err_free; + } + queues[i].head = 0; + queues[i].pushed = 0; + queues[i].tail = 0; + } + + npc->flow_queues = queues; + npc->nb_flow_queues = nb_queues; + return 0; + +err_free: + for (i = 0; i < nb_queues; i++) + plt_free(queues[i].ops); + + plt_free(queues); + return rc; +} + +struct roc_npc_pattern_template * +roc_npc_pattern_template_create(struct roc_npc *roc_npc, + const struct roc_npc_pattern_template_attr *attr, + const struct roc_npc_item_info pattern[], int *errcode) +{ + struct roc_npc_pattern_template *tmpl = NULL; + int i, nb_items = 0, rc = -ENOMEM; + uint16_t j; + + PLT_SET_USED(roc_npc); + + if (attr == NULL || pattern == NULL) { + *errcode = -EINVAL; + return NULL; + } + + while (pattern[nb_items].type != ROC_NPC_ITEM_TYPE_END) + nb_items++; + + /* Bound template size to the fixed-size rule arrays. */ + if (nb_items > ROC_NPC_ITEM_TYPE_END) { + *errcode = -ENOTSUP; + return NULL; + } + + tmpl = plt_zmalloc(sizeof(*tmpl), 0); + if (tmpl == NULL) + goto err; + + tmpl->attr = *attr; + tmpl->nb_items = nb_items; + + tmpl->pattern = plt_zmalloc((nb_items + 1) * sizeof(struct roc_npc_item_info), 0); + if (tmpl->pattern == NULL) + goto err; + + /* Reserve up to 3 payload copies per item. */ + tmpl->copies = plt_zmalloc(3 * (nb_items + 1) * sizeof(void *), 0); + if (tmpl->copies == NULL) + goto err; + + for (i = 0; i < nb_items; i++) { + enum roc_npc_item_type t = pattern[i].type; + uint32_t sz = pattern[i].size; + + /* RAW is not flat-copyable. */ + if (t == ROC_NPC_ITEM_TYPE_RAW) { + rc = -ENOTSUP; + goto err; + } + + tmpl->pattern[i].type = t; + tmpl->pattern[i].size = sz; + /* Track each copy for the error path. */ + if (roc_npc_dup_item_field(pattern[i].spec, sz, &tmpl->pattern[i].spec)) + goto err; + if (tmpl->pattern[i].spec != NULL) + tmpl->copies[tmpl->nb_copies++] = (void *)(uintptr_t)tmpl->pattern[i].spec; + if (roc_npc_dup_item_field(pattern[i].last, sz, &tmpl->pattern[i].last)) + goto err; + if (tmpl->pattern[i].last != NULL) + tmpl->copies[tmpl->nb_copies++] = (void *)(uintptr_t)tmpl->pattern[i].last; + if (roc_npc_dup_item_field(pattern[i].mask, sz, &tmpl->pattern[i].mask)) + goto err; + if (tmpl->pattern[i].mask != NULL) + tmpl->copies[tmpl->nb_copies++] = (void *)(uintptr_t)tmpl->pattern[i].mask; + } + tmpl->pattern[nb_items].type = ROC_NPC_ITEM_TYPE_END; + tmpl->refcnt = 0; + + return tmpl; + +err: + if (tmpl != NULL) { + for (j = 0; j < tmpl->nb_copies; j++) + plt_free(tmpl->copies[j]); + plt_free(tmpl->copies); + plt_free(tmpl->pattern); + plt_free(tmpl); + } + *errcode = rc; + return NULL; +} + +int +roc_npc_pattern_template_destroy(struct roc_npc *roc_npc, struct roc_npc_pattern_template *tmpl) +{ + uint16_t i; + + PLT_SET_USED(roc_npc); + + if (tmpl == NULL) + return 0; + + if (tmpl->refcnt != 0) + return -EBUSY; + + for (i = 0; i < tmpl->nb_copies; i++) + plt_free(tmpl->copies[i]); + + plt_free(tmpl->copies); + plt_free(tmpl->pattern); + plt_free(tmpl); + return 0; +} + +struct roc_npc_actions_template * +roc_npc_actions_template_create(struct roc_npc *roc_npc, + const struct roc_npc_actions_template_attr *attr, + const struct roc_npc_action actions[], + const struct roc_npc_action masks[], int *errcode) +{ + struct roc_npc_actions_template *tmpl = NULL; + int i, nb_actions = 0, rc = -ENOMEM; + uint16_t j; + + PLT_SET_USED(roc_npc); + + if (attr == NULL || actions == NULL) { + *errcode = -EINVAL; + return NULL; + } + + while (actions[nb_actions].type != ROC_NPC_ACTION_TYPE_END) + nb_actions++; + + /* Bound template size to the fixed-size rule arrays. */ + if (nb_actions > ROC_NPC_MAX_ACTION_COUNT - 1) { + *errcode = -ENOTSUP; + return NULL; + } + + tmpl = plt_zmalloc(sizeof(*tmpl), 0); + if (tmpl == NULL) + goto err; + + tmpl->attr = *attr; + tmpl->nb_actions = nb_actions; + + tmpl->actions = plt_zmalloc((nb_actions + 1) * sizeof(struct roc_npc_action), 0); + if (tmpl->actions == NULL) + goto err; + + tmpl->masks = plt_zmalloc((nb_actions + 1) * sizeof(struct roc_npc_action), 0); + if (tmpl->masks == NULL) + goto err; + + /* Reserve up to 2 conf copies per action. */ + tmpl->copies = plt_zmalloc(2 * (nb_actions + 1) * sizeof(void *), 0); + if (tmpl->copies == NULL) + goto err; + + for (i = 0; i < nb_actions; i++) { + enum roc_npc_action_type t = actions[i].type; + int sz = roc_npc_action_conf_size(t); + + if (sz < 0 && t != ROC_NPC_ACTION_TYPE_RSS) { + rc = -ENOTSUP; + goto err; + } + + tmpl->actions[i].type = t; + tmpl->actions[i].rss_repte_pf_func = actions[i].rss_repte_pf_func; + + /* A malformed RSS conf is invalid input, not an allocation failure. */ + if (t == ROC_NPC_ACTION_TYPE_RSS && actions[i].conf != NULL) { + const struct roc_npc_action_rss *rss = actions[i].conf; + + if ((rss->queue_num && rss->queue == NULL) || + (rss->key_len && rss->key == NULL)) { + rc = -EINVAL; + goto err; + } + } + + tmpl->actions[i].conf = roc_npc_dup_action_conf(&actions[i]); + if (tmpl->actions[i].conf == NULL && actions[i].conf != NULL && + (sz > 0 || t == ROC_NPC_ACTION_TYPE_RSS)) + goto err; + if (tmpl->actions[i].conf != NULL) + tmpl->copies[tmpl->nb_copies++] = (void *)(uintptr_t)tmpl->actions[i].conf; + + if (masks != NULL) { + if (masks[i].type != t) { + rc = -EINVAL; + goto err; + } + tmpl->masks[i].type = t; + /* RSS masks cannot carry a nested pointer. */ + if (t == ROC_NPC_ACTION_TYPE_RSS) { + if (masks[i].conf != NULL) { + rc = -ENOTSUP; + goto err; + } + tmpl->masks[i].conf = NULL; + } else { + tmpl->masks[i].conf = roc_npc_dup_action_conf(&masks[i]); + if (tmpl->masks[i].conf == NULL && masks[i].conf != NULL && sz > 0) + goto err; + if (tmpl->masks[i].conf != NULL) + tmpl->copies[tmpl->nb_copies++] = + (void *)(uintptr_t)tmpl->masks[i].conf; + } + } + } + + tmpl->actions[nb_actions].type = ROC_NPC_ACTION_TYPE_END; + tmpl->masks[nb_actions].type = ROC_NPC_ACTION_TYPE_END; + tmpl->refcnt = 0; + + return tmpl; + +err: + if (tmpl != NULL) { + for (j = 0; j < tmpl->nb_copies; j++) + plt_free(tmpl->copies[j]); + plt_free(tmpl->copies); + plt_free(tmpl->masks); + plt_free(tmpl->actions); + plt_free(tmpl); + } + *errcode = rc; + return NULL; +} + +int +roc_npc_actions_template_destroy(struct roc_npc *roc_npc, struct roc_npc_actions_template *tmpl) +{ + uint16_t i; + + PLT_SET_USED(roc_npc); + + if (tmpl == NULL) + return 0; + + if (tmpl->refcnt != 0) + return -EBUSY; + + for (i = 0; i < tmpl->nb_copies; i++) + plt_free(tmpl->copies[i]); + + plt_free(tmpl->copies); + plt_free(tmpl->masks); + plt_free(tmpl->actions); + plt_free(tmpl); + return 0; +} + +struct roc_npc_template_table * +roc_npc_template_table_create(struct roc_npc *roc_npc, + const struct roc_npc_template_table_attr *attr, + struct roc_npc_pattern_template *pattern_templates[], + uint8_t nb_pattern_templates, + struct roc_npc_actions_template *actions_templates[], + uint8_t nb_actions_templates, int *errcode) +{ + struct npc *npc = roc_npc_to_npc_priv(roc_npc); + struct roc_npc_template_table *table; + uint32_t nb_flows; + uint8_t i; + + if (attr == NULL) { + *errcode = -EINVAL; + return NULL; + } + + nb_flows = attr->nb_flows; + + table = plt_zmalloc(sizeof(*table), 0); + if (table == NULL) + goto enomem; + + table->insertion_type = attr->insertion_type; + table->nb_flows = nb_flows; + table->nb_live = 0; + table->cookie = attr->cookie; + plt_spinlock_init(&table->lock); + + table->flow_attr = attr->flow_attr; + table->transfer = attr->transfer; + + if (attr->insertion_type == ROC_NPC_TEMPLATE_INSERTION_INDEX_WITH_PATTERN) { + int resp_count = 0; + uint32_t j; + int rc; + + table->mcam_ids = plt_zmalloc(nb_flows * sizeof(int), 0); + if (table->mcam_ids == NULL) + goto free_index; + for (j = 0; j < nb_flows; j++) + table->mcam_ids[j] = -1; + + table->flow_pool = plt_zmalloc(nb_flows * sizeof(struct roc_npc_template_flow), 0); + if (table->flow_pool == NULL) + goto free_index; + + rc = roc_npc_mcam_alloc_entries(roc_npc, 0, table->mcam_ids, nb_flows, + NPC_MCAM_ANY_PRIO, &resp_count, false); + if (rc != 0 || resp_count < (int)nb_flows) { + for (j = 0; j < (uint32_t)resp_count; j++) { + if (table->mcam_ids[j] != -1) + roc_npc_mcam_free_entry(roc_npc, table->mcam_ids[j]); + } + + *errcode = -ENOSPC; + goto free_index; + } + + for (j = 0; j < nb_flows; j++) { + table->flow_pool[j].table = table; + table->flow_pool[j].nb_items = 0; + table->flow_pool[j].nb_actions = 0; + } + } + /* PATTERN: nothing reserved; each push allocates at the right priority. */ + + if (nb_pattern_templates != 0) { + table->pattern_templates = + plt_zmalloc(nb_pattern_templates * sizeof(*table->pattern_templates), 0); + if (table->pattern_templates == NULL) + goto free_table; + } + if (nb_actions_templates != 0) { + table->actions_templates = + plt_zmalloc(nb_actions_templates * sizeof(*table->actions_templates), 0); + if (table->actions_templates == NULL) + goto free_table; + } + + for (i = 0; i < nb_pattern_templates; i++) { + table->pattern_templates[i] = pattern_templates[i]; + table->pattern_templates[i]->refcnt++; + } + table->nb_pattern_templates = nb_pattern_templates; + for (i = 0; i < nb_actions_templates; i++) { + table->actions_templates[i] = actions_templates[i]; + table->actions_templates[i]->refcnt++; + } + table->nb_actions_templates = nb_actions_templates; + + /* Track the table on the port so a flush can reconcile it. */ + TAILQ_INIT(&table->live_flows); + table->next = npc->flow_tables; + npc->flow_tables = table; + + return table; + +free_index: + plt_free(table->mcam_ids); + plt_free(table->flow_pool); + table->mcam_ids = NULL; + table->flow_pool = NULL; + plt_free(table->pattern_templates); + plt_free(table->actions_templates); + plt_free(table); + if (*errcode == 0) + *errcode = -ENOMEM; + return NULL; + +free_table: + if (attr->insertion_type == ROC_NPC_TEMPLATE_INSERTION_INDEX_WITH_PATTERN) { + uint32_t j; + + if (table->mcam_ids != NULL) { + for (j = 0; j < nb_flows; j++) { + if (table->mcam_ids[j] != -1) + roc_npc_mcam_free_entry(roc_npc, table->mcam_ids[j]); + } + + plt_free(table->mcam_ids); + } + plt_free(table->flow_pool); + } + plt_free(table->pattern_templates); + plt_free(table->actions_templates); + plt_free(table); + +enomem: + *errcode = -ENOMEM; + return NULL; +} + +int +roc_npc_template_table_destroy(struct roc_npc *roc_npc, struct roc_npc_template_table *table) +{ + struct npc *npc = roc_npc_to_npc_priv(roc_npc); + struct roc_npc_template_table **pp; + uint8_t i; + + if (table == NULL) + return 0; + + if (table->insertion_type == ROC_NPC_TEMPLATE_INSERTION_PATTERN) { + if (table->nb_live != 0) + return -EBUSY; + } else { + uint32_t j; + + /* A live (enabled) index rule still owns its MCAM entry. */ + if (!table->hw_released && table->flow_pool != NULL) { + for (j = 0; j < table->nb_flows; j++) { + if (table->flow_pool[j].flow.enable != 0) + return -EBUSY; + } + } + + if (table->mcam_ids != NULL) { + /* Skip if a flush already freed the entries. */ + if (!table->hw_released) { + for (j = 0; j < table->nb_flows; j++) { + int mcam_id = table->mcam_ids[j]; + + if (mcam_id != -1) + roc_npc_mcam_free_entry(roc_npc, mcam_id); + } + } + plt_free(table->mcam_ids); + } + plt_free(table->flow_pool); + } + + for (i = 0; i < table->nb_pattern_templates; i++) + table->pattern_templates[i]->refcnt--; + + for (i = 0; i < table->nb_actions_templates; i++) + table->actions_templates[i]->refcnt--; + + /* Unlink from the port's table list. */ + pp = &npc->flow_tables; + while (*pp != NULL) { + if (*pp == table) { + *pp = table->next; + break; + } + pp = &(*pp)->next; + } + + plt_free(table->pattern_templates); + plt_free(table->actions_templates); + plt_free(table); + return 0; +} + +struct roc_npc_template_flow * +roc_npc_async_flow_create(struct roc_npc *roc_npc, uint32_t queue_id, + struct roc_npc_template_table *table, + const struct roc_npc_item_info pattern[], + const struct roc_npc_action actions[], uint16_t dst_pf_func, + uint64_t npc_default_action, void *user_data, int *errcode) +{ + struct npc *npc = roc_npc_to_npc_priv(roc_npc); + struct roc_npc_template_flow *flow = NULL; + int nb_items = 0, nb_actions = 0; + struct roc_npc_flow_queue *q; + struct roc_npc_async_op *op; + int k; + + if (table == NULL || pattern == NULL || actions == NULL || npc->flow_queues == NULL || + queue_id >= npc->nb_flow_queues) { + *errcode = -EINVAL; + return NULL; + } + if (table->insertion_type != ROC_NPC_TEMPLATE_INSERTION_PATTERN) { + *errcode = -EINVAL; + return NULL; + } + + while (pattern[nb_items].type != ROC_NPC_ITEM_TYPE_END) + nb_items++; + while (actions[nb_actions].type != ROC_NPC_ACTION_TYPE_END) + nb_actions++; + if (nb_items > ROC_NPC_ITEM_TYPE_END || nb_actions > ROC_NPC_MAX_ACTION_COUNT - 1) { + *errcode = -ENOTSUP; + return NULL; + } + + q = &npc->flow_queues[queue_id]; + if (q->head - q->tail >= q->size) { + *errcode = -ENOSPC; + return NULL; + } + + flow = plt_zmalloc(sizeof(*flow), 0); + if (flow == NULL) { + *errcode = -ENOMEM; + return NULL; + } + flow->table = table; + flow->nb_items = nb_items; + flow->nb_actions = nb_actions; + flow->items = plt_zmalloc((nb_items + 1) * sizeof(struct roc_npc_item_info), 0); + if (flow->items == NULL) { + *errcode = -ENOMEM; + goto err; + } + + flow->actions = plt_zmalloc((nb_actions + 1) * sizeof(struct roc_npc_action), 0); + if (flow->actions == NULL) { + *errcode = -ENOMEM; + goto err; + } + + /* Deep-copy each item so push() owns a self-contained rule. */ + for (k = 0; k < nb_items; k++) { + uint32_t sz = pattern[k].size; + + flow->items[k].type = pattern[k].type; + flow->items[k].size = sz; + if (roc_npc_dup_item_field(pattern[k].spec, sz, &flow->items[k].spec) || + roc_npc_dup_item_field(pattern[k].last, sz, &flow->items[k].last) || + roc_npc_dup_item_field(pattern[k].mask, sz, &flow->items[k].mask)) { + *errcode = -ENOMEM; + goto err; + } + } + flow->items[nb_items].type = ROC_NPC_ITEM_TYPE_END; + + /* Deep-copy each action conf. */ + for (k = 0; k < nb_actions; k++) { + int sz = roc_npc_action_conf_size(actions[k].type); + + flow->actions[k].type = actions[k].type; + flow->actions[k].rss_repte_pf_func = actions[k].rss_repte_pf_func; + + /* A malformed RSS conf (non-zero length with a NULL nested + * pointer) is invalid input, not an allocation failure. + */ + if (actions[k].type == ROC_NPC_ACTION_TYPE_RSS && actions[k].conf != NULL) { + const struct roc_npc_action_rss *rss = actions[k].conf; + + if ((rss->queue_num && rss->queue == NULL) || + (rss->key_len && rss->key == NULL)) { + *errcode = -EINVAL; + goto err; + } + } + + flow->actions[k].conf = roc_npc_dup_action_conf(&actions[k]); + /* A NULL copy from a non-NULL conf now means the deep copy failed. */ + if (flow->actions[k].conf == NULL && actions[k].conf != NULL && + (sz > 0 || actions[k].type == ROC_NPC_ACTION_TYPE_RSS)) { + *errcode = -ENOMEM; + goto err; + } + } + flow->actions[nb_actions].type = ROC_NPC_ACTION_TYPE_END; + + /* Capture the rule attributes used by push() to program hardware. */ + flow->attr = table->flow_attr; + if (table->transfer) + flow->attr.ingress = 1; + flow->dst_pf_func = dst_pf_func; + flow->npc_default_action = npc_default_action; + /* Capture the caller's RSS flow key so batched rules keep their own. */ + flow->flowkey_cfg = roc_npc->flowkey_cfg_state; + + op = &q->ops[q->head & (q->size - 1)]; + op->flow = flow; + op->is_create = true; + op->is_update = false; + op->user_data = user_data; + op->done = false; + op->rc = 0; + q->head++; + + return flow; + +err: + roc_npc_template_flow_free_rule(flow); + plt_free(flow); + return NULL; +} + +int +roc_npc_async_flow_destroy(struct roc_npc *roc_npc, uint32_t queue_id, + struct roc_npc_template_flow *flow, void *user_data) +{ + struct npc *npc = roc_npc_to_npc_priv(roc_npc); + struct roc_npc_flow_queue *q; + struct roc_npc_async_op *op; + + if (flow == NULL || npc->flow_queues == NULL || queue_id >= npc->nb_flow_queues) + return -EINVAL; + + q = &npc->flow_queues[queue_id]; + if (q->head - q->tail >= q->size) + return -ENOSPC; + + op = &q->ops[q->head & (q->size - 1)]; + op->flow = flow; + op->is_create = false; + op->is_update = false; + op->user_data = user_data; + op->done = false; + op->rc = 0; + q->head++; + + return 0; +} + +/* Create a rule at an app-chosen index in an INDEX_WITH_PATTERN table. */ +struct roc_npc_template_flow * +roc_npc_async_flow_create_by_index_with_pattern(struct roc_npc *roc_npc, uint32_t queue_id, + struct roc_npc_template_table *table, + uint32_t rule_index, + const struct roc_npc_item_info pattern[], + const struct roc_npc_action actions[], + void *user_data, int *errcode) +{ + struct npc *npc = roc_npc_to_npc_priv(roc_npc); + struct roc_npc_template_flow *flow; + uint32_t mark_save, vtag_save; + struct roc_npc_flow_queue *q; + struct roc_npc_async_op *op; + struct roc_npc_attr attr; + int rc; + + if (table == NULL || pattern == NULL || actions == NULL || npc->flow_queues == NULL || + queue_id >= npc->nb_flow_queues) { + *errcode = -EINVAL; + return NULL; + } + if (table->insertion_type != ROC_NPC_TEMPLATE_INSERTION_INDEX_WITH_PATTERN) { + *errcode = -EINVAL; + return NULL; + } + if (rule_index >= table->nb_flows) { + *errcode = -EINVAL; + return NULL; + } + + /* Reject creates into a flushed table. */ + if (table->hw_released) { + *errcode = -EINVAL; + return NULL; + } + + q = &npc->flow_queues[queue_id]; + if (q->head - q->tail >= q->size) { + *errcode = -ENOSPC; + return NULL; + } + + /* App owns the index: use the reserved pool slot directly. */ + flow = &table->flow_pool[rule_index]; + flow->table = table; + flow->slot = rule_index; + memset(&flow->flow, 0, sizeof(flow->flow)); + + attr = table->flow_attr; + if (table->transfer) + attr.ingress = 1; + + /* Snapshot MARK/VLAN refcounts before parsing. */ + mark_save = npc->mark_actions; + vtag_save = npc->vtag_strip_actions; + + rc = roc_npc_flow_parse(roc_npc, &attr, pattern, actions, &flow->flow); + if (rc != 0) { + npc->mark_actions = mark_save; + npc->vtag_strip_actions = vtag_save; + *errcode = rc; + return NULL; + } + + roc_npc_mcam_write_rx_finalize(roc_npc, &flow->flow); + flow->flow.mcam_id = table->mcam_ids[rule_index]; + flow->flow.enable = 1; + + op = &q->ops[q->head & (q->size - 1)]; + op->flow = flow; + op->is_create = true; + op->is_update = false; + op->user_data = user_data; + op->done = false; + op->rc = 0; + q->head++; + + return flow; +} + +/* Update a rule's actions in place and swap them in at push. */ +int +roc_npc_async_flow_actions_update(struct roc_npc *roc_npc, uint32_t queue_id, + struct roc_npc_template_flow *flow, + const struct roc_npc_action actions[], void *user_data) +{ + struct npc *npc = roc_npc_to_npc_priv(roc_npc); + struct roc_npc_item_info end_pattern[1]; + struct roc_npc_template_table *table; + struct roc_npc_flow *fptr, tmpflow; + uint32_t mark_save, vtag_save; + struct roc_npc_flow_queue *q; + struct roc_npc_async_op *op; + struct roc_npc_attr attr; + uint16_t old_match_id; + bool old_vtag; + int rc; + + if (flow == NULL || actions == NULL || npc->flow_queues == NULL || + queue_id >= npc->nb_flow_queues) + return -EINVAL; + + table = flow->table; + if (table == NULL) + return -EINVAL; + /* PATTERN rules have roc_flow only after create commit. */ + if (table->insertion_type == ROC_NPC_TEMPLATE_INSERTION_PATTERN && flow->roc_flow == NULL) + return -EINVAL; + + /* Reject a second update until the pending one is flushed. */ + if (flow->pending_update.pending) + return -EBUSY; + + q = &npc->flow_queues[queue_id]; + if (q->head - q->tail >= q->size) + return -ENOSPC; + + end_pattern[0].type = ROC_NPC_ITEM_TYPE_END; + + attr = table->flow_attr; + if (table->transfer) + attr.ingress = 1; + + memset(&tmpflow, 0, sizeof(tmpflow)); + + /* Resolve the live flow before parsing the new actions. */ + fptr = flow->roc_flow ? flow->roc_flow : &flow->flow; + old_match_id = (fptr->npc_action >> NPC_RX_ACT_MATCH_OFFSET) & NPC_RX_ACT_MATCH_MASK; + old_vtag = fptr->nix_intf == ROC_NPC_INTF_RX && fptr->vtag_action != 0; + mark_save = npc->mark_actions; + vtag_save = npc->vtag_strip_actions; + + rc = roc_npc_flow_parse(roc_npc, &attr, end_pattern, actions, &tmpflow); + if (rc != 0) { + /* Undo any refcount bumps a partial parse made on failure. */ + npc->mark_actions = mark_save; + npc->vtag_strip_actions = vtag_save; + return rc; + } + + /* Stash the parsed update so push can commit or roll it back. */ + flow->pending_update.npc_action = tmpflow.npc_action; + flow->pending_update.npc_action2 = tmpflow.npc_action2; + flow->pending_update.vtag_action = tmpflow.vtag_action; + flow->pending_update.recv_queue = tmpflow.recv_queue; + flow->pending_update.old_match_id = old_match_id != 0; + flow->pending_update.old_vtag = old_vtag; + flow->pending_update.new_match_id = + ((tmpflow.npc_action >> NPC_RX_ACT_MATCH_OFFSET) & NPC_RX_ACT_MATCH_MASK) != 0; + flow->pending_update.new_vtag = + tmpflow.nix_intf == ROC_NPC_INTF_RX && tmpflow.vtag_action != 0; + flow->pending_update.pending = true; + + op = &q->ops[q->head & (q->size - 1)]; + op->flow = flow; + op->is_create = false; + op->is_update = true; + op->user_data = user_data; + op->done = false; + op->rc = 0; + q->head++; + + return 0; +} + +/* Commit one retained create to hardware. */ +static int +roc_npc_flow_commit_create(struct roc_npc *roc_npc, struct roc_npc_template_flow *flow) +{ + struct npc *npc = roc_npc_to_npc_priv(roc_npc); + uint32_t mark_save, vtag_save; + struct roc_npc_flow *rflow; + int errcode = 0; + + /* Restore this rule's RSS flow key before programming (see create). */ + roc_npc->flowkey_cfg_state = flow->flowkey_cfg; + + /* Restore MARK/VLAN refcounts if create fails. */ + mark_save = npc->mark_actions; + vtag_save = npc->vtag_strip_actions; + + rflow = roc_npc_flow_create(roc_npc, &flow->attr, flow->items, flow->actions, + flow->dst_pf_func, flow->npc_default_action, &errcode); + if (rflow == NULL) { + npc->mark_actions = mark_save; + npc->vtag_strip_actions = vtag_save; + return errcode ? errcode : -EINVAL; + } + + flow->roc_flow = rflow; + return 0; +} + +/* Apply a pending actions update and roll it back on failure. */ +static int +roc_npc_flow_apply_pattern_update(struct roc_npc *roc_npc, struct npc *npc, + struct roc_npc_template_flow *flow) +{ + struct roc_npc_flow *fptr = flow->roc_flow; + uint64_t save_action2, save_action, save_vtag; + uint32_t save_rq; + int rc; + + if (fptr == NULL) + return -EINVAL; + + save_action2 = fptr->npc_action2; + save_action = fptr->npc_action; + save_vtag = fptr->vtag_action; + save_rq = fptr->recv_queue; + + fptr->npc_action = flow->pending_update.npc_action; + fptr->npc_action2 = flow->pending_update.npc_action2; + fptr->vtag_action = flow->pending_update.vtag_action; + fptr->recv_queue = flow->pending_update.recv_queue; + fptr->enable = 1; + roc_npc_mcam_write_rx_finalize(roc_npc, fptr); + rc = roc_npc_mcam_write_entry(roc_npc, fptr); + if (rc == 0) { + /* Drop the old action set's refcount contribution. */ + if (flow->pending_update.old_match_id && npc->mark_actions > 0) + npc->mark_actions--; + if (flow->pending_update.old_vtag && npc->vtag_strip_actions > 0) + npc->vtag_strip_actions--; + } else { + /* Restore the live entry and undo the new contribution. */ + fptr->npc_action = save_action; + fptr->npc_action2 = save_action2; + fptr->vtag_action = save_vtag; + fptr->recv_queue = save_rq; + if (flow->pending_update.new_match_id && npc->mark_actions > 0) + npc->mark_actions--; + if (flow->pending_update.new_vtag && npc->vtag_strip_actions > 0) + npc->vtag_strip_actions--; + } + return rc; +} + +int +roc_npc_flow_push(struct roc_npc *roc_npc, uint32_t queue_id) +{ + struct npc *npc = roc_npc_to_npc_priv(roc_npc); + struct roc_npc_flow_queue *q; + uint32_t i; + + if (npc->flow_queues == NULL || queue_id >= npc->nb_flow_queues) + return -EINVAL; + + q = &npc->flow_queues[queue_id]; + + for (i = q->pushed; i != q->head; i++) { + struct roc_npc_async_op *op = &q->ops[i & (q->size - 1)]; + struct roc_npc_template_flow *flow = op->flow; + struct roc_npc_template_table *table = flow->table; + int rc; + + if (table->insertion_type == ROC_NPC_TEMPLATE_INSERTION_PATTERN) { + if (op->is_update) { + rc = roc_npc_flow_apply_pattern_update(roc_npc, npc, flow); + /* Clear the pending update after push. */ + flow->pending_update.pending = false; + } else if (op->is_create) { + rc = roc_npc_flow_commit_create(roc_npc, flow); + /* Rule arrays are consumed on push. */ + roc_npc_template_flow_free_rule(flow); + if (rc == 0) { + plt_spinlock_lock(&table->lock); + table->nb_live++; + TAILQ_INSERT_TAIL(&table->live_flows, flow, link); + plt_spinlock_unlock(&table->lock); + } + /* Keep the handle valid on failure. */ + } else if (flow->roc_flow == NULL) { + /* Never committed (or a failed create): just free it. */ + rc = 0; + roc_npc_template_flow_free_rule(flow); + plt_free(flow); + } else { + uint16_t match_id; + uint8_t nix_intf; + bool had_vtag; + + /* Capture match_id and interface before destroy. */ + match_id = (flow->roc_flow->npc_action >> NPC_RX_ACT_MATCH_OFFSET) & + NPC_RX_ACT_MATCH_MASK; + nix_intf = flow->roc_flow->nix_intf; + had_vtag = flow->roc_flow->vtag_action != 0; + + rc = roc_npc_flow_destroy(roc_npc, flow->roc_flow); + if (rc == 0 && match_id) + roc_npc_mark_actions_sub_return(roc_npc, 1); + if (rc == 0 && nix_intf == ROC_NPC_INTF_RX && had_vtag) + roc_npc_vtag_actions_sub_return(roc_npc, 1); + if (rc == 0) { + plt_spinlock_lock(&table->lock); + table->nb_live--; + TAILQ_REMOVE(&table->live_flows, flow, link); + plt_spinlock_unlock(&table->lock); + flow->roc_flow = NULL; + roc_npc_template_flow_free_rule(flow); + plt_free(flow); + } + /* Keep the handle on failure so the caller can retry. */ + } + } else { + /* INDEX_WITH_PATTERN: entry fixed at the app-chosen index. */ + struct roc_npc_flow *fptr = &flow->flow; + bool destroy = !op->is_create && !op->is_update; + uint16_t match_id = (fptr->npc_action >> NPC_RX_ACT_MATCH_OFFSET) & + NPC_RX_ACT_MATCH_MASK; + uint64_t save_action2 = fptr->npc_action2; + uint64_t save_action = fptr->npc_action; + uint64_t save_vtag = fptr->vtag_action; + uint32_t save_rq = fptr->recv_queue; + + /* Swap in the stashed update before writing hardware. */ + if (op->is_update) { + fptr->npc_action = flow->pending_update.npc_action; + fptr->npc_action2 = flow->pending_update.npc_action2; + fptr->vtag_action = flow->pending_update.vtag_action; + fptr->recv_queue = flow->pending_update.recv_queue; + } + + /* Set enable before writing hardware. */ + fptr->enable = destroy ? 0 : 1; + if (op->is_update) + roc_npc_mcam_write_rx_finalize(roc_npc, fptr); + rc = roc_npc_mcam_write_entry(roc_npc, fptr); + if (rc != 0) { + /* Restore the previous enable state on failure. */ + fptr->enable = destroy ? 1 : 0; + /* Roll back the swapped-in action words too. */ + if (op->is_update) { + fptr->npc_action = save_action; + fptr->npc_action2 = save_action2; + fptr->vtag_action = save_vtag; + fptr->recv_queue = save_rq; + } + } + + /* Commit or roll back the update refcount change. */ + if (op->is_update) { + if (rc == 0) { + if (flow->pending_update.old_match_id && + npc->mark_actions > 0) + npc->mark_actions--; + if (flow->pending_update.old_vtag && + npc->vtag_strip_actions > 0) + npc->vtag_strip_actions--; + } else { + if (flow->pending_update.new_match_id && + npc->mark_actions > 0) + npc->mark_actions--; + if (flow->pending_update.new_vtag && + npc->vtag_strip_actions > 0) + npc->vtag_strip_actions--; + } + flow->pending_update.pending = false; + } + + /* Release the COUNT counter on destroy. */ + if (destroy && rc == 0 && fptr->use_ctr && + fptr->ctr_id != NPC_COUNTER_NONE) { + roc_npc_mcam_free_counter(roc_npc, fptr->ctr_id); + fptr->ctr_id = NPC_COUNTER_NONE; + } + + /* Balance the mark_actions refcount for MARK/FLAG rules. */ + if (destroy && rc == 0 && match_id) + roc_npc_mark_actions_sub_return(roc_npc, 1); + + /* Balance the vtag_strip_actions refcount for RX VLAN strip rules. */ + if (destroy && rc == 0 && fptr->nix_intf == ROC_NPC_INTF_RX && + fptr->vtag_action != 0) + roc_npc_vtag_actions_sub_return(roc_npc, 1); + } + + op->rc = rc; + op->done = true; + } + + q->pushed = q->head; + return 0; +} + +int +roc_npc_flow_pull(struct roc_npc *roc_npc, uint32_t queue_id, struct roc_npc_flow_op_result res[], + uint16_t n_res) +{ + struct npc *npc = roc_npc_to_npc_priv(roc_npc); + struct roc_npc_flow_queue *q; + uint16_t cnt = 0; + + if (npc->flow_queues == NULL || queue_id >= npc->nb_flow_queues) + return -EINVAL; + + q = &npc->flow_queues[queue_id]; + + /* Return ops that were pushed but not yet reported. */ + while (q->tail != q->pushed && cnt < n_res) { + struct roc_npc_async_op *op = &q->ops[q->tail & (q->size - 1)]; + + res[cnt].user_data = op->user_data; + res[cnt].rc = op->rc; + cnt++; + q->tail++; + } + + return cnt; +} + +/* Resolve a candidate handle to a template flow or a sync flow. */ +bool +roc_npc_async_flow_resolve(struct roc_npc *roc_npc, void *handle, struct roc_npc_flow **flow) +{ + struct npc *npc = roc_npc_to_npc_priv(roc_npc); + struct roc_npc_template_flow *cand = handle; + struct roc_npc_template_table *table; + uint16_t i; + + if (handle == NULL || flow == NULL) + return false; + + for (table = npc->flow_tables; table != NULL; table = table->next) { + struct roc_npc_template_flow *match; + + /* Integer range check: pointer comparison across objects is UB. */ + if (table->flow_pool != NULL) { + uintptr_t p = (uintptr_t)cand; + uintptr_t base = (uintptr_t)table->flow_pool; + uintptr_t end = (uintptr_t)(table->flow_pool + table->nb_flows); + + if (p >= base && p < end) { + *flow = &cand->flow; + return true; + } + } + + plt_spinlock_lock(&table->lock); + /* clang-format off */ + TAILQ_FOREACH(match, &table->live_flows, link) { + if (match == cand) { + plt_spinlock_unlock(&table->lock); + *flow = cand->roc_flow; + return true; + } + } + /* clang-format on */ + plt_spinlock_unlock(&table->lock); + } + + /* Handle still queued: a PATTERN rule not yet committed. */ + for (i = 0; i < npc->nb_flow_queues; i++) { + struct roc_npc_flow_queue *q = &npc->flow_queues[i]; + uint32_t j; + + for (j = q->tail; j != q->head; j++) { + struct roc_npc_async_op *op = &q->ops[j & (q->size - 1)]; + + if (op->flow == cand) { + *flow = cand->roc_flow; + return true; + } + } + } + + return false; +} + +/* Return the table cookie used to recover update context. */ +void * +roc_npc_async_flow_table_cookie(struct roc_npc_template_flow *flow) +{ + if (flow == NULL || flow->table == NULL) + return NULL; + + return flow->table->cookie; +} + +/* Reconcile template-table bookkeeping after flush or port close. */ +void +npc_template_tables_flush_reconcile(struct npc *npc) +{ + struct roc_npc_template_table *table; + struct roc_npc_template_flow *flow; + struct roc_npc_flow *fptr; + uint16_t match_id; + uint32_t j; + + for (table = npc->flow_tables; table != NULL; table = table->next) { + plt_spinlock_lock(&table->lock); + while ((flow = TAILQ_FIRST(&table->live_flows)) != NULL) { + TAILQ_REMOVE(&table->live_flows, flow, link); + /* roc_flow is already freed by free_all_resources. */ + roc_npc_template_flow_free_rule(flow); + plt_free(flow); + } + + /* INDEX_WITH_PATTERN rules live in flow_pool[]; clear stale state. */ + if (table->flow_pool != NULL) { + for (j = 0; j < table->nb_flows; j++) { + fptr = &table->flow_pool[j].flow; + + if (fptr->enable == 0) + continue; + + match_id = (fptr->npc_action >> NPC_RX_ACT_MATCH_OFFSET) & + NPC_RX_ACT_MATCH_MASK; + if (match_id && npc->mark_actions > 0) + npc->mark_actions--; + + if (fptr->nix_intf == ROC_NPC_INTF_RX && fptr->vtag_action != 0 && + npc->vtag_strip_actions > 0) + npc->vtag_strip_actions--; + + memset(fptr, 0, sizeof(*fptr)); + fptr->ctr_id = NPC_COUNTER_NONE; + } + } + + table->nb_live = 0; + table->hw_released = true; + plt_spinlock_unlock(&table->lock); + } +} diff --git a/drivers/common/cnxk/roc_npc_mcam.c b/drivers/common/cnxk/roc_npc_mcam.c index 286cfd2dbd..34665f1bc3 100644 --- a/drivers/common/cnxk/roc_npc_mcam.c +++ b/drivers/common/cnxk/roc_npc_mcam.c @@ -714,7 +714,6 @@ npc_mcam_fetch_kex_cfg(struct npc *npc) struct mbox *mbox = mbox_get(npc->mbox); int rc = 0; - if (!roc_model_is_cn20k()) { mbox_alloc_msg_npc_get_kex_cfg(mbox); rc = mbox_process_msg(mbox, (void *)&kex_rsp); @@ -786,6 +785,46 @@ npc_mcam_set_channel(struct roc_npc_flow *flow, struct npc_cn20k_mcam_write_entr flow->mcam_mask[0] |= (uint64_t)mask; } +/* Fold RQ into the action word and program the RX channel into kw[0], + * matching the RX steps the sync path does in npc_mcam_alloc_and_write(). + */ +void +roc_npc_mcam_write_rx_finalize(struct roc_npc *roc_npc, struct roc_npc_flow *flow) +{ + struct npc *npc = roc_npc_to_npc_priv(roc_npc); + uint16_t chan, mask; + uint64_t op; + + if (flow->nix_intf != NIX_INTF_RX) + return; + + /* Set the RQ in the action index field, but only for UCAST* ops that + * use it as a receive queue. Clear then set so a re-finalize on an + * async QUEUE update leaves no stale bits. + */ + op = flow->npc_action & 0xFULL; + if (op == NIX_RX_ACTIONOP_UCAST || op == NIX_RX_ACTIONOP_UCAST_IPSEC || + op == NIX_RX_ACTIONOP_UCAST_CPT) { + flow->npc_action &= ~GENMASK_ULL(39, 20); + flow->npc_action |= ((uint64_t)flow->recv_queue << 20) & GENMASK_ULL(39, 20); + } + + /* Program the RX channel into kw[0]. */ + chan = npc->channel; + mask = BIT_ULL(12) - 1; + + flow->mcam_data[0] &= ~(GENMASK(11, 0)); + flow->mcam_mask[0] &= ~(GENMASK(11, 0)); + + if (roc_model_runtime_is_cn10k() && !roc_npc_action_is_rx_inline(flow->npc_action)) { + chan = (chan & NIX_CHAN_CPT_X2P_MASK); + mask = (mask & NIX_CHAN_CPT_X2P_MASK); + } + + flow->mcam_data[0] |= (uint64_t)chan; + flow->mcam_mask[0] |= (uint64_t)mask; +} + #define NPC_PF_FUNC_WIDTH 2 #define NPC_KEX_PF_FUNC_MASK 0xFFFF @@ -1430,5 +1469,9 @@ npc_flow_free_all_resources(struct npc *npc) plt_free(flow); } } + + /* Reconcile template-table bookkeeping for all teardown paths. */ + npc_template_tables_flush_reconcile(npc); + return rc; } diff --git a/drivers/common/cnxk/roc_npc_priv.h b/drivers/common/cnxk/roc_npc_priv.h index c7e7d2d8f6..c6ab033d9f 100644 --- a/drivers/common/cnxk/roc_npc_priv.h +++ b/drivers/common/cnxk/roc_npc_priv.h @@ -427,6 +427,99 @@ struct npc_age_flow_entry { TAILQ_HEAD(npc_age_flow_list_head, npc_age_flow_entry); +/* Queue-based async template flow engine types. */ + +/* One in-flight async op in a queue ring. */ +struct roc_npc_async_op { + struct roc_npc_template_flow *flow; /**< Rule the op acts on. */ + void *user_data; /**< Echoed back by flow_pull. */ + bool is_create; /**< true = create, false = destroy. */ + bool is_update; /**< true = in-place actions rewrite. */ + bool done; /**< Set once pushed to hardware. */ + int rc; /**< Result captured at push time. */ +}; + +/* Fixed-size op ring; head/pushed/tail are free-running indices. */ +struct roc_npc_flow_queue { + struct roc_npc_async_op *ops; /**< size entries, power-of-two. */ + uint32_t size; /**< Ring capacity (power of two). */ + uint32_t head; /**< Next slot to enqueue into. */ + uint32_t pushed; /**< Ops up to here are committed. */ + uint32_t tail; /**< Next slot to return via pull. */ +}; + +/* A deep-copied, retained rule awaiting (or holding) a hardware entry. */ +struct roc_npc_template_flow { + struct roc_npc_template_table *table; /**< Owning table. */ + struct roc_npc_flow *roc_flow; /**< HW flow (PATTERN, once committed). */ + struct roc_npc_item_info *items; /**< Deep-copied merged pattern. */ + struct roc_npc_action *actions; /**< Deep-copied merged actions. */ + struct roc_npc_flow flow; /**< Embedded HW flow (INDEX path). */ + struct roc_npc_attr attr; /**< Priority / ingress / egress. */ + uint64_t npc_default_action; /**< Resolved port default action. */ + uint32_t flowkey_cfg; /**< RSS flow key captured at create. */ + uint32_t slot; /**< Pre-reserved index (INDEX path). */ + uint16_t dst_pf_func; /**< Resolved destination pf_func. */ + uint16_t nb_items; /**< Item count (excluding END). */ + uint16_t nb_actions; /**< Action count (excluding END). */ + /* Pending in-place actions update, applied at push. */ + struct { + uint64_t npc_action; + uint64_t npc_action2; + uint64_t vtag_action; + uint32_t recv_queue; + bool pending; /**< An update is enqueued, awaiting push. */ + bool old_match_id; /**< Old action set carried a mark/match_id. */ + bool old_vtag; /**< Old action set carried RX vtag strip. */ + bool new_match_id; /**< New action set carries a mark/match_id. */ + bool new_vtag; /**< New action set carries RX vtag strip. */ + } pending_update; + TAILQ_ENTRY(roc_npc_template_flow) link; +}; + +TAILQ_HEAD(roc_npc_template_flow_list, roc_npc_template_flow); + +/* Pattern template blueprint. */ +struct roc_npc_pattern_template { + struct roc_npc_pattern_template_attr attr; + struct roc_npc_item_info *pattern; /**< nb_items + END terminator. */ + void **copies; /**< Tracked payload allocations. */ + uint16_t nb_copies; /**< Entries used in copies[]. */ + uint16_t nb_items; /**< Item count (excluding END). */ + uint32_t refcnt; /**< Tables referencing this template. */ +}; + +/* Actions template blueprint. */ +struct roc_npc_actions_template { + struct roc_npc_actions_template_attr attr; + struct roc_npc_action *actions; /**< nb_actions + END terminator. */ + struct roc_npc_action *masks; /**< Per-action masks (optional). */ + void **copies; /**< Tracked conf allocations. */ + uint16_t nb_copies; /**< Entries used in copies[]. */ + uint16_t nb_actions; /**< Action count (excluding END). */ + uint32_t refcnt; /**< Tables referencing this template. */ +}; + +/* Template table and rule storage. */ +struct roc_npc_template_table { + int *mcam_ids; /**< Reserved entries (index). */ + struct roc_npc_template_flow *flow_pool; /**< Per-index rule slots. */ + struct roc_npc_pattern_template **pattern_templates; + struct roc_npc_actions_template **actions_templates; + struct roc_npc_template_flow_list live_flows; /**< Live pattern rules. */ + struct roc_npc_template_table *next; /**< Port table list link. */ + void *cookie; /**< Opaque consumer context. */ + struct roc_npc_attr flow_attr; /**< Priority/ingress/egress. */ + enum roc_npc_template_table_insertion_type insertion_type; + plt_spinlock_t lock; /**< Guards live_flows/nb_live. */ + uint32_t nb_flows; /**< Max rules the table holds. */ + uint32_t nb_live; /**< Live pattern rules count. */ + uint32_t transfer; /**< Transfer (port) rule. */ + uint8_t nb_pattern_templates; /**< pattern_templates entries. */ + uint8_t nb_actions_templates; /**< actions_templates entries. */ + bool hw_released; /**< Entries freed by a flush. */ +}; + struct npc { struct mbox *mbox; /* Mbox */ uint64_t keyx_supp_nmask[NPC_MAX_INTF]; /* nibble mask */ @@ -459,6 +552,10 @@ struct npc { struct plt_bitmap *rss_grp_entries; struct npc_flow_list ipsec_list; uint8_t enable_debug; + /* Queue-based async template flow state owned by roc. */ + struct roc_npc_flow_queue *flow_queues; + uint16_t nb_flow_queues; + struct roc_npc_template_table *flow_tables; }; #define NPC_HASH_FIELD_LEN 16 @@ -533,6 +630,7 @@ int npc_mcam_fetch_hw_cap(struct npc *npc, uint8_t *npc_hw_cap); int npc_get_free_mcam_entry(struct mbox *mbox, struct roc_npc_flow *flow, struct npc *npc); void npc_delete_prio_list_entry(struct npc *npc, struct roc_npc_flow *flow); int npc_flow_free_all_resources(struct npc *npc); +void npc_template_tables_flush_reconcile(struct npc *npc); const struct roc_npc_item_info * npc_parse_skip_void_and_any_items(const struct roc_npc_item_info *pattern); int npc_program_mcam(struct npc *npc, struct npc_parse_state *pst, bool mcam_alloc); diff --git a/drivers/common/cnxk/roc_npc_utils.c b/drivers/common/cnxk/roc_npc_utils.c index 8e83b8662d..7ae877aa98 100644 --- a/drivers/common/cnxk/roc_npc_utils.c +++ b/drivers/common/cnxk/roc_npc_utils.c @@ -1088,7 +1088,7 @@ npc_get_free_mcam_entry(struct mbox *mbox, struct roc_npc_flow *flow, struct npc flow->key_type); if (rc) { plt_npc_dbg("npc: failed to allocate MCAM entry."); - return rc; + goto err; } new_entry->flow = flow; @@ -1096,7 +1096,7 @@ npc_get_free_mcam_entry(struct mbox *mbox, struct roc_npc_flow *flow, struct npc rc = npc_alloc_mcam_by_ref_entry(mbox, flow, npc, &rsp_local); if (rc) - return rc; + goto err; new_entry->flow = flow; diff --git a/drivers/common/cnxk/roc_platform_base_symbols.c b/drivers/common/cnxk/roc_platform_base_symbols.c index 00746abc06..de8de910f6 100644 --- a/drivers/common/cnxk/roc_platform_base_symbols.c +++ b/drivers/common/cnxk/roc_platform_base_symbols.c @@ -487,6 +487,7 @@ RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_mcam_enable_all_entries) RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_mcam_alloc_entry) RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_mcam_ena_dis_entry) RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_mcam_write_entry) +RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_mcam_write_rx_finalize) RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_get_low_priority_mcam) RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_profile_name_get) RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_kex_capa_get) @@ -505,6 +506,21 @@ RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_aged_flow_ctx_get) RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_defrag_mcam_banks) RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_get_key_type) RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_flow_mcam_dump) +RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_flow_configure) +RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_pattern_template_create) +RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_pattern_template_destroy) +RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_actions_template_create) +RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_actions_template_destroy) +RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_template_table_create) +RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_template_table_destroy) +RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_async_flow_create) +RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_async_flow_destroy) +RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_async_flow_create_by_index_with_pattern) +RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_async_flow_actions_update) +RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_flow_push) +RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_flow_pull) +RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_async_flow_resolve) +RTE_EXPORT_INTERNAL_SYMBOL(roc_npc_async_flow_table_cookie) RTE_EXPORT_INTERNAL_SYMBOL(roc_re_ml_zeta_get) RTE_EXPORT_INTERNAL_SYMBOL(roc_re_ml_zeta_put) RTE_EXPORT_INTERNAL_SYMBOL(roc_ree_queues_attach) diff --git a/drivers/net/cnxk/cn10k_ethdev.c b/drivers/net/cnxk/cn10k_ethdev.c index 23a2341c8b..bcf4e23286 100644 --- a/drivers/net/cnxk/cn10k_ethdev.c +++ b/drivers/net/cnxk/cn10k_ethdev.c @@ -5,6 +5,7 @@ #include "cn10k_flow.h" #include "cn10k_rx.h" #include "cn10k_tx.h" +#include "cnxk_flow.h" static uint16_t nix_rx_offload_flags(struct rte_eth_dev *eth_dev) @@ -916,8 +917,27 @@ npc_flow_ops_override(void) cnxk_flow_ops.create = cn10k_flow_create; cnxk_flow_ops.destroy = cn10k_flow_destroy; cnxk_flow_ops.info_get = cn10k_flow_info_get; + + cnxk_flow_ops.configure = cnxk_flow_configure; + cnxk_flow_ops.pattern_template_create = cnxk_flow_pattern_template_create; + cnxk_flow_ops.pattern_template_destroy = cnxk_flow_pattern_template_destroy; + cnxk_flow_ops.actions_template_create = cnxk_flow_actions_template_create; + cnxk_flow_ops.actions_template_destroy = cnxk_flow_actions_template_destroy; + cnxk_flow_ops.template_table_create = cnxk_flow_template_table_create; + cnxk_flow_ops.template_table_destroy = cnxk_flow_template_table_destroy; } +/* Fast-path (async) flow ops. Installed per-port in cn10k_nix_probe(). */ +static const struct rte_flow_fp_ops cn10k_flow_fp_ops = { + .async_create = cnxk_flow_async_create, + .async_create_by_index = cnxk_flow_async_create_by_index, + .async_create_by_index_with_pattern = cnxk_flow_async_create_by_index_with_pattern, + .async_actions_update = cnxk_flow_async_actions_update, + .async_destroy = cnxk_flow_async_destroy, + .push = cn10k_flow_push, + .pull = cnxk_flow_pull, +}; + static int cn10k_nix_remove(struct rte_pci_device *pci_dev) { @@ -966,6 +986,9 @@ cn10k_nix_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) dev = cnxk_eth_pmd_priv(eth_dev); + /* Install fast-path (async) flow ops for this port. */ + eth_dev->flow_fp_ops = &cn10k_flow_fp_ops; + /* DROP_RE is not supported with inline IPSec for CN10K A0 and * when vector mode is enabled. */ diff --git a/drivers/net/cnxk/cn10k_flow.c b/drivers/net/cnxk/cn10k_flow.c index b95fb83f08..02bc002507 100644 --- a/drivers/net/cnxk/cn10k_flow.c +++ b/drivers/net/cnxk/cn10k_flow.c @@ -48,6 +48,49 @@ cn10k_flow_info_get(struct rte_eth_dev *dev, struct rte_flow_port_info *port_inf return cnxk_flow_info_get_common(dev, port_info, queue_info, err); } +int +cn10k_flow_push(struct rte_eth_dev *eth_dev, uint32_t queue, struct rte_flow_error *error) +{ + struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); + struct roc_npc *npc = &dev->npc; + bool reset_rx = false; + int rc; + + /* Flush the pending async ops to hardware first. */ + rc = cnxk_flow_push(eth_dev, queue, error); + if (rc) + return rc; + + /* Async push, unlike sync create/destroy, does not toggle the MARK and + * VLAN-strip RX offloads; reconcile each with its refcount so the Rx + * function matches the live rules. + */ + if (roc_npc_mark_actions_get(npc) && + !(dev->rx_offload_flags & NIX_RX_OFFLOAD_MARK_UPDATE_F)) { + dev->rx_offload_flags |= NIX_RX_OFFLOAD_MARK_UPDATE_F; + reset_rx = true; + } else if (!roc_npc_mark_actions_get(npc) && + (dev->rx_offload_flags & NIX_RX_OFFLOAD_MARK_UPDATE_F)) { + dev->rx_offload_flags &= ~NIX_RX_OFFLOAD_MARK_UPDATE_F; + reset_rx = true; + } + + if (roc_npc_vtag_actions_get(npc) && + !(dev->rx_offload_flags & NIX_RX_OFFLOAD_VLAN_STRIP_F)) { + dev->rx_offload_flags |= NIX_RX_OFFLOAD_VLAN_STRIP_F; + reset_rx = true; + } else if (!roc_npc_vtag_actions_get(npc) && + (dev->rx_offload_flags & NIX_RX_OFFLOAD_VLAN_STRIP_F)) { + dev->rx_offload_flags &= ~NIX_RX_OFFLOAD_VLAN_STRIP_F; + reset_rx = true; + } + + if (reset_rx) + cn10k_eth_set_rx_function(eth_dev); + + return 0; +} + int cn10k_flow_destroy(struct rte_eth_dev *eth_dev, struct rte_flow *rte_flow, struct rte_flow_error *error) diff --git a/drivers/net/cnxk/cn10k_flow.h b/drivers/net/cnxk/cn10k_flow.h index c9e8c86e96..c0d239c20f 100644 --- a/drivers/net/cnxk/cn10k_flow.h +++ b/drivers/net/cnxk/cn10k_flow.h @@ -16,6 +16,8 @@ int cn10k_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow, int cn10k_flow_info_get(struct rte_eth_dev *dev, struct rte_flow_port_info *port_info, struct rte_flow_queue_info *queue_info, struct rte_flow_error *err); +int cn10k_flow_push(struct rte_eth_dev *dev, uint32_t queue, struct rte_flow_error *error); + #define CNXK_NPC_COUNTERS_MAX 512 #endif /* __CN10K_FLOW_H__ */ diff --git a/drivers/net/cnxk/cn20k_ethdev.c b/drivers/net/cnxk/cn20k_ethdev.c index f08079f612..9fac027650 100644 --- a/drivers/net/cnxk/cn20k_ethdev.c +++ b/drivers/net/cnxk/cn20k_ethdev.c @@ -5,7 +5,7 @@ #include "cn20k_flow.h" #include "cn20k_rx.h" #include "cn20k_tx.h" - +#include "cnxk_flow.h" static uint16_t nix_rx_offload_flags(struct rte_eth_dev *eth_dev) { @@ -962,8 +962,27 @@ npc_flow_ops_override(void) cnxk_flow_ops.create = cn20k_flow_create; cnxk_flow_ops.destroy = cn20k_flow_destroy; cnxk_flow_ops.info_get = cn20k_flow_info_get; + + cnxk_flow_ops.configure = cnxk_flow_configure; + cnxk_flow_ops.pattern_template_create = cnxk_flow_pattern_template_create; + cnxk_flow_ops.pattern_template_destroy = cnxk_flow_pattern_template_destroy; + cnxk_flow_ops.actions_template_create = cnxk_flow_actions_template_create; + cnxk_flow_ops.actions_template_destroy = cnxk_flow_actions_template_destroy; + cnxk_flow_ops.template_table_create = cnxk_flow_template_table_create; + cnxk_flow_ops.template_table_destroy = cnxk_flow_template_table_destroy; } +/* Fast-path (async) flow ops. Installed per-port in cn20k_nix_probe(). */ +static const struct rte_flow_fp_ops cn20k_flow_fp_ops = { + .async_create = cnxk_flow_async_create, + .async_create_by_index = cnxk_flow_async_create_by_index, + .async_create_by_index_with_pattern = cnxk_flow_async_create_by_index_with_pattern, + .async_actions_update = cnxk_flow_async_actions_update, + .async_destroy = cnxk_flow_async_destroy, + .push = cn20k_flow_push, + .pull = cnxk_flow_pull, +}; + static int cn20k_nix_remove(struct rte_pci_device *pci_dev) { @@ -1012,6 +1031,9 @@ cn20k_nix_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) dev = cnxk_eth_pmd_priv(eth_dev); + /* Install fast-path (async) flow ops for this port. */ + eth_dev->flow_fp_ops = &cn20k_flow_fp_ops; + /* Register up msg callbacks for PTP information */ roc_nix_ptp_info_cb_register(&dev->nix, cn20k_nix_ptp_info_update_cb); diff --git a/drivers/net/cnxk/cn20k_flow.c b/drivers/net/cnxk/cn20k_flow.c index fd50f516ee..ff6e220103 100644 --- a/drivers/net/cnxk/cn20k_flow.c +++ b/drivers/net/cnxk/cn20k_flow.c @@ -48,6 +48,49 @@ cn20k_flow_info_get(struct rte_eth_dev *dev, struct rte_flow_port_info *port_inf return cnxk_flow_info_get_common(dev, port_info, queue_info, err); } +int +cn20k_flow_push(struct rte_eth_dev *eth_dev, uint32_t queue, struct rte_flow_error *error) +{ + struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); + struct roc_npc *npc = &dev->npc; + bool reset_rx = false; + int rc; + + /* Flush the pending async ops to hardware first. */ + rc = cnxk_flow_push(eth_dev, queue, error); + if (rc) + return rc; + + /* Async push, unlike sync create/destroy, does not toggle the MARK and + * VLAN-strip RX offloads; reconcile each with its refcount so the Rx + * function matches the live rules. + */ + if (roc_npc_mark_actions_get(npc) && + !(dev->rx_offload_flags & NIX_RX_OFFLOAD_MARK_UPDATE_F)) { + dev->rx_offload_flags |= NIX_RX_OFFLOAD_MARK_UPDATE_F; + reset_rx = true; + } else if (!roc_npc_mark_actions_get(npc) && + (dev->rx_offload_flags & NIX_RX_OFFLOAD_MARK_UPDATE_F)) { + dev->rx_offload_flags &= ~NIX_RX_OFFLOAD_MARK_UPDATE_F; + reset_rx = true; + } + + if (roc_npc_vtag_actions_get(npc) && + !(dev->rx_offload_flags & NIX_RX_OFFLOAD_VLAN_STRIP_F)) { + dev->rx_offload_flags |= NIX_RX_OFFLOAD_VLAN_STRIP_F; + reset_rx = true; + } else if (!roc_npc_vtag_actions_get(npc) && + (dev->rx_offload_flags & NIX_RX_OFFLOAD_VLAN_STRIP_F)) { + dev->rx_offload_flags &= ~NIX_RX_OFFLOAD_VLAN_STRIP_F; + reset_rx = true; + } + + if (reset_rx) + cn20k_eth_set_rx_function(eth_dev); + + return 0; +} + int cn20k_flow_destroy(struct rte_eth_dev *eth_dev, struct rte_flow *rte_flow, struct rte_flow_error *error) diff --git a/drivers/net/cnxk/cn20k_flow.h b/drivers/net/cnxk/cn20k_flow.h index 1e4bd6214a..ed8a2aa1f3 100644 --- a/drivers/net/cnxk/cn20k_flow.h +++ b/drivers/net/cnxk/cn20k_flow.h @@ -16,6 +16,8 @@ int cn20k_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow, int cn20k_flow_info_get(struct rte_eth_dev *dev, struct rte_flow_port_info *port_info, struct rte_flow_queue_info *queue_info, struct rte_flow_error *err); +int cn20k_flow_push(struct rte_eth_dev *dev, uint32_t queue, struct rte_flow_error *error); + #define CNXK_NPC_COUNTERS_MAX 512 #endif /* __CN20K_FLOW_H__ */ diff --git a/drivers/net/cnxk/cnxk_ethdev.h b/drivers/net/cnxk/cnxk_ethdev.h index fc66135c66..342d249cf4 100644 --- a/drivers/net/cnxk/cnxk_ethdev.h +++ b/drivers/net/cnxk/cnxk_ethdev.h @@ -328,6 +328,50 @@ struct cnxk_macsec_sess { }; TAILQ_HEAD(cnxk_macsec_sess_list, cnxk_macsec_sess); +/* Pattern template: owns deep copies tracked in copies[]. */ +struct cnxk_flow_pattern_template { + struct rte_flow_pattern_template_attr attr; + struct rte_flow_item *pattern; + /* Mirrored roc handle for table-level ownership/lifecycle in roc. */ + struct roc_npc_pattern_template *roc_tmpl; + uint16_t nb_items; + void **copies; + uint16_t nb_copies; + uint32_t refcnt; +}; + +/* Actions template: owns deep copies tracked in copies[]. */ +struct cnxk_flow_actions_template { + struct rte_flow_actions_template_attr attr; + struct rte_flow_action *actions; + struct rte_flow_action *masks; + /* Mirrored roc handle for table-level ownership/lifecycle in roc. */ + struct roc_npc_actions_template *roc_tmpl; + uint16_t nb_actions; + void **copies; + uint16_t nb_copies; + uint32_t refcnt; +}; + +/* Template table wrapper. The roc layer owns rule storage, MCAM reservation + * and rule lifecycle; cnxk keeps rte templates for per-rule translation. + */ +struct cnxk_flow_table { + /* roc engine table handle: storage, MCAM, rule lifecycle. */ + struct roc_npc_template_table *roc_table; + + struct cnxk_flow_pattern_template **pattern_templates; + struct cnxk_flow_actions_template **actions_templates; + uint8_t nb_pattern_templates; + uint8_t nb_actions_templates; + + uint32_t nb_flows; + struct roc_npc_attr flow_attr; + /* roc_npc_attr has no transfer field; replay it into each rule attr. */ + bool transfer; + enum rte_flow_table_insertion_type insertion_type; +}; + struct cnxk_eth_dev { /* ROC NIX */ struct roc_nix nix; diff --git a/drivers/net/cnxk/cnxk_flow.c b/drivers/net/cnxk/cnxk_flow.c index c1c48eb7ab..1562b7389e 100644 --- a/drivers/net/cnxk/cnxk_flow.c +++ b/drivers/net/cnxk/cnxk_flow.c @@ -8,6 +8,7 @@ #define TNL_DCP_MATCH_ID 5 #define NRML_MATCH_ID 1 + const struct cnxk_rte_flow_term_info term[] = { [RTE_FLOW_ITEM_TYPE_ETH] = {ROC_NPC_ITEM_TYPE_ETH, sizeof(struct rte_flow_item_eth)}, [RTE_FLOW_ITEM_TYPE_VLAN] = {ROC_NPC_ITEM_TYPE_VLAN, sizeof(struct rte_flow_item_vlan)}, @@ -49,8 +50,7 @@ const struct cnxk_rte_flow_term_info term[] = { [RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT] = {ROC_NPC_ITEM_TYPE_REPRESENTED_PORT, sizeof(struct rte_flow_item_ethdev)}, [RTE_FLOW_ITEM_TYPE_PPPOES] = {ROC_NPC_ITEM_TYPE_PPPOES, - sizeof(struct rte_flow_item_pppoe)} -}; + sizeof(struct rte_flow_item_pppoe)}}; static int npc_rss_action_validate(struct rte_eth_dev *eth_dev, const struct rte_flow_attr *attr, @@ -239,7 +239,7 @@ append_rss_action(struct cnxk_eth_dev *dev, struct roc_npc_action *in_actions, u /* Add RSS action */ rss_conf->queue_num = nb_rxq; - queue_arr = calloc(1, rss_conf->queue_num * sizeof(uint16_t)); + queue_arr = plt_zmalloc(rss_conf->queue_num * sizeof(uint16_t), 0); if (!queue_arr) { plt_err("Failed to allocate memory for rss queue"); rc = -ENOMEM; @@ -266,10 +266,12 @@ append_rss_action(struct cnxk_eth_dev *dev, struct roc_npc_action *in_actions, u while (free_allocs[j] != 0) j++; free_allocs[j] = (uint64_t)rss_conf; + j++; + free_allocs[j] = (uint64_t)queue_arr; return 0; free_rss: - rte_free(rss_conf); + plt_free(rss_conf); fail: return rc; } @@ -1070,22 +1072,18 @@ cnxk_flow_query_common(struct rte_eth_dev *eth_dev, struct rte_flow *flow, const struct rte_flow_action *action, void *data, struct rte_flow_error *error, bool is_rep) { - struct roc_npc_flow *in_flow = (struct roc_npc_flow *)flow; struct rte_flow_query_count *query = data; + struct roc_npc_flow *in_flow; struct cnxk_rep_dev *rep_dev; - struct cnxk_eth_dev *dev; - struct roc_npc *npc; const char *errmsg = NULL; + struct cnxk_eth_dev *dev; int errcode = ENOTSUP; + struct roc_npc *npc; int rc; - if (action->type != RTE_FLOW_ACTION_TYPE_COUNT) { - errmsg = "Only COUNT is supported in query"; - goto err_exit; - } - - if (in_flow->ctr_id == NPC_COUNTER_NONE) { - errmsg = "Counter is not available"; + if (flow == NULL) { + errmsg = "Invalid flow handle"; + errcode = EINVAL; goto err_exit; } @@ -1098,6 +1096,29 @@ cnxk_flow_query_common(struct rte_eth_dev *eth_dev, struct rte_flow *flow, npc = &rep_dev->parent_dev->npc; } + /* Let the roc layer resolve async/template handles to the underlying + * roc_npc_flow (NULL until committed); else treat as a sync flow. + */ + in_flow = (struct roc_npc_flow *)flow; + if (!is_rep) + roc_npc_async_flow_resolve(npc, flow, &in_flow); + + if (in_flow == NULL) { + errmsg = "Flow is not committed yet"; + errcode = EINVAL; + goto err_exit; + } + + if (action->type != RTE_FLOW_ACTION_TYPE_COUNT) { + errmsg = "Only COUNT is supported in query"; + goto err_exit; + } + + if (in_flow->ctr_id == NPC_COUNTER_NONE) { + errmsg = "Counter is not available"; + goto err_exit; + } + if (in_flow->use_pre_alloc) { rc = roc_npc_inl_mcam_read_counter(in_flow->ctr_id, &query->hits); } else { @@ -1149,8 +1170,8 @@ cnxk_flow_isolate(struct rte_eth_dev *eth_dev __rte_unused, int enable __rte_unu * entry for this port. */ - rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, "Flow isolation not supported"); + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Flow isolation not supported"); return -rte_errno; } @@ -1159,8 +1180,9 @@ int cnxk_flow_dev_dump_common(struct rte_eth_dev *eth_dev, struct rte_flow *flow, FILE *file, struct rte_flow_error *error, bool is_rep) { + struct roc_npc_flow *in_flow = NULL; + struct cnxk_eth_dev *dev = NULL; struct cnxk_rep_dev *rep_dev; - struct cnxk_eth_dev *dev; struct roc_npc *npc; /* is_rep set for operation performed via representor ports */ @@ -1179,7 +1201,17 @@ cnxk_flow_dev_dump_common(struct rte_eth_dev *eth_dev, struct rte_flow *flow, FI } if (flow != NULL) { - roc_npc_flow_mcam_dump(file, npc, (struct roc_npc_flow *)flow); + /* Resolve async/template handles to the underlying roc_npc_flow; + * else treat as a sync flow. + */ + if (is_rep || !roc_npc_async_flow_resolve(npc, flow, &in_flow)) + in_flow = (struct roc_npc_flow *)flow; + + if (in_flow == NULL) + return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "flow is not committed yet"); + + roc_npc_flow_mcam_dump(file, npc, in_flow); return 0; } @@ -1187,6 +1219,1276 @@ cnxk_flow_dev_dump_common(struct rte_eth_dev *eth_dev, struct rte_flow *flow, FI return 0; } +/* Configure async flow queues for the port. */ +int +cnxk_flow_configure(struct rte_eth_dev *eth_dev, const struct rte_flow_port_attr *port_attr, + uint16_t nb_queue, const struct rte_flow_queue_attr *queue_attr[], + struct rte_flow_error *err) +{ + struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); + const struct roc_npc_flow_queue_attr **rptrs = NULL; + struct roc_npc_flow_queue_attr *rattr = NULL; + struct roc_npc *npc = &dev->npc; + uint16_t i; + int rc; + + RTE_SET_USED(port_attr); + + /* Translate the rte queue attributes into roc_npc types; the roc + * engine validates nb_queue == 0 and a NULL queue_attr. + */ + if (nb_queue != 0 && queue_attr != NULL) { + rattr = plt_zmalloc(nb_queue * sizeof(*rattr), 0); + if (rattr == NULL) + return rte_flow_error_set(err, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Failed to allocate flow queues"); + + rptrs = plt_zmalloc(nb_queue * sizeof(*rptrs), 0); + if (rptrs == NULL) { + plt_free(rattr); + return rte_flow_error_set(err, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Failed to allocate flow queues"); + } + for (i = 0; i < nb_queue; i++) { + if (queue_attr[i] != NULL) { + rattr[i].size = queue_attr[i]->size; + rptrs[i] = &rattr[i]; + } else { + rptrs[i] = NULL; + } + } + } + + rc = roc_npc_flow_configure(npc, nb_queue, rptrs); + + plt_free(rattr); + plt_free(rptrs); + + if (rc) + return rte_flow_error_set(err, -rc, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Failed to configure flow queues"); + return 0; +} + +/* Deep-copy a spec/last/mask/conf payload into template-owned memory. */ +static int +cnxk_flow_dup_conf(void **copies, uint16_t *nb_copies, const void *src, size_t sz, const void **dst) +{ + void *copy; + + if (src == NULL || sz == 0) { + *dst = NULL; + return 0; + } + copy = plt_zmalloc(sz, 0); + if (copy == NULL) + return -ENOMEM; + memcpy(copy, src, sz); + copies[(*nb_copies)++] = copy; + *dst = copy; + return 0; +} + +struct rte_flow_pattern_template * +cnxk_flow_pattern_template_create(struct rte_eth_dev *eth_dev, + const struct rte_flow_pattern_template_attr *attr, + const struct rte_flow_item pattern[], + struct rte_flow_error *error) +{ + const char *errmsg = "Failed to allocate pattern template"; + struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); + struct cnxk_flow_pattern_template *tmpl = NULL; + struct roc_npc_pattern_template_attr roc_attr; + struct roc_npc_item_info *roc_pattern = NULL; + int i, nb_items = 0, errcode = ENOMEM; + struct roc_npc *npc = &dev->npc; + int roc_err; + + if (attr == NULL) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "NULL template attr"); + return NULL; + } + + if (pattern == NULL) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "NULL pattern"); + return NULL; + } + + while (pattern[nb_items].type != RTE_FLOW_ITEM_TYPE_END) + nb_items++; + + /* Reject oversized templates; rules are built into fixed-size stack + * arrays (combined_items[ROC_NPC_ITEM_TYPE_END + 1]). + */ + if (nb_items > ROC_NPC_ITEM_TYPE_END) { + errcode = ENOTSUP; + errmsg = "too many pattern items for hardware"; + goto err_nomem; + } + + tmpl = plt_zmalloc(sizeof(*tmpl), 0); + if (tmpl == NULL) + goto err_nomem; + + tmpl->attr = *attr; + tmpl->nb_items = nb_items; + + tmpl->pattern = plt_zmalloc((nb_items + 1) * sizeof(struct rte_flow_item), 0); + if (tmpl->pattern == NULL) + goto err_free; + + /* At most 3 payload copies (spec/last/mask) per item. */ + tmpl->copies = plt_zmalloc(3 * (nb_items + 1) * sizeof(void *), 0); + if (tmpl->copies == NULL) + goto err_free; + + for (i = 0; i < nb_items; i++) { + enum rte_flow_item_type t = pattern[i].type; + size_t sz = (t < (int)RTE_DIM(term)) ? term[t].item_size : 0; + + /* Reject unrepresentable item types (sz 0), which would drop + * spec/last/mask. VOID/ANY legitimately carry no payload. + */ + if (t != RTE_FLOW_ITEM_TYPE_VOID && t != RTE_FLOW_ITEM_TYPE_ANY && sz == 0) { + errcode = ENOTSUP; + errmsg = "unsupported pattern item type in template"; + goto err_free; + } + + /* Reject items with nested variable-length payloads; a flat + * memcpy would retain a dangling pointer into caller memory. + */ + if (t == RTE_FLOW_ITEM_TYPE_RAW) { + errcode = ENOTSUP; + errmsg = "Item type with nested pointers is not supported " + "in pattern template (e.g. RAW)"; + goto err_free; + } + + tmpl->pattern[i].type = t; + if (cnxk_flow_dup_conf(tmpl->copies, &tmpl->nb_copies, pattern[i].spec, sz, + &tmpl->pattern[i].spec) || + cnxk_flow_dup_conf(tmpl->copies, &tmpl->nb_copies, pattern[i].last, sz, + &tmpl->pattern[i].last) || + cnxk_flow_dup_conf(tmpl->copies, &tmpl->nb_copies, pattern[i].mask, sz, + &tmpl->pattern[i].mask)) + goto err_free; + } + tmpl->pattern[nb_items].type = RTE_FLOW_ITEM_TYPE_END; + tmpl->roc_tmpl = NULL; + tmpl->refcnt = 0; + + roc_pattern = plt_zmalloc((nb_items + 1) * sizeof(*roc_pattern), 0); + if (roc_pattern == NULL) + goto err_free; + + for (i = 0; i < nb_items; i++) { + enum rte_flow_item_type t = tmpl->pattern[i].type; + size_t sz = (t < (int)RTE_DIM(term)) ? term[t].item_size : 0; + + roc_pattern[i].type = + (t < (int)RTE_DIM(term)) ? term[t].item_type : ROC_NPC_ITEM_TYPE_VOID; + roc_pattern[i].size = sz; + roc_pattern[i].spec = tmpl->pattern[i].spec; + roc_pattern[i].last = tmpl->pattern[i].last; + roc_pattern[i].mask = tmpl->pattern[i].mask; + } + roc_pattern[nb_items].type = ROC_NPC_ITEM_TYPE_END; + + memset(&roc_attr, 0, sizeof(roc_attr)); + roc_attr.relaxed_matching = attr->relaxed_matching; + roc_attr.ingress = attr->ingress; + roc_attr.egress = attr->egress; + roc_attr.transfer = attr->transfer; + + roc_err = 0; + tmpl->roc_tmpl = roc_npc_pattern_template_create(npc, &roc_attr, roc_pattern, &roc_err); + if (tmpl->roc_tmpl == NULL) { + errcode = (roc_err < 0) ? -roc_err : ENOMEM; + errmsg = "Failed to create ROC pattern template"; + goto err_free; + } + + plt_free(roc_pattern); + + return (struct rte_flow_pattern_template *)tmpl; + +err_free: + plt_free(roc_pattern); + if (tmpl != NULL) { + uint16_t j; + if (tmpl->roc_tmpl != NULL) + roc_npc_pattern_template_destroy(npc, tmpl->roc_tmpl); + for (j = 0; j < tmpl->nb_copies; j++) + plt_free(tmpl->copies[j]); + plt_free(tmpl->copies); + plt_free(tmpl->pattern); + plt_free(tmpl); + } +err_nomem: + rte_flow_error_set(error, errcode, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, errmsg); + return NULL; +} + +int +cnxk_flow_pattern_template_destroy(struct rte_eth_dev *eth_dev, + struct rte_flow_pattern_template *templ, + struct rte_flow_error *error) +{ + struct cnxk_flow_pattern_template *tmpl = (struct cnxk_flow_pattern_template *)templ; + struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); + struct roc_npc *npc = &dev->npc; + uint16_t i; + int rc; + + if (tmpl == NULL) + return 0; + + if (tmpl->refcnt != 0) + return rte_flow_error_set(error, EBUSY, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "pattern template still in use by a table"); + + rc = roc_npc_pattern_template_destroy(npc, tmpl->roc_tmpl); + if (rc) + return rte_flow_error_set(error, -rc, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "failed to destroy ROC pattern template"); + + for (i = 0; i < tmpl->nb_copies; i++) + plt_free(tmpl->copies[i]); + plt_free(tmpl->copies); + plt_free(tmpl->pattern); + plt_free(tmpl); + return 0; +} +/* Size of an action's conf for the v1-supported set; -1 = unsupported. */ +static int +cnxk_flow_action_conf_size(enum rte_flow_action_type type) +{ + switch (type) { + case RTE_FLOW_ACTION_TYPE_VOID: + case RTE_FLOW_ACTION_TYPE_DROP: + case RTE_FLOW_ACTION_TYPE_PF: + case RTE_FLOW_ACTION_TYPE_FLAG: + case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: + case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: + return 0; /* no conf */ + case RTE_FLOW_ACTION_TYPE_QUEUE: + return sizeof(struct rte_flow_action_queue); + case RTE_FLOW_ACTION_TYPE_VF: + return sizeof(struct rte_flow_action_vf); + case RTE_FLOW_ACTION_TYPE_MARK: + return sizeof(struct rte_flow_action_mark); + case RTE_FLOW_ACTION_TYPE_COUNT: + return sizeof(struct rte_flow_action_count); + case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: + return sizeof(struct rte_flow_action_of_push_vlan); + case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: + return sizeof(struct rte_flow_action_of_set_vlan_vid); + case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: + return sizeof(struct rte_flow_action_of_set_vlan_pcp); + case RTE_FLOW_ACTION_TYPE_AGE: + return sizeof(struct rte_flow_action_age); + case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: + case RTE_FLOW_ACTION_TYPE_PORT_REPRESENTOR: + return sizeof(struct rte_flow_action_ethdev); + case RTE_FLOW_ACTION_TYPE_PORT_ID: + return sizeof(struct rte_flow_action_port_id); + default: + return -1; + } +} + +/* Deep-copy an action conf. RSS nests queue[]/key[] pointers, so pack them + * into one blob and fix the pointers; other confs use a flat memcpy. + */ +static void * +cnxk_flow_dup_action_conf(const struct rte_flow_action *act) +{ + if (act->conf == NULL) + return NULL; + + if (act->type == RTE_FLOW_ACTION_TYPE_RSS) { + const struct rte_flow_action_rss *rss = act->conf; + size_t rss_blob_bytes = sizeof(*rss); + size_t rss_queue_bytes = (size_t)rss->queue_num * sizeof(uint16_t); + size_t rss_key_bytes = rss->key_len; + struct rte_flow_action_rss *rss_copy; + uint8_t *rss_blob; + + /* Reject a non-zero length paired with a NULL pointer, which + * would make the memcpy below dereference NULL. + */ + if ((rss_queue_bytes && rss->queue == NULL) || (rss_key_bytes && rss->key == NULL)) + return NULL; + + rss_blob = plt_zmalloc(rss_blob_bytes + rss_queue_bytes + rss_key_bytes, 0); + if (rss_blob == NULL) + return NULL; + rss_copy = (struct rte_flow_action_rss *)rss_blob; + *rss_copy = *rss; + if (rss_queue_bytes) { + memcpy(rss_blob + rss_blob_bytes, rss->queue, rss_queue_bytes); + rss_copy->queue = (const uint16_t *)(rss_blob + rss_blob_bytes); + } else { + rss_copy->queue = NULL; + } + if (rss_key_bytes) { + memcpy(rss_blob + rss_blob_bytes + rss_queue_bytes, rss->key, + rss_key_bytes); + rss_copy->key = + (const uint8_t *)(rss_blob + rss_blob_bytes + rss_queue_bytes); + } else { + rss_copy->key = NULL; + } + return rss_copy; + } + + { + int conf_size = cnxk_flow_action_conf_size(act->type); + void *copy; + + if (conf_size <= 0) + return NULL; + copy = plt_zmalloc(conf_size, 0); + if (copy != NULL) + memcpy(copy, act->conf, conf_size); + return copy; + } +} + +/* Map a supported rte_flow action type to its roc_npc action type, to mirror + * the actions template into the roc engine for lifecycle/refcount ownership. + */ +static int +cnxk_flow_roc_action_type(enum rte_flow_action_type type, enum roc_npc_action_type *roc_type) +{ + switch (type) { + case RTE_FLOW_ACTION_TYPE_VOID: + case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: + *roc_type = ROC_NPC_ACTION_TYPE_VOID; + return 0; + case RTE_FLOW_ACTION_TYPE_DROP: + *roc_type = ROC_NPC_ACTION_TYPE_DROP; + return 0; + case RTE_FLOW_ACTION_TYPE_PF: + *roc_type = ROC_NPC_ACTION_TYPE_PF; + return 0; + case RTE_FLOW_ACTION_TYPE_VF: + *roc_type = ROC_NPC_ACTION_TYPE_VF; + return 0; + case RTE_FLOW_ACTION_TYPE_FLAG: + *roc_type = ROC_NPC_ACTION_TYPE_FLAG; + return 0; + case RTE_FLOW_ACTION_TYPE_MARK: + *roc_type = ROC_NPC_ACTION_TYPE_MARK; + return 0; + case RTE_FLOW_ACTION_TYPE_COUNT: + *roc_type = ROC_NPC_ACTION_TYPE_COUNT; + return 0; + case RTE_FLOW_ACTION_TYPE_QUEUE: + *roc_type = ROC_NPC_ACTION_TYPE_QUEUE; + return 0; + case RTE_FLOW_ACTION_TYPE_RSS: + *roc_type = ROC_NPC_ACTION_TYPE_RSS; + return 0; + case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: + *roc_type = ROC_NPC_ACTION_TYPE_VLAN_STRIP; + return 0; + case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: + *roc_type = ROC_NPC_ACTION_TYPE_VLAN_ETHTYPE_INSERT; + return 0; + case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: + *roc_type = ROC_NPC_ACTION_TYPE_VLAN_INSERT; + return 0; + case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: + *roc_type = ROC_NPC_ACTION_TYPE_VLAN_PCP_INSERT; + return 0; + case RTE_FLOW_ACTION_TYPE_AGE: + *roc_type = ROC_NPC_ACTION_TYPE_AGE; + return 0; + case RTE_FLOW_ACTION_TYPE_PORT_ID: + case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: + case RTE_FLOW_ACTION_TYPE_PORT_REPRESENTOR: + *roc_type = ROC_NPC_ACTION_TYPE_PORT_ID; + return 0; + default: + return -1; + } +} + +struct rte_flow_actions_template * +cnxk_flow_actions_template_create(struct rte_eth_dev *eth_dev, + const struct rte_flow_actions_template_attr *attr, + const struct rte_flow_action actions[], + const struct rte_flow_action masks[], + struct rte_flow_error *error) +{ + const char *errmsg = "Failed to allocate actions template"; + struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); + struct cnxk_flow_actions_template *tmpl = NULL; + struct roc_npc_actions_template_attr roc_attr; + struct roc_npc_action *roc_actions = NULL; + int i, nb_actions = 0, errcode = ENOMEM; + struct roc_npc *npc = &dev->npc; + int roc_err = 0; + + if (attr == NULL) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "NULL template attr"); + return NULL; + } + + if (actions == NULL) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "NULL actions"); + return NULL; + } + + while (actions[nb_actions].type != RTE_FLOW_ACTION_TYPE_END) + nb_actions++; + + /* Reject oversized templates; rules are built into fixed-size stack + * arrays (combined_actions[ROC_NPC_MAX_ACTION_COUNT] plus END). + */ + if (nb_actions > ROC_NPC_MAX_ACTION_COUNT - 1) { + errcode = ENOTSUP; + errmsg = "too many actions for hardware"; + goto err; + } + + tmpl = plt_zmalloc(sizeof(*tmpl), 0); + if (tmpl == NULL) + goto err; + + tmpl->attr = *attr; + tmpl->nb_actions = nb_actions; + + tmpl->actions = plt_zmalloc((nb_actions + 1) * sizeof(struct rte_flow_action), 0); + if (tmpl->actions == NULL) + goto err; + + tmpl->masks = plt_zmalloc((nb_actions + 1) * sizeof(struct rte_flow_action), 0); + if (tmpl->masks == NULL) + goto err; + + /* up to 2 conf copies (action + mask) per action */ + tmpl->copies = plt_zmalloc(2 * (nb_actions + 1) * sizeof(void *), 0); + if (tmpl->copies == NULL) + goto err; + + for (i = 0; i < nb_actions; i++) { + enum rte_flow_action_type t = actions[i].type; + int sz = cnxk_flow_action_conf_size(t); + + if (sz < 0 && t != RTE_FLOW_ACTION_TYPE_RSS) { + errcode = ENOTSUP; + errmsg = "Unsupported action in template " + "(supports VOID/DROP/QUEUE/PF/VF/MARK/FLAG/COUNT/RSS/" + "OF_POP_VLAN/OF_PUSH_VLAN/OF_SET_VLAN_VID/" + "OF_SET_VLAN_PCP/AGE/REPRESENTED_PORT/" + "PORT_REPRESENTOR/PORT_ID/VXLAN_DECAP)"; + goto err; + } + + tmpl->actions[i].type = t; + + if (t == RTE_FLOW_ACTION_TYPE_RSS) { + /* RSS has nested pointers — use deep-copy helper. */ + void *copy = cnxk_flow_dup_action_conf(&actions[i]); + + if (copy == NULL && actions[i].conf != NULL) { + const struct rte_flow_action_rss *rss = actions[i].conf; + + /* Malformed conf vs allocation failure: report a + * precise errno instead of the default ENOMEM. + */ + if ((rss->queue_num && rss->queue == NULL) || + (rss->key_len && rss->key == NULL)) { + errcode = EINVAL; + errmsg = "Malformed RSS action conf in template"; + } + goto err; + } + tmpl->actions[i].conf = copy; + if (copy) + tmpl->copies[tmpl->nb_copies++] = copy; + } else { + if (cnxk_flow_dup_conf(tmpl->copies, &tmpl->nb_copies, actions[i].conf, sz, + &tmpl->actions[i].conf)) + goto err; + } + + if (masks != NULL) { + if (masks[i].type != t) { + errcode = EINVAL; + errmsg = "actions/masks type mismatch in template"; + goto err; + } + tmpl->masks[i].type = t; + if (t == RTE_FLOW_ACTION_TYPE_RSS) { + /* RSS nests pointers and cannot be copied by the flat + * helper; reject a non-NULL RSS mask. + */ + if (masks[i].conf != NULL) { + errcode = ENOTSUP; + errmsg = "Non-NULL RSS mask conf is not " + "supported in template"; + goto err; + } + tmpl->masks[i].conf = NULL; + } else if (cnxk_flow_dup_conf(tmpl->copies, &tmpl->nb_copies, masks[i].conf, + sz > 0 ? sz : 0, &tmpl->masks[i].conf)) { + goto err; + } + } + } + + tmpl->actions[nb_actions].type = RTE_FLOW_ACTION_TYPE_END; + tmpl->masks[nb_actions].type = RTE_FLOW_ACTION_TYPE_END; + tmpl->roc_tmpl = NULL; + tmpl->refcnt = 0; + + /* Mirror only action types into the roc engine for template lifecycle + * and refcount ownership; per-rule conf translation stays in cnxk. + */ + roc_actions = plt_zmalloc((nb_actions + 1) * sizeof(*roc_actions), 0); + if (roc_actions == NULL) + goto err; + + for (i = 0; i < nb_actions; i++) { + enum roc_npc_action_type rt; + + if (cnxk_flow_roc_action_type(tmpl->actions[i].type, &rt)) { + errcode = ENOTSUP; + errmsg = "Unsupported action in template"; + goto err; + } + roc_actions[i].type = rt; + roc_actions[i].conf = NULL; + } + roc_actions[nb_actions].type = ROC_NPC_ACTION_TYPE_END; + + memset(&roc_attr, 0, sizeof(roc_attr)); + roc_attr.ingress = attr->ingress; + roc_attr.egress = attr->egress; + roc_attr.transfer = attr->transfer; + + tmpl->roc_tmpl = + roc_npc_actions_template_create(npc, &roc_attr, roc_actions, NULL, &roc_err); + plt_free(roc_actions); + roc_actions = NULL; + if (tmpl->roc_tmpl == NULL) { + errcode = (roc_err < 0) ? -roc_err : ENOMEM; + errmsg = "Failed to create ROC actions template"; + goto err; + } + + return (struct rte_flow_actions_template *)tmpl; + +err: + plt_free(roc_actions); + if (tmpl != NULL) { + uint16_t j; + if (tmpl->roc_tmpl != NULL) + roc_npc_actions_template_destroy(npc, tmpl->roc_tmpl); + for (j = 0; j < tmpl->nb_copies; j++) + plt_free(tmpl->copies[j]); + plt_free(tmpl->copies); + plt_free(tmpl->masks); + plt_free(tmpl->actions); + plt_free(tmpl); + } + rte_flow_error_set(error, errcode, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, errmsg); + return NULL; +} + +int +cnxk_flow_actions_template_destroy(struct rte_eth_dev *eth_dev, + struct rte_flow_actions_template *templ, + struct rte_flow_error *error) +{ + struct cnxk_flow_actions_template *tmpl = (struct cnxk_flow_actions_template *)templ; + struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); + struct roc_npc *npc = &dev->npc; + uint16_t i; + int rc; + + if (tmpl == NULL) + return 0; + + if (tmpl->refcnt != 0) + return rte_flow_error_set(error, EBUSY, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "actions template still in use by a table"); + + rc = roc_npc_actions_template_destroy(npc, tmpl->roc_tmpl); + if (rc) + return rte_flow_error_set(error, -rc, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "failed to destroy ROC actions template"); + + for (i = 0; i < tmpl->nb_copies; i++) + plt_free(tmpl->copies[i]); + plt_free(tmpl->copies); + plt_free(tmpl->masks); + plt_free(tmpl->actions); + plt_free(tmpl); + return 0; +} + +struct rte_flow_template_table * +cnxk_flow_template_table_create(struct rte_eth_dev *eth_dev, + const struct rte_flow_template_table_attr *table_attr, + struct rte_flow_pattern_template *pattern_templates[], + uint8_t nb_pattern_templates, + struct rte_flow_actions_template *actions_templates[], + uint8_t nb_actions_templates, struct rte_flow_error *error) +{ + struct roc_npc_pattern_template **roc_pattern_templates = NULL; + struct roc_npc_actions_template **roc_actions_templates = NULL; + struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); + struct roc_npc_template_table_attr roc_attr; + enum rte_flow_table_insertion_type itype; + const struct rte_flow_attr *fattr; + struct roc_npc *npc = &dev->npc; + struct cnxk_flow_table *table; + uint32_t nb_flows; + int errcode = 0; + uint8_t i; + + if (table_attr == NULL) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "NULL table_attr"); + return NULL; + } + + itype = table_attr->insertion_type; + fattr = &table_attr->flow_attr; + nb_flows = table_attr->nb_flows; + + if (itype == RTE_FLOW_TABLE_INSERTION_TYPE_INDEX) { + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "pure-index insertion not supported; NPC is match-based"); + return NULL; + } + + /* Validate the template handles before allocating anything. */ + for (i = 0; i < nb_pattern_templates; i++) { + if (pattern_templates[i] == NULL) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "invalid template handle"); + return NULL; + } + } + for (i = 0; i < nb_actions_templates; i++) { + if (actions_templates[i] == NULL) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "invalid template handle"); + return NULL; + } + } + + table = plt_zmalloc(sizeof(*table), 0); + if (table == NULL) + goto enomem; + + table->insertion_type = itype; + table->nb_flows = nb_flows; + + memset(&table->flow_attr, 0, sizeof(table->flow_attr)); + table->flow_attr.priority = fattr->priority; + table->flow_attr.ingress = fattr->ingress; + table->flow_attr.egress = fattr->egress; + table->transfer = fattr->transfer; + + if (nb_pattern_templates != 0) { + table->pattern_templates = + plt_zmalloc(nb_pattern_templates * sizeof(*table->pattern_templates), 0); + if (table->pattern_templates == NULL) + goto free_table; + roc_pattern_templates = + plt_zmalloc(nb_pattern_templates * sizeof(*roc_pattern_templates), 0); + if (roc_pattern_templates == NULL) + goto free_table; + } + if (nb_actions_templates != 0) { + table->actions_templates = + plt_zmalloc(nb_actions_templates * sizeof(*table->actions_templates), 0); + if (table->actions_templates == NULL) + goto free_table; + roc_actions_templates = + plt_zmalloc(nb_actions_templates * sizeof(*roc_actions_templates), 0); + if (roc_actions_templates == NULL) + goto free_table; + } + + /* Hand the table to the roc engine, which owns rule storage, MCAM + * reservation and the rule lifecycle; cnxk keeps per-rule translation. + */ + memset(&roc_attr, 0, sizeof(roc_attr)); + roc_attr.flow_attr = table->flow_attr; + roc_attr.transfer = table->transfer; + roc_attr.nb_flows = nb_flows; + roc_attr.insertion_type = (itype == RTE_FLOW_TABLE_INSERTION_TYPE_INDEX_WITH_PATTERN) ? + ROC_NPC_TEMPLATE_INSERTION_INDEX_WITH_PATTERN : + ROC_NPC_TEMPLATE_INSERTION_PATTERN; + /* Stash this wrapper as the table cookie so async_actions_update can + * recover the rte templates from just the flow handle. + */ + roc_attr.cookie = table; + + for (i = 0; i < nb_pattern_templates; i++) { + table->pattern_templates[i] = + (struct cnxk_flow_pattern_template *)pattern_templates[i]; + roc_pattern_templates[i] = table->pattern_templates[i]->roc_tmpl; + if (roc_pattern_templates[i] == NULL) + goto bad_handle; + } + for (i = 0; i < nb_actions_templates; i++) { + table->actions_templates[i] = + (struct cnxk_flow_actions_template *)actions_templates[i]; + roc_actions_templates[i] = table->actions_templates[i]->roc_tmpl; + if (roc_actions_templates[i] == NULL) + goto bad_handle; + } + + /* clang-format off */ + table->roc_table = roc_npc_template_table_create(npc, &roc_attr, + roc_pattern_templates, nb_pattern_templates, + roc_actions_templates, nb_actions_templates, &errcode); + /* clang-format on */ + plt_free(roc_pattern_templates); + plt_free(roc_actions_templates); + roc_pattern_templates = NULL; + roc_actions_templates = NULL; + if (table->roc_table == NULL) { + plt_free(table->pattern_templates); + plt_free(table->actions_templates); + plt_free(table); + rte_flow_error_set(error, errcode < 0 ? -errcode : ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "could not create template table"); + return NULL; + } + + for (i = 0; i < nb_pattern_templates; i++) + table->pattern_templates[i]->refcnt++; + table->nb_pattern_templates = nb_pattern_templates; + for (i = 0; i < nb_actions_templates; i++) + table->actions_templates[i]->refcnt++; + table->nb_actions_templates = nb_actions_templates; + + return (struct rte_flow_template_table *)table; + +bad_handle: + plt_free(roc_pattern_templates); + plt_free(roc_actions_templates); + plt_free(table->pattern_templates); + plt_free(table->actions_templates); + plt_free(table); + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "invalid template handle"); + return NULL; + +free_table: + plt_free(roc_pattern_templates); + plt_free(roc_actions_templates); + plt_free(table->pattern_templates); + plt_free(table->actions_templates); + plt_free(table); + +enomem: + rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "out of memory"); + return NULL; +} + +int +cnxk_flow_template_table_destroy(struct rte_eth_dev *eth_dev, + struct rte_flow_template_table *tbl_handle, + struct rte_flow_error *error) +{ + struct cnxk_flow_table *table = (struct cnxk_flow_table *)tbl_handle; + struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); + struct roc_npc *npc = &dev->npc; + uint8_t i; + int rc; + + if (table == NULL) + return 0; + + /* The roc engine guards teardown: a busy table returns -EBUSY and + * nothing is freed, so the cnxk wrapper and refcounts stay intact. + */ + rc = roc_npc_template_table_destroy(npc, table->roc_table); + if (rc == -EBUSY) + return rte_flow_error_set(error, EBUSY, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "table still has live flows; destroy them first"); + if (rc) + return rte_flow_error_set(error, -rc, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "failed to destroy template table"); + + for (i = 0; i < table->nb_pattern_templates; i++) + table->pattern_templates[i]->refcnt--; + for (i = 0; i < table->nb_actions_templates; i++) + table->actions_templates[i]->refcnt--; + + plt_free(table->pattern_templates); + plt_free(table->actions_templates); + plt_free(table); + return 0; +} + +/* Reject per-rule pattern/actions arrays whose types/order do not mirror the + * chosen templates; the merge takes the type from the template but the + * spec/conf from the per-rule array. A NULL array uses template defaults. + */ +static int +cnxk_flow_check_items(const struct cnxk_flow_pattern_template *pt, + const struct rte_flow_item items[]) +{ + uint16_t k; + + if (items == NULL) + return 0; + for (k = 0; k < pt->nb_items; k++) + if (items[k].type != pt->pattern[k].type) + return -EINVAL; + if (items[pt->nb_items].type != RTE_FLOW_ITEM_TYPE_END) + return -EINVAL; + return 0; +} + +static int +cnxk_flow_check_actions(const struct cnxk_flow_actions_template *at, + const struct rte_flow_action actions[]) +{ + uint16_t k; + + if (actions == NULL) + return 0; + for (k = 0; k < at->nb_actions; k++) + if (actions[k].type != at->actions[k].type) + return -EINVAL; + if (actions[at->nb_actions].type != RTE_FLOW_ACTION_TYPE_END) + return -EINVAL; + return 0; +} + +/* Enqueue a PATTERN create: merge the per-rule spec with the templates, + * translate to roc_npc types, and hand to the roc engine (programmed at push). + */ +struct rte_flow * +cnxk_flow_async_create(struct rte_eth_dev *eth_dev, uint32_t queue, + const struct rte_flow_op_attr *attr, + struct rte_flow_template_table *tbl_handle, + const struct rte_flow_item items[], uint8_t pattern_template_index, + const struct rte_flow_action actions[], uint8_t action_template_index, + void *user_data, struct rte_flow_error *error) +{ + struct cnxk_flow_table *table = (struct cnxk_flow_table *)tbl_handle; + struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); + struct cnxk_flow_pattern_template *pt; + struct cnxk_flow_actions_template *at; + struct roc_npc_template_flow *flow; + struct roc_npc *npc = &dev->npc; + + uint64_t free_allocs[ROC_NPC_MAX_ACTION_COUNT + ROC_NPC_ITEM_TYPE_END + 1] = {0}; + struct roc_npc_item_info in_pattern[ROC_NPC_ITEM_TYPE_END + 1] = {0}; + struct rte_flow_action combined_actions[ROC_NPC_MAX_ACTION_COUNT]; + struct roc_npc_action in_actions[ROC_NPC_MAX_ACTION_COUNT] = {0}; + struct rte_flow_item combined_items[ROC_NPC_ITEM_TYPE_END + 1]; + struct roc_npc_action_sample in_sample = {0}; + struct rte_flow_attr rte_attr; + struct roc_npc_attr in_attr; + uint16_t dst_pf_func = 0; + uint64_t def_action = 0; + int errcode = 0; + uint16_t k; + int rc, j; + + RTE_SET_USED(attr); + + if (table == NULL) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "bad table"); + return NULL; + } + if (table->insertion_type != RTE_FLOW_TABLE_INSERTION_TYPE_PATTERN) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "table is index-insertion; use create_by_index_with_pattern"); + return NULL; + } + if (pattern_template_index >= table->nb_pattern_templates || + action_template_index >= table->nb_actions_templates) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "template index out of range"); + return NULL; + } + + pt = table->pattern_templates[pattern_template_index]; + at = table->actions_templates[action_template_index]; + + if (cnxk_flow_check_items(pt, items) != 0 || cnxk_flow_check_actions(at, actions) != 0) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "pattern/actions do not match the selected templates"); + return NULL; + } + + /* Merge: spec comes from the per-rule call, last/mask from the template. */ + for (k = 0; k < pt->nb_items; k++) { + combined_items[k].type = pt->pattern[k].type; + combined_items[k].spec = items ? items[k].spec : NULL; + combined_items[k].last = pt->pattern[k].last; + combined_items[k].mask = pt->pattern[k].mask; + } + combined_items[pt->nb_items].type = RTE_FLOW_ITEM_TYPE_END; + + /* Merge: per-rule conf overrides the template conf. */ + for (k = 0; k < at->nb_actions; k++) { + combined_actions[k].type = at->actions[k].type; + combined_actions[k].conf = + (actions && actions[k].conf) ? actions[k].conf : at->actions[k].conf; + } + combined_actions[at->nb_actions].type = RTE_FLOW_ACTION_TYPE_END; + + memset(&rte_attr, 0, sizeof(rte_attr)); + rte_attr.priority = table->flow_attr.priority; + rte_attr.ingress = table->flow_attr.ingress; + rte_attr.egress = table->flow_attr.egress; + rte_attr.transfer = table->transfer; + + /* Translate into roc_npc types. The RSS flow key is stashed in + * npc->flowkey_cfg_state so the engine can restore it at push(). + */ + memset(&in_attr, 0, sizeof(in_attr)); + rc = cnxk_map_flow_data(eth_dev, &rte_attr, combined_items, combined_actions, &in_attr, + in_pattern, in_actions, &in_sample, &npc->flowkey_cfg_state, + &dst_pf_func, &def_action, false, free_allocs, 0); + if (rc == 0) + flow = roc_npc_async_flow_create(npc, queue, table->roc_table, in_pattern, + in_actions, dst_pf_func, def_action, user_data, + &errcode); + else + flow = NULL; + + for (j = 0; j < (int)RTE_DIM(free_allocs) && free_allocs[j]; j++) + plt_free((void *)free_allocs[j]); + + if (rc != 0) { + rte_flow_error_set(error, rc < 0 ? -rc : rc, RTE_FLOW_ERROR_TYPE_ACTION_NUM, NULL, + "Failed to map flow data"); + return NULL; + } + if (flow == NULL) { + rte_flow_error_set(error, errcode < 0 ? -errcode : EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "async flow create failed"); + return NULL; + } + + return (struct rte_flow *)flow; +} + +/* Enqueue a destroy operation. */ +int +cnxk_flow_async_destroy(struct rte_eth_dev *eth_dev, uint32_t queue, + const struct rte_flow_op_attr *attr, struct rte_flow *flow, void *user_data, + struct rte_flow_error *error) +{ + struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); + struct roc_npc *npc = &dev->npc; + int rc; + + RTE_SET_USED(attr); + + rc = roc_npc_async_flow_destroy(npc, queue, (struct roc_npc_template_flow *)flow, + user_data); + if (rc) + return rte_flow_error_set(error, -rc, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "bad queue or flow handle"); + return 0; +} + +/* Flush all pending async operations to hardware in one batch. */ +int +cnxk_flow_push(struct rte_eth_dev *eth_dev, uint32_t queue, struct rte_flow_error *error) +{ + struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); + struct roc_npc *npc = &dev->npc; + int rc; + + rc = roc_npc_flow_push(npc, queue); + if (rc) + return rte_flow_error_set(error, -rc, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "bad queue"); + return 0; +} + +/* Return results for completed ops back to the application. */ +int +cnxk_flow_pull(struct rte_eth_dev *eth_dev, uint32_t queue, struct rte_flow_op_result res[], + uint16_t n_res, struct rte_flow_error *error) +{ + struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); + struct roc_npc *npc = &dev->npc; + uint16_t done = 0; + + /* Drain roc engine results in fixed-size batches, translating each + * roc op result into the rte op result without a per-pull allocation. + */ + while (done < n_res) { + struct roc_npc_flow_op_result rres[64]; + uint16_t batch = RTE_MIN((uint16_t)(n_res - done), (uint16_t)RTE_DIM(rres)); + int got = roc_npc_flow_pull(npc, queue, rres, batch); + uint16_t k; + + if (got < 0) { + if (done != 0) + break; + return rte_flow_error_set(error, -got, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "bad queue"); + } + + for (k = 0; k < (uint16_t)got; k++) { + res[done + k].status = + (rres[k].rc == 0) ? RTE_FLOW_OP_SUCCESS : RTE_FLOW_OP_ERROR; + res[done + k].user_data = rres[k].user_data; + } + done += (uint16_t)got; + + if (got < batch) + break; + } + + return done; +} + +struct rte_flow * +cnxk_flow_async_create_by_index(struct rte_eth_dev *eth_dev, uint32_t queue, + const struct rte_flow_op_attr *attr, + struct rte_flow_template_table *table, uint32_t rule_index, + const struct rte_flow_action actions[], + uint8_t action_template_index, void *user_data, + struct rte_flow_error *error) +{ + RTE_SET_USED(eth_dev); + RTE_SET_USED(queue); + RTE_SET_USED(attr); + RTE_SET_USED(table); + RTE_SET_USED(rule_index); + RTE_SET_USED(actions); + RTE_SET_USED(action_template_index); + RTE_SET_USED(user_data); + + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "index-only insertion needs a match key; NPC is match-based"); + return NULL; +} + +/* Like async_create but with an app-specified index; the app must avoid + * concurrent creates to the same index and use a valid table index. + */ +struct rte_flow * +/* clang-format off */ +cnxk_flow_async_create_by_index_with_pattern(struct rte_eth_dev *eth_dev, + uint32_t queue, const struct rte_flow_op_attr *attr, + struct rte_flow_template_table *tbl_handle, uint32_t rule_index, + const struct rte_flow_item items[], uint8_t pattern_template_index, + const struct rte_flow_action actions[], + uint8_t action_template_index, void *user_data, + struct rte_flow_error *error) +/* clang-format on */ +{ + struct cnxk_flow_table *table = (struct cnxk_flow_table *)tbl_handle; + struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); + struct cnxk_flow_pattern_template *pt; + struct cnxk_flow_actions_template *at; + struct roc_npc_template_flow *flow; + struct roc_npc *npc = &dev->npc; + + uint64_t free_allocs[ROC_NPC_MAX_ACTION_COUNT + ROC_NPC_ITEM_TYPE_END + 1] = {0}; + struct roc_npc_item_info in_pattern[ROC_NPC_ITEM_TYPE_END + 1] = {0}; + struct rte_flow_action combined_actions[ROC_NPC_MAX_ACTION_COUNT]; + struct roc_npc_action in_actions[ROC_NPC_MAX_ACTION_COUNT] = {0}; + struct rte_flow_item combined_items[ROC_NPC_ITEM_TYPE_END + 1]; + struct roc_npc_action_sample in_sample = {0}; + struct rte_flow_attr rte_attr; + struct roc_npc_attr in_attr; + uint32_t flowkey_cfg = 0; + uint16_t dst_pf_func = 0; + uint64_t def_action = 0; + int errcode = 0; + uint16_t k; + int rc, j; + + RTE_SET_USED(attr); + + if (table == NULL) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "bad table"); + return NULL; + } + if (table->insertion_type != RTE_FLOW_TABLE_INSERTION_TYPE_INDEX_WITH_PATTERN) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "table is not index-with-pattern insertion"); + return NULL; + } + if (pattern_template_index >= table->nb_pattern_templates || + action_template_index >= table->nb_actions_templates) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "template index out of range"); + return NULL; + } + if (rule_index >= table->nb_flows) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "rule_index out of range"); + return NULL; + } + + pt = table->pattern_templates[pattern_template_index]; + at = table->actions_templates[action_template_index]; + + if (cnxk_flow_check_items(pt, items) != 0 || cnxk_flow_check_actions(at, actions) != 0) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "pattern/actions do not match the selected templates"); + return NULL; + } + + for (k = 0; k < pt->nb_items; k++) { + combined_items[k].type = pt->pattern[k].type; + combined_items[k].spec = items ? items[k].spec : NULL; + combined_items[k].last = pt->pattern[k].last; + combined_items[k].mask = pt->pattern[k].mask; + } + combined_items[pt->nb_items].type = RTE_FLOW_ITEM_TYPE_END; + + for (k = 0; k < at->nb_actions; k++) { + combined_actions[k].type = at->actions[k].type; + combined_actions[k].conf = + (actions && actions[k].conf) ? actions[k].conf : at->actions[k].conf; + } + combined_actions[at->nb_actions].type = RTE_FLOW_ACTION_TYPE_END; + + memset(&rte_attr, 0, sizeof(rte_attr)); + rte_attr.priority = table->flow_attr.priority; + rte_attr.ingress = table->flow_attr.ingress; + rte_attr.egress = table->flow_attr.egress; + rte_attr.transfer = table->transfer; + + memset(&in_attr, 0, sizeof(in_attr)); + rc = cnxk_map_flow_data(eth_dev, &rte_attr, combined_items, combined_actions, &in_attr, + in_pattern, in_actions, &in_sample, &flowkey_cfg, &dst_pf_func, + &def_action, false, free_allocs, 0); + if (rc == 0) + /* clang-format off */ + flow = roc_npc_async_flow_create_by_index_with_pattern(npc, queue, + table->roc_table, rule_index, in_pattern, in_actions, + user_data, &errcode); + /* clang-format on */ + else + flow = NULL; + + for (j = 0; j < (int)RTE_DIM(free_allocs) && free_allocs[j]; j++) + plt_free((void *)free_allocs[j]); + + if (rc != 0) { + rte_flow_error_set(error, rc < 0 ? -rc : rc, RTE_FLOW_ERROR_TYPE_ACTION_NUM, NULL, + "Failed to map flow data"); + return NULL; + } + if (flow == NULL) { + rte_flow_error_set(error, errcode < 0 ? -errcode : EINVAL, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, "flow parse failed"); + return NULL; + } + + return (struct rte_flow *)flow; +} + +/* Update the actions of an existing flow, keeping key/mask/mcam_id/enable. */ +int +cnxk_flow_async_actions_update(struct rte_eth_dev *eth_dev, uint32_t queue, + const struct rte_flow_op_attr *attr, struct rte_flow *flow, + const struct rte_flow_action actions[], + uint8_t action_template_index, void *user_data, + struct rte_flow_error *error) +{ + struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); + struct cnxk_flow_actions_template *at; + struct roc_npc *npc = &dev->npc; + struct cnxk_flow_table *table; + + uint64_t free_allocs[ROC_NPC_MAX_ACTION_COUNT + ROC_NPC_ITEM_TYPE_END + 1] = {0}; + struct roc_npc_item_info in_pattern[ROC_NPC_ITEM_TYPE_END + 1] = {0}; + struct rte_flow_action combined_actions[ROC_NPC_MAX_ACTION_COUNT]; + struct roc_npc_action in_actions[ROC_NPC_MAX_ACTION_COUNT] = {0}; + struct roc_npc_action_sample in_sample = {0}; + struct rte_flow_item end_pattern[1]; + struct rte_flow_attr rte_attr; + struct roc_npc_attr in_attr; + uint32_t flowkey_cfg = 0; + uint16_t dst_pf_func = 0; + uint64_t def_action = 0; + uint16_t k; + int rc, j; + + RTE_SET_USED(attr); + + if (flow == NULL) + return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "bad flow handle"); + + /* Recover the cnxk table context from the cookie the engine kept for + * this rule's table. + */ + table = roc_npc_async_flow_table_cookie((struct roc_npc_template_flow *)flow); + if (table == NULL) + return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "flow handle has no associated table"); + if (action_template_index >= table->nb_actions_templates) + return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "actions template index out of range"); + + at = table->actions_templates[action_template_index]; + + if (cnxk_flow_check_actions(at, actions) != 0) + return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "actions do not match the selected template"); + + for (k = 0; k < at->nb_actions; k++) { + combined_actions[k].type = at->actions[k].type; + combined_actions[k].conf = + (actions && actions[k].conf) ? actions[k].conf : at->actions[k].conf; + } + combined_actions[at->nb_actions].type = RTE_FLOW_ACTION_TYPE_END; + + end_pattern[0].type = RTE_FLOW_ITEM_TYPE_END; + + memset(&rte_attr, 0, sizeof(rte_attr)); + rte_attr.priority = table->flow_attr.priority; + rte_attr.ingress = table->flow_attr.ingress; + rte_attr.egress = table->flow_attr.egress; + rte_attr.transfer = table->transfer; + + memset(&in_attr, 0, sizeof(in_attr)); + rc = cnxk_map_flow_data(eth_dev, &rte_attr, end_pattern, combined_actions, &in_attr, + in_pattern, in_actions, &in_sample, &flowkey_cfg, &dst_pf_func, + &def_action, false, free_allocs, 0); + if (rc == 0) + /* clang-format off */ + rc = roc_npc_async_flow_actions_update(npc, queue, + (struct roc_npc_template_flow *)flow, in_actions, user_data); + /* clang-format on */ + + for (j = 0; j < (int)RTE_DIM(free_allocs) && free_allocs[j]; j++) + plt_free((void *)free_allocs[j]); + + if (rc != 0) + return rte_flow_error_set(error, rc < 0 ? -rc : EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "failed to update flow actions"); + + return 0; +} static int cnxk_flow_dev_dump(struct rte_eth_dev *eth_dev, struct rte_flow *flow, FILE *file, diff --git a/drivers/net/cnxk/cnxk_flow.h b/drivers/net/cnxk/cnxk_flow.h index 2986ea81d1..2202f4f3b8 100644 --- a/drivers/net/cnxk/cnxk_flow.h +++ b/drivers/net/cnxk/cnxk_flow.h @@ -48,4 +48,81 @@ int cnxk_flow_dev_dump_common(struct rte_eth_dev *eth_dev, struct rte_flow *flow struct rte_flow_error *error, bool is_rep); int cnxk_mtr_destroy(struct rte_eth_dev *eth_dev, uint32_t mtr_id); +int cnxk_flow_configure(struct rte_eth_dev *eth_dev, const struct rte_flow_port_attr *port_attr, + uint16_t nb_queue, const struct rte_flow_queue_attr *queue_attr[], + struct rte_flow_error *err); + +/* clang-format off */ +struct rte_flow_pattern_template *cnxk_flow_pattern_template_create(struct rte_eth_dev *eth_dev, + const struct rte_flow_pattern_template_attr *attr, + const struct rte_flow_item pattern[], + struct rte_flow_error *error); +/* clang-format on */ + +int cnxk_flow_pattern_template_destroy(struct rte_eth_dev *eth_dev, + struct rte_flow_pattern_template *templ, + struct rte_flow_error *error); + +/* clang-format off */ +struct rte_flow_actions_template *cnxk_flow_actions_template_create(struct rte_eth_dev *eth_dev, + const struct rte_flow_actions_template_attr *attr, + const struct rte_flow_action actions[], + const struct rte_flow_action masks[], + struct rte_flow_error *error); +/* clang-format on */ + +int cnxk_flow_actions_template_destroy(struct rte_eth_dev *eth_dev, + struct rte_flow_actions_template *templ, + struct rte_flow_error *error); + +/* clang-format off */ +struct rte_flow_template_table *cnxk_flow_template_table_create(struct rte_eth_dev *eth_dev, + const struct rte_flow_template_table_attr *table_attr, + struct rte_flow_pattern_template *pattern_templates[], + uint8_t nb_pattern_templates, + struct rte_flow_actions_template *actions_templates[], + uint8_t nb_actions_templates, + struct rte_flow_error *error); +/* clang-format on */ + +int cnxk_flow_template_table_destroy(struct rte_eth_dev *eth_dev, + struct rte_flow_template_table *tbl_handle, + struct rte_flow_error *error); +struct rte_flow * +cnxk_flow_async_create(struct rte_eth_dev *eth_dev, uint32_t queue, + const struct rte_flow_op_attr *attr, struct rte_flow_template_table *table, + const struct rte_flow_item items[], uint8_t pattern_template_index, + const struct rte_flow_action actions[], uint8_t action_template_index, + void *user_data, struct rte_flow_error *error); +int cnxk_flow_async_destroy(struct rte_eth_dev *eth_dev, uint32_t queue, + const struct rte_flow_op_attr *attr, struct rte_flow *flow, + void *user_data, struct rte_flow_error *error); + +int cnxk_flow_push(struct rte_eth_dev *eth_dev, uint32_t queue, struct rte_flow_error *error); +int cnxk_flow_pull(struct rte_eth_dev *eth_dev, uint32_t queue, struct rte_flow_op_result res[], + uint16_t n_res, struct rte_flow_error *error); + +struct rte_flow *cnxk_flow_async_create_by_index(struct rte_eth_dev *eth_dev, uint32_t queue, + const struct rte_flow_op_attr *attr, + struct rte_flow_template_table *table, + uint32_t rule_index, + const struct rte_flow_action actions[], + uint8_t action_template_index, void *user_data, + struct rte_flow_error *error); + +/* clang-format off */ +struct rte_flow *cnxk_flow_async_create_by_index_with_pattern(struct rte_eth_dev *eth_dev, + uint32_t queue, const struct rte_flow_op_attr *attr, + struct rte_flow_template_table *tbl_handle, uint32_t rule_index, + const struct rte_flow_item items[], uint8_t pattern_template_index, + const struct rte_flow_action actions[], + uint8_t action_template_index, void *user_data, + struct rte_flow_error *error); +/* clang-format on */ + +int cnxk_flow_async_actions_update(struct rte_eth_dev *eth_dev, uint32_t queue, + const struct rte_flow_op_attr *attr, struct rte_flow *flow, + const struct rte_flow_action actions[], + uint8_t action_template_index, void *user_data, + struct rte_flow_error *error); #endif /* __CNXK_RTE_FLOW_H__ */ -- 2.43.0
