From: Oleksandr Kolomeiets <okl-...@napatech.com>
Add basic API for initialization resources required by flow filter API
Signed-off-by: Oleksandr Kolomeiets <okl-...@napatech.com>
---
drivers/net/ntnic/include/flow_api.h | 82 ++++++
drivers/net/ntnic/include/flow_api_engine.h | 43 +++
drivers/net/ntnic/include/hw_mod_backend.h | 3 +
.../ntnic/include/stream_binary_flow_api.h | 22 ++
drivers/net/ntnic/meson.build | 2 +
drivers/net/ntnic/nthw/flow_api/flow_api.c | 245 ++++++++++++++++++
.../ntnic/nthw/flow_api/flow_api_nic_setup.h | 6 +
drivers/net/ntnic/nthw/flow_api/flow_filter.c | 17 +-
drivers/net/ntnic/nthw/flow_api/flow_kcc.c | 19 ++
drivers/net/ntnic/nthw/flow_api/flow_km.c | 19 ++
10 files changed, 453 insertions(+), 5 deletions(-)
create mode 100644 drivers/net/ntnic/include/flow_api_engine.h
create mode 100644 drivers/net/ntnic/include/stream_binary_flow_api.h
create mode 100644 drivers/net/ntnic/nthw/flow_api/flow_kcc.c
create mode 100644 drivers/net/ntnic/nthw/flow_api/flow_km.c
diff --git a/drivers/net/ntnic/include/flow_api.h
b/drivers/net/ntnic/include/flow_api.h
index 6a2277c2ca..bad1f72868 100644
--- a/drivers/net/ntnic/include/flow_api.h
+++ b/drivers/net/ntnic/include/flow_api.h
@@ -6,14 +6,96 @@
#ifndef _FLOW_API_H_
#define _FLOW_API_H_
+#include <pthread.h>
+
#include "ntlog.h"
+#include "flow_api_engine.h"
#include "hw_mod_backend.h"
+#include "stream_binary_flow_api.h"
+
+/*
+ * Flow NIC and Eth port device management
+ */
+
+struct hw_mod_resource_s {
+ uint8_t *alloc_bm; /* allocation bitmap */
+ uint32_t *ref; /* reference counter for each resource element */
+ uint32_t resource_count;/* number of total available entries */
+};
+
+/*
+ * Device Management API
+ */
+int flow_delete_eth_dev(struct flow_eth_dev *eth_dev);
+
+struct flow_eth_dev {
+ /* NIC that owns this port device */
+ struct flow_nic_dev *ndev;
+ /* NIC port id */
+ uint8_t port;
+
+ /* 0th for exception */
+ struct flow_queue_id_s rx_queue[FLOW_MAX_QUEUES + 1];
+
+ /* VSWITCH has exceptions sent on queue 0 per design */
+ int num_queues;
+
+ struct flow_eth_dev *next;
+};
/* registered NIC backends */
struct flow_nic_dev {
+ struct hw_mod_resource_s res[RES_COUNT];/* raw NIC resource allocation
table */
+ void *km_res_handle;
+ void *kcc_res_handle;
+
+ uint32_t flow_unique_id_counter;
+ /* linked list of all flows created on this NIC */
+ struct flow_handle *flow_base;
+
/* NIC backend API */
struct flow_api_backend_s be;
+ /* linked list of created eth-port devices on this NIC */
+ struct flow_eth_dev *eth_base;
+ pthread_mutex_t mtx;
+
+ /* next NIC linked list */
+ struct flow_nic_dev *next;
};
+/*
+ * Resources
+ */
+
+extern const char *dbg_res_descr[];
+
+#define flow_nic_unset_bit(arr, x)
\
+ do {
\
+ size_t _temp_x = (x);
\
+ arr[_temp_x / 8] &= (uint8_t)(~(1 << (_temp_x % 8)));
\
+ } while (0)
+
+#define flow_nic_is_bit_set(arr, x)
\
+ ({
\
+ size_t _temp_x = (x);
\
+ (arr[_temp_x / 8] & (uint8_t)(1 << (_temp_x % 8)));
\
+ })
+
+#define flow_nic_mark_resource_unused(_ndev, res_type, index)
\
+ do {
\
+ typeof(res_type) _temp_res_type = (res_type);
\
+ size_t _temp_index = (index);
\
+ NT_LOG(DBG, FILTER, "mark resource unused: %s idx %zu\n",
\
+ dbg_res_descr[_temp_res_type], _temp_index);
\
+ flow_nic_unset_bit((_ndev)->res[_temp_res_type].alloc_bm,
_temp_index); \
+ } while (0)
+
+#define flow_nic_is_resource_used(_ndev, res_type, index)
\
+ (!!flow_nic_is_bit_set((_ndev)->res[res_type].alloc_bm, index))
+
+void flow_nic_free_resource(struct flow_nic_dev *ndev, enum res_type_e
res_type, int idx);
+
+int flow_nic_deref_resource(struct flow_nic_dev *ndev, enum res_type_e
res_type, int index);
+
#endif
diff --git a/drivers/net/ntnic/include/flow_api_engine.h
b/drivers/net/ntnic/include/flow_api_engine.h
new file mode 100644
index 0000000000..724b68c3e8
--- /dev/null
+++ b/drivers/net/ntnic/include/flow_api_engine.h
@@ -0,0 +1,43 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Napatech A/S
+ */
+
+#ifndef _FLOW_API_ENGINE_H_
+#define _FLOW_API_ENGINE_H_
+
+/*
+ * Resource management
+ * These are free resources in FPGA
+ * Other FPGA memory lists are linked to one of these
+ * and will implicitly follow them
+ */
+enum res_type_e {
+ RES_QUEUE,
+ RES_CAT_CFN,
+ RES_CAT_COT,
+ RES_CAT_EXO,
+ RES_CAT_LEN,
+ RES_KM_FLOW_TYPE,
+ RES_KM_CATEGORY,
+ RES_HSH_RCP,
+ RES_PDB_RCP,
+ RES_QSL_RCP,
+ RES_QSL_QST,
+ RES_SLC_LR_RCP,
+
+ RES_FLM_FLOW_TYPE,
+ RES_FLM_RCP,
+ RES_TPE_RCP,
+ RES_TPE_EXT,
+ RES_TPE_RPL,
+ RES_SCRUB_RCP,
+ RES_COUNT,
+ RES_INVALID
+};
+
+void km_free_ndev_resource_management(void **handle);
+
+void kcc_free_ndev_resource_management(void **handle);
+
+#endif /* _FLOW_API_ENGINE_H_ */
diff --git a/drivers/net/ntnic/include/hw_mod_backend.h
b/drivers/net/ntnic/include/hw_mod_backend.h
index 29c33306d1..3c6c15c896 100644
--- a/drivers/net/ntnic/include/hw_mod_backend.h
+++ b/drivers/net/ntnic/include/hw_mod_backend.h
@@ -8,6 +8,8 @@
#include <stdbool.h>
+#include "ntlog.h"
+
#include "hw_mod_cat_v18.h"
#include "hw_mod_cat_v21.h"
#include "hw_mod_flm_v25.h"
@@ -275,6 +277,7 @@ struct flow_api_backend_ops {
struct flow_api_backend_s {
void *be_dev;
+ const struct flow_api_backend_ops *iface;
};
#endif /* _HW_MOD_BACKEND_H_ */
diff --git a/drivers/net/ntnic/include/stream_binary_flow_api.h
b/drivers/net/ntnic/include/stream_binary_flow_api.h
new file mode 100644
index 0000000000..10529b8843
--- /dev/null
+++ b/drivers/net/ntnic/include/stream_binary_flow_api.h
@@ -0,0 +1,22 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Napatech A/S
+ */
+
+#ifndef _STREAM_BINARY_FLOW_API_H_
+#define _STREAM_BINARY_FLOW_API_H_
+
+/*
+ * Flow frontend for binary programming interface
+ */
+
+#define FLOW_MAX_QUEUES 128
+
+struct flow_queue_id_s {
+ int id;
+ int hw_id;
+};
+
+struct flow_eth_dev; /* port device */
+
+#endif /* _STREAM_BINARY_FLOW_API_H_ */
diff --git a/drivers/net/ntnic/meson.build b/drivers/net/ntnic/meson.build
index e236b82b36..15549e1c94 100644
--- a/drivers/net/ntnic/meson.build
+++ b/drivers/net/ntnic/meson.build
@@ -46,6 +46,8 @@ sources = files(
'nthw/flow_api/flow_api.c',
'nthw/flow_api/flow_backend/flow_backend.c',
'nthw/flow_api/flow_filter.c',
+ 'nthw/flow_api/flow_kcc.c',
+ 'nthw/flow_api/flow_km.c',
'nthw/flow_filter/flow_nthw_cat.c',
'nthw/flow_filter/flow_nthw_csu.c',
'nthw/flow_filter/flow_nthw_flm.c',
diff --git a/drivers/net/ntnic/nthw/flow_api/flow_api.c
b/drivers/net/ntnic/nthw/flow_api/flow_api.c
index 9671a20e0b..92b8d083e0 100644
--- a/drivers/net/ntnic/nthw/flow_api/flow_api.c
+++ b/drivers/net/ntnic/nthw/flow_api/flow_api.c
@@ -3,11 +3,256 @@
* Copyright(c) 2023 Napatech A/S
*/
+#include "flow_api_engine.h"
#include "flow_api_nic_setup.h"
#include "ntnic_mod_reg.h"
#include "flow_filter.h"
+const char *dbg_res_descr[] = {
+ /* RES_QUEUE */ "RES_QUEUE",
+ /* RES_CAT_CFN */ "RES_CAT_CFN",
+ /* RES_CAT_COT */ "RES_CAT_COT",
+ /* RES_CAT_EXO */ "RES_CAT_EXO",
+ /* RES_CAT_LEN */ "RES_CAT_LEN",
+ /* RES_KM_FLOW_TYPE */ "RES_KM_FLOW_TYPE",
+ /* RES_KM_CATEGORY */ "RES_KM_CATEGORY",
+ /* RES_HSH_RCP */ "RES_HSH_RCP",
+ /* RES_PDB_RCP */ "RES_PDB_RCP",
+ /* RES_QSL_RCP */ "RES_QSL_RCP",
+ /* RES_QSL_LTX */ "RES_QSL_LTX",
+ /* RES_QSL_QST */ "RES_QSL_QST",
+ /* RES_SLC_LR_RCP */ "RES_SLC_LR_RCP",
+ /* RES_FLM_FLOW_TYPE */ "RES_FLM_FLOW_TYPE",
+ /* RES_FLM_RCP */ "RES_FLM_RCP",
+ /* RES_TPE_RCP */ "RES_TPE_RCP",
+ /* RES_TPE_EXT */ "RES_TPE_EXT",
+ /* RES_TPE_RPL */ "RES_TPE_RPL",
+ /* RES_COUNT */ "RES_COUNT",
+ /* RES_INVALID */ "RES_INVALID"
+};
+
+static struct flow_nic_dev *dev_base;
+static pthread_mutex_t base_mtx = PTHREAD_MUTEX_INITIALIZER;
+
+void flow_nic_free_resource(struct flow_nic_dev *ndev, enum res_type_e
res_type, int idx)
+{
+ flow_nic_mark_resource_unused(ndev, res_type, idx);
+}
+
+int flow_nic_deref_resource(struct flow_nic_dev *ndev, enum res_type_e
res_type, int index)
+{
+ NT_LOG(DBG, FILTER, "De-reference resource %s idx %i (before ref cnt
%i)\n",
+ dbg_res_descr[res_type], index, ndev->res[res_type].ref[index]);
+ assert(flow_nic_is_resource_used(ndev, res_type, index));
+ assert(ndev->res[res_type].ref[index]);
+ /* deref */
+ ndev->res[res_type].ref[index]--;
+
+ if (!ndev->res[res_type].ref[index])
+ flow_nic_free_resource(ndev, res_type, index);
+
+ return !!ndev->res[res_type].ref[index];/* if 0 resource has been freed
*/
+}
+
+/*
+ * Device Management API
+ */
+
+static int nic_remove_eth_port_dev(struct flow_nic_dev *ndev, struct
flow_eth_dev *eth_dev)
+{
+ struct flow_eth_dev *dev = ndev->eth_base, *prev = NULL;
+
+ while (dev) {
+ if (dev == eth_dev) {
+ if (prev)
+ prev->next = dev->next;
+
+ else
+ ndev->eth_base = dev->next;
+
+ return 0;
+ }
+
+ prev = dev;
+ dev = dev->next;
+ }
+
+ return -1;
+}
+
+static void flow_ndev_reset(struct flow_nic_dev *ndev)
+{
+ /* Delete all eth-port devices created on this NIC device */
+ while (ndev->eth_base)
+ flow_delete_eth_dev(ndev->eth_base);
+
+ km_free_ndev_resource_management(&ndev->km_res_handle);
+ kcc_free_ndev_resource_management(&ndev->kcc_res_handle);
+
+ ndev->flow_unique_id_counter = 0;
+
+#ifdef FLOW_DEBUG
+ /*
+ * free all resources default allocated, initially for this NIC DEV
+ * Is not really needed since the bitmap will be freed in a sec.
Therefore
+ * only in debug mode
+ */
+
+ /* Check if all resources has been released */
+ NT_LOG(DBG, FILTER, "Delete NIC DEV Adaptor %i\n", ndev->adapter_no);
+
+ for (unsigned int i = 0; i < RES_COUNT; i++) {
+ int err = 0;
+#if defined(FLOW_DEBUG)
+ NT_LOG(DBG, FILTER, "RES state for: %s\n", dbg_res_descr[i]);
+#endif
+
+ for (unsigned int ii = 0; ii < ndev->res[i].resource_count;
ii++) {
+ int ref = ndev->res[i].ref[ii];
+ int used = flow_nic_is_resource_used(ndev, i, ii);
+
+ if (ref || used) {
+ NT_LOG(DBG, FILTER, " [%i]: ref cnt %i, used
%i\n", ii, ref,
+ used);
+ err = 1;
+ }
+ }
+
+ if (err)
+ NT_LOG(DBG, FILTER, "ERROR - some resources not
freed\n");
+ }
+
+#endif
+}
+
+int flow_delete_eth_dev(struct flow_eth_dev *eth_dev)
+{
+ struct flow_nic_dev *ndev = eth_dev->ndev;
+
+ if (!ndev) {
+ /* Error invalid nic device */
+ return -1;
+ }
+
+ NT_LOG(DBG, FILTER, "Delete eth-port device %p, port %i\n", eth_dev,
eth_dev->port);
+
+#ifdef FLOW_DEBUG
+ ndev->be.iface->set_debug_mode(ndev->be.be_dev,
FLOW_BACKEND_DEBUG_MODE_WRITE);
+#endif
+
+ /* delete all created flows from this device */
+ pthread_mutex_lock(&ndev->mtx);
+
+#ifdef FLOW_DEBUG
+ ndev->be.iface->set_debug_mode(ndev->be.be_dev,
FLOW_BACKEND_DEBUG_MODE_NONE);
+#endif
+
+#ifndef SCATTER_GATHER
+
+ /* free rx queues */
+ for (int i = 0; i < eth_dev->num_queues; i++) {
+ ndev->be.iface->free_rx_queue(ndev->be.be_dev,
eth_dev->rx_queue[i].hw_id);
+ flow_nic_deref_resource(ndev, RES_QUEUE,
eth_dev->rx_queue[i].id);
+ }
+
+#endif
+
+ /* take eth_dev out of ndev list */
+ if (nic_remove_eth_port_dev(ndev, eth_dev) != 0)
+ NT_LOG(ERR, FILTER, "ERROR : eth_dev %p not found\n", eth_dev);
+
+ pthread_mutex_unlock(&ndev->mtx);
+
+ /* free eth_dev */
+ free(eth_dev);
+
+ return 0;
+}
+
+/*
+ * Flow API NIC Setup
+ * Flow backend creation function - register and initialize common backend API
to FPA modules
+ */
+
+static void done_resource_elements(struct flow_nic_dev *ndev, enum res_type_e
res_type)
+{
+ assert(ndev);
+
+ if (ndev->res[res_type].alloc_bm)
+ free(ndev->res[res_type].alloc_bm);
+}
+
+static int list_remove_flow_nic(struct flow_nic_dev *ndev)
+{
+ pthread_mutex_lock(&base_mtx);
+ struct flow_nic_dev *nic_dev = dev_base, *prev = NULL;
+
+ while (nic_dev) {
+ if (nic_dev == ndev) {
+ if (prev)
+ prev->next = nic_dev->next;
+
+ else
+ dev_base = nic_dev->next;
+
+ pthread_mutex_unlock(&base_mtx);
+ return 0;
+ }
+
+ prev = nic_dev;
+ nic_dev = nic_dev->next;
+ }
+
+ pthread_mutex_unlock(&base_mtx);
+ return -1;
+}
+
+struct flow_nic_dev *flow_api_create(uint8_t adapter_no, const struct
flow_api_backend_ops *be_if,
+ void *be_dev)
+{
+ (void)adapter_no;
+
+ if (!be_if || be_if->version != 1) {
+ NT_LOG(DBG, FILTER, "ERR: %s\n", __func__);
+ return NULL;
+ }
+
+ struct flow_nic_dev *ndev = calloc(1, sizeof(struct flow_nic_dev));
+
+ if (!ndev) {
+ NT_LOG(ERR, FILTER, "ERROR: calloc failed\n");
+ return NULL;
+ }
+
+ /*
+ * To dump module initialization writes use
+ * FLOW_BACKEND_DEBUG_MODE_WRITE
+ * then remember to set it ...NONE afterwards again
+ */
+ be_if->set_debug_mode(be_dev, FLOW_BACKEND_DEBUG_MODE_NONE);
+
+ return ndev;
+}
+
+int flow_api_done(struct flow_nic_dev *ndev)
+{
+ NT_LOG(DBG, FILTER, "FLOW API DONE\n");
+
+ if (ndev) {
+ flow_ndev_reset(ndev);
+
+ /* delete resource management allocations for this ndev */
+ for (int i = 0; i < RES_COUNT; i++)
+ done_resource_elements(ndev, i);
+
+ list_remove_flow_nic(ndev);
+ free(ndev);
+ }
+
+ return 0;
+}
+
void *flow_api_get_be_dev(struct flow_nic_dev *ndev)
{
if (!ndev) {
diff --git a/drivers/net/ntnic/nthw/flow_api/flow_api_nic_setup.h
b/drivers/net/ntnic/nthw/flow_api/flow_api_nic_setup.h
index da083f050a..eeb070c569 100644
--- a/drivers/net/ntnic/nthw/flow_api/flow_api_nic_setup.h
+++ b/drivers/net/ntnic/nthw/flow_api/flow_api_nic_setup.h
@@ -9,6 +9,12 @@
#include "hw_mod_backend.h"
#include "flow_api.h"
+/*
+ * Flow capable NIC backend - creating flow api instance for adapter nr
(backend)
+ */
+struct flow_nic_dev *flow_api_create(uint8_t adapter_no, const struct
flow_api_backend_ops *be_if,
+ void *be_dev);
+int flow_api_done(struct flow_nic_dev *dev);
void *flow_api_get_be_dev(struct flow_nic_dev *dev);
#endif /* __FLOW_API_NIC_SETUP_H__ */
diff --git a/drivers/net/ntnic/nthw/flow_api/flow_filter.c
b/drivers/net/ntnic/nthw/flow_api/flow_filter.c
index e822ba7df9..69e5972f5d 100644
--- a/drivers/net/ntnic/nthw/flow_api/flow_filter.c
+++ b/drivers/net/ntnic/nthw/flow_api/flow_filter.c
@@ -9,10 +9,8 @@
int flow_filter_init(nthw_fpga_t *p_fpga, struct flow_nic_dev **p_flow_device,
int adapter_no)
{
- (void)p_flow_device;
- (void)adapter_no;
-
void *be_dev = NULL;
+ struct flow_nic_dev *flow_nic;
const struct flow_backend_ops *flow_backend_ops =
get_flow_backend_ops();
@@ -22,8 +20,17 @@ int flow_filter_init(nthw_fpga_t *p_fpga, struct
flow_nic_dev **p_flow_device, i
}
NT_LOG(DBG, FILTER, "Initializing flow filter api\n");
- flow_backend_ops->bin_flow_backend_init(p_fpga, &be_dev);
+ const struct flow_api_backend_ops *iface =
+ flow_backend_ops->bin_flow_backend_init(p_fpga, &be_dev);
+
+ flow_nic = flow_api_create((uint8_t)adapter_no, iface, be_dev);
+
+ if (!flow_nic) {
+ *p_flow_device = NULL;
+ return -1;
+ }
+ *p_flow_device = flow_nic;
return 0;
}
@@ -31,7 +38,7 @@ int flow_filter_done(struct flow_nic_dev *dev)
{
void *be_dev = flow_api_get_be_dev(dev);
- int res = 0;
+ int res = flow_api_done(dev);
if (be_dev) {
const struct flow_backend_ops *flow_backend_ops =
get_flow_backend_ops();
diff --git a/drivers/net/ntnic/nthw/flow_api/flow_kcc.c
b/drivers/net/ntnic/nthw/flow_api/flow_kcc.c
new file mode 100644
index 0000000000..dc5e5474c1
--- /dev/null
+++ b/drivers/net/ntnic/nthw/flow_api/flow_kcc.c
@@ -0,0 +1,19 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Napatech A/S
+ */
+
+#include <stdlib.h>
+
+#include "hw_mod_backend.h"
+#include "flow_api_engine.h"
+
+void kcc_free_ndev_resource_management(void **handle)
+{
+ if (*handle) {
+ free(*handle);
+ NT_LOG(DBG, FILTER, "Free NIC DEV KCC-CAM record manager\n");
+ }
+
+ *handle = NULL;
+}
diff --git a/drivers/net/ntnic/nthw/flow_api/flow_km.c
b/drivers/net/ntnic/nthw/flow_api/flow_km.c
new file mode 100644
index 0000000000..1ba7ed7da2
--- /dev/null
+++ b/drivers/net/ntnic/nthw/flow_api/flow_km.c
@@ -0,0 +1,19 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Napatech A/S
+ */
+
+#include <stdlib.h>
+
+#include "hw_mod_backend.h"
+#include "flow_api_engine.h"
+
+void km_free_ndev_resource_management(void **handle)
+{
+ if (*handle) {
+ free(*handle);
+ NT_LOG(DBG, FILTER, "Free NIC DEV CAM and TCAM record
manager\n");
+ }
+
+ *handle = NULL;
+}
--
2.45.0
