Split control and datapath to make datapath substitutable and
possibly reusable with alternative control path.
libefx-based Tx datapath is bound to libefx control path, but
other datapaths should be possible to use with alternative
control path(s).
Signed-off-by: Andrew Rybchenko <arybche...@solarflare.com>
---
doc/guides/nics/sfc_efx.rst | 8 ++
drivers/net/sfc/sfc.h | 1 +
drivers/net/sfc/sfc_dp.c | 4 +-
drivers/net/sfc/sfc_dp.h | 1 +
drivers/net/sfc/sfc_dp_tx.h | 155 ++++++++++++++++++++++
drivers/net/sfc/sfc_ethdev.c | 41 +++++-
drivers/net/sfc/sfc_ev.c | 50 ++++++--
drivers/net/sfc/sfc_ev.h | 8 +-
drivers/net/sfc/sfc_kvargs.c | 1 +
drivers/net/sfc/sfc_kvargs.h | 4 +
drivers/net/sfc/sfc_tso.c | 22 ++--
drivers/net/sfc/sfc_tx.c | 298 ++++++++++++++++++++++++++++++++-----------
drivers/net/sfc/sfc_tx.h | 95 +++++++++-----
13 files changed, 557 insertions(+), 131 deletions(-)
create mode 100644 drivers/net/sfc/sfc_dp_tx.h
diff --git a/doc/guides/nics/sfc_efx.rst b/doc/guides/nics/sfc_efx.rst
index 0aa6740..e864ccc 100644
--- a/doc/guides/nics/sfc_efx.rst
+++ b/doc/guides/nics/sfc_efx.rst
@@ -191,6 +191,14 @@ boolean parameters value.
more efficient than libefx-based and provides richer packet type
classification, but lacks Rx scatter support.
+- ``tx_datapath`` [auto|efx] (default **auto**)
+
+ Choose transmit datapath implementation.
+ **auto** allows the driver itself to make a choice based on firmware
+ features available and required by the datapath implementation.
+ **efx** chooses libefx-based datapath which supports VLAN insertion
+ (full-feature firmware variant only), TSO and multi-segment mbufs.
+
- ``perf_profile`` [auto|throughput|low-latency] (default **throughput**)
Choose hardware tunning to be optimized for either throughput or
diff --git a/drivers/net/sfc/sfc.h b/drivers/net/sfc/sfc.h
index 2512f2e..893eafe 100644
--- a/drivers/net/sfc/sfc.h
+++ b/drivers/net/sfc/sfc.h
@@ -212,6 +212,7 @@ struct sfc_adapter {
#endif
const struct sfc_dp_rx *dp_rx;
+ const struct sfc_dp_tx *dp_tx;
};
/*
diff --git a/drivers/net/sfc/sfc_dp.c b/drivers/net/sfc/sfc_dp.c
index c4d5fb3..47bd500 100644
--- a/drivers/net/sfc/sfc_dp.c
+++ b/drivers/net/sfc/sfc_dp.c
@@ -76,7 +76,9 @@ struct sfc_dp *
if (sfc_dp_find_by_name(head, entry->type, entry->name) != NULL) {
rte_log(RTE_LOG_ERR, RTE_LOGTYPE_PMD,
"sfc %s dapapath '%s' already registered\n",
- entry->type == SFC_DP_RX ? "Rx" : "unknown",
+ entry->type == SFC_DP_RX ? "Rx" :
+ entry->type == SFC_DP_TX ? "Tx" :
+ "unknown",
entry->name);
return EEXIST;
}
diff --git a/drivers/net/sfc/sfc_dp.h b/drivers/net/sfc/sfc_dp.h
index 39e1e70..d3e7007 100644
--- a/drivers/net/sfc/sfc_dp.h
+++ b/drivers/net/sfc/sfc_dp.h
@@ -52,6 +52,7 @@
enum sfc_dp_type {
SFC_DP_RX = 0, /**< Receive datapath */
+ SFC_DP_TX, /**< Transmit datapath */
};
/** Datapath definition */
diff --git a/drivers/net/sfc/sfc_dp_tx.h b/drivers/net/sfc/sfc_dp_tx.h
new file mode 100644
index 0000000..4879db5
--- /dev/null
+++ b/drivers/net/sfc/sfc_dp_tx.h
@@ -0,0 +1,155 @@
+/*-
+ * Copyright (c) 2016 Solarflare Communications Inc.
+ * All rights reserved.
+ *
+ * This software was jointly developed between OKTET Labs (under contract
+ * for Solarflare) and Solarflare Communications, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _SFC_DP_TX_H
+#define _SFC_DP_TX_H
+
+#include <rte_ethdev.h>
+
+#include "sfc_dp.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct sfc_dp_txq;
+
+/**
+ * Callback to get control path transmit queue by datapath transmit queue
+ * handle.
+ */
+typedef void * (sfc_dp_txq_get_ctrl_t)(struct sfc_dp_txq *dp_txq);
+
+/** Datapath transmit queue operations */
+struct sfc_dp_txq_ops {
+ sfc_dp_txq_get_ctrl_t *get_ctrl;
+};
+
+/**
+ * Generic transmit queue information used on data path.
+ * It must be kept as small as it is possible since it is built into
+ * the structure used on datapath.
+ */
+struct sfc_dp_txq {
+ const struct sfc_dp_txq_ops *ops;
+};
+
+/**
+ * Datapath transmit queue creation arguments.
+ *
+ * The structure is used just to pass information from control path to
+ * datapath. It could be just function arguments, but it would be hardly
+ * readable.
+ */
+struct sfc_dp_tx_qcreate_args {
+ /** Minimum number of unused Tx descriptors to do reap */
+ unsigned int free_thresh;
+ /** Transmit queue configuration flags */
+ unsigned int flags;
+ /** Tx queue size */
+ unsigned int txq_entries;
+};
+
+/**
+ * Allocate and initalize datapath transmit queue.
+ *
+ * @param ctrl Control path Tx queue opaque handle
+ * @param exception Datapath exception handler to bail out to control path
+ * @param socket_id Socket ID to allocate memory
+ * @param args Tx queue details wrapped in structure
+ * @param dp_txqp Location for generic datapath transmit queue pointer
+ *
+ * @return 0 or positive errno.
+ */
+typedef int (sfc_dp_tx_qcreate_t)(void *ctrl, sfc_dp_exception_t *exception,
+ int socket_id,
+ const struct sfc_dp_tx_qcreate_args *args,
+ struct sfc_dp_txq **dp_txqp);
+
+/**
+ * Free resources allocated for datapath transmit queue.
+ */
+typedef void (sfc_dp_tx_qdestroy_t)(struct sfc_dp_txq *dp_txq);
+
+/**
+ * Transmit queue start callback.
+ *
+ * It handovers EvQ to the datapath.
+ */
+typedef int (sfc_dp_tx_qstart_t)(struct sfc_dp_txq *dp_txq,
+ unsigned int evq_read_ptr,
+ unsigned int txq_desc_index);
+
+/**
+ * Transmit queue stop function called before the queue flush.
+ *
+ * It returns EvQ to the control path.
+ */
+typedef void (sfc_dp_tx_qstop_t)(struct sfc_dp_txq *dp_txq,
+ unsigned int *evq_read_ptr);
+
+/**
+ * Transmit queue function called after the queue flush.
+ */
+typedef void (sfc_dp_tx_qreap_t)(struct sfc_dp_txq *dp_txq);
+
+/** Transmit datapath definition */
+struct sfc_dp_tx {
+ struct sfc_dp dp;
+
+ sfc_dp_tx_qcreate_t *qcreate;
+ sfc_dp_tx_qdestroy_t *qdestroy;
+ sfc_dp_tx_qstart_t *qstart;
+ sfc_dp_tx_qstop_t *qstop;
+ sfc_dp_tx_qreap_t *qreap;
+ eth_tx_burst_t pkt_burst;
+};
+
+static inline struct sfc_dp_tx *
+sfc_dp_find_tx_by_name(struct sfc_dp_list *head, const char *name)
+{
+ struct sfc_dp *p = sfc_dp_find_by_name(head, SFC_DP_TX, name);
+
+ return (p == NULL) ? NULL : container_of(p, struct sfc_dp_tx, dp);
+}
+
+static inline struct sfc_dp_tx *
+sfc_dp_find_tx_by_caps(struct sfc_dp_list *head, unsigned int avail_caps)
+{
+ struct sfc_dp *p = sfc_dp_find_by_caps(head, SFC_DP_TX, avail_caps);
+
+ return (p == NULL) ? NULL : container_of(p, struct sfc_dp_tx, dp);
+}
+
+extern struct sfc_dp_tx sfc_efx_tx;
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* _SFC_DP_TX_H */
diff --git a/drivers/net/sfc/sfc_ethdev.c b/drivers/net/sfc/sfc_ethdev.c
index f6562f9..c6db730 100644
--- a/drivers/net/sfc/sfc_ethdev.c
+++ b/drivers/net/sfc/sfc_ethdev.c
@@ -416,7 +416,7 @@
if (rc != 0)
goto fail_tx_qinit;
- dev->data->tx_queues[tx_queue_id] = sa->txq_info[tx_queue_id].txq;
+ dev->data->tx_queues[tx_queue_id] = sa->txq_info[tx_queue_id].txq->dp;
sfc_adapter_unlock(sa);
return 0;
@@ -1232,6 +1232,7 @@
struct sfc_adapter *sa = dev->data->dev_private;
unsigned int avail_caps = 0;
const char *rx_name = NULL;
+ const char *tx_name = NULL;
int rc;
if (sa == NULL || sa->state == SFC_ADAPTER_UNINITIALIZED)
@@ -1279,12 +1280,45 @@
dev->rx_pkt_burst = sa->dp_rx->pkt_burst;
- dev->tx_pkt_burst = sfc_xmit_pkts;
+ rc = sfc_kvargs_process(sa, SFC_KVARG_TX_DATAPATH,
+ sfc_kvarg_string_handler, &tx_name);
+ if (rc != 0)
+ goto fail_kvarg_tx_datapath;
+
+ if (tx_name != NULL) {
+ sa->dp_tx = sfc_dp_find_tx_by_name(&sfc_dp_head, tx_name);
+ if (sa->dp_tx == NULL) {
+ sfc_err(sa, "Tx datapath %s not found", tx_name);
+ rc = ENOENT;
+ goto fail_dp_tx;
+ }
+ if (!sfc_dp_match_hw_fw_caps(&sa->dp_tx->dp, avail_caps)) {
+ sfc_err(sa,
+ "Insufficient Hw/FW capabilities to use Tx
datapath %s",
+ tx_name);
+ rc = EINVAL;
+ goto fail_dp_tx;
+ }
+ } else {
+ sa->dp_tx = sfc_dp_find_tx_by_caps(&sfc_dp_head, avail_caps);
+ if (sa->dp_tx == NULL) {
+ sfc_err(sa, "Tx datapath by caps %#x not found",
+ avail_caps);
+ rc = ENOENT;
+ goto fail_dp_tx;
+ }
+ }
+
+ sfc_info(sa, "use %s Tx datapath", sa->dp_tx->dp.name);
+
+ dev->tx_pkt_burst = sa->dp_tx->pkt_burst;
dev->dev_ops = &sfc_eth_dev_ops;
return 0;
+fail_dp_tx:
+fail_kvarg_tx_datapath:
fail_dp_rx:
fail_kvarg_rx_datapath:
return rc;
@@ -1298,6 +1332,8 @@
/* Prefer EF10 datapath */
sfc_dp_register(&sfc_dp_head, &sfc_ef10_rx.dp);
sfc_dp_register(&sfc_dp_head, &sfc_efx_rx.dp);
+
+ sfc_dp_register(&sfc_dp_head, &sfc_efx_tx.dp);
}
}
@@ -1431,6 +1467,7 @@
RTE_PMD_REGISTER_KMOD_DEP(net_sfc_efx, "* igb_uio | uio_pci_generic | vfio");
RTE_PMD_REGISTER_PARAM_STRING(net_sfc_efx,
SFC_KVARG_RX_DATAPATH "=" SFC_KVARG_VALUES_RX_DATAPATH " "
+ SFC_KVARG_TX_DATAPATH "=" SFC_KVARG_VALUES_TX_DATAPATH " "
SFC_KVARG_PERF_PROFILE "=" SFC_KVARG_VALUES_PERF_PROFILE " "
SFC_KVARG_MCDI_LOGGING "=" SFC_KVARG_VALUES_BOOL " "
SFC_KVARG_DEBUG_INIT "=" SFC_KVARG_VALUES_BOOL);
diff --git a/drivers/net/sfc/sfc_ev.c b/drivers/net/sfc/sfc_ev.c
index 64694f5..04e923f 100644
--- a/drivers/net/sfc/sfc_ev.c
+++ b/drivers/net/sfc/sfc_ev.c
@@ -184,16 +184,18 @@
sfc_ev_tx(void *arg, __rte_unused uint32_t label, uint32_t id)
{
struct sfc_evq *evq = arg;
- struct sfc_txq *txq;
+ struct sfc_dp_txq *dp_txq;
+ struct sfc_efx_txq *txq;
unsigned int stop;
unsigned int delta;
- txq = evq->txq;
+ dp_txq = evq->dp_txq;
+ SFC_ASSERT(dp_txq != NULL);
- SFC_ASSERT(txq != NULL);
+ txq = sfc_efx_txq_by_dp_txq(dp_txq);
SFC_ASSERT(txq->evq == evq);
- if (unlikely((txq->state & SFC_TXQ_STARTED) == 0))
+ if (unlikely((txq->flags & SFC_EFX_TXQ_FLAG_STARTED) == 0))
goto done;
stop = (id + 1) & txq->ptr_mask;
@@ -306,9 +308,13 @@
sfc_ev_txq_flush_done(void *arg, __rte_unused uint32_t txq_hw_index)
{
struct sfc_evq *evq = arg;
+ struct sfc_dp_txq *dp_txq;
struct sfc_txq *txq;
- txq = evq->txq;
+ dp_txq = evq->dp_txq;
+ SFC_ASSERT(dp_txq != NULL);
+
+ txq = dp_txq->ops->get_ctrl(dp_txq);
SFC_ASSERT(txq != NULL);
SFC_ASSERT(txq->hw_index == txq_hw_index);
SFC_ASSERT(txq->evq == evq);
@@ -442,7 +448,7 @@
.eec_link_change = sfc_ev_nop_link_change,
};
-static const efx_ev_callbacks_t sfc_ev_callbacks_tx = {
+static const efx_ev_callbacks_t sfc_ev_callbacks_efx_tx = {
.eec_initialized = sfc_ev_initialized,
.eec_rx = sfc_ev_nop_rx,
.eec_tx = sfc_ev_tx,
@@ -457,6 +463,21 @@
.eec_link_change = sfc_ev_nop_link_change,
};
+static const efx_ev_callbacks_t sfc_ev_callbacks_dp_tx = {
+ .eec_initialized = sfc_ev_initialized,
+ .eec_rx = sfc_ev_nop_rx,
+ .eec_tx = sfc_ev_nop_tx,
+ .eec_exception = sfc_ev_exception,
+ .eec_rxq_flush_done = sfc_ev_nop_rxq_flush_done,
+ .eec_rxq_flush_failed = sfc_ev_nop_rxq_flush_failed,
+ .eec_txq_flush_done = sfc_ev_txq_flush_done,
+ .eec_software = sfc_ev_software,
+ .eec_sram = sfc_ev_sram,
+ .eec_wake_up = sfc_ev_wake_up,
+ .eec_timer = sfc_ev_timer,
+ .eec_link_change = sfc_ev_nop_link_change,
+};
+
void
sfc_ev_qpoll(struct sfc_evq *evq)
@@ -490,8 +511,12 @@
rxq_sw_index);
}
- if (evq->txq != NULL) {
- unsigned int txq_sw_index = sfc_txq_sw_index(evq->txq);
+ if (evq->dp_txq != NULL) {
+ struct sfc_txq *txq;
+ unsigned int txq_sw_index;
+
+ txq = evq->dp_txq->ops->get_ctrl(evq->dp_txq);
+ txq_sw_index = sfc_txq_sw_index(txq);
sfc_warn(sa,
"restart TxQ %u because of exception on its
EvQ %u",
@@ -561,14 +586,17 @@
if (rc != 0)
goto fail_ev_qcreate;
- SFC_ASSERT(evq->dp_rxq == NULL || evq->txq == NULL);
+ SFC_ASSERT(evq->dp_rxq == NULL || evq->dp_txq == NULL);
if (evq->dp_rxq != 0) {
if (strcmp(sa->dp_rx->dp.name, SFC_KVARG_DATAPATH_EFX) == 0)
evq->callbacks = &sfc_ev_callbacks_efx_rx;
else
evq->callbacks = &sfc_ev_callbacks_dp_rx;
- } else if (evq->txq != 0) {
- evq->callbacks = &sfc_ev_callbacks_tx;
+ } else if (evq->dp_txq != 0) {
+ if (strcmp(sa->dp_tx->dp.name, SFC_KVARG_DATAPATH_EFX) == 0)
+ evq->callbacks = &sfc_ev_callbacks_efx_tx;
+ else
+ evq->callbacks = &sfc_ev_callbacks_dp_tx;
} else {
evq->callbacks = &sfc_ev_callbacks;
}
diff --git a/drivers/net/sfc/sfc_ev.h b/drivers/net/sfc/sfc_ev.h
index e99cd74..f7434a3 100644
--- a/drivers/net/sfc/sfc_ev.h
+++ b/drivers/net/sfc/sfc_ev.h
@@ -30,8 +30,12 @@
#ifndef _SFC_EV_H_
#define _SFC_EV_H_
+#include <rte_ethdev.h>
+
#include "efx.h"
+#include "sfc.h"
+
#ifdef __cplusplus
extern "C" {
#endif
@@ -41,7 +45,7 @@
struct sfc_adapter;
struct sfc_dp_rxq;
-struct sfc_txq;
+struct sfc_dp_txq;
enum sfc_evq_state {
SFC_EVQ_UNINITIALIZED = 0,
@@ -60,7 +64,7 @@ struct sfc_evq {
boolean_t exception;
efsys_mem_t mem;
struct sfc_dp_rxq *dp_rxq;
- struct sfc_txq *txq;
+ struct sfc_dp_txq *dp_txq;
/* Not used on datapath */
struct sfc_adapter *sa;
diff --git a/drivers/net/sfc/sfc_kvargs.c b/drivers/net/sfc/sfc_kvargs.c
index d8529fa..f23bc1a 100644
--- a/drivers/net/sfc/sfc_kvargs.c
+++ b/drivers/net/sfc/sfc_kvargs.c
@@ -46,6 +46,7 @@
SFC_KVARG_MCDI_LOGGING,
SFC_KVARG_PERF_PROFILE,
SFC_KVARG_RX_DATAPATH,
+ SFC_KVARG_TX_DATAPATH,
NULL,
};
diff --git a/drivers/net/sfc/sfc_kvargs.h b/drivers/net/sfc/sfc_kvargs.h
index b57439b..14f46db 100644
--- a/drivers/net/sfc/sfc_kvargs.h
+++ b/drivers/net/sfc/sfc_kvargs.h
@@ -60,6 +60,10 @@
"[" SFC_KVARG_DATAPATH_EFX "|" \
SFC_KVARG_DATAPATH_EF10 "]"
+#define SFC_KVARG_TX_DATAPATH "tx_datapath"
+#define SFC_KVARG_VALUES_TX_DATAPATH \
+ "[" SFC_KVARG_DATAPATH_EFX "]"
+
struct sfc_adapter;
int sfc_kvargs_parse(struct sfc_adapter *sa);
diff --git a/drivers/net/sfc/sfc_tso.c b/drivers/net/sfc/sfc_tso.c
index 68d84c9..317c805 100644
--- a/drivers/net/sfc/sfc_tso.c
+++ b/drivers/net/sfc/sfc_tso.c
@@ -42,13 +42,13 @@
#define SFC_TSO_OPDESCS_IDX_SHIFT 2
int
-sfc_tso_alloc_tsoh_objs(struct sfc_tx_sw_desc *sw_ring,
- unsigned int txq_entries, unsigned int socket_id)
+sfc_efx_tso_alloc_tsoh_objs(struct sfc_efx_tx_sw_desc *sw_ring,
+ unsigned int txq_entries, unsigned int socket_id)
{
unsigned int i;
for (i = 0; i < txq_entries; ++i) {
- sw_ring[i].tsoh = rte_malloc_socket("sfc-txq-tsoh-obj",
+ sw_ring[i].tsoh = rte_malloc_socket("sfc-efx-txq-tsoh-obj",
SFC_TSOH_STD_LEN,
SFC_TX_SEG_BOUNDARY,
socket_id);
@@ -66,7 +66,8 @@
}
void
-sfc_tso_free_tsoh_objs(struct sfc_tx_sw_desc *sw_ring, unsigned int
txq_entries)
+sfc_efx_tso_free_tsoh_objs(struct sfc_efx_tx_sw_desc *sw_ring,
+ unsigned int txq_entries)
{
unsigned int i;
@@ -77,8 +78,8 @@
}
static void
-sfc_tso_prepare_header(struct sfc_txq *txq, struct rte_mbuf **in_seg,
- size_t *in_off, unsigned int idx, size_t bytes_left)
+sfc_efx_tso_prepare_header(struct sfc_efx_txq *txq, struct rte_mbuf **in_seg,
+ size_t *in_off, unsigned int idx, size_t bytes_left)
{
struct rte_mbuf *m = *in_seg;
size_t bytes_to_copy = 0;
@@ -109,9 +110,9 @@
}
int
-sfc_tso_do(struct sfc_txq *txq, unsigned int idx, struct rte_mbuf **in_seg,
- size_t *in_off, efx_desc_t **pend, unsigned int *pkt_descs,
- size_t *pkt_len)
+sfc_efx_tso_do(struct sfc_efx_txq *txq, unsigned int idx,
+ struct rte_mbuf **in_seg, size_t *in_off, efx_desc_t **pend,
+ unsigned int *pkt_descs, size_t *pkt_len)
{
uint8_t *tsoh;
const struct tcp_hdr *th;
@@ -151,7 +152,8 @@
*/
if ((m->data_len < header_len) ||
((paddr_next_frag - header_paddr) < header_len)) {
- sfc_tso_prepare_header(txq, in_seg, in_off, idx, header_len);
+ sfc_efx_tso_prepare_header(txq, in_seg, in_off, idx,
+ header_len);
tsoh = txq->sw_ring[idx & txq->ptr_mask].tsoh;
header_paddr = rte_malloc_virt2phy((void *)tsoh);
diff --git a/drivers/net/sfc/sfc_tx.c b/drivers/net/sfc/sfc_tx.c
index 5a6282c..31bccac 100644
--- a/drivers/net/sfc/sfc_tx.c
+++ b/drivers/net/sfc/sfc_tx.c
@@ -33,6 +33,7 @@
#include "sfc_ev.h"
#include "sfc_tx.h"
#include "sfc_tweak.h"
+#include "sfc_kvargs.h"
/*
* Maximum number of TX queue flush attempts in case of
@@ -109,27 +110,29 @@
txq->state &= ~SFC_TXQ_FLUSHING;
}
+static sfc_dp_exception_t sfc_tx_dp_exception;
static void
-sfc_tx_reap(struct sfc_txq *txq)
+sfc_tx_dp_exception(void *ctrl)
{
- unsigned int completed;
+ struct sfc_txq *txq = ctrl;
+ struct sfc_adapter *sa = txq->evq->sa;
+ unsigned int txq_sw_index;
+ int rc;
+ if (!sfc_adapter_trylock(sa))
+ return;
- sfc_ev_qpoll(txq->evq);
-
- for (completed = txq->completed;
- completed != txq->pending; completed++) {
- struct sfc_tx_sw_desc *txd;
+ txq_sw_index = sfc_txq_sw_index(txq);
- txd = &txq->sw_ring[completed & txq->ptr_mask];
+ sfc_warn(sa, "restart TxQ %u because of datapath exception",
+ txq_sw_index);
- if (txd->mbuf != NULL) {
- rte_pktmbuf_free(txd->mbuf);
- txd->mbuf = NULL;
- }
- }
+ sfc_tx_qstop(sa, txq_sw_index);
+ rc = sfc_tx_qstart(sa, txq_sw_index);
+ if (rc != 0)
+ sfc_err(sa, "cannot restart TxQ %u", txq_sw_index);
- txq->completed = completed;
+ sfc_adapter_unlock(sa);
}
int
@@ -142,6 +145,7 @@
struct sfc_txq *txq;
unsigned int evq_index = sfc_evq_index_by_txq_sw_index(sa, sw_index);
int rc = 0;
+ struct sfc_dp_tx_qcreate_args args;
sfc_log_init(sa, "TxQ = %u", sw_index);
@@ -166,56 +170,43 @@
if (txq == NULL)
goto fail_txq_alloc;
+ txq_info->txq = txq;
+
+ txq->hw_index = sw_index;
+ txq->evq = evq;
+ txq->free_thresh =
+ (tx_conf->tx_free_thresh) ? tx_conf->tx_free_thresh :
+ SFC_TX_DEFAULT_FREE_THRESH;
+ txq->flags = tx_conf->txq_flags;
+
rc = sfc_dma_alloc(sa, "txq", sw_index, EFX_TXQ_SIZE(txq_info->entries),
socket_id, &txq->mem);
if (rc != 0)
goto fail_dma_alloc;
- rc = ENOMEM;
- txq->pend_desc = rte_calloc_socket("sfc-txq-pend-desc",
- EFX_TXQ_LIMIT(txq_info->entries),
- sizeof(efx_desc_t), 0, socket_id);
- if (txq->pend_desc == NULL)
- goto fail_pend_desc_alloc;
+ memset(&args, 0, sizeof(args));
+ args.free_thresh = txq->free_thresh;
+ args.flags = tx_conf->txq_flags;
+ args.txq_entries = txq_info->entries;
- rc = ENOMEM;
- txq->sw_ring = rte_calloc_socket("sfc-txq-desc", txq_info->entries,
- sizeof(*txq->sw_ring), 0, socket_id);
- if (txq->sw_ring == NULL)
- goto fail_desc_alloc;
+ rc = sa->dp_tx->qcreate(txq, sfc_tx_dp_exception, socket_id, &args,
+ &txq->dp);
+ if (rc != 0)
+ goto fail_dp_tx_qinit;
- if (sa->tso) {
- rc = sfc_tso_alloc_tsoh_objs(txq->sw_ring, txq_info->entries,
- socket_id);
- if (rc != 0)
- goto fail_alloc_tsoh_objs;
- }
+ evq->dp_txq = txq->dp;
txq->state = SFC_TXQ_INITIALIZED;
- txq->ptr_mask = txq_info->entries - 1;
- txq->free_thresh = (tx_conf->tx_free_thresh) ? tx_conf->tx_free_thresh :
- SFC_TX_DEFAULT_FREE_THRESH;
- txq->hw_index = sw_index;
- txq->flags = tx_conf->txq_flags;
- txq->evq = evq;
-
- evq->txq = txq;
- txq_info->txq = txq;
txq_info->deferred_start = (tx_conf->tx_deferred_start != 0);
return 0;
-fail_alloc_tsoh_objs:
- rte_free(txq->sw_ring);
-
-fail_desc_alloc:
- rte_free(txq->pend_desc);
-
-fail_pend_desc_alloc:
+fail_dp_tx_qinit:
sfc_dma_free(sa, &txq->mem);
fail_dma_alloc:
+ txq_info->txq = NULL;
rte_free(txq);
fail_txq_alloc:
@@ -244,13 +235,12 @@
SFC_ASSERT(txq != NULL);
SFC_ASSERT(txq->state == SFC_TXQ_INITIALIZED);
- sfc_tso_free_tsoh_objs(txq->sw_ring, txq_info->entries);
+ sa->dp_tx->qdestroy(txq->dp);
+ txq->dp = NULL;
txq_info->txq = NULL;
txq_info->entries = 0;
- rte_free(txq->sw_ring);
- rte_free(txq->pend_desc);
sfc_dma_free(sa, &txq->mem);
rte_free(txq);
}
@@ -405,12 +395,13 @@
goto fail_tx_qcreate;
}
- txq->added = txq->pending = txq->completed = desc_index;
- txq->hw_vlan_tci = 0;
-
efx_tx_qenable(txq->common);
- txq->state |= (SFC_TXQ_STARTED | SFC_TXQ_RUNNING);
+ txq->state |= SFC_TXQ_STARTED;
+
+ rc = sa->dp_tx->qstart(txq->dp, evq->read_ptr, desc_index);
+ if (rc != 0)
+ goto fail_dp_qstart;
/*
* It seems to be used by DPDK for debug purposes only ('rte_ether')
@@ -420,6 +411,10 @@
return 0;
+fail_dp_qstart:
+ txq->state = SFC_TXQ_INITIALIZED;
+ efx_tx_qdestroy(txq->common);
+
fail_tx_qcreate:
sfc_ev_qstop(sa, evq->evq_index);
@@ -435,7 +430,6 @@
struct sfc_txq *txq;
unsigned int retry_count;
unsigned int wait_count;
- unsigned int txds;
sfc_log_init(sa, "TxQ = %u", sw_index);
@@ -449,7 +443,7 @@
SFC_ASSERT(txq->state & SFC_TXQ_STARTED);
- txq->state &= ~SFC_TXQ_RUNNING;
+ sa->dp_tx->qstop(txq->dp, &txq->evq->read_ptr);
/*
* Retry TX queue flushing in case of flush failed or
@@ -484,14 +478,7 @@
sfc_info(sa, "TxQ %u flushed", sw_index);
}
- sfc_tx_reap(txq);
-
- for (txds = 0; txds < txq_info->entries; txds++) {
- if (txq->sw_ring[txds].mbuf != NULL) {
- rte_pktmbuf_free(txq->sw_ring[txds].mbuf);
- txq->sw_ring[txds].mbuf = NULL;
- }
- }
+ sa->dp_tx->qreap(txq->dp);
txq->state = SFC_TXQ_INITIALIZED;
@@ -563,6 +550,28 @@
efx_tx_fini(sa->nic);
}
+static void
+sfc_efx_tx_reap(struct sfc_efx_txq *txq)
+{
+ unsigned int completed;
+
+ sfc_ev_qpoll(txq->evq);
+
+ for (completed = txq->completed;
+ completed != txq->pending; completed++) {
+ struct sfc_efx_tx_sw_desc *txd;
+
+ txd = &txq->sw_ring[completed & txq->ptr_mask];
+
+ if (txd->mbuf != NULL) {
+ rte_pktmbuf_free(txd->mbuf);
+ txd->mbuf = NULL;
+ }
+ }
+
+ txq->completed = completed;
+}
+
/*
* The function is used to insert or update VLAN tag;
* the firmware has state of the firmware tag to insert per TxQ
@@ -571,8 +580,8 @@
* the function will update it
*/
static unsigned int
-sfc_tx_maybe_insert_tag(struct sfc_txq *txq, struct rte_mbuf *m,
- efx_desc_t **pend)
+sfc_efx_tx_maybe_insert_tag(struct sfc_efx_txq *txq, struct rte_mbuf *m,
+ efx_desc_t **pend)
{
uint16_t this_tag = ((m->ol_flags & PKT_TX_VLAN_PKT) ?
m->vlan_tci : 0);
@@ -594,10 +603,11 @@
return 1;
}
-uint16_t
-sfc_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+static uint16_t
+sfc_efx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
{
- struct sfc_txq *txq = (struct sfc_txq *)tx_queue;
+ struct sfc_dp_txq *dp_txq = (struct sfc_dp_txq *)tx_queue;
+ struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq);
unsigned int added = txq->added;
unsigned int pushed = added;
unsigned int pkts_sent = 0;
@@ -609,7 +619,7 @@
int rc __rte_unused;
struct rte_mbuf **pktp;
- if (unlikely((txq->state & SFC_TXQ_RUNNING) == 0))
+ if (unlikely((txq->flags & SFC_EFX_TXQ_FLAG_RUNNING) == 0))
goto done;
/*
@@ -620,7 +630,7 @@
reap_done = (fill_level > soft_max_fill);
if (reap_done) {
- sfc_tx_reap(txq);
+ sfc_efx_tx_reap(txq);
/*
* Recalculate fill level since 'txq->completed'
* might have changed on reap
@@ -643,15 +653,16 @@
* DEV_TX_VLAN_OFFLOAD and pushes VLAN TCI, then
* TX_ERROR will occur
*/
- pkt_descs += sfc_tx_maybe_insert_tag(txq, m_seg, &pend);
+ pkt_descs += sfc_efx_tx_maybe_insert_tag(txq, m_seg, &pend);
+#ifdef RTE_LIBRTE_SFC_EFX_TSO
if (m_seg->ol_flags & PKT_TX_TCP_SEG) {
/*
* We expect correct 'pkt->l[2, 3, 4]_len' values
* to be set correctly by the caller
*/
- if (sfc_tso_do(txq, added, &m_seg, &in_off, &pend,
- &pkt_descs, &pkt_len) != 0) {
+ if (sfc_efx_tso_do(txq, added, &m_seg, &in_off, &pend,
+ &pkt_descs, &pkt_len) != 0) {
/* We may have reached this place for
* one of the following reasons:
*
@@ -682,6 +693,7 @@
* as for the usual non-TSO path
*/
}
+#endif /* RTE_LIBRTE_SFC_EFX_TSO */
for (; m_seg != NULL; m_seg = m_seg->next) {
efsys_dma_addr_t next_frag;
@@ -729,7 +741,7 @@
* Try to reap (if we haven't yet).
*/
if (!reap_done) {
- sfc_tx_reap(txq);
+ sfc_efx_tx_reap(txq);
reap_done = B_TRUE;
fill_level = added - txq->completed;
if (fill_level > hard_max_fill) {
@@ -758,9 +770,143 @@
#if SFC_TX_XMIT_PKTS_REAP_AT_LEAST_ONCE
if (!reap_done)
- sfc_tx_reap(txq);
+ sfc_efx_tx_reap(txq);
#endif
done:
return pkts_sent;
}
+
+static sfc_dp_tx_qcreate_t sfc_efx_tx_qcreate;
+static int
+sfc_efx_tx_qcreate(void *ctrl, __rte_unused sfc_dp_exception_t *exception,
+ int socket_id, const struct sfc_dp_tx_qcreate_args *args,
+ struct sfc_dp_txq **dp_txqp)
+{
+ struct sfc_txq *ctrl_txq = ctrl;
+ struct sfc_efx_txq *txq;
+ int rc;
+
+ rc = ENOMEM;
+ txq = rte_zmalloc_socket("sfc-efx-txq", sizeof(*txq),
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (txq == NULL)
+ goto fail_txq_alloc;
+
+ rc = ENOMEM;
+ txq->pend_desc = rte_calloc_socket("sfc-efx-txq-pend-desc",
+ EFX_TXQ_LIMIT(args->txq_entries),
+ sizeof(*txq->pend_desc), 0,
+ socket_id);
+ if (txq->pend_desc == NULL)
+ goto fail_pend_desc_alloc;
+
+ rc = ENOMEM;
+ txq->sw_ring = rte_calloc_socket("sfc-efx-txq-sw_ring",
+ args->txq_entries,
+ sizeof(*txq->sw_ring),
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (txq->sw_ring == NULL)
+ goto fail_sw_ring_alloc;
+
+ if (ctrl_txq->evq->sa->tso) {
+ rc = sfc_efx_tso_alloc_tsoh_objs(txq->sw_ring,
+ args->txq_entries, socket_id);
+ if (rc != 0)
+ goto fail_alloc_tsoh_objs;
+ }
+
+ txq->ctrl = ctrl_txq;
+ txq->evq = ctrl_txq->evq;
+ txq->ptr_mask = args->txq_entries - 1;
+ txq->free_thresh = args->free_thresh;
+
+ *dp_txqp = &txq->dp;
+ return 0;
+
+fail_alloc_tsoh_objs:
+ rte_free(txq->sw_ring);
+
+fail_sw_ring_alloc:
+ rte_free(txq->pend_desc);
+
+fail_pend_desc_alloc:
+ rte_free(txq);
+
+fail_txq_alloc:
+ return rc;
+}
+
+static sfc_dp_tx_qdestroy_t sfc_efx_tx_qdestroy;
+static void
+sfc_efx_tx_qdestroy(struct sfc_dp_txq *dp_txq)
+{
+ struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq);
+
+ sfc_efx_tso_free_tsoh_objs(txq->sw_ring, txq->ptr_mask + 1);
+ rte_free(txq->sw_ring);
+ rte_free(txq->pend_desc);
+ rte_free(txq);
+}
+
+static sfc_dp_tx_qstart_t sfc_efx_tx_qstart;
+static int
+sfc_efx_tx_qstart(struct sfc_dp_txq *dp_txq,
+ __rte_unused unsigned int evq_read_ptr,
+ unsigned int txq_desc_index)
+{
+ /* libefx-based datapath is specific to libefx-based PMD */
+ struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq);
+
+ txq->common = txq->ctrl->common;
+
+ txq->pending = txq->completed = txq->added = txq_desc_index;
+ txq->hw_vlan_tci = 0;
+
+ txq->flags |= (SFC_EFX_TXQ_FLAG_STARTED | SFC_EFX_TXQ_FLAG_RUNNING);
+
+ return 0;
+}
+
+static sfc_dp_tx_qstop_t sfc_efx_tx_qstop;
+static void
+sfc_efx_tx_qstop(struct sfc_dp_txq *dp_txq,
+ __rte_unused unsigned int *evq_read_ptr)
+{
+ struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq);
+
+ txq->flags &= ~SFC_EFX_TXQ_FLAG_RUNNING;
+}
+
+static sfc_dp_tx_qreap_t sfc_efx_tx_qreap;
+static void
+sfc_efx_tx_qreap(struct sfc_dp_txq *dp_txq)
+{
+ struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq);
+ unsigned int txds;
+
+ sfc_efx_tx_reap(txq);
+
+ for (txds = 0; txds <= txq->ptr_mask; txds++) {
+ if (txq->sw_ring[txds].mbuf != NULL) {
+ rte_pktmbuf_free(txq->sw_ring[txds].mbuf);
+ txq->sw_ring[txds].mbuf = NULL;
+ }
+ }
+
+ txq->flags &= ~SFC_EFX_TXQ_FLAG_STARTED;
+}
+
+struct sfc_dp_tx sfc_efx_tx = {
+ .dp = {
+ .name = SFC_KVARG_DATAPATH_EFX,
+ .type = SFC_DP_TX,
+ .hw_fw_caps = 0,
+ },
+ .qcreate = sfc_efx_tx_qcreate,
+ .qdestroy = sfc_efx_tx_qdestroy,
+ .qstart = sfc_efx_tx_qstart,
+ .qstop = sfc_efx_tx_qstop,
+ .qreap = sfc_efx_tx_qreap,
+ .pkt_burst = sfc_efx_xmit_pkts,
+};
diff --git a/drivers/net/sfc/sfc_tx.h b/drivers/net/sfc/sfc_tx.h
index 39977a5..63c86a1 100644
--- a/drivers/net/sfc/sfc_tx.h
+++ b/drivers/net/sfc/sfc_tx.h
@@ -35,6 +35,8 @@
#include "efx.h"
+#include "sfc_dp_tx.h"
+
#ifdef __cplusplus
extern "C" {
#endif
@@ -48,7 +50,11 @@
struct sfc_adapter;
struct sfc_evq;
-struct sfc_tx_sw_desc {
+/**
+ * Software Tx descriptor information associated with hardware Tx
+ * descriptor.
+ */
+struct sfc_efx_tx_sw_desc {
struct rte_mbuf *mbuf;
uint8_t *tsoh; /* Buffer to store TSO header */
};
@@ -58,36 +64,69 @@ enum sfc_txq_state_bit {
#define SFC_TXQ_INITIALIZED (1 << SFC_TXQ_INITIALIZED_BIT)
SFC_TXQ_STARTED_BIT,
#define SFC_TXQ_STARTED (1 << SFC_TXQ_STARTED_BIT)
- SFC_TXQ_RUNNING_BIT,
-#define SFC_TXQ_RUNNING (1 << SFC_TXQ_RUNNING_BIT)
SFC_TXQ_FLUSHING_BIT,
#define SFC_TXQ_FLUSHING (1 << SFC_TXQ_FLUSHING_BIT)
SFC_TXQ_FLUSHED_BIT,
#define SFC_TXQ_FLUSHED (1 << SFC_TXQ_FLUSHED_BIT)
};
+/**
+ * Transmit queue control information. Not used on datapath.
+ * Allocated on the socket specified on the queue setup.
+ */
struct sfc_txq {
- struct sfc_evq *evq;
- struct sfc_tx_sw_desc *sw_ring;
- unsigned int state;
- unsigned int ptr_mask;
- efx_desc_t *pend_desc;
- efx_txq_t *common;
- efsys_mem_t mem;
- unsigned int added;
- unsigned int pending;
- unsigned int completed;
- unsigned int free_thresh;
- uint16_t hw_vlan_tci;
-
- unsigned int hw_index;
- unsigned int flags;
+ unsigned int state;
+ unsigned int hw_index;
+ struct sfc_evq *evq;
+ efsys_mem_t mem;
+ struct sfc_dp_txq *dp;
+ efx_txq_t *common;
+ unsigned int free_thresh;
+ unsigned int flags;
};
static inline unsigned int
+sfc_txq_sw_index_by_hw_index(unsigned int hw_index)
+{
+ return hw_index;
+}
+
+static inline unsigned int
sfc_txq_sw_index(const struct sfc_txq *txq)
{
- return txq->hw_index;
+ return sfc_txq_sw_index_by_hw_index(txq->hw_index);
+}
+
+/**
+ * Transmit queue information used on libefx-based data path.
+ * Allocated on the socket specified on the queue setup.
+ */
+struct sfc_efx_txq {
+ struct sfc_evq *evq;
+ struct sfc_efx_tx_sw_desc *sw_ring;
+ unsigned int ptr_mask;
+ efx_desc_t *pend_desc;
+ efx_txq_t *common;
+ unsigned int added;
+ unsigned int pending;
+ unsigned int completed;
+ unsigned int free_thresh;
+ uint16_t hw_vlan_tci;
+
+ unsigned int hw_index;
+ unsigned int flags;
+#define SFC_EFX_TXQ_FLAG_STARTED 0x1
+#define SFC_EFX_TXQ_FLAG_RUNNING 0x2
+
+ /* Datapath transmit queue anchor */
+ struct sfc_dp_txq dp;
+ struct sfc_txq *ctrl;
+};
+
+static inline struct sfc_efx_txq *
+sfc_efx_txq_by_dp_txq(struct sfc_dp_txq *dp_txq)
+{
+ return container_of(dp_txq, struct sfc_efx_txq, dp);
}
struct sfc_txq_info {
@@ -111,17 +150,15 @@ int sfc_tx_qinit(struct sfc_adapter *sa, unsigned int
sw_index,
int sfc_tx_start(struct sfc_adapter *sa);
void sfc_tx_stop(struct sfc_adapter *sa);
-uint16_t sfc_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
- uint16_t nb_pkts);
-
/* From 'sfc_tso.c' */
-int sfc_tso_alloc_tsoh_objs(struct sfc_tx_sw_desc *sw_ring,
- unsigned int txq_entries, unsigned int socket_id);
-void sfc_tso_free_tsoh_objs(struct sfc_tx_sw_desc *sw_ring,
- unsigned int txq_entries);
-int sfc_tso_do(struct sfc_txq *txq, unsigned int idx, struct rte_mbuf **in_seg,
- size_t *in_off, efx_desc_t **pend, unsigned int *pkt_descs,
- size_t *pkt_len);
+int sfc_efx_tso_alloc_tsoh_objs(struct sfc_efx_tx_sw_desc *sw_ring,
+ unsigned int txq_entries,
+ unsigned int socket_id);
+void sfc_efx_tso_free_tsoh_objs(struct sfc_efx_tx_sw_desc *sw_ring,
+ unsigned int txq_entries);
+int sfc_efx_tso_do(struct sfc_efx_txq *txq, unsigned int idx,
+ struct rte_mbuf **in_seg, size_t *in_off, efx_desc_t **pend,
+ unsigned int *pkt_descs, size_t *pkt_len);
#ifdef __cplusplus
}
--
1.8.2.3
