- Implement hairpin queue setup/confige/enable/disable.
- Cross-vport hairpin queue implemented via hairpin_bind/unbind API.
Test step:
1. Make sure no bug on CP side.
2. Add rule on IMC.
- devmem 0x202920C100 64 0x804
- opcode=0x1303 prof_id=0x34 sub_prof_id=0x0 cookie=0xa2b87 key=0x18,\
0x0,00,00,00,00,de,0xad,0xbe,0xef,0x20,0x24,0x0,0x0,0x0,0x0,00,00,\
00,00,00,00,0xa,0x2,0x1d,0x64,00,00,00,00,00,00,00,00,00,00,00,00,\
0xa,0x2,0x1d,0x2,00,00,00,00,00,00,00,00,00,00,00,00 act=set_vsi{\
act_val=0 val_type=2 dst_pe=0 slot=0x0} act=set_q{\
qnum=0x142 no_implicit_vsi=1 prec=5}
3. Send packets on ixia side
UDP packets with dmac=de:ad:be:ef:20:24 sip=10.2.29.100
dip=10.2.29.2
Signed-off-by: Beilei Xing <beilei.x...@intel.com>
Signed-off-by: Xiao Wang <xiao.w.w...@intel.com>
Signed-off-by: Junfeng Guo <junfeng....@intel.com>
Signed-off-by: Mingxia Liu <mingxia....@intel.com>
---
drivers/common/idpf/idpf_common_device.c | 50 ++
drivers/common/idpf/idpf_common_device.h | 2 +
drivers/common/idpf/idpf_common_virtchnl.c | 100 ++-
drivers/common/idpf/idpf_common_virtchnl.h | 12 +
drivers/common/idpf/version.map | 5 +
drivers/net/cpfl/cpfl_ethdev.c | 374 +++++++--
drivers/net/cpfl/cpfl_ethdev.h | 8 +-
drivers/net/cpfl/cpfl_logs.h | 2 +
drivers/net/cpfl/cpfl_rxtx.c | 851 +++++++++++++++++++--
drivers/net/cpfl/cpfl_rxtx.h | 58 ++
drivers/net/cpfl/cpfl_rxtx_vec_common.h | 18 +-
11 files changed, 1347 insertions(+), 133 deletions(-)
diff --git a/drivers/common/idpf/idpf_common_device.c
b/drivers/common/idpf/idpf_common_device.c
index b90b20d0f2..be2ec19650 100644
--- a/drivers/common/idpf/idpf_common_device.c
+++ b/drivers/common/idpf/idpf_common_device.c
@@ -362,6 +362,56 @@ idpf_adapter_init(struct idpf_adapter *adapter)
return ret;
}
+int
+idpf_adapter_common_init(struct idpf_adapter *adapter)
+{
+ struct idpf_hw *hw = &adapter->hw;
+ int ret;
+
+ idpf_reset_pf(hw);
+ ret = idpf_check_pf_reset_done(hw);
+ if (ret != 0) {
+ DRV_LOG(ERR, "IDPF is still resetting");
+ goto err_check_reset;
+ }
+
+ ret = idpf_init_mbx(hw);
+ if (ret != 0) {
+ DRV_LOG(ERR, "Failed to init mailbox");
+ goto err_check_reset;
+ }
+
+ adapter->mbx_resp = rte_zmalloc("idpf_adapter_mbx_resp",
+ IDPF_DFLT_MBX_BUF_SIZE, 0);
+ if (adapter->mbx_resp == NULL) {
+ DRV_LOG(ERR, "Failed to allocate idpf_adapter_mbx_resp memory");
+ ret = -ENOMEM;
+ goto err_mbx_resp;
+ }
+
+ ret = idpf_vc_check_api_version(adapter);
+ if (ret != 0) {
+ DRV_LOG(ERR, "Failed to check api version");
+ goto err_check_api;
+ }
+
+ ret = idpf_get_pkt_type(adapter);
+ if (ret != 0) {
+ DRV_LOG(ERR, "Failed to set ptype table");
+ goto err_check_api;
+ }
+
+ return 0;
+
+err_check_api:
+ rte_free(adapter->mbx_resp);
+ adapter->mbx_resp = NULL;
+err_mbx_resp:
+ idpf_ctlq_deinit(hw);
+err_check_reset:
+ return ret;
+}
+
int
idpf_adapter_deinit(struct idpf_adapter *adapter)
{
diff --git a/drivers/common/idpf/idpf_common_device.h
b/drivers/common/idpf/idpf_common_device.h
index 2adeeff37e..9eef9cf73c 100644
--- a/drivers/common/idpf/idpf_common_device.h
+++ b/drivers/common/idpf/idpf_common_device.h
@@ -183,6 +183,8 @@ atomic_set_cmd(struct idpf_adapter *adapter, uint32_t ops)
__rte_internal
int idpf_adapter_init(struct idpf_adapter *adapter);
__rte_internal
+int idpf_adapter_common_init(struct idpf_adapter *adapter);
+__rte_internal
int idpf_adapter_deinit(struct idpf_adapter *adapter);
__rte_internal
int idpf_vport_init(struct idpf_vport *vport,
diff --git a/drivers/common/idpf/idpf_common_virtchnl.c
b/drivers/common/idpf/idpf_common_virtchnl.c
index f36aae8a93..b0cd405f24 100644
--- a/drivers/common/idpf/idpf_common_virtchnl.c
+++ b/drivers/common/idpf/idpf_common_virtchnl.c
@@ -338,6 +338,34 @@ idpf_vc_get_caps(struct idpf_adapter *adapter)
return 0;
}
+int idpf_vc_get_caps_by_caps_info(struct idpf_adapter *adapter,
+ struct virtchnl2_get_capabilities *caps_info)
+{
+ struct virtchnl2_get_capabilities caps_msg;
+ struct idpf_cmd_info args;
+ int err;
+
+ memset(&caps_msg, 0, sizeof(struct virtchnl2_get_capabilities));
+ rte_memcpy(&caps_msg, caps_info, sizeof(caps_msg));
+
+ args.ops = VIRTCHNL2_OP_GET_CAPS;
+ args.in_args = (uint8_t *)&caps_msg;
+ args.in_args_size = sizeof(caps_msg);
+ args.out_buffer = adapter->mbx_resp;
+ args.out_size = IDPF_DFLT_MBX_BUF_SIZE;
+
+ err = idpf_execute_vc_cmd(adapter, &args);
+ if (err != 0) {
+ DRV_LOG(ERR,
+ "Failed to execute command of VIRTCHNL2_OP_GET_CAPS");
+ return err;
+ }
+
+ rte_memcpy(&adapter->caps, args.out_buffer, sizeof(caps_msg));
+
+ return 0;
+}
+
int
idpf_vc_create_vport(struct idpf_vport *vport,
struct virtchnl2_create_vport *create_vport_info)
@@ -706,7 +734,7 @@ idpf_vc_dealloc_vectors(struct idpf_vport *vport)
return err;
}
-static int
+int
idpf_vc_ena_dis_one_queue(struct idpf_vport *vport, uint16_t qid,
uint32_t type, bool on)
{
@@ -1022,6 +1050,41 @@ idpf_vc_config_rxq(struct idpf_vport *vport, struct
idpf_rx_queue *rxq)
return err;
}
+int idpf_vc_config_rxq_by_info(struct idpf_vport *vport, struct
virtchnl2_rxq_info *rxq_info,
+ uint16_t num_qs)
+{
+ struct idpf_adapter *adapter = vport->adapter;
+ struct virtchnl2_config_rx_queues *vc_rxqs = NULL;
+ struct idpf_cmd_info args;
+ int size, err, i;
+
+ size = sizeof(*vc_rxqs) + (num_qs - 1) *
+ sizeof(struct virtchnl2_rxq_info);
+ vc_rxqs = rte_zmalloc("cfg_rxqs", size, 0);
+ if (vc_rxqs == NULL) {
+ DRV_LOG(ERR, "Failed to allocate virtchnl2_config_rx_queues");
+ err = -ENOMEM;
+ return err;
+ }
+ vc_rxqs->vport_id = vport->vport_id;
+ vc_rxqs->num_qinfo = num_qs;
+ memcpy(vc_rxqs->qinfo, rxq_info, num_qs * sizeof(struct
virtchnl2_rxq_info));
+
+ memset(&args, 0, sizeof(args));
+ args.ops = VIRTCHNL2_OP_CONFIG_RX_QUEUES;
+ args.in_args = (uint8_t *)vc_rxqs;
+ args.in_args_size = size;
+ args.out_buffer = adapter->mbx_resp;
+ args.out_size = IDPF_DFLT_MBX_BUF_SIZE;
+
+ err = idpf_execute_vc_cmd(adapter, &args);
+ rte_free(vc_rxqs);
+ if (err != 0)
+ DRV_LOG(ERR, "Failed to execute command of
VIRTCHNL2_OP_CONFIG_RX_QUEUES");
+
+ return err;
+}
+
int
idpf_vc_config_txq(struct idpf_vport *vport, struct idpf_tx_queue *txq)
{
@@ -1092,3 +1155,38 @@ idpf_vc_config_txq(struct idpf_vport *vport, struct
idpf_tx_queue *txq)
return err;
}
+
+int
+idpf_vc_config_txq_by_info(struct idpf_vport *vport, struct virtchnl2_txq_info
*txq_info,
+ uint16_t num_qs)
+{
+ struct idpf_adapter *adapter = vport->adapter;
+ struct virtchnl2_config_tx_queues *vc_txqs = NULL;
+ struct idpf_cmd_info args;
+ int size, err;
+
+ size = sizeof(*vc_txqs) + (num_qs - 1) * sizeof(struct
virtchnl2_txq_info);
+ vc_txqs = rte_zmalloc("cfg_txqs", size, 0);
+ if (vc_txqs == NULL) {
+ DRV_LOG(ERR, "Failed to allocate virtchnl2_config_tx_queues");
+ err = -ENOMEM;
+ return err;
+ }
+ vc_txqs->vport_id = vport->vport_id;
+ vc_txqs->num_qinfo = num_qs;
+ memcpy(vc_txqs->qinfo, txq_info, num_qs * sizeof(struct
virtchnl2_txq_info));
+
+ memset(&args, 0, sizeof(args));
+ args.ops = VIRTCHNL2_OP_CONFIG_TX_QUEUES;
+ args.in_args = (uint8_t *)vc_txqs;
+ args.in_args_size = size;
+ args.out_buffer = adapter->mbx_resp;
+ args.out_size = IDPF_DFLT_MBX_BUF_SIZE;
+
+ err = idpf_execute_vc_cmd(adapter, &args);
+ rte_free(vc_txqs);
+ if (err != 0)
+ DRV_LOG(ERR, "Failed to execute command of
VIRTCHNL2_OP_CONFIG_TX_QUEUES");
+
+ return err;
+}
diff --git a/drivers/common/idpf/idpf_common_virtchnl.h
b/drivers/common/idpf/idpf_common_virtchnl.h
index 86a8dfcece..7a6b4d6294 100644
--- a/drivers/common/idpf/idpf_common_virtchnl.h
+++ b/drivers/common/idpf/idpf_common_virtchnl.h
@@ -50,6 +50,12 @@ int idpf_vc_config_rxq(struct idpf_vport *vport, struct
idpf_rx_queue *rxq);
__rte_internal
int idpf_vc_config_txq(struct idpf_vport *vport, struct idpf_tx_queue *txq);
__rte_internal
+int idpf_vc_config_rxq_by_info(struct idpf_vport *vport, struct
virtchnl2_rxq_info *rxq_info,
+ uint16_t num_qs);
+__rte_internal
+int idpf_vc_config_txq_by_info(struct idpf_vport *vport, struct
virtchnl2_txq_info *txq_info,
+ uint16_t num_qs);
+__rte_internal
int idpf_query_stats(struct idpf_vport *vport,
struct virtchnl2_vport_stats **pstats);
__rte_internal
@@ -58,4 +64,10 @@ __rte_internal
int idpf_vc_get_rss_lut(struct idpf_vport *vport);
__rte_internal
int idpf_vc_get_rss_hash(struct idpf_vport *vport);
+__rte_internal
+int idpf_vc_ena_dis_one_queue(struct idpf_vport *vport, uint16_t qid,
+ uint32_t type, bool on);
+__rte_internal
+int idpf_vc_get_caps_by_caps_info(struct idpf_adapter *adapter,
+ struct virtchnl2_get_capabilities *caps_info);
#endif /* _IDPF_COMMON_VIRTCHNL_H_ */
diff --git a/drivers/common/idpf/version.map b/drivers/common/idpf/version.map
index 591af6b046..cc9a88972b 100644
--- a/drivers/common/idpf/version.map
+++ b/drivers/common/idpf/version.map
@@ -67,6 +67,11 @@ INTERNAL {
idpf_vc_get_rss_key;
idpf_vc_get_rss_lut;
idpf_vc_get_rss_hash;
+ idpf_vc_ena_dis_one_queue;
+ idpf_vc_config_rxq_by_info;
+ idpf_vc_config_txq_by_info;
+ idpf_vc_get_caps_by_caps_info;
+ idpf_adapter_common_init;
local: *;
};
diff --git a/drivers/net/cpfl/cpfl_ethdev.c b/drivers/net/cpfl/cpfl_ethdev.c
index f178f3fbb8..e464d76b60 100644
--- a/drivers/net/cpfl/cpfl_ethdev.c
+++ b/drivers/net/cpfl/cpfl_ethdev.c
@@ -108,7 +108,9 @@ static int
cpfl_dev_link_update(struct rte_eth_dev *dev,
__rte_unused int wait_to_complete)
{
- struct idpf_vport *vport = dev->data->dev_private;
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
struct rte_eth_link new_link;
memset(&new_link, 0, sizeof(new_link));
@@ -157,10 +159,24 @@ cpfl_dev_link_update(struct rte_eth_dev *dev,
return rte_eth_linkstatus_set(dev, &new_link);
}
+static int
+cpfl_hairpin_cap_get(__rte_unused struct rte_eth_dev *dev,
+ struct rte_eth_hairpin_cap *cap)
+{
+ cap->max_nb_queues = 1;
+ cap->max_rx_2_tx = 1;
+ cap->max_tx_2_rx = 1;
+ cap->max_nb_desc = 1024;
+
+ return 0;
+}
+
static int
cpfl_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
{
- struct idpf_vport *vport = dev->data->dev_private;
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
struct idpf_adapter *adapter = vport->adapter;
dev_info->max_rx_queues = adapter->caps.max_rx_q;
@@ -274,8 +290,9 @@ cpfl_get_mbuf_alloc_failed_stats(struct rte_eth_dev *dev)
static int
cpfl_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
{
- struct idpf_vport *vport =
- (struct idpf_vport *)dev->data->dev_private;
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
struct virtchnl2_vport_stats *pstats = NULL;
int ret;
@@ -319,8 +336,9 @@ cpfl_reset_mbuf_alloc_failed_stats(struct rte_eth_dev *dev)
static int
cpfl_dev_stats_reset(struct rte_eth_dev *dev)
{
- struct idpf_vport *vport =
- (struct idpf_vport *)dev->data->dev_private;
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
struct virtchnl2_vport_stats *pstats = NULL;
int ret;
@@ -345,8 +363,9 @@ static int cpfl_dev_xstats_reset(struct rte_eth_dev *dev)
static int cpfl_dev_xstats_get(struct rte_eth_dev *dev,
struct rte_eth_xstat *xstats, unsigned int n)
{
- struct idpf_vport *vport =
- (struct idpf_vport *)dev->data->dev_private;
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
struct virtchnl2_vport_stats *pstats = NULL;
unsigned int i;
int ret;
@@ -442,7 +461,8 @@ cpfl_init_rss(struct idpf_vport *vport)
{
struct rte_eth_rss_conf *rss_conf;
struct rte_eth_dev_data *dev_data;
- uint16_t i, nb_q;
+ struct cpfl_rx_queue *cpfl_rxq;
+ uint16_t i, nb_q, max_nb_data_q;
int ret = 0;
dev_data = vport->dev_data;
@@ -461,8 +481,16 @@ cpfl_init_rss(struct idpf_vport *vport)
vport->rss_key_size);
}
+ /* RSS only to the data queues */
+ max_nb_data_q = nb_q;
+ if (nb_q > 1) {
+ cpfl_rxq = dev_data->rx_queues[nb_q - 1];
+ if (cpfl_rxq && cpfl_rxq->hairpin_info.hairpin_q)
+ max_nb_data_q = nb_q - 1;
+ }
+
for (i = 0; i < vport->rss_lut_size; i++)
- vport->rss_lut[i] = i % nb_q;
+ vport->rss_lut[i] = i % max_nb_data_q;
vport->rss_hf = IDPF_DEFAULT_RSS_HASH_EXPANDED;
@@ -478,7 +506,9 @@ cpfl_rss_reta_update(struct rte_eth_dev *dev,
struct rte_eth_rss_reta_entry64 *reta_conf,
uint16_t reta_size)
{
- struct idpf_vport *vport = dev->data->dev_private;
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
struct idpf_adapter *adapter = vport->adapter;
uint16_t idx, shift;
uint32_t *lut;
@@ -534,7 +564,9 @@ cpfl_rss_reta_query(struct rte_eth_dev *dev,
struct rte_eth_rss_reta_entry64 *reta_conf,
uint16_t reta_size)
{
- struct idpf_vport *vport = dev->data->dev_private;
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
struct idpf_adapter *adapter = vport->adapter;
uint16_t idx, shift;
int ret = 0;
@@ -572,7 +604,9 @@ static int
cpfl_rss_hash_update(struct rte_eth_dev *dev,
struct rte_eth_rss_conf *rss_conf)
{
- struct idpf_vport *vport = dev->data->dev_private;
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
struct idpf_adapter *adapter = vport->adapter;
int ret = 0;
@@ -637,7 +671,9 @@ static int
cpfl_rss_hash_conf_get(struct rte_eth_dev *dev,
struct rte_eth_rss_conf *rss_conf)
{
- struct idpf_vport *vport = dev->data->dev_private;
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
struct idpf_adapter *adapter = vport->adapter;
int ret = 0;
@@ -674,10 +710,10 @@ cpfl_rss_hash_conf_get(struct rte_eth_dev *dev,
static int
cpfl_dev_configure(struct rte_eth_dev *dev)
{
- struct idpf_vport *vport = dev->data->dev_private;
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
struct rte_eth_conf *conf = &dev->data->dev_conf;
- struct idpf_adapter *adapter = vport->adapter;
- int ret;
if (conf->link_speeds & RTE_ETH_LINK_SPEED_FIXED) {
PMD_INIT_LOG(ERR, "Setting link speed is not supported");
@@ -716,17 +752,6 @@ cpfl_dev_configure(struct rte_eth_dev *dev)
return -ENOTSUP;
}
- if (adapter->caps.rss_caps != 0 && dev->data->nb_rx_queues != 0) {
- ret = cpfl_init_rss(vport);
- if (ret != 0) {
- PMD_INIT_LOG(ERR, "Failed to init rss");
- return ret;
- }
- } else {
- PMD_INIT_LOG(ERR, "RSS is not supported.");
- return -1;
- }
-
vport->max_pkt_len =
(dev->data->mtu == 0) ? CPFL_DEFAULT_MTU : dev->data->mtu +
CPFL_ETH_OVERHEAD;
@@ -737,7 +762,9 @@ cpfl_dev_configure(struct rte_eth_dev *dev)
static int
cpfl_config_rx_queues_irqs(struct rte_eth_dev *dev)
{
- struct idpf_vport *vport = dev->data->dev_private;
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
uint16_t nb_rx_queues = dev->data->nb_rx_queues;
return idpf_config_irq_map(vport, nb_rx_queues);
@@ -746,30 +773,92 @@ cpfl_config_rx_queues_irqs(struct rte_eth_dev *dev)
static int
cpfl_start_queues(struct rte_eth_dev *dev)
{
- struct idpf_rx_queue *rxq;
- struct idpf_tx_queue *txq;
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
+ struct idpf_adapter *adapter = vport->adapter;
+ struct cpfl_rx_queue *cpfl_rxq;
+ struct cpfl_tx_queue *cpfl_txq;
int err = 0;
int i;
- for (i = 0; i < dev->data->nb_tx_queues; i++) {
- txq = dev->data->tx_queues[i];
- if (txq == NULL || txq->tx_deferred_start)
- continue;
- err = cpfl_tx_queue_start(dev, i);
+ if (adapter->caps.rss_caps != 0 && dev->data->nb_rx_queues != 0) {
+ err = cpfl_init_rss(vport);
if (err != 0) {
- PMD_DRV_LOG(ERR, "Fail to start Tx queue %u", i);
+ PMD_INIT_LOG(ERR, "Failed to init rss");
return err;
}
+ } else {
+ PMD_INIT_LOG(ERR, "RSS is not supported.");
+ return -1;
+ }
+
+ for (i = 0; i < dev->data->nb_tx_queues; i++) {
+ cpfl_txq = dev->data->tx_queues[i];
+ if (cpfl_txq == NULL || cpfl_txq->base.tx_deferred_start)
+ continue;
+
+ if (!cpfl_txq->hairpin_info.hairpin_q) {
+ err = cpfl_tx_queue_start(dev, i);
+ if (err != 0) {
+ PMD_DRV_LOG(ERR, "Fail to start Tx queue %u",
i);
+ return err;
+ }
+ } else if (!cpfl_txq->hairpin_info.hairpin_cv) {
+ err = cpfl_set_hairpin_txqinfo(vport, cpfl_txq);
+ if (err) {
+ PMD_DRV_LOG(ERR, "Fail to configure hairpin Tx
queue %u", i);
+ return err;
+ }
+ }
}
for (i = 0; i < dev->data->nb_rx_queues; i++) {
- rxq = dev->data->rx_queues[i];
- if (rxq == NULL || rxq->rx_deferred_start)
+ cpfl_rxq = dev->data->rx_queues[i];
+ if (cpfl_rxq == NULL || cpfl_rxq->base.rx_deferred_start)
continue;
- err = cpfl_rx_queue_start(dev, i);
- if (err != 0) {
- PMD_DRV_LOG(ERR, "Fail to start Rx queue %u", i);
- return err;
+ if (!cpfl_rxq->hairpin_info.hairpin_q) {
+ err = cpfl_rx_queue_start(dev, i);
+ if (err != 0) {
+ PMD_DRV_LOG(ERR, "Fail to start Rx queue %u",
i);
+ return err;
+ }
+ } else if (!cpfl_rxq->hairpin_info.hairpin_cv) {
+ err = cpfl_set_hairpin_rxqinfo(vport, cpfl_rxq);
+ if (err) {
+ PMD_DRV_LOG(ERR, "Fail to configure hairpin Rx
queue %u", i);
+ return err;
+ }
+ err = cpfl_rx_queue_init(dev, i);
+ if (err) {
+ PMD_DRV_LOG(ERR, "Fail to init hairpin Rx queue
%u", i);
+ return err;
+ }
+ }
+ }
+
+ /* For non-cross vport hairpin queues, enable Txq and Rxq at last. */
+ for (i = 0; i < dev->data->nb_tx_queues; i++) {
+ cpfl_txq = dev->data->tx_queues[i];
+ if (cpfl_txq->hairpin_info.hairpin_q &&
!cpfl_txq->hairpin_info.hairpin_cv) {
+ err = cpfl_switch_hairpin_queue(vport, i, false, true);
+ if (err)
+ PMD_DRV_LOG(ERR, "Failed to switch hairpin TX
queue %u on",
+ i);
+ else
+ cpfl_txq->base.q_started = true;
+ }
+ }
+
+ for (i = 0; i < dev->data->nb_rx_queues; i++) {
+ cpfl_rxq = dev->data->rx_queues[i];
+ if (cpfl_rxq->hairpin_info.hairpin_q &&
!cpfl_rxq->hairpin_info.hairpin_cv) {
+ err = cpfl_switch_hairpin_queue(vport, i, true, true);
+ if (err)
+ PMD_DRV_LOG(ERR, "Failed to switch hairpin RX
queue %u on",
+ i);
+ else
+ cpfl_rxq->base.q_started = true;
}
}
@@ -779,7 +868,9 @@ cpfl_start_queues(struct rte_eth_dev *dev)
static int
cpfl_dev_start(struct rte_eth_dev *dev)
{
- struct idpf_vport *vport = dev->data->dev_private;
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
struct idpf_adapter *base = vport->adapter;
struct cpfl_adapter_ext *adapter = CPFL_ADAPTER_TO_EXT(base);
uint16_t num_allocated_vectors = base->caps.num_allocated_vectors;
@@ -841,10 +932,106 @@ cpfl_dev_start(struct rte_eth_dev *dev)
return ret;
}
+static int
+cpfl_hairpin_get_peer_ports(struct rte_eth_dev *dev, uint16_t *peer_ports,
+ __rte_unused size_t len, uint32_t tx)
+{
+ /* Assume the last queue is used by app as hairpin */
+ int qid = dev->data->nb_tx_queues - 1;
+ struct cpfl_txq_hairpin_info *txq_hairpin_info;
+ struct cpfl_rxq_hairpin_info *rxq_hairpin_info;
+ struct cpfl_tx_queue *cpfl_txq = dev->data->tx_queues[qid];
+ struct cpfl_rx_queue *cpfl_rxq = dev->data->rx_queues[qid];
+
+ PMD_INIT_FUNC_TRACE();
+
+ txq_hairpin_info = &(cpfl_txq->hairpin_info);
+ rxq_hairpin_info = &(cpfl_rxq->hairpin_info);
+
+ if (tx && txq_hairpin_info->hairpin_cv) {
+ peer_ports[0] = txq_hairpin_info->peer_rxp;
+ return 1;
+ } else if (!tx && rxq_hairpin_info->hairpin_cv) {
+ peer_ports[0] = rxq_hairpin_info->peer_txp;
+ return 1;
+ }
+
+ return 0;
+}
+
+static int
+cpfl_hairpin_bind(struct rte_eth_dev *dev, uint16_t rx_port)
+{
+ struct cpfl_vport *cpfl_vport, *peer_cpfl_vport;
+ struct idpf_vport *vport, *peer_vport;
+ /* Assume the last queue is used by app as hairpin */
+ int qid = dev->data->nb_tx_queues - 1;
+ struct cpfl_tx_queue *cpfl_txq = dev->data->tx_queues[qid];
+ struct cpfl_rx_queue *cpfl_rxq;
+ struct rte_eth_dev *peer_dev;
+ int err;
+
+ PMD_INIT_FUNC_TRACE();
+ if (rx_port >= RTE_MAX_ETHPORTS)
+ return 0;
+
+ if (cpfl_txq->hairpin_info.bound) {
+ PMD_DRV_LOG(INFO, "port %u already hairpin bound",
+ dev->data->port_id);
+ return 0;
+ }
+
+ cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ vport = &(cpfl_vport->base);
+ err = cpfl_set_hairpin_txqinfo(vport, cpfl_txq);
+ if (err) {
+ PMD_DRV_LOG(ERR, "Fail to configure hairpin Tx queue %u of port
%u",
+ qid, dev->data->port_id);
+ return err;
+ }
+
+ peer_dev = &rte_eth_devices[rx_port];
+ peer_cpfl_vport = (struct cpfl_vport *)peer_dev->data->dev_private;
+ peer_vport = &(peer_cpfl_vport->base);
+ cpfl_rxq = peer_dev->data->rx_queues[qid];
+ err = cpfl_set_hairpin_rxqinfo(peer_vport, cpfl_rxq);
+ if (err) {
+ PMD_DRV_LOG(ERR, "Fail to configure hairpin Rx queue %u of port
%u",
+ qid, peer_dev->data->port_id);
+ return err;
+ }
+ err = cpfl_rx_queue_init(peer_dev, qid);
+ if (err) {
+ PMD_DRV_LOG(ERR, "Fail to init hairpin Rx queue %u of port %u",
+ qid, peer_dev->data->port_id);
+ return err;
+ }
+
+ err = cpfl_switch_hairpin_queue(vport, qid, false, true);
+ if (err) {
+ PMD_DRV_LOG(ERR, "Fail to enable hairpin Tx queue %u of port
%u",
+ qid, dev->data->port_id);
+ return err;
+ }
+
+ err = cpfl_switch_hairpin_queue(peer_vport, qid, true, true);
+ if (err) {
+ PMD_DRV_LOG(ERR, "Fail to enable hairpin Rx queue %u of port
%u",
+ qid, peer_dev->data->port_id);
+ return err;
+ }
+
+ cpfl_txq->hairpin_info.bound = true;
+ return 0;
+}
+
static int
cpfl_dev_stop(struct rte_eth_dev *dev)
{
- struct idpf_vport *vport = dev->data->dev_private;
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
if (vport->stopped == 1)
return 0;
@@ -865,17 +1052,23 @@ cpfl_dev_stop(struct rte_eth_dev *dev)
static int
cpfl_dev_close(struct rte_eth_dev *dev)
{
- struct idpf_vport *vport = dev->data->dev_private;
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
struct cpfl_adapter_ext *adapter = CPFL_ADAPTER_TO_EXT(vport->adapter);
cpfl_dev_stop(dev);
+ if (cpfl_vport->p2p_mp) {
+ rte_mempool_free(cpfl_vport->p2p_mp);
+ cpfl_vport->p2p_mp = NULL;
+ }
idpf_vport_deinit(vport);
adapter->cur_vports &= ~RTE_BIT32(vport->devarg_id);
adapter->cur_vport_nb--;
dev->data->dev_private = NULL;
adapter->vports[vport->sw_idx] = NULL;
- rte_free(vport);
+ rte_free(cpfl_vport);
return 0;
}
@@ -1048,7 +1241,7 @@ cpfl_find_vport(struct cpfl_adapter_ext *adapter,
uint32_t vport_id)
int i;
for (i = 0; i < adapter->cur_vport_nb; i++) {
- vport = adapter->vports[i];
+ vport = &(adapter->vports[i]->base);
if (vport->vport_id != vport_id)
continue;
else
@@ -1162,6 +1355,72 @@ cpfl_dev_alarm_handler(void *param)
rte_eal_alarm_set(CPFL_ALARM_INTERVAL, cpfl_dev_alarm_handler, adapter);
}
+static int
+cpfl_get_caps(struct idpf_adapter *adapter)
+{
+ struct virtchnl2_get_capabilities caps_msg = {0};
+
+ caps_msg.csum_caps =
+ VIRTCHNL2_CAP_TX_CSUM_L3_IPV4 |
+ VIRTCHNL2_CAP_TX_CSUM_L4_IPV4_TCP |
+ VIRTCHNL2_CAP_TX_CSUM_L4_IPV4_UDP |
+ VIRTCHNL2_CAP_TX_CSUM_L4_IPV4_SCTP |
+ VIRTCHNL2_CAP_TX_CSUM_L4_IPV6_TCP |
+ VIRTCHNL2_CAP_TX_CSUM_L4_IPV6_UDP |
+ VIRTCHNL2_CAP_TX_CSUM_L4_IPV6_SCTP |
+ VIRTCHNL2_CAP_TX_CSUM_GENERIC |
+ VIRTCHNL2_CAP_RX_CSUM_L3_IPV4 |
+ VIRTCHNL2_CAP_RX_CSUM_L4_IPV4_TCP |
+ VIRTCHNL2_CAP_RX_CSUM_L4_IPV4_UDP |
+ VIRTCHNL2_CAP_RX_CSUM_L4_IPV4_SCTP |
+ VIRTCHNL2_CAP_RX_CSUM_L4_IPV6_TCP |
+ VIRTCHNL2_CAP_RX_CSUM_L4_IPV6_UDP |
+ VIRTCHNL2_CAP_RX_CSUM_L4_IPV6_SCTP |
+ VIRTCHNL2_CAP_RX_CSUM_GENERIC;
+
+ caps_msg.rss_caps =
+ VIRTCHNL2_CAP_RSS_IPV4_TCP |
+ VIRTCHNL2_CAP_RSS_IPV4_UDP |
+ VIRTCHNL2_CAP_RSS_IPV4_SCTP |
+ VIRTCHNL2_CAP_RSS_IPV4_OTHER |
+ VIRTCHNL2_CAP_RSS_IPV6_TCP |
+ VIRTCHNL2_CAP_RSS_IPV6_UDP |
+ VIRTCHNL2_CAP_RSS_IPV6_SCTP |
+ VIRTCHNL2_CAP_RSS_IPV6_OTHER |
+ VIRTCHNL2_CAP_RSS_IPV4_AH |
+ VIRTCHNL2_CAP_RSS_IPV4_ESP |
+ VIRTCHNL2_CAP_RSS_IPV4_AH_ESP |
+ VIRTCHNL2_CAP_RSS_IPV6_AH |
+ VIRTCHNL2_CAP_RSS_IPV6_ESP |
+ VIRTCHNL2_CAP_RSS_IPV6_AH_ESP;
+
+ caps_msg.other_caps = VIRTCHNL2_CAP_WB_ON_ITR |
+ VIRTCHNL2_CAP_PTP |
+ VIRTCHNL2_CAP_RX_FLEX_DESC;
+
+ return idpf_vc_get_caps_by_caps_info(adapter, &caps_msg);
+}
+
+static int
+cpfl_adapter_init(struct idpf_adapter *adapter)
+{
+ int ret = 0;
+
+ ret = idpf_adapter_common_init(adapter);
+ if (ret != 0) {
+ PMD_DRV_LOG(ERR, "Failed to init idpf common adapter");
+ return ret;
+ }
+
+ ret = cpfl_get_caps(adapter);
+ if (ret != 0) {
+ PMD_DRV_LOG(ERR, "Failed to get capabilities");
+ return ret;
+ }
+
+ return ret;
+}
+
static int
cpfl_adapter_ext_init(struct rte_pci_device *pci_dev, struct cpfl_adapter_ext
*adapter)
{
@@ -1178,7 +1437,7 @@ cpfl_adapter_ext_init(struct rte_pci_device *pci_dev,
struct cpfl_adapter_ext *a
strncpy(adapter->name, pci_dev->device.name, PCI_PRI_STR_SIZE);
- ret = idpf_adapter_init(base);
+ ret = cpfl_adapter_init(base);
if (ret != 0) {
PMD_INIT_LOG(ERR, "Failed to init adapter");
goto err_adapter_init;
@@ -1237,6 +1496,11 @@ static const struct eth_dev_ops cpfl_eth_dev_ops = {
.xstats_get = cpfl_dev_xstats_get,
.xstats_get_names = cpfl_dev_xstats_get_names,
.xstats_reset = cpfl_dev_xstats_reset,
+ .hairpin_cap_get = cpfl_hairpin_cap_get,
+ .rx_hairpin_queue_setup = cpfl_rx_hairpin_queue_setup,
+ .tx_hairpin_queue_setup = cpfl_tx_hairpin_queue_setup,
+ .hairpin_get_peer_ports = cpfl_hairpin_get_peer_ports,
+ .hairpin_bind = cpfl_hairpin_bind,
};
static uint16_t
@@ -1261,7 +1525,9 @@ cpfl_vport_idx_alloc(struct cpfl_adapter_ext *ad)
static int
cpfl_dev_vport_init(struct rte_eth_dev *dev, void *init_params)
{
- struct idpf_vport *vport = dev->data->dev_private;
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
struct cpfl_vport_param *param = init_params;
struct cpfl_adapter_ext *adapter = param->adapter;
/* for sending create vport virtchnl msg prepare */
@@ -1287,7 +1553,7 @@ cpfl_dev_vport_init(struct rte_eth_dev *dev, void
*init_params)
goto err;
}
- adapter->vports[param->idx] = vport;
+ adapter->vports[param->idx] = cpfl_vport;
adapter->cur_vports |= RTE_BIT32(param->devarg_id);
adapter->cur_vport_nb++;
@@ -1370,7 +1636,7 @@ cpfl_pci_probe(struct rte_pci_driver *pci_drv
__rte_unused,
if (adapter == NULL) {
first_probe = true;
adapter = rte_zmalloc("cpfl_adapter_ext",
- sizeof(struct
cpfl_adapter_ext), 0);
+ sizeof(struct cpfl_adapter_ext), 0);
if (adapter == NULL) {
PMD_INIT_LOG(ERR, "Failed to allocate adapter.");
return -ENOMEM;
@@ -1405,7 +1671,7 @@ cpfl_pci_probe(struct rte_pci_driver *pci_drv
__rte_unused,
snprintf(name, sizeof(name), "cpfl_%s_vport_0",
pci_dev->device.name);
retval = rte_eth_dev_create(&pci_dev->device, name,
- sizeof(struct idpf_vport),
+ sizeof(struct cpfl_vport),
NULL, NULL, cpfl_dev_vport_init,
&vport_param);
if (retval != 0)
@@ -1423,7 +1689,7 @@ cpfl_pci_probe(struct rte_pci_driver *pci_drv
__rte_unused,
pci_dev->device.name,
devargs.req_vports[i]);
retval = rte_eth_dev_create(&pci_dev->device, name,
- sizeof(struct idpf_vport),
+ sizeof(struct cpfl_vport),
NULL, NULL,
cpfl_dev_vport_init,
&vport_param);
if (retval != 0)
diff --git a/drivers/net/cpfl/cpfl_ethdev.h b/drivers/net/cpfl/cpfl_ethdev.h
index 0d60ee3aed..65c10c0c64 100644
--- a/drivers/net/cpfl/cpfl_ethdev.h
+++ b/drivers/net/cpfl/cpfl_ethdev.h
@@ -70,13 +70,19 @@ struct cpfl_devargs {
uint16_t req_vport_nb;
};
+struct cpfl_vport {
+ /* p2p mbuf pool */
+ struct rte_mempool *p2p_mp;
+ struct idpf_vport base;
+};
+
struct cpfl_adapter_ext {
TAILQ_ENTRY(cpfl_adapter_ext) next;
struct idpf_adapter base;
char name[CPFL_ADAPTER_NAME_LEN];
- struct idpf_vport **vports;
+ struct cpfl_vport **vports;
uint16_t max_vport_nb;
uint16_t cur_vports; /* bit mask of created vport */
diff --git a/drivers/net/cpfl/cpfl_logs.h b/drivers/net/cpfl/cpfl_logs.h
index 451bdfbd1d..ddfeb5434c 100644
--- a/drivers/net/cpfl/cpfl_logs.h
+++ b/drivers/net/cpfl/cpfl_logs.h
@@ -18,6 +18,8 @@ extern int cpfl_logtype_driver;
__func__, \
RTE_FMT_TAIL(__VA_ARGS__,)))
+#define PMD_INIT_FUNC_TRACE() PMD_DRV_LOG(DEBUG, " >>")
+
#define PMD_DRV_LOG_RAW(level, ...) \
rte_log(RTE_LOG_ ## level, \
cpfl_logtype_driver, \
diff --git a/drivers/net/cpfl/cpfl_rxtx.c b/drivers/net/cpfl/cpfl_rxtx.c
index b15323a4f4..8af2245a61 100644
--- a/drivers/net/cpfl/cpfl_rxtx.c
+++ b/drivers/net/cpfl/cpfl_rxtx.c
@@ -10,6 +10,79 @@
#include "cpfl_rxtx.h"
#include "cpfl_rxtx_vec_common.h"
+#define CPFL_NB_MBUF 4096
+#define CPFL_CACHE_SIZE 250
+#define CPFL_MBUF_SIZE 2048
+#define CPFL_P2P_RING_BUF 128
+
+static void
+cpfl_tx_queue_release(void *txq);
+static void
+cpfl_rx_queue_release(void *txq);
+static int
+cpfl_alloc_split_p2p_rxq_mbufs(struct idpf_rx_queue *rxq);
+
+static inline void
+reset_tx_hairpin_descq(struct idpf_tx_queue *txq)
+{
+ uint32_t i, size;
+
+ if (!txq) {
+ PMD_DRV_LOG(DEBUG, "Pointer to txq is NULL");
+ return;
+ }
+
+ size = txq->nb_tx_desc * CPFL_P2P_DESC_LEN;
+ for (i = 0; i < size; i++)
+ ((volatile char *)txq->desc_ring)[i] = 0;
+}
+
+static inline void
+reset_tx_hairpin_complq(struct idpf_tx_queue *cq)
+{
+ uint32_t i, size;
+
+ if (!cq) {
+ PMD_DRV_LOG(DEBUG, "Pointer to complq is NULL");
+ return;
+ }
+
+ size = cq->nb_tx_desc * CPFL_P2P_DESC_LEN;
+ for (i = 0; i < size; i++)
+ ((volatile char *)cq->compl_ring)[i] = 0;
+}
+
+static inline void
+reset_rx_hairpin_descq(struct idpf_rx_queue *rxq)
+{
+ uint16_t len;
+ uint32_t i;
+
+ if (!rxq)
+ return;
+
+ len = rxq->nb_rx_desc;
+ for (i = 0; i < len * CPFL_P2P_DESC_LEN; i++)
+ ((volatile char *)rxq->rx_ring)[i] = 0;
+}
+
+static inline void
+reset_rx_hairpin_bufq(struct idpf_rx_queue *rxbq)
+{
+ uint16_t len;
+ uint32_t i;
+
+ if (!rxbq)
+ return;
+
+ len = rxbq->nb_rx_desc;
+ for (i = 0; i < len * CPFL_P2P_DESC_LEN; i++)
+ ((volatile char *)rxbq->rx_ring)[i] = 0;
+
+ rxbq->bufq1 = NULL;
+ rxbq->bufq2 = NULL;
+}
+
static uint64_t
cpfl_rx_offload_convert(uint64_t offload)
{
@@ -128,7 +201,9 @@ cpfl_rx_split_bufq_setup(struct rte_eth_dev *dev, struct
idpf_rx_queue *rxq,
uint16_t nb_desc, unsigned int socket_id,
struct rte_mempool *mp, uint8_t bufq_id)
{
- struct idpf_vport *vport = dev->data->dev_private;
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
struct idpf_adapter *adapter = vport->adapter;
struct idpf_hw *hw = &adapter->hw;
const struct rte_memzone *mz;
@@ -225,9 +300,12 @@ cpfl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t
queue_idx,
const struct rte_eth_rxconf *rx_conf,
struct rte_mempool *mp)
{
- struct idpf_vport *vport = dev->data->dev_private;
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
struct idpf_adapter *adapter = vport->adapter;
struct idpf_hw *hw = &adapter->hw;
+ struct cpfl_rx_queue *cpfl_rxq;
const struct rte_memzone *mz;
struct idpf_rx_queue *rxq;
uint16_t rx_free_thresh;
@@ -247,21 +325,23 @@ cpfl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t
queue_idx,
/* Free memory if needed */
if (dev->data->rx_queues[queue_idx] != NULL) {
- idpf_rx_queue_release(dev->data->rx_queues[queue_idx]);
+ cpfl_rx_queue_release(dev->data->rx_queues[queue_idx]);
dev->data->rx_queues[queue_idx] = NULL;
}
/* Setup Rx queue */
- rxq = rte_zmalloc_socket("cpfl rxq",
- sizeof(struct idpf_rx_queue),
+ cpfl_rxq = rte_zmalloc_socket("cpfl rxq",
+ sizeof(struct cpfl_rx_queue),
RTE_CACHE_LINE_SIZE,
socket_id);
- if (rxq == NULL) {
+ if (cpfl_rxq == NULL) {
PMD_INIT_LOG(ERR, "Failed to allocate memory for rx queue data
structure");
ret = -ENOMEM;
goto err_rxq_alloc;
}
+ rxq = &(cpfl_rxq->base);
+
is_splitq = !!(vport->rxq_model == VIRTCHNL2_QUEUE_MODEL_SPLIT);
rxq->mp = mp;
@@ -328,7 +408,7 @@ cpfl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t
queue_idx,
}
rxq->q_set = true;
- dev->data->rx_queues[queue_idx] = rxq;
+ dev->data->rx_queues[queue_idx] = cpfl_rxq;
return 0;
@@ -348,7 +428,9 @@ cpfl_tx_complq_setup(struct rte_eth_dev *dev, struct
idpf_tx_queue *txq,
uint16_t queue_idx, uint16_t nb_desc,
unsigned int socket_id)
{
- struct idpf_vport *vport = dev->data->dev_private;
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
const struct rte_memzone *mz;
struct idpf_tx_queue *cq;
int ret;
@@ -396,15 +478,18 @@ cpfl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t
queue_idx,
uint16_t nb_desc, unsigned int socket_id,
const struct rte_eth_txconf *tx_conf)
{
- struct idpf_vport *vport = dev->data->dev_private;
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
struct idpf_adapter *adapter = vport->adapter;
uint16_t tx_rs_thresh, tx_free_thresh;
struct idpf_hw *hw = &adapter->hw;
+ struct cpfl_tx_queue *cpfl_txq;
const struct rte_memzone *mz;
struct idpf_tx_queue *txq;
uint64_t offloads;
- uint16_t len;
bool is_splitq;
+ uint16_t len;
int ret;
offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
@@ -418,21 +503,23 @@ cpfl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t
queue_idx,
/* Free memory if needed. */
if (dev->data->tx_queues[queue_idx] != NULL) {
- idpf_tx_queue_release(dev->data->tx_queues[queue_idx]);
+ cpfl_tx_queue_release(dev->data->tx_queues[queue_idx]);
dev->data->tx_queues[queue_idx] = NULL;
}
/* Allocate the TX queue data structure. */
- txq = rte_zmalloc_socket("cpfl txq",
- sizeof(struct idpf_tx_queue),
+ cpfl_txq = rte_zmalloc_socket("cpfl txq",
+ sizeof(struct cpfl_tx_queue),
RTE_CACHE_LINE_SIZE,
socket_id);
- if (txq == NULL) {
+ if (cpfl_txq == NULL) {
PMD_INIT_LOG(ERR, "Failed to allocate memory for tx queue
structure");
ret = -ENOMEM;
goto err_txq_alloc;
}
+ txq = &(cpfl_txq->base);
+
is_splitq = !!(vport->txq_model == VIRTCHNL2_QUEUE_MODEL_SPLIT);
txq->nb_tx_desc = nb_desc;
@@ -486,7 +573,7 @@ cpfl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t
queue_idx,
queue_idx * vport->chunks_info.tx_qtail_spacing);
txq->ops = &def_txq_ops;
txq->q_set = true;
- dev->data->tx_queues[queue_idx] = txq;
+ dev->data->tx_queues[queue_idx] = cpfl_txq;
return 0;
@@ -502,6 +589,7 @@ cpfl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t
queue_idx,
int
cpfl_rx_queue_init(struct rte_eth_dev *dev, uint16_t rx_queue_id)
{
+ struct cpfl_rx_queue *cpfl_rxq;
struct idpf_rx_queue *rxq;
uint16_t max_pkt_len;
uint32_t frame_size;
@@ -510,7 +598,8 @@ cpfl_rx_queue_init(struct rte_eth_dev *dev, uint16_t
rx_queue_id)
if (rx_queue_id >= dev->data->nb_rx_queues)
return -EINVAL;
- rxq = dev->data->rx_queues[rx_queue_id];
+ cpfl_rxq = dev->data->rx_queues[rx_queue_id];
+ rxq = &(cpfl_rxq->base);
if (rxq == NULL || !rxq->q_set) {
PMD_DRV_LOG(ERR, "RX queue %u not available or setup",
@@ -550,36 +639,97 @@ cpfl_rx_queue_init(struct rte_eth_dev *dev, uint16_t
rx_queue_id)
IDPF_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
} else {
/* Split queue */
- err = idpf_alloc_split_rxq_mbufs(rxq->bufq1);
- if (err != 0) {
- PMD_DRV_LOG(ERR, "Failed to allocate RX buffer queue
mbuf");
- return err;
- }
- err = idpf_alloc_split_rxq_mbufs(rxq->bufq2);
- if (err != 0) {
- PMD_DRV_LOG(ERR, "Failed to allocate RX buffer queue
mbuf");
- return err;
+ if (cpfl_rxq->hairpin_info.hairpin_q) {
+ err = cpfl_alloc_split_p2p_rxq_mbufs(rxq->bufq1);
+ if (err != 0) {
+ PMD_DRV_LOG(ERR, "Failed to allocate p2p RX
buffer queue mbuf");
+ return err;
+ }
+ } else {
+ err = idpf_alloc_split_rxq_mbufs(rxq->bufq1);
+ if (err != 0) {
+ PMD_DRV_LOG(ERR, "Failed to allocate RX buffer
queue mbuf");
+ return err;
+ }
+
+ err = idpf_alloc_split_rxq_mbufs(rxq->bufq2);
+ if (err != 0) {
+ PMD_DRV_LOG(ERR, "Failed to allocate RX buffer
queue mbuf");
+ return err;
+ }
}
rte_wmb();
/* Init the RX tail register. */
IDPF_PCI_REG_WRITE(rxq->bufq1->qrx_tail, rxq->bufq1->rx_tail);
- IDPF_PCI_REG_WRITE(rxq->bufq2->qrx_tail, rxq->bufq2->rx_tail);
+ if (rxq->bufq2)
+ IDPF_PCI_REG_WRITE(rxq->bufq2->qrx_tail,
rxq->bufq2->rx_tail);
}
return err;
}
+static bool cpfl_is_hairpin_txq(u32 txq_model, bool hairpin_txq)
+{
+ return (txq_model == VIRTCHNL2_QUEUE_MODEL_SPLIT) && hairpin_txq;
+}
+
+static bool cpfl_is_hairpin_rxq(u32 rxq_model, bool hairpin_rxq)
+{
+ return (rxq_model == VIRTCHNL2_QUEUE_MODEL_SPLIT) && hairpin_rxq;
+}
+
+int
+cpfl_set_hairpin_rxqinfo(struct idpf_vport *vport, struct cpfl_rx_queue
*cpfl_rxq)
+{
+ struct virtchnl2_rxq_info rxq_info[IDPF_RXQ_PER_GRP + 1] = {0};
+ struct idpf_rx_queue *rxq = &(cpfl_rxq->base);
+ struct idpf_rx_queue *bufq = rxq->bufq1;
+
+ rxq_info->type = VIRTCHNL2_QUEUE_TYPE_P2P_RX;
+ rxq_info->queue_id = rxq->queue_id;
+ rxq_info->ring_len = rxq->nb_rx_desc;
+ rxq_info->dma_ring_addr = rxq->rx_ring_phys_addr;
+ rxq_info->rx_bufq1_id = bufq->queue_id;
+ rxq_info->max_pkt_size = vport->max_pkt_len;
+ rxq_info->desc_ids = VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M;
+ rxq_info->qflags |= VIRTCHNL2_RX_DESC_SIZE_32BYTE;
+
+ rxq_info->data_buffer_size = rxq->rx_buf_len;
+ rxq_info->model = VIRTCHNL2_QUEUE_MODEL_SPLIT;
+ rxq_info->rx_buffer_low_watermark = 64;
+
+ /* Buffer queue */
+ rxq_info->type = VIRTCHNL2_QUEUE_TYPE_P2P_RX_BUFFER;
+ rxq_info->queue_id = bufq->queue_id;
+ rxq_info->ring_len = bufq->nb_rx_desc;
+ rxq_info->dma_ring_addr = bufq->rx_ring_phys_addr;
+ rxq_info->desc_ids = VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M;
+ rxq_info->rx_buffer_low_watermark = 64;
+ rxq_info->model = VIRTCHNL2_QUEUE_MODEL_SPLIT;
+ rxq_info->data_buffer_size = bufq->rx_buf_len;
+ rxq_info->buffer_notif_stride = CPFL_RX_BUF_STRIDE;
+ PMD_DRV_LOG(NOTICE, "hairpin: vport %u, Rxbufq id 0x%x",
+ vport->vport_id, rxq_info->queue_id);
+ return idpf_vc_config_rxq_by_info(vport, rxq_info, CPFL_RXQ_PER_GRP +
1);
+}
+
int
cpfl_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
{
- struct idpf_vport *vport = dev->data->dev_private;
- struct idpf_rx_queue *rxq =
- dev->data->rx_queues[rx_queue_id];
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
+ struct cpfl_rx_queue *cpfl_rxq = dev->data->rx_queues[rx_queue_id];
+ struct idpf_rx_queue *rxq = &(cpfl_rxq->base);
int err = 0;
- err = idpf_vc_config_rxq(vport, rxq);
+ if (cpfl_is_hairpin_rxq(vport->rxq_model,
cpfl_rxq->hairpin_info.hairpin_q))
+ err = cpfl_set_hairpin_rxqinfo(vport, cpfl_rxq);
+ else
+ err = idpf_vc_config_rxq(vport, rxq);
+
if (err != 0) {
PMD_DRV_LOG(ERR, "Fail to configure Rx queue %u", rx_queue_id);
return err;
@@ -609,28 +759,61 @@ cpfl_rx_queue_start(struct rte_eth_dev *dev, uint16_t
rx_queue_id)
int
cpfl_tx_queue_init(struct rte_eth_dev *dev, uint16_t tx_queue_id)
{
- struct idpf_tx_queue *txq;
+ struct cpfl_tx_queue *cpfl_txq;
if (tx_queue_id >= dev->data->nb_tx_queues)
return -EINVAL;
- txq = dev->data->tx_queues[tx_queue_id];
-
+ cpfl_txq = dev->data->tx_queues[tx_queue_id];
/* Init the RX tail register. */
- IDPF_PCI_REG_WRITE(txq->qtx_tail, 0);
+ IDPF_PCI_REG_WRITE(cpfl_txq->base.qtx_tail, 0);
return 0;
}
+int
+cpfl_set_hairpin_txqinfo(struct idpf_vport *vport, struct cpfl_tx_queue
*cpfl_txq)
+{
+ struct idpf_tx_queue *txq = &(cpfl_txq->base);
+ struct virtchnl2_txq_info txq_info[CPFL_RXQ_PER_GRP + 1] = {0};
+
+ /* txq info */
+ txq_info[0].dma_ring_addr = txq->tx_ring_phys_addr;
+ txq_info[0].type = VIRTCHNL2_QUEUE_TYPE_P2P_TX;
+ txq_info[0].queue_id = txq->queue_id;
+ txq_info[0].ring_len = txq->nb_tx_desc;
+ txq_info[0].tx_compl_queue_id = txq->complq->queue_id;
+ txq_info[0].relative_queue_id = txq->queue_id;
+ txq_info[0].peer_rx_queue_id = cpfl_txq->hairpin_info.peer_rxq_id;
+ txq_info[0].model = VIRTCHNL2_QUEUE_MODEL_SPLIT;
+ txq_info[0].sched_mode = VIRTCHNL2_TXQ_SCHED_MODE_FLOW;
+
+ /* tx completion queue info */
+ txq_info[1].dma_ring_addr = txq->complq->tx_ring_phys_addr;
+ txq_info[1].type = VIRTCHNL2_QUEUE_TYPE_P2P_TX_COMPLETION;
+ txq_info[1].queue_id = txq->complq->queue_id;
+ txq_info[1].ring_len = txq->complq->nb_tx_desc;
+ txq_info[1].peer_rx_queue_id =
cpfl_txq->hairpin_info.complq_peer_rxq_id;
+ txq_info[1].model = VIRTCHNL2_QUEUE_MODEL_SPLIT;
+ txq_info[1].sched_mode = VIRTCHNL2_TXQ_SCHED_MODE_FLOW;
+
+ return idpf_vc_config_txq_by_info(vport, txq_info, CPFL_RXQ_PER_GRP +
1);
+}
+
int
cpfl_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
{
- struct idpf_vport *vport = dev->data->dev_private;
- struct idpf_tx_queue *txq =
- dev->data->tx_queues[tx_queue_id];
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
+ struct cpfl_tx_queue *cpfl_txq =
+ dev->data->tx_queues[tx_queue_id];
int err = 0;
- err = idpf_vc_config_txq(vport, txq);
+ if (cpfl_is_hairpin_txq(vport->txq_model,
cpfl_txq->hairpin_info.hairpin_q))
+ err = cpfl_set_hairpin_txqinfo(vport, cpfl_txq);
+ else
+ err = idpf_vc_config_txq(vport, &(cpfl_txq->base));
if (err != 0) {
PMD_DRV_LOG(ERR, "Fail to configure Tx queue %u", tx_queue_id);
return err;
@@ -649,7 +832,7 @@ cpfl_tx_queue_start(struct rte_eth_dev *dev, uint16_t
tx_queue_id)
PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on",
tx_queue_id);
} else {
- txq->q_started = true;
+ cpfl_txq->base.q_started = true;
dev->data->tx_queue_state[tx_queue_id] =
RTE_ETH_QUEUE_STATE_STARTED;
}
@@ -660,30 +843,46 @@ cpfl_tx_queue_start(struct rte_eth_dev *dev, uint16_t
tx_queue_id)
int
cpfl_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
{
- struct idpf_vport *vport = dev->data->dev_private;
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
+ struct cpfl_rx_queue *cpfl_rxq;
struct idpf_rx_queue *rxq;
int err;
if (rx_queue_id >= dev->data->nb_rx_queues)
return -EINVAL;
- err = idpf_switch_queue(vport, rx_queue_id, true, false);
+ cpfl_rxq = dev->data->rx_queues[rx_queue_id];
+ if (cpfl_rxq->hairpin_info.hairpin_q)
+ err = cpfl_switch_hairpin_queue(vport, rx_queue_id, true,
false);
+ else
+ err = idpf_switch_queue(vport, rx_queue_id, true, false);
+
if (err != 0) {
PMD_DRV_LOG(ERR, "Failed to switch RX queue %u off",
rx_queue_id);
return err;
}
- rxq = dev->data->rx_queues[rx_queue_id];
+ rxq = &(cpfl_rxq->base);
if (vport->rxq_model == VIRTCHNL2_QUEUE_MODEL_SINGLE) {
rxq->ops->release_mbufs(rxq);
idpf_reset_single_rx_queue(rxq);
} else {
rxq->bufq1->ops->release_mbufs(rxq->bufq1);
- rxq->bufq2->ops->release_mbufs(rxq->bufq2);
- idpf_reset_split_rx_queue(rxq);
+ if (rxq->bufq2)
+ rxq->bufq2->ops->release_mbufs(rxq->bufq2);
+
+ if (cpfl_rxq->hairpin_info.hairpin_q) {
+ reset_rx_hairpin_descq(rxq);
+ reset_rx_hairpin_bufq(rxq->bufq1);
+ } else {
+ idpf_reset_split_rx_queue(rxq);
+ }
}
- dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
+ if (!cpfl_rxq->hairpin_info.hairpin_q)
+ dev->data->rx_queue_state[rx_queue_id] =
RTE_ETH_QUEUE_STATE_STOPPED;
return 0;
}
@@ -691,64 +890,141 @@ cpfl_rx_queue_stop(struct rte_eth_dev *dev, uint16_t
rx_queue_id)
int
cpfl_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
{
- struct idpf_vport *vport = dev->data->dev_private;
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
+ struct cpfl_tx_queue *cpfl_txq;
struct idpf_tx_queue *txq;
int err;
if (tx_queue_id >= dev->data->nb_tx_queues)
return -EINVAL;
- err = idpf_switch_queue(vport, tx_queue_id, false, false);
+ cpfl_txq = dev->data->tx_queues[tx_queue_id];
+ if (cpfl_txq->hairpin_info.hairpin_q)
+ err = cpfl_switch_hairpin_queue(vport, tx_queue_id, false,
false);
+ else
+ err = idpf_switch_queue(vport, tx_queue_id, false, false);
if (err != 0) {
PMD_DRV_LOG(ERR, "Failed to switch TX queue %u off",
tx_queue_id);
return err;
}
- txq = dev->data->tx_queues[tx_queue_id];
+ txq = &(cpfl_txq->base);
txq->ops->release_mbufs(txq);
if (vport->txq_model == VIRTCHNL2_QUEUE_MODEL_SINGLE) {
idpf_reset_single_tx_queue(txq);
} else {
- idpf_reset_split_tx_descq(txq);
- idpf_reset_split_tx_complq(txq->complq);
+ if (cpfl_txq->hairpin_info.hairpin_q) {
+ reset_tx_hairpin_descq(txq);
+ reset_tx_hairpin_complq(txq->complq);
+ } else {
+ idpf_reset_split_tx_descq(txq);
+ idpf_reset_split_tx_complq(txq->complq);
+ }
}
- dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
+ if (!cpfl_txq->hairpin_info.hairpin_q)
+ dev->data->tx_queue_state[tx_queue_id] =
RTE_ETH_QUEUE_STATE_STOPPED;
return 0;
}
+void
+cpfl_rx_queue_release(void *rxq)
+{
+ struct cpfl_rx_queue *cpfl_rxq = rxq;
+ struct idpf_rx_queue *q = NULL;
+
+ if (cpfl_rxq == NULL)
+ return;
+
+ q = &(cpfl_rxq->base);
+
+ /* Split queue */
+ if (q->bufq1 != NULL) {
+ /* the mz is shared between Tx/Rx hairpin, let Tx_release
+ * free the buf.
+ */
+ if (!cpfl_rxq->hairpin_info.hairpin_q) {
+ rte_memzone_free(q->bufq1->mz);
+ if (q->bufq2 != NULL)
+ rte_memzone_free(q->bufq2->mz);
+ rte_memzone_free(q->mz);
+ }
+ q->bufq1->ops->release_mbufs(q->bufq1);
+ rte_free(q->bufq1->sw_ring);
+ rte_free(q->bufq1);
+
+ if (q->bufq2 != NULL) {
+ q->bufq2->ops->release_mbufs(q->bufq2);
+ rte_free(q->bufq2->sw_ring);
+ rte_free(q->bufq2);
+ }
+ rte_free(cpfl_rxq);
+ return;
+ }
+
+ /* Single queue */
+ q->ops->release_mbufs(q);
+ rte_free(q->sw_ring);
+ rte_memzone_free(q->mz);
+ rte_free(cpfl_rxq);
+}
+
void
cpfl_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
{
- idpf_rx_queue_release(dev->data->rx_queues[qid]);
+ cpfl_rx_queue_release(dev->data->rx_queues[qid]);
+}
+
+void
+cpfl_tx_queue_release(void *txq)
+{
+ struct cpfl_tx_queue *cpfl_txq = txq;
+ struct idpf_tx_queue *q = NULL;
+
+ if (cpfl_txq == NULL)
+ return;
+
+ q = &(cpfl_txq->base);
+
+ if (q->complq) {
+ rte_memzone_free(q->complq->mz);
+ rte_free(q->complq);
+ }
+
+ q->ops->release_mbufs(q);
+ rte_free(q->sw_ring);
+ rte_memzone_free(q->mz);
+ rte_free(cpfl_txq);
}
void
cpfl_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
{
- idpf_tx_queue_release(dev->data->tx_queues[qid]);
+ cpfl_tx_queue_release(dev->data->tx_queues[qid]);
}
void
cpfl_stop_queues(struct rte_eth_dev *dev)
{
- struct idpf_rx_queue *rxq;
- struct idpf_tx_queue *txq;
+ struct cpfl_rx_queue *cpfl_rxq;
+ struct cpfl_tx_queue *cpfl_txq;
int i;
for (i = 0; i < dev->data->nb_rx_queues; i++) {
- rxq = dev->data->rx_queues[i];
- if (rxq == NULL)
+ cpfl_rxq = dev->data->rx_queues[i];
+ if (cpfl_rxq == NULL)
continue;
-
+ /* hairpin queue is also stopped here. */
if (cpfl_rx_queue_stop(dev, i) != 0)
PMD_DRV_LOG(WARNING, "Fail to stop Rx queue %d", i);
}
for (i = 0; i < dev->data->nb_tx_queues; i++) {
- txq = dev->data->tx_queues[i];
- if (txq == NULL)
+ cpfl_txq = dev->data->tx_queues[i];
+ if (cpfl_txq == NULL)
continue;
if (cpfl_tx_queue_stop(dev, i) != 0)
@@ -759,9 +1035,11 @@ cpfl_stop_queues(struct rte_eth_dev *dev)
void
cpfl_set_rx_function(struct rte_eth_dev *dev)
{
- struct idpf_vport *vport = dev->data->dev_private;
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
#ifdef RTE_ARCH_X86
- struct idpf_rx_queue *rxq;
+ struct cpfl_rx_queue *cpfl_rxq;
int i;
if (cpfl_rx_vec_dev_check_default(dev) == CPFL_VECTOR_PATH &&
@@ -788,8 +1066,10 @@ cpfl_set_rx_function(struct rte_eth_dev *dev)
#ifdef RTE_ARCH_X86
if (vport->rx_vec_allowed) {
for (i = 0; i < dev->data->nb_rx_queues; i++) {
- rxq = dev->data->rx_queues[i];
- (void)idpf_splitq_rx_vec_setup(rxq);
+ cpfl_rxq = dev->data->rx_queues[i];
+ if (cpfl_rxq->hairpin_info.hairpin_q)
+ continue;
+
(void)idpf_splitq_rx_vec_setup(&(cpfl_rxq->base));
}
#ifdef CC_AVX512_SUPPORT
if (vport->rx_use_avx512) {
@@ -803,8 +1083,8 @@ cpfl_set_rx_function(struct rte_eth_dev *dev)
} else {
if (vport->rx_vec_allowed) {
for (i = 0; i < dev->data->nb_rx_queues; i++) {
- rxq = dev->data->rx_queues[i];
- (void)idpf_singleq_rx_vec_setup(rxq);
+ cpfl_rxq = dev->data->rx_queues[i];
+
(void)idpf_singleq_rx_vec_setup(&(cpfl_rxq->base));
}
#ifdef CC_AVX512_SUPPORT
if (vport->rx_use_avx512) {
@@ -836,10 +1116,12 @@ cpfl_set_rx_function(struct rte_eth_dev *dev)
void
cpfl_set_tx_function(struct rte_eth_dev *dev)
{
- struct idpf_vport *vport = dev->data->dev_private;
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
#ifdef RTE_ARCH_X86
#ifdef CC_AVX512_SUPPORT
- struct idpf_tx_queue *txq;
+ struct cpfl_tx_queue *cpfl_txq;
int i;
#endif /* CC_AVX512_SUPPORT */
@@ -854,8 +1136,8 @@ cpfl_set_tx_function(struct rte_eth_dev *dev)
vport->tx_use_avx512 = true;
if (vport->tx_use_avx512) {
for (i = 0; i < dev->data->nb_tx_queues; i++) {
- txq = dev->data->tx_queues[i];
- idpf_tx_vec_setup_avx512(txq);
+ cpfl_txq = dev->data->tx_queues[i];
+
idpf_tx_vec_setup_avx512(&(cpfl_txq->base));
}
}
}
@@ -898,3 +1180,432 @@ cpfl_set_tx_function(struct rte_eth_dev *dev)
dev->tx_pkt_prepare = idpf_prep_pkts;
}
}
+
+int
+cpfl_switch_hairpin_queue(struct idpf_vport *vport, uint16_t qid,
+ bool rx, bool on)
+{
+ uint32_t type;
+ int err, queue_id;
+
+ type = rx ? VIRTCHNL2_QUEUE_TYPE_P2P_RX : VIRTCHNL2_QUEUE_TYPE_P2P_TX;
+
+ /* switch p2p txq/rxq */
+ if (type == VIRTCHNL2_QUEUE_TYPE_P2P_RX)
+ queue_id = vport->chunks_info.rx_start_qid + qid;
+ else
+ queue_id = vport->chunks_info.tx_start_qid + qid;
+ err = idpf_vc_ena_dis_one_queue(vport, queue_id, type, on);
+ if (err)
+ return err;
+
+ /* switch p2p tx completion queue */
+ if (!rx && vport->txq_model == VIRTCHNL2_QUEUE_MODEL_SPLIT) {
+ type = VIRTCHNL2_QUEUE_TYPE_P2P_TX_COMPLETION;
+ queue_id = vport->chunks_info.tx_compl_start_qid + qid;
+ err = idpf_vc_ena_dis_one_queue(vport, queue_id, type, on);
+ if (err)
+ return err;
+ }
+
+ /* switch p2p rx buffer queue */
+ if (rx && vport->rxq_model == VIRTCHNL2_QUEUE_MODEL_SPLIT) {
+ type = VIRTCHNL2_QUEUE_TYPE_P2P_RX_BUFFER;
+ queue_id = vport->chunks_info.rx_buf_start_qid + 2 * qid;
+ err = idpf_vc_ena_dis_one_queue(vport, queue_id, type, on);
+ if (err)
+ return err;
+ }
+
+ return err;
+}
+
+static int
+idpf_rx_hairpin_bufq_setup(struct rte_eth_dev *dev, struct idpf_rx_queue *bufq,
+ uint16_t queue_idx, uint16_t nb_desc,
+ struct idpf_tx_queue *peer_txq)
+{
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
+ struct idpf_adapter *adapter = vport->adapter;
+ struct idpf_hw *hw = &adapter->hw;
+ const struct rte_memzone *mz;
+ struct rte_mempool *mp;
+ uint32_t ring_size;
+ char pool_name[RTE_MEMPOOL_NAMESIZE];
+
+ mp = cpfl_vport->p2p_mp;
+ if (!mp) {
+ snprintf(pool_name, RTE_MEMPOOL_NAMESIZE, "p2p_mb_pool_%u",
+ dev->data->port_id);
+ mp = rte_pktmbuf_pool_create(pool_name, CPFL_NB_MBUF,
CPFL_CACHE_SIZE,
+ 0, CPFL_MBUF_SIZE,
dev->device->numa_node);
+ if (!mp) {
+ PMD_INIT_LOG(ERR, "Failed to allocate mbuf pool for
p2p");
+ return -ENOMEM;
+ }
+ cpfl_vport->p2p_mp = mp;
+ }
+
+ bufq->mp = mp;
+ bufq->nb_rx_desc = nb_desc;
+ bufq->queue_id = vport->chunks_info.rx_buf_start_qid + queue_idx;
+ bufq->port_id = dev->data->port_id;
+ bufq->adapter = adapter;
+ bufq->rx_buf_len = CPFL_MBUF_SIZE - RTE_PKTMBUF_HEADROOM;
+
+ bufq->sw_ring = rte_zmalloc("sw ring",
+ sizeof(struct rte_mbuf *) * nb_desc,
+ RTE_CACHE_LINE_SIZE);
+ if (!bufq->sw_ring) {
+ PMD_INIT_LOG(ERR, "Failed to allocate memory for SW ring");
+ return -ENOMEM;
+ }
+
+ /* Hairpin Rx buffer queue and Tx completion queue share the same HW
ring */
+ if (peer_txq && peer_txq->complq->mz) {
+ mz = peer_txq->complq->mz;
+ bufq->rx_ring_phys_addr = mz->iova;
+ bufq->rx_ring = mz->addr;
+ bufq->mz = mz;
+ } else {
+ ring_size = RTE_ALIGN(bufq->nb_rx_desc * CPFL_P2P_DESC_LEN,
+ CPFL_DMA_MEM_ALIGN);
+ mz = rte_eth_dma_zone_reserve(dev, "hairpin_rx_buf_ring",
queue_idx,
+ ring_size + CPFL_P2P_RING_BUF,
+ CPFL_RING_BASE_ALIGN,
+ dev->device->numa_node);
+ if (!mz) {
+ PMD_INIT_LOG(ERR, "Failed to reserve DMA memory"
+ "for hairpin RX buffer queue.");
+ rte_free(bufq->sw_ring);
+ return -ENOMEM;
+ }
+
+ bufq->rx_ring_phys_addr = mz->iova;
+ bufq->rx_ring = mz->addr;
+ bufq->mz = mz;
+ }
+ reset_rx_hairpin_bufq(bufq);
+ bufq->q_set = true;
+ bufq->qrx_tail = hw->hw_addr + (vport->chunks_info.rx_buf_qtail_start +
+ queue_idx * vport->chunks_info.rx_buf_qtail_spacing);
+ bufq->ops = &def_rxq_ops;
+
+ return 0;
+}
+
+int
+cpfl_rx_hairpin_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
+ uint16_t nb_desc,
+ const struct rte_eth_hairpin_conf *conf)
+{
+ struct cpfl_vport *cpfl_vport = (struct cpfl_vport
*)dev->data->dev_private;
+ struct idpf_vport *vport = &(cpfl_vport->base);
+ struct idpf_adapter *adapter_base = vport->adapter;
+ struct cpfl_adapter_ext *adapter = CPFL_ADAPTER_TO_EXT(adapter_base);
+ struct cpfl_rxq_hairpin_info *hairpin_info;
+ struct cpfl_vport *peer_cpfl_vport;
+ struct rte_eth_dev_data *dev_data;
+ struct cpfl_rx_queue *cpfl_rxq;
+ struct cpfl_tx_queue *peer_txq = NULL;
+ struct idpf_vport *peer_vport;
+ struct idpf_rx_queue *bufq1 = NULL;
+ struct idpf_rx_queue *rxq;
+ uint16_t peer_port = conf->peers[0].port;
+ uint16_t peer_q = conf->peers[0].queue;
+ const struct rte_memzone *mz;
+ uint32_t ring_size;
+ uint16_t qid;
+ int ret;
+
+ if (vport->rxq_model == VIRTCHNL2_QUEUE_MODEL_SINGLE) {
+ PMD_INIT_LOG(ERR, "Only spilt queue model supports hairpin
queue.");
+ return -EINVAL;
+ }
+
+ if (nb_desc % CPFL_ALIGN_RING_DESC != 0 ||
+ nb_desc > CPFL_MAX_RING_DESC ||
+ nb_desc < CPFL_MIN_RING_DESC) {
+ PMD_INIT_LOG(ERR, "Number (%u) of receive descriptors is
invalid", nb_desc);
+ return -EINVAL;
+ }
+
+ /* Free memory if needed */
+ if (dev->data->rx_queues[queue_idx]) {
+ idpf_rx_queue_release(dev->data->rx_queues[queue_idx]);
+ dev->data->rx_queues[queue_idx] = NULL;
+ }
+
+ /* Setup Rx description queue */
+ cpfl_rxq = rte_zmalloc_socket("cpfl hairpin txq",
+ sizeof(struct cpfl_tx_queue),
+ RTE_CACHE_LINE_SIZE,
+ SOCKET_ID_ANY);
+ if (!cpfl_rxq) {
+ PMD_INIT_LOG(ERR, "Failed to allocate memory for rx queue data
structure");
+ return -ENOMEM;
+ }
+
+ rxq = &(cpfl_rxq->base);
+ hairpin_info = &(cpfl_rxq->hairpin_info);
+ rxq->nb_rx_desc = nb_desc * 2;
+ rxq->queue_id = vport->chunks_info.rx_start_qid + queue_idx;
+ rxq->port_id = dev->data->port_id;
+ rxq->adapter = adapter_base;
+ hairpin_info->hairpin_q = true;
+ rxq->rx_buf_len = CPFL_MBUF_SIZE - RTE_PKTMBUF_HEADROOM;
+
+ if (peer_port != dev->data->port_id)
+ hairpin_info->hairpin_cv = true;
+ hairpin_info->peer_txp = peer_port;
+ peer_cpfl_vport = adapter->vports[peer_port];
+ peer_vport = &(peer_cpfl_vport->base);
+ dev_data = peer_vport->dev_data;
+ if (peer_q < dev_data->nb_tx_queues)
+ peer_txq = dev_data->tx_queues[peer_q];
+
+ /* Hairpin Rxq and Txq share the same HW ring */
+ if (peer_txq && peer_txq->base.mz) {
+ mz = peer_txq->base.mz;
+ rxq->rx_ring_phys_addr = mz->iova;
+ rxq->rx_ring = mz->addr;
+ rxq->mz = mz;
+ } else {
+ ring_size = RTE_ALIGN(rxq->nb_rx_desc * CPFL_P2P_DESC_LEN,
+ CPFL_DMA_MEM_ALIGN);
+ mz = rte_eth_dma_zone_reserve(dev, "hairpin_rx_ring", queue_idx,
+ ring_size + CPFL_P2P_RING_BUF,
+ CPFL_RING_BASE_ALIGN,
+ dev->device->numa_node);
+ if (!mz) {
+ PMD_INIT_LOG(ERR, "Failed to reserve DMA memory for
RX");
+ ret = -ENOMEM;
+ goto free_rxq;
+ }
+
+ rxq->rx_ring_phys_addr = mz->iova;
+ rxq->rx_ring = mz->addr;
+ rxq->mz = mz;
+ }
+ reset_rx_hairpin_descq(rxq);
+
+ /* setup 1 Rx buffer queue for 1 hairpin rxq */
+ bufq1 = rte_zmalloc_socket("hairpin rx bufq1",
+ sizeof(struct idpf_rx_queue),
+ RTE_CACHE_LINE_SIZE,
+ SOCKET_ID_ANY);
+ if (!bufq1) {
+ PMD_INIT_LOG(ERR, "Failed to allocate memory for hairpin Rx
buffer queue 1.");
+ ret = -ENOMEM;
+ goto free_mz;
+ }
+ qid = 2 * queue_idx;
+ ret = idpf_rx_hairpin_bufq_setup(dev, bufq1, qid, nb_desc,
&(peer_txq->base));
+ if (ret) {
+ PMD_INIT_LOG(ERR, "Failed to setup hairpin Rx buffer queue 1");
+ ret = -EINVAL;
+ goto free_bufq1;
+ }
+ rxq->bufq1 = bufq1;
+ rxq->bufq2 = NULL;
+ rxq->q_set = true;
+ dev->data->rx_queues[queue_idx] = cpfl_rxq;
+
+ return 0;
+free_bufq1:
+ rte_free(bufq1);
+free_mz:
+ rte_memzone_free(mz);
+free_rxq:
+ rte_free(rxq);
+
+ return ret;
+}
+
+int
+cpfl_tx_hairpin_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
+ uint16_t nb_desc,
+ const struct rte_eth_hairpin_conf *conf)
+{
+ struct cpfl_vport *cpfl_vport =
+ (struct cpfl_vport *)dev->data->dev_private;
+
+ struct idpf_vport *vport = &(cpfl_vport->base);
+ struct idpf_adapter *adapter_base = vport->adapter;
+ struct cpfl_adapter_ext *adapter = CPFL_ADAPTER_TO_EXT(adapter_base);
+ struct cpfl_txq_hairpin_info *hairpin_info;
+ struct cpfl_vport *peer_cpfl_vport;
+ struct rte_eth_dev_data *dev_data;
+ struct idpf_vport *peer_vport;
+ struct idpf_hw *hw = &adapter_base->hw;
+ struct cpfl_tx_queue *cpfl_txq;
+ struct idpf_tx_queue *txq, *cq;
+ struct idpf_rx_queue *peer_rxq = NULL;
+ const struct rte_memzone *mz;
+ uint32_t ring_size;
+ uint16_t peer_port = conf->peers[0].port;
+ uint16_t peer_q = conf->peers[0].queue;
+
+ if (vport->txq_model == VIRTCHNL2_QUEUE_MODEL_SINGLE) {
+ PMD_INIT_LOG(ERR, "Only spilt queue model supports hairpin
queue.");
+ return -EINVAL;
+ }
+
+ if (nb_desc % CPFL_ALIGN_RING_DESC != 0 ||
+ nb_desc > CPFL_MAX_RING_DESC ||
+ nb_desc < CPFL_MIN_RING_DESC) {
+ PMD_INIT_LOG(ERR, "Number (%u) of transmit descriptors is
invalid",
+ nb_desc);
+ return -EINVAL;
+ }
+
+ /* Free memory if needed. */
+ if (dev->data->tx_queues[queue_idx]) {
+ idpf_tx_queue_release(dev->data->tx_queues[queue_idx]);
+ dev->data->tx_queues[queue_idx] = NULL;
+ }
+
+ /* Allocate the TX queue data structure. */
+ cpfl_txq = rte_zmalloc_socket("cpfl hairpin txq",
+ sizeof(struct cpfl_tx_queue),
+ RTE_CACHE_LINE_SIZE,
+ SOCKET_ID_ANY);
+ if (!cpfl_txq) {
+ PMD_INIT_LOG(ERR, "Failed to allocate memory for tx queue
structure");
+ return -ENOMEM;
+ }
+
+ txq = &(cpfl_txq->base);
+ hairpin_info = &(cpfl_txq->hairpin_info);
+ /* Txq ring length should be 2 times of Tx completion queue size. */
+ txq->nb_tx_desc = nb_desc * 2;
+ txq->queue_id = vport->chunks_info.tx_start_qid + queue_idx;
+ txq->port_id = dev->data->port_id;
+ hairpin_info->hairpin_q = true;
+
+ if (peer_port != dev->data->port_id)
+ cpfl_txq->hairpin_info.hairpin_cv = true;
+ hairpin_info->peer_rxp = peer_port;
+ peer_cpfl_vport = adapter->vports[peer_port];
+ peer_vport = &(peer_cpfl_vport->base);
+ hairpin_info->peer_rxq_id = peer_vport->chunks_info.rx_start_qid +
conf->peers[0].queue;
+ dev_data = peer_vport->dev_data;
+ if (peer_q < dev_data->nb_rx_queues)
+ peer_rxq = dev_data->rx_queues[peer_q];
+
+ /* Hairpin Rxq and Txq share the same HW ring */
+ if (peer_rxq && peer_rxq->mz) {
+ mz = peer_rxq->mz;
+ txq->tx_ring_phys_addr = mz->iova;
+ txq->desc_ring = mz->addr;
+ txq->mz = mz;
+ } else {
+ ring_size = RTE_ALIGN(txq->nb_tx_desc * CPFL_P2P_DESC_LEN,
+ CPFL_DMA_MEM_ALIGN);
+ mz = rte_eth_dma_zone_reserve(dev, "hairpin_tx_ring", queue_idx,
+ ring_size + CPFL_P2P_RING_BUF,
+ CPFL_RING_BASE_ALIGN,
+ dev->device->numa_node);
+ if (!mz) {
+ PMD_INIT_LOG(ERR, "Failed to reserve DMA memory for
TX");
+ rte_free(txq->sw_ring);
+ rte_free(txq);
+ return -ENOMEM;
+ }
+
+ txq->tx_ring_phys_addr = mz->iova;
+ txq->desc_ring = mz->addr;
+ txq->mz = mz;
+ }
+
+ reset_tx_hairpin_descq(txq);
+ txq->qtx_tail = hw->hw_addr + (vport->chunks_info.tx_qtail_start +
+ queue_idx * vport->chunks_info.tx_qtail_spacing);
+ txq->ops = &def_txq_ops;
+
+ /* Allocate the TX completion queue data structure. */
+ txq->complq = rte_zmalloc_socket("cpfl hairpin cq",
+ sizeof(struct idpf_tx_queue),
+ RTE_CACHE_LINE_SIZE,
+ dev->device->numa_node);
+ cq = txq->complq;
+ if (!cq) {
+ PMD_INIT_LOG(ERR, "Failed to allocate memory for tx queue
structure");
+ return -ENOMEM;
+ }
+
+ cq->nb_tx_desc = nb_desc;
+ cq->queue_id = vport->chunks_info.tx_compl_start_qid + queue_idx;
+ cq->port_id = dev->data->port_id;
+ hairpin_info->complq_peer_rxq_id =
+ peer_vport->chunks_info.rx_buf_start_qid + conf->peers[0].queue * 2;
+
+ /* Hairpin Rx buffer queue and Tx completion queue share the same HW
ring */
+ if (peer_rxq && peer_rxq->bufq1->mz) {
+ mz = peer_rxq->bufq1->mz;
+ cq->tx_ring_phys_addr = mz->iova;
+ cq->compl_ring = mz->addr;
+ cq->mz = mz;
+ } else {
+ ring_size = RTE_ALIGN(cq->nb_tx_desc * CPFL_P2P_DESC_LEN,
+ CPFL_DMA_MEM_ALIGN);
+ mz = rte_eth_dma_zone_reserve(dev, "hairpin_tx_compl_ring",
queue_idx,
+ ring_size + CPFL_P2P_RING_BUF,
+ CPFL_RING_BASE_ALIGN,
+ dev->device->numa_node);
+ if (!mz) {
+ PMD_INIT_LOG(ERR, "Failed to reserve DMA memory for TX
completion queue");
+ rte_free(txq->sw_ring);
+ rte_free(txq);
+ return -ENOMEM;
+ }
+ cq->tx_ring_phys_addr = mz->iova;
+ cq->compl_ring = mz->addr;
+ cq->mz = mz;
+ }
+
+ reset_tx_hairpin_complq(cq);
+
+ txq->q_set = true;
+ dev->data->tx_queues[queue_idx] = cpfl_txq;
+
+ return 0;
+}
+
+static int
+cpfl_alloc_split_p2p_rxq_mbufs(struct idpf_rx_queue *rxq)
+{
+ volatile struct virtchnl2_p2p_rx_buf_desc *rxd;
+ struct rte_mbuf *mbuf = NULL;
+ uint64_t dma_addr;
+ uint16_t i;
+
+ for (i = 0; i < rxq->nb_rx_desc; i++) {
+ mbuf = rte_mbuf_raw_alloc(rxq->mp);
+ if (unlikely(!mbuf)) {
+ PMD_DRV_LOG(ERR, "Failed to allocate mbuf for RX");
+ return -ENOMEM;
+ }
+
+ rte_mbuf_refcnt_set(mbuf, 1);
+ mbuf->next = NULL;
+ mbuf->data_off = RTE_PKTMBUF_HEADROOM;
+ mbuf->nb_segs = 1;
+ mbuf->port = rxq->port_id;
+ dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
+
+ rxd = &((volatile struct virtchnl2_p2p_rx_buf_desc
*)(rxq->rx_ring))[i];
+ rxd->reserve0 = 0;
+ rxd->pkt_addr = dma_addr;
+
+ rxq->sw_ring[i] = mbuf;
+ }
+
+ rxq->nb_rx_hold = 0;
+ /* The value written in the RX buffer queue tail register, must be a
multiple of 8.*/
+ rxq->rx_tail = rxq->nb_rx_desc - CPFL_HAIRPIN_Q_TAIL_AUX_VALUE;
+
+ return 0;
+}
diff --git a/drivers/net/cpfl/cpfl_rxtx.h b/drivers/net/cpfl/cpfl_rxtx.h
index 2d55f58455..853b1c9d45 100644
--- a/drivers/net/cpfl/cpfl_rxtx.h
+++ b/drivers/net/cpfl/cpfl_rxtx.h
@@ -13,6 +13,7 @@
#define CPFL_MIN_RING_DESC 32
#define CPFL_MAX_RING_DESC 4096
#define CPFL_DMA_MEM_ALIGN 4096
+#define CPFL_P2P_DESC_LEN 16
/* Base address of the HW descriptor ring should be 128B aligned. */
#define CPFL_RING_BASE_ALIGN 128
@@ -23,6 +24,50 @@
#define CPFL_SUPPORT_CHAIN_NUM 5
+#define CPFL_RX_BUF_STRIDE 64
+
+#define CPFL_RXQ_PER_GRP 1
+
+/* The value written in the RX buffer queue tail register,
+ * and in WritePTR field in the TX completion queue context,
+ * must be a multiple of 8.
+ */
+#define CPFL_HAIRPIN_Q_TAIL_AUX_VALUE 8
+
+struct virtchnl2_p2p_rx_buf_desc {
+ __le64 reserve0;
+ __le64 pkt_addr; /* Packet buffer address */
+};
+struct cpfl_rxq_hairpin_info {
+ bool hairpin_q; /* if rx queue is a hairpin queue */
+ /* only valid if the hairpin queue pair crosses vport */
+ bool hairpin_cv;
+ uint16_t peer_txp;
+};
+
+struct cpfl_rx_queue {
+ struct idpf_rx_queue base;
+ struct cpfl_rxq_hairpin_info hairpin_info;
+
+};
+
+struct cpfl_txq_hairpin_info {
+ /* only valid for hairpin queue */
+ bool hairpin_q;
+ /* only valid if the hairpin queue pair crosses vport */
+ bool hairpin_cv;
+ uint16_t peer_rxq_id;
+ uint16_t peer_rxp;
+ bool bound;
+
+ uint16_t complq_peer_rxq_id;
+};
+
+struct cpfl_tx_queue {
+ struct idpf_tx_queue base;
+ struct cpfl_txq_hairpin_info hairpin_info;
+};
+
int cpfl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
uint16_t nb_desc, unsigned int socket_id,
const struct rte_eth_txconf *tx_conf);
@@ -41,4 +86,17 @@ void cpfl_dev_tx_queue_release(struct rte_eth_dev *dev,
uint16_t qid);
void cpfl_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid);
void cpfl_set_rx_function(struct rte_eth_dev *dev);
void cpfl_set_tx_function(struct rte_eth_dev *dev);
+int cpfl_switch_hairpin_queue(struct idpf_vport *vport, uint16_t qid,
+ bool rx, bool on);
+int
+cpfl_set_hairpin_txqinfo(struct idpf_vport *vport, struct cpfl_tx_queue
*cpfl_txq);
+int
+cpfl_set_hairpin_rxqinfo(struct idpf_vport *vport, struct cpfl_rx_queue
*cpfl_rxq);
+int
+cpfl_rx_hairpin_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
+ uint16_t nb_desc, const struct rte_eth_hairpin_conf
*conf);
+int
+cpfl_tx_hairpin_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
+ uint16_t nb_desc,
+ const struct rte_eth_hairpin_conf *conf);
#endif /* _CPFL_RXTX_H_ */
diff --git a/drivers/net/cpfl/cpfl_rxtx_vec_common.h
b/drivers/net/cpfl/cpfl_rxtx_vec_common.h
index 1f01cd40c5..6680771e16 100644
--- a/drivers/net/cpfl/cpfl_rxtx_vec_common.h
+++ b/drivers/net/cpfl/cpfl_rxtx_vec_common.h
@@ -77,14 +77,16 @@ static inline int
cpfl_rx_vec_dev_check_default(struct rte_eth_dev *dev)
{
struct idpf_vport *vport = dev->data->dev_private;
- struct idpf_rx_queue *rxq;
+ struct cpfl_rx_queue *cpfl_rxq;
int i, default_ret, splitq_ret, ret = CPFL_SCALAR_PATH;
for (i = 0; i < dev->data->nb_rx_queues; i++) {
- rxq = dev->data->rx_queues[i];
- default_ret = cpfl_rx_vec_queue_default(rxq);
+ cpfl_rxq = dev->data->rx_queues[i];
+ default_ret = cpfl_rx_vec_queue_default(&cpfl_rxq->base);
if (vport->rxq_model == VIRTCHNL2_QUEUE_MODEL_SPLIT) {
- splitq_ret = cpfl_rx_splitq_vec_default(rxq);
+ if (cpfl_rxq->hairpin_info.hairpin_q)
+ continue;
+ splitq_ret =
cpfl_rx_splitq_vec_default(&cpfl_rxq->base);
ret = splitq_ret && default_ret;
} else
ret = default_ret;
@@ -99,12 +101,14 @@ static inline int
cpfl_tx_vec_dev_check_default(struct rte_eth_dev *dev)
{
int i;
- struct idpf_tx_queue *txq;
+ struct cpfl_tx_queue *cpfl_txq;
int ret = 0;
for (i = 0; i < dev->data->nb_tx_queues; i++) {
- txq = dev->data->tx_queues[i];
- ret = cpfl_tx_vec_queue_default(txq);
+ cpfl_txq = dev->data->tx_queues[i];
+ if (cpfl_txq->hairpin_info.hairpin_q)
+ continue;
+ ret = cpfl_tx_vec_queue_default(&cpfl_txq->base);
if (ret == CPFL_SCALAR_PATH)
return CPFL_SCALAR_PATH;
}
--
2.25.1
