add mbox initialization and handling to enable virtual
functions queue setup.
Signed-off-by: Aman Kumar <aman.ku...@vvdntech.in>
---
drivers/net/qdma/qdma.h | 1 +
drivers/net/qdma/qdma_devops.c | 196 ++++++++++++++++++++++++++++++---
drivers/net/qdma/qdma_ethdev.c | 47 ++++++++
3 files changed, 226 insertions(+), 18 deletions(-)
diff --git a/drivers/net/qdma/qdma.h b/drivers/net/qdma/qdma.h
index 8fb64c21b0..20a1b72dd1 100644
--- a/drivers/net/qdma/qdma.h
+++ b/drivers/net/qdma/qdma.h
@@ -303,6 +303,7 @@ struct qdma_pci_dev {
void qdma_dev_ops_init(struct rte_eth_dev *dev);
void qdma_txq_pidx_update(void *arg);
int qdma_pf_csr_read(struct rte_eth_dev *dev);
+int qdma_vf_csr_read(struct rte_eth_dev *dev);
uint8_t qmda_get_desc_sz_idx(enum rte_pmd_qdma_bypass_desc_len);
diff --git a/drivers/net/qdma/qdma_devops.c b/drivers/net/qdma/qdma_devops.c
index 7f525773d0..e6803dd86f 100644
--- a/drivers/net/qdma/qdma_devops.c
+++ b/drivers/net/qdma/qdma_devops.c
@@ -22,6 +22,8 @@
#include "qdma.h"
#include "qdma_access_common.h"
+#include "qdma_mbox_protocol.h"
+#include "qdma_mbox.h"
#include "qdma_reg_dump.h"
#include "qdma_platform.h"
#include "qdma_devops.h"
@@ -64,6 +66,39 @@ static void qdma_sort_c2h_cntr_th_values(struct qdma_pci_dev
*qdma_dev)
}
#endif /* QDMA_LATENCY_OPTIMIZED */
+int qdma_vf_csr_read(struct rte_eth_dev *dev)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct qdma_mbox_msg *m = qdma_mbox_msg_alloc();
+ int rv, i;
+ struct qdma_csr_info csr_info;
+
+ if (!m)
+ return -ENOMEM;
+
+ qdma_mbox_compose_csr_read(qdma_dev->func_id, m->raw_data);
+ rv = qdma_mbox_msg_send(dev, m, MBOX_OP_RSP_TIMEOUT);
+ if (rv < 0)
+ goto free_msg;
+
+ rv = qdma_mbox_vf_csr_get(m->raw_data, &csr_info);
+ if (rv < 0)
+ goto free_msg;
+ for (i = 0; i < QDMA_NUM_RING_SIZES; i++) {
+ qdma_dev->g_ring_sz[i] = (uint32_t)csr_info.ringsz[i];
+ qdma_dev->g_c2h_buf_sz[i] = (uint32_t)csr_info.bufsz[i];
+ qdma_dev->g_c2h_timer_cnt[i] = (uint32_t)csr_info.timer_cnt[i];
+ qdma_dev->g_c2h_cnt_th[i] = (uint32_t)csr_info.cnt_thres[i];
+ #ifdef QDMA_LATENCY_OPTIMIZED
+ qdma_sort_c2h_cntr_th_values(qdma_dev);
+ #endif /* QDMA_LATENCY_OPTIMIZED */
+ }
+
+free_msg:
+ qdma_mbox_msg_free(m);
+ return rv;
+}
+
int qdma_pf_csr_read(struct rte_eth_dev *dev)
{
int ret = 0;
@@ -131,6 +166,44 @@ static int qdma_pf_fmap_prog(struct rte_eth_dev *dev)
return ret;
}
+int qdma_dev_notify_qadd(struct rte_eth_dev *dev, uint32_t qidx_hw,
+ enum qdma_dev_q_type q_type)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct qdma_mbox_msg *m;
+ int rv = 0;
+
+ m = qdma_mbox_msg_alloc();
+ if (!m)
+ return -ENOMEM;
+
+ qdma_mbox_compose_vf_notify_qadd(qdma_dev->func_id, qidx_hw,
+ q_type, m->raw_data);
+ rv = qdma_mbox_msg_send(dev, m, MBOX_OP_RSP_TIMEOUT);
+
+ qdma_mbox_msg_free(m);
+ return rv;
+}
+
+int qdma_dev_notify_qdel(struct rte_eth_dev *dev, uint32_t qidx_hw,
+ enum qdma_dev_q_type q_type)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct qdma_mbox_msg *m;
+ int rv = 0;
+
+ m = qdma_mbox_msg_alloc();
+ if (!m)
+ return -ENOMEM;
+
+ qdma_mbox_compose_vf_notify_qdel(qdma_dev->func_id, qidx_hw,
+ q_type, m->raw_data);
+ rv = qdma_mbox_msg_send(dev, m, MBOX_OP_RSP_TIMEOUT);
+
+ qdma_mbox_msg_free(m);
+ return rv;
+}
+
uint8_t qmda_get_desc_sz_idx(enum rte_pmd_qdma_bypass_desc_len size)
{
uint8_t ret;
@@ -243,9 +316,33 @@ int qdma_dev_rx_queue_setup(struct rte_eth_dev *dev,
uint16_t rx_queue_id,
return -EINVAL;
}
}
+ } else {
+ err = qdma_dev_notify_qadd(dev, rx_queue_id +
+ qdma_dev->queue_base,
+ QDMA_DEV_Q_TYPE_C2H);
+ if (err < 0)
+ return -EINVAL;
+
+ if (qdma_dev->q_info[rx_queue_id].queue_mode ==
+ RTE_PMD_QDMA_STREAMING_MODE) {
+ err = qdma_dev_notify_qadd(dev, rx_queue_id +
+ qdma_dev->queue_base,
+ QDMA_DEV_Q_TYPE_CMPT);
+ if (err < 0) {
+ qdma_dev_notify_qdel(dev, rx_queue_id +
+ qdma_dev->queue_base,
+ QDMA_DEV_Q_TYPE_C2H);
+ return -EINVAL;
+ }
+ }
}
+
if (!qdma_dev->init_q_range) {
- if (!qdma_dev->is_vf) {
+ if (qdma_dev->is_vf) {
+ err = qdma_vf_csr_read(dev);
+ if (err < 0)
+ goto rx_setup_err;
+ } else {
err = qdma_pf_csr_read(dev);
if (err < 0)
goto rx_setup_err;
@@ -534,18 +631,27 @@ int qdma_dev_rx_queue_setup(struct rte_eth_dev *dev,
uint16_t rx_queue_id,
QDMA_DEV_Q_TYPE_C2H);
if (qdma_dev->q_info[rx_queue_id].queue_mode ==
- RTE_PMD_QDMA_STREAMING_MODE)
+ RTE_PMD_QDMA_STREAMING_MODE) {
qdma_dev_decrement_active_queue
(qdma_dev->dma_device_index,
qdma_dev->func_id,
QDMA_DEV_Q_TYPE_CMPT);
- }
- if (rxq) {
- if (rxq->rx_mz)
- rte_memzone_free(rxq->rx_mz);
- if (rxq->sw_ring)
- rte_free(rxq->sw_ring);
- rte_free(rxq);
+ } else {
+ qdma_dev_notify_qdel(dev, rx_queue_id +
+ qdma_dev->queue_base, QDMA_DEV_Q_TYPE_C2H);
+
+ if (qdma_dev->q_info[rx_queue_id].queue_mode ==
+ RTE_PMD_QDMA_STREAMING_MODE)
+ qdma_dev_notify_qdel(dev, rx_queue_id +
+ qdma_dev->queue_base,
QDMA_DEV_Q_TYPE_CMPT);
+ }
+ if (rxq) {
+ if (rxq->rx_mz)
+ rte_memzone_free(rxq->rx_mz);
+ if (rxq->sw_ring)
+ rte_free(rxq->sw_ring);
+ rte_free(rxq);
+ }
}
return err;
}
@@ -591,9 +697,21 @@ int qdma_dev_tx_queue_setup(struct rte_eth_dev *dev,
uint16_t tx_queue_id,
QDMA_DEV_Q_TYPE_H2C);
if (err != QDMA_SUCCESS)
return -EINVAL;
+ } else {
+ err = qdma_dev_notify_qadd(dev, tx_queue_id +
+ qdma_dev->queue_base,
+ QDMA_DEV_Q_TYPE_H2C);
+ if (err < 0)
+ return -EINVAL;
}
if (!qdma_dev->init_q_range) {
- if (!qdma_dev->is_vf) {
+ if (qdma_dev->is_vf) {
+ err = qdma_vf_csr_read(dev);
+ if (err < 0) {
+ PMD_DRV_LOG(ERR, "CSR read failed\n");
+ goto tx_setup_err;
+ }
+ } else {
err = qdma_pf_csr_read(dev);
if (err < 0) {
PMD_DRV_LOG(ERR, "CSR read failed\n");
@@ -751,16 +869,28 @@ int qdma_dev_tx_queue_setup(struct rte_eth_dev *dev,
uint16_t tx_queue_id,
tx_setup_err:
PMD_DRV_LOG(ERR, " Tx queue setup failed");
- if (!qdma_dev->is_vf)
+ if (!qdma_dev->is_vf) {
qdma_dev_decrement_active_queue(qdma_dev->dma_device_index,
qdma_dev->func_id,
QDMA_DEV_Q_TYPE_H2C);
+ } else {
+ qdma_dev_notify_qdel(dev, tx_queue_id +
+ qdma_dev->queue_base, QDMA_DEV_Q_TYPE_H2C);
+ }
if (txq) {
- if (txq->tx_mz)
- rte_memzone_free(txq->tx_mz);
- if (txq->sw_ring)
- rte_free(txq->sw_ring);
- rte_free(txq);
+ if (qdma_dev->is_vf) {
+ err = qdma_vf_csr_read(dev);
+ if (err < 0) {
+ PMD_DRV_LOG(ERR, "CSR read failed\n");
+ goto tx_setup_err;
+ }
+ } else {
+ if (txq->tx_mz)
+ rte_memzone_free(txq->tx_mz);
+ if (txq->sw_ring)
+ rte_free(txq->sw_ring);
+ rte_free(txq);
+ }
}
return err;
}
@@ -802,11 +932,16 @@ void qdma_dev_tx_queue_release(struct rte_eth_dev *dev,
uint16_t q_id)
PMD_DRV_LOG(INFO, "Remove H2C queue: %d", txq->queue_id);
qdma_dev = txq->dev->data->dev_private;
- if (!qdma_dev->is_vf)
+ if (!qdma_dev->is_vf) {
qdma_dev_decrement_active_queue
(qdma_dev->dma_device_index,
qdma_dev->func_id,
- QDMA_DEV_Q_TYPE_H2C);
+ QDMA_DEV_Q_TYPE_H2C);
+ } else {
+ qdma_dev_notify_qdel(txq->dev, txq->queue_id +
+ qdma_dev->queue_base,
+ QDMA_DEV_Q_TYPE_H2C);
+ }
if (txq->sw_ring)
rte_free(txq->sw_ring);
if (txq->tx_mz)
@@ -837,6 +972,15 @@ void qdma_dev_rx_queue_release(struct rte_eth_dev *dev,
uint16_t q_id)
(qdma_dev->dma_device_index,
qdma_dev->func_id,
QDMA_DEV_Q_TYPE_CMPT);
+ } else {
+ qdma_dev_notify_qdel(rxq->dev, rxq->queue_id +
+ qdma_dev->queue_base,
+ QDMA_DEV_Q_TYPE_C2H);
+
+ if (rxq->st_mode)
+ qdma_dev_notify_qdel(rxq->dev, rxq->queue_id +
+ qdma_dev->queue_base,
+ QDMA_DEV_Q_TYPE_CMPT);
}
if (rxq->sw_ring)
@@ -1111,6 +1255,7 @@ int qdma_dev_reset(struct rte_eth_dev *dev)
{
struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct qdma_mbox_msg *m = NULL;
uint32_t vf_device_count = 0;
uint32_t i = 0;
int ret = 0;
@@ -1141,6 +1286,21 @@ int qdma_dev_reset(struct rte_eth_dev *dev)
for (i = 0; i < pci_dev->max_vfs; i++) {
if (qdma_dev->vfinfo[i].func_id == QDMA_FUNC_ID_INVALID)
continue;
+
+ m = qdma_mbox_msg_alloc();
+ if (!m)
+ return -ENOMEM;
+ qdma_mbox_compose_pf_reset_done_message(m->raw_data,
qdma_dev->func_id,
+ qdma_dev->vfinfo[i].func_id);
+ ret = qdma_mbox_msg_send(dev, m, 0);
+ if (ret < 0)
+ PMD_DRV_LOG(ERR, "Sending reset failed from PF:%d to
VF:%d\n",
+ qdma_dev->func_id,
qdma_dev->vfinfo[i].func_id);
+
+ /* Mark VFs with invalid function id mapping,
+ * and this gets updated when VF comes online again
+ */
+ qdma_dev->vfinfo[i].func_id = QDMA_FUNC_ID_INVALID;
}
/* Start waiting for a maximum of 60 secs to get all its VFs
diff --git a/drivers/net/qdma/qdma_ethdev.c b/drivers/net/qdma/qdma_ethdev.c
index cc1e8eee71..466a9e9284 100644
--- a/drivers/net/qdma/qdma_ethdev.c
+++ b/drivers/net/qdma/qdma_ethdev.c
@@ -25,6 +25,7 @@
#include "qdma_version.h"
#include "qdma_access_common.h"
#include "qdma_access_export.h"
+#include "qdma_mbox.h"
#include "qdma_devops.h"
/* Poll for QDMA errors every 1 second */
@@ -546,6 +547,8 @@ int qdma_eth_dev_init(struct rte_eth_dev *dev)
}
pcie_perf_enable(pci_dev);
+ if (dma_priv->dev_cap.mailbox_en && pci_dev->max_vfs)
+ qdma_mbox_init(dev);
if (!dma_priv->reset_in_progress) {
num_vfs = pci_dev->max_vfs;
@@ -581,13 +584,57 @@ int qdma_eth_dev_init(struct rte_eth_dev *dev)
int qdma_eth_dev_uninit(struct rte_eth_dev *dev)
{
struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct qdma_mbox_msg *m = NULL;
+ int i, rv;
/* only uninitialize in the primary process */
if (rte_eal_process_type() != RTE_PROC_PRIMARY)
return -EPERM;
+ if (qdma_dev->vf_online_count) {
+ for (i = 0; i < pci_dev->max_vfs; i++) {
+ if (qdma_dev->vfinfo[i].func_id == QDMA_FUNC_ID_INVALID)
+ continue;
+ m = qdma_mbox_msg_alloc();
+ if (!m)
+ return -ENOMEM;
+ if (!qdma_dev->reset_in_progress)
+ qdma_mbox_compose_pf_offline(m->raw_data,
+ qdma_dev->func_id,
+ qdma_dev->vfinfo[i].func_id);
+ else
+ qdma_mbox_compose_vf_reset_message(m->raw_data,
+ qdma_dev->func_id,
+ qdma_dev->vfinfo[i].func_id);
+ rv = qdma_mbox_msg_send(dev, m, 0);
+ if (rv < 0)
+ PMD_DRV_LOG(ERR, "Send bye failed from PF:%d to
VF:%d\n",
+ qdma_dev->func_id,
+ qdma_dev->vfinfo[i].func_id);
+ }
+ PMD_DRV_LOG(INFO, "%s: Wait till all VFs shutdown for
PF-%d(DEVFN)\n",
+ __func__,
qdma_dev->func_id);
+ i = 0;
+ while (i < SHUTDOWN_TIMEOUT) {
+ if (!qdma_dev->vf_online_count) {
+ PMD_DRV_LOG(INFO, "%s: VFs shutdown completed
for PF-%d(DEVFN)\n",
+ __func__, qdma_dev->func_id);
+ break;
+ }
+ rte_delay_ms(1);
+ i++;
+ }
+
+ if (i >= SHUTDOWN_TIMEOUT) {
+ PMD_DRV_LOG(ERR, "%s: Failed VFs shutdown for
PF-%d(DEVFN)\n",
+ __func__, qdma_dev->func_id);
+ }
+ }
if (qdma_dev->dev_configured)
qdma_dev_close(dev);
+ if (qdma_dev->dev_cap.mailbox_en && pci_dev->max_vfs)
+ qdma_mbox_uninit(dev);
/* cancel pending polls */
if (qdma_dev->is_master)
--
2.36.1
