> -----Original Message-----
> From: Qiu, Michael
> Sent: Tuesday, December 22, 2015 3:28 PM
> To: He, Shaopeng; dev at dpdk.org
> Subject: Re: [dpdk-dev] [PATCH v4 2/6] fm10k: setup rx queue interrupts for
> PF and VF
>
> On 12/21/2015 6:20 PM, Shaopeng He wrote:
> > In interrupt mode, each rx queue can have one interrupt to notify the
> > up layer application when packets are available in that queue. Some
> > queues also can share one interrupt.
> > Currently, fm10k needs one separate interrupt for mailbox. So, only
> > those drivers which support multiple interrupt vectors e.g. vfio-pci
> > can work in fm10k interrupt mode.
> > This patch uses the RXINT/INT_MAP registers to map interrupt causes
> > (rx queue and other events) to vectors, and enable these interrupts
> > through kernel drivers like vfio-pci.
> >
> > Signed-off-by: Shaopeng He <shaopeng.he at intel.com>
> > Acked-by: Jing Chen <jing.d.chen at intel.com>
> > ---
> > doc/guides/rel_notes/release_2_3.rst | 2 +
> > drivers/net/fm10k/fm10k.h | 3 ++
> > drivers/net/fm10k/fm10k_ethdev.c | 101
> +++++++++++++++++++++++++++++++----
> > 3 files changed, 95 insertions(+), 11 deletions(-)
> >
> > diff --git a/doc/guides/rel_notes/release_2_3.rst
> > b/doc/guides/rel_notes/release_2_3.rst
> > index 99de186..2cb5ebd 100644
> > --- a/doc/guides/rel_notes/release_2_3.rst
> > +++ b/doc/guides/rel_notes/release_2_3.rst
> > @@ -4,6 +4,8 @@ DPDK Release 2.3
> > New Features
> > ------------
> >
> > +* **Added fm10k Rx interrupt support.**
> > +
> >
> > Resolved Issues
> > ---------------
> > diff --git a/drivers/net/fm10k/fm10k.h b/drivers/net/fm10k/fm10k.h
> > index e2f677a..770d6ba 100644
> > --- a/drivers/net/fm10k/fm10k.h
> > +++ b/drivers/net/fm10k/fm10k.h
> > @@ -129,6 +129,9 @@
> > #define RTE_FM10K_TX_MAX_FREE_BUF_SZ 64
> > #define RTE_FM10K_DESCS_PER_LOOP 4
> >
> > +#define FM10K_MISC_VEC_ID RTE_INTR_VEC_ZERO_OFFSET
> > +#define FM10K_RX_VEC_START RTE_INTR_VEC_RXTX_OFFSET
> > +
> > #define FM10K_SIMPLE_TX_FLAG
> ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \
> > ETH_TXQ_FLAGS_NOOFFLOADS)
> >
> > diff --git a/drivers/net/fm10k/fm10k_ethdev.c
> > b/drivers/net/fm10k/fm10k_ethdev.c
> > index d39c33b..a34c5e2 100644
> > --- a/drivers/net/fm10k/fm10k_ethdev.c
> > +++ b/drivers/net/fm10k/fm10k_ethdev.c
> > @@ -54,6 +54,8 @@
> > /* Number of chars per uint32 type */ #define CHARS_PER_UINT32
> > (sizeof(uint32_t)) #define BIT_MASK_PER_UINT32 ((1 <<
> > CHARS_PER_UINT32) - 1)
> > +/* default 1:1 map from queue ID to interrupt vector ID */ #define
> > +Q2V(dev, queue_id) (dev->pci_dev->intr_handle.intr_vec[queue_id])
> >
> > static void fm10k_close_mbx_service(struct fm10k_hw *hw); static
> > void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev); @@ -
> 109,6
> > +111,8 @@ struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[]
> =
> > {
> >
> > #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS +
> FM10K_MAX_QUEUES_PF * \
> > (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
> > +static int
> > +fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
> >
> > static void
> > fm10k_mbx_initlock(struct fm10k_hw *hw) @@ -687,6 +691,7 @@ static
> > int fm10k_dev_rx_init(struct rte_eth_dev *dev) {
> > struct fm10k_hw *hw =
> > FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
> > + struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
> > int i, ret;
> > struct fm10k_rx_queue *rxq;
> > uint64_t base_addr;
> > @@ -694,10 +699,23 @@ fm10k_dev_rx_init(struct rte_eth_dev *dev)
> > uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
> > uint16_t buf_size;
> >
> > - /* Disable RXINT to avoid possible interrupt */
> > - for (i = 0; i < hw->mac.max_queues; i++)
> > + /* enable RXINT for interrupt mode */
> > + i = 0;
> > + if (rte_intr_dp_is_en(intr_handle)) {
> > + for (; i < dev->data->nb_rx_queues; i++) {
> > + FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(dev,
> i));
> > + if (hw->mac.type == fm10k_mac_pf)
> > + FM10K_WRITE_REG(hw,
> FM10K_ITR(Q2V(dev, i)),
> > + FM10K_ITR_AUTOMASK |
> FM10K_ITR_MASK_CLEAR);
> > + else
> > + FM10K_WRITE_REG(hw,
> FM10K_VFITR(Q2V(dev, i)),
> > + FM10K_ITR_AUTOMASK |
> FM10K_ITR_MASK_CLEAR);
> > + }
> > + }
> > + /* Disable other RXINT to avoid possible interrupt */
> > + for (; i < hw->mac.max_queues; i++)
> > FM10K_WRITE_REG(hw, FM10K_RXINT(i),
> > - 3 << FM10K_RXINT_TIMER_SHIFT);
> > + 3 << FM10K_RXINT_TIMER_SHIFT);
> >
> > /* Setup RX queues */
> > for (i = 0; i < dev->data->nb_rx_queues; ++i) { @@ -1053,6 +1071,9
> > @@ fm10k_dev_start(struct rte_eth_dev *dev)
> > return diag;
> > }
> >
> > + if (fm10k_dev_rxq_interrupt_setup(dev))
> > + return -EIO;
> > +
> > diag = fm10k_dev_rx_init(dev);
> > if (diag) {
> > PMD_INIT_LOG(ERR, "RX init failed: %d", diag); @@ -2072,7
> +2093,7
> > @@ fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
> > uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
> >
> > /* Bind all local non-queue interrupt to vector 0 */
> > - int_map |= 0;
> > + int_map |= FM10K_MISC_VEC_ID;
> >
> > FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox),
> int_map);
> > FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault),
> int_map); @@
> > -2103,7 +2124,7 @@ fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
> > struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data-
> >dev_private);
> > uint32_t int_map = FM10K_INT_MAP_DISABLE;
> >
> > - int_map |= 0;
> > + int_map |= FM10K_MISC_VEC_ID;
> >
> > FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox),
> int_map);
> > FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault),
> int_map); @@
> > -2134,7 +2155,7 @@ fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
> > uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
> >
> > /* Bind all local non-queue interrupt to vector 0 */
> > - int_map |= 0;
> > + int_map |= FM10K_MISC_VEC_ID;
> >
> > /* Only INT 0 available, other 15 are reserved. */
> > FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map); @@ -2151,7
> +2172,7 @@
> > fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
> > struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data-
> >dev_private);
> > uint32_t int_map = FM10K_INT_MAP_DISABLE;
> >
> > - int_map |= 0;
> > + int_map |= FM10K_MISC_VEC_ID;
> >
> > /* Only INT 0 available, other 15 are reserved. */
> > FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map); @@ -2162,6
> +2183,64
> > @@ fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev) }
> >
> > static int
> > +fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev) {
> > + struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data-
> >dev_private);
> > + struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
> > + uint32_t intr_vector, vec;
> > + uint16_t queue_id;
> > + int result = 0;
> > +
> > + /* fm10k needs one separate interrupt for mailbox,
> > + * so only drivers which support multiple interrupt vectors
> > + * e.g. vfio-pci can work for fm10k interrupt mode
> > + */
> > + if (!rte_intr_cap_multiple(intr_handle) ||
> > + dev->data->dev_conf.intr_conf.rxq == 0)
> > + return result;
> > +
> > + intr_vector = dev->data->nb_rx_queues;
> > +
> > + /* disable interrupt first */
> > + rte_intr_disable(&dev->pci_dev->intr_handle);
> > + if (hw->mac.type == fm10k_mac_pf)
> > + fm10k_dev_disable_intr_pf(dev);
> > + else
> > + fm10k_dev_disable_intr_vf(dev);
> > +
> > + if (rte_intr_efd_enable(intr_handle, intr_vector)) {
> > + PMD_INIT_LOG(ERR, "Failed to init event fd");
> > + result = -EIO;
> > + }
> > +
> > + if (rte_intr_dp_is_en(intr_handle) && !result) {
> > + intr_handle->intr_vec = rte_zmalloc("intr_vec",
> > + dev->data->nb_rx_queues * sizeof(int), 0);
> > + if (intr_handle->intr_vec) {
> > + for (queue_id = 0, vec = FM10K_RX_VEC_START;
> > + queue_id < dev->data-
> >nb_rx_queues;
> > + queue_id++) {
> > + intr_handle->intr_vec[queue_id] = vec;
> > + if (vec < intr_handle->nb_efd - 1 +
> FM10K_RX_VEC_START)
> > + vec++;
> > + }
> > + } else {
> > + PMD_INIT_LOG(ERR, "Failed to allocate %d
> rx_queues"
> > + " intr_vec", dev->data->nb_rx_queues);
>
> Here if failure, should we call rte_intr_efd_disable()?
Thanks for the comments, you are right, better to do more clean up here, will
update in next version
Thanks,
--Shaopeng
>
> Thanks,
> Michael
> > + result = -ENOMEM;
> > + }
> > + }
> > +
> > + if (hw->mac.type == fm10k_mac_pf)
> > + fm10k_dev_enable_intr_pf(dev);
> > + else
> > + fm10k_dev_enable_intr_vf(dev);
> > + rte_intr_enable(&dev->pci_dev->intr_handle);
> > + hw->mac.ops.update_int_moderator(hw);
> > + return result;
> > +}
> > +
> > +static int
> > fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr) {
> > struct fm10k_fault fault;
> > @@ -2531,7 +2610,7 @@ static int
> > eth_fm10k_dev_init(struct rte_eth_dev *dev) {
> > struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data-
> >dev_private);
> > - int diag;
> > + int diag, i;
> > struct fm10k_macvlan_filter_info *macvlan;
> >
> > PMD_INIT_FUNC_TRACE();
> > @@ -2637,7 +2716,7 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)
> > fm10k_dev_enable_intr_vf(dev);
> > }
> >
> > - /* Enable uio intr after callback registered */
> > + /* Enable intr after callback registered */
> > rte_intr_enable(&(dev->pci_dev->intr_handle));
> >
> > hw->mac.ops.update_int_moderator(hw);
> > @@ -2645,7 +2724,6 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)
> > /* Make sure Switch Manager is ready before going forward. */
> > if (hw->mac.type == fm10k_mac_pf) {
> > int switch_ready = 0;
> > - int i;
> >
> > for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
> > fm10k_mbx_lock(hw);
> > @@ -2752,7 +2830,8 @@ static struct eth_driver rte_pmd_fm10k = {
> > .pci_drv = {
> > .name = "rte_pmd_fm10k",
> > .id_table = pci_id_fm10k_map,
> > - .drv_flags = RTE_PCI_DRV_NEED_MAPPING |
> RTE_PCI_DRV_DETACHABLE,
> > + .drv_flags = RTE_PCI_DRV_NEED_MAPPING |
> RTE_PCI_DRV_INTR_LSC |
> > + RTE_PCI_DRV_DETACHABLE,
> > },
> > .eth_dev_init = eth_fm10k_dev_init,
> > .eth_dev_uninit = eth_fm10k_dev_uninit,
