> The 'options:n_rxq_desc' and 'n_txq_desc' fields allow the number of rx
> and tx descriptors for dpdk ports to be modified. By default the values
> are set to 2048, but can be modified to an integer between 1 and 4096
> that is a power of two. The values can be modified at runtime, however
> require the NIC to restart when changed.
>
> Signed-off-by: Ciara Loftus <ciara.loftus at intel.com>
> Acked-by: Yunhong Jiang <yunhong.jiang at linux.intel.com>
> Acked-by: Ilya Maximets <i.maximets at samsung.com>
> ---
>
> v5:
> * Rebase
> * If invalid value is specified, request a default value rather than the
> previous value.
> * Added requested values to get_config
>
> v4:
> * Fix typos in vswitchd.xml
> * Use internal OVS function is_pow2 rather than DPDK function
> * Additional documentation
>
> v3:
> * Make queue sizes per-port rather than global
> * Check if queue size is power of 2 - fail if so.
>
> v2:
> * Rebase
> ---
> INSTALL.DPDK-ADVANCED.md | 20 ++++++++++++++--
> NEWS | 2 ++
> lib/netdev-dpdk.c | 60
> ++++++++++++++++++++++++++++++++++++++++++------
> vswitchd/vswitch.xml | 22 ++++++++++++++++++
> 4 files changed, 95 insertions(+), 9 deletions(-)
>
> diff --git a/INSTALL.DPDK-ADVANCED.md b/INSTALL.DPDK-ADVANCED.md
> index d7b9873..555f026 100644
> --- a/INSTALL.DPDK-ADVANCED.md
> +++ b/INSTALL.DPDK-ADVANCED.md
> @@ -257,7 +257,23 @@ needs to be affinitized accordingly.
> The rx queues are assigned to pmd threads on the same NUMA node in a
> round-robin fashion.
>
> -### 4.4 Exact Match Cache
> +### 4.4 DPDK Physical Port Queue Sizes
> + `ovs-vsctl set Interface dpdk0 options:n_rxq_desc=<integer>`
> + `ovs-vsctl set Interface dpdk0 options:n_txq_desc=<integer>`
> +
> + The command above sets the number of rx/tx descriptors that the NIC
> + associated with dpdk0 will be initialised with.
> +
> + Different 'n_rxq_desc' and 'n_txq_desc' configurations yield different
> + benefits in terms of throughput and latency for different scenarios.
> + Generally, smaller queue sizes can have a positive impact for latency at
> the
> + expense of throughput. The opposite is often true for larger queue sizes.
> + Note: increasing the number of rx descriptors eg. to 4096 may have a
> + negative impact on performance due to the fact that non-vectorised DPDK rx
> + functions may be used. This is dependant on the driver in use, but is true
> + for the commonly used i40e and ixgbe DPDK drivers.
> +
> +### 4.5 Exact Match Cache
>
> Each pmd thread contains one EMC. After initial flow setup in the
> datapath, the EMC contains a single table and provides the lowest level
> @@ -274,7 +290,7 @@ needs to be affinitized accordingly.
> avoiding datapath classifier lookups is to have multiple pmd threads
> running. This can be done as described in section 4.2.
>
> -### 4.5 Rx Mergeable buffers
> +### 4.6 Rx Mergeable buffers
>
> Rx Mergeable buffers is a virtio feature that allows chaining of multiple
> virtio descriptors to handle large packet sizes. As such, large packets
> diff --git a/NEWS b/NEWS
> index 069ab42..725754a 100644
> --- a/NEWS
> +++ b/NEWS
> @@ -127,6 +127,8 @@ v2.6.0 - xx xxx xxxx
> * Remove dpdkvhostcuse port type.
> * OVS client mode for vHost and vHost reconnect (Requires QEMU 2.7)
> * 'dpdkvhostuserclient' port type.
> + * New option 'n_rxq_desc' and 'n_txq_desc' fields for DPDK interfaces
> + which set the number of rx and tx descriptors to use for the given
> port.
> - Increase number of registers to 16.
> - ovs-benchmark: This utility has been removed due to lack of use and
> bitrot.
> diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
> index 3847b1f..87bddd3 100644
> --- a/lib/netdev-dpdk.c
> +++ b/lib/netdev-dpdk.c
> @@ -134,10 +134,12 @@ BUILD_ASSERT_DECL((MAX_NB_MBUF /
> ROUND_DOWN_POW2(MAX_NB_MBUF/MIN_NB_MBUF))
>
> #define SOCKET0 0
>
> -/* Size of Physical NIC RX Queue, Max (n + 32 <= 4096) */
> -#define NIC_PORT_RX_Q_SIZE 2048
> -/* Size of Physical NIC TX Queue, Max (n + 32 <= 4096) */
> -#define NIC_PORT_TX_Q_SIZE 2048
> +/* Default size of Physical NIC RXQ */
> +#define NIC_PORT_DEFAULT_RXQ_SIZE 2048
> +/* Default size of Physical NIC TXQ */
> +#define NIC_PORT_DEFAULT_TXQ_SIZE 2048
> +/* Maximum size of Physical NIC Queues */
> +#define NIC_PORT_MAX_Q_SIZE 4096
>
> #define OVS_VHOST_MAX_QUEUE_NUM 1024 /* Maximum number of vHost TX queues.
> */
> #define OVS_VHOST_QUEUE_MAP_UNKNOWN (-1) /* Mapping not initialized. */
> @@ -379,6 +381,12 @@ struct netdev_dpdk {
> int requested_mtu;
> int requested_n_txq;
> int requested_n_rxq;
> + int requested_rxq_size;
> + int requested_txq_size;
> +
> + /* Number of rx/tx descriptors for physical devices */
> + int rxq_size;
> + int txq_size;
>
> /* Socket ID detected when vHost device is brought up */
> int requested_socket_id;
> @@ -662,7 +670,7 @@ dpdk_eth_dev_queue_setup(struct netdev_dpdk *dev, int
> n_rxq, int n_txq)
> }
>
> for (i = 0; i < n_txq; i++) {
> - diag = rte_eth_tx_queue_setup(dev->port_id, i,
> NIC_PORT_TX_Q_SIZE,
> + diag = rte_eth_tx_queue_setup(dev->port_id, i, dev->txq_size,
> dev->socket_id, NULL);
> if (diag) {
> VLOG_INFO("Interface %s txq(%d) setup error: %s",
> @@ -678,7 +686,7 @@ dpdk_eth_dev_queue_setup(struct netdev_dpdk *dev, int
> n_rxq, int n_txq)
> }
>
> for (i = 0; i < n_rxq; i++) {
> - diag = rte_eth_rx_queue_setup(dev->port_id, i,
> NIC_PORT_RX_Q_SIZE,
> + diag = rte_eth_rx_queue_setup(dev->port_id, i, dev->rxq_size,
> dev->socket_id, NULL,
> dev->dpdk_mp->mp);
> if (diag) {
> @@ -854,6 +862,10 @@ netdev_dpdk_init(struct netdev *netdev, unsigned int
> port_no,
> netdev->n_txq = NR_QUEUE;
> dev->requested_n_rxq = netdev->n_rxq;
> dev->requested_n_txq = netdev->n_txq;
> + dev->rxq_size = NIC_PORT_DEFAULT_RXQ_SIZE;
> + dev->txq_size = NIC_PORT_DEFAULT_TXQ_SIZE;
> + dev->requested_rxq_size = dev->rxq_size;
> + dev->requested_txq_size = dev->txq_size;
>
> /* Initialize the flow control to NULL */
> memset(&dev->fc_conf, 0, sizeof dev->fc_conf);
> @@ -1069,6 +1081,12 @@ netdev_dpdk_get_config(const struct netdev *netdev,
> struct smap *args)
> smap_add_format(args, "configured_rx_queues", "%d", netdev->n_rxq);
> smap_add_format(args, "requested_tx_queues", "%d", dev->requested_n_txq);
> smap_add_format(args, "configured_tx_queues", "%d", netdev->n_txq);
> + smap_add_format(args, "requested_rxq_descriptors", "%d",
> + dev->requested_rxq_size);
> + smap_add_format(args, "configured_rxq_descriptors", "%d", dev->rxq_size);
> + smap_add_format(args, "requested_txq_descriptors", "%d",
> + dev->requested_txq_size);
> + smap_add_format(args, "configured_txq_descriptors", "%d", dev->txq_size);
> smap_add_format(args, "mtu", "%d", dev->mtu);
> ovs_mutex_unlock(&dev->mutex);
>
> @@ -1087,6 +1105,22 @@ dpdk_set_rxq_config(struct netdev_dpdk *dev, const
> struct smap *args)
> }
> }
>
> +static void
> +dpdk_process_queue_size(struct netdev *netdev, const struct smap *args,
> + char *flag, int default_size, int *new_size)
> +{
> + int queue_size = smap_get_int(args, flag, default_size);
> +
> + if (queue_size > NIC_PORT_MAX_Q_SIZE || !is_pow2(queue_size)) {
check for 'queue_size <= 0' still required because DB doesn't check values
of options.
> + queue_size = default_size;
> + }
> +
> + if (queue_size != *new_size) {
> + *new_size = queue_size;
> + netdev_request_reconfigure(netdev);
> + }
> +}
> +
> static int
> netdev_dpdk_set_config(struct netdev *netdev, const struct smap *args)
> {
> @@ -1096,6 +1130,13 @@ netdev_dpdk_set_config(struct netdev *netdev, const
> struct smap *args)
>
> dpdk_set_rxq_config(dev, args);
>
> + dpdk_process_queue_size(netdev, args, "n_rxq_desc",
> + NIC_PORT_DEFAULT_RXQ_SIZE,
> + &dev->requested_rxq_size);
> + dpdk_process_queue_size(netdev, args, "n_txq_desc",
> + NIC_PORT_DEFAULT_TXQ_SIZE,
> + &dev->requested_txq_size);
> +
> /* Flow control support is only available for DPDK Ethernet ports. */
> bool rx_fc_en = false;
> bool tx_fc_en = false;
> @@ -2945,7 +2986,9 @@ netdev_dpdk_reconfigure(struct netdev *netdev)
>
> if (netdev->n_txq == dev->requested_n_txq
> && netdev->n_rxq == dev->requested_n_rxq
> - && dev->mtu == dev->requested_mtu) {
> + && dev->mtu == dev->requested_mtu
> + && dev->rxq_size == dev->requested_rxq_size
> + && dev->txq_size == dev->requested_txq_size) {
> /* Reconfiguration is unnecessary */
>
> goto out;
> @@ -2960,6 +3003,9 @@ netdev_dpdk_reconfigure(struct netdev *netdev)
> netdev->n_txq = dev->requested_n_txq;
> netdev->n_rxq = dev->requested_n_rxq;
>
> + dev->rxq_size = dev->requested_rxq_size;
> + dev->txq_size = dev->requested_txq_size;
> +
> rte_free(dev->tx_q);
> err = dpdk_eth_dev_init(dev);
> netdev_dpdk_alloc_txq(dev, netdev->n_txq);
> diff --git a/vswitchd/vswitch.xml b/vswitchd/vswitch.xml
> index 8ff3853..da793c3 100644
> --- a/vswitchd/vswitch.xml
> +++ b/vswitchd/vswitch.xml
> @@ -2332,6 +2332,28 @@
> Only supported by dpdkvhostuserclient interfaces.
> </p>
> </column>
> +
> + <column name="options" key="n_rxq_desc"
> + type='{"type": "integer", "minInteger": 1, "maxInteger":
> 4096}'>
> + <p>
> + Specifies the rx queue size (number rx descriptors) for dpdk ports.
> + The value must be a power of 2, less than 4096 and supported
> + by the hardware of the device being configured.
> + If not specified or an incorrect value is specified, 2048 rx
> + descriptors will be used by default.
> + </p>
> + </column>
> +
> + <column name="options" key="n_txq_desc"
> + type='{"type": "integer", "minInteger": 1, "maxInteger":
> 4096}'>
> + <p>
> + Specifies the tx queue size (number tx descriptors) for dpdk ports.
> + The value must be a power of 2, less than 4096 and supported
> + by the hardware of the device being configured.
> + If not specified or an incorrect value is specified, 2048 tx
> + descriptors will be used by default.
> + </p>
> + </column>
> </group>
>
> <group title="MTU">
> --
> 2.4.3
_______________________________________________
dev mailing list
dev@openvswitch.org
http://openvswitch.org/mailman/listinfo/dev
