Thanks for your detailed comments, replies inline.
I'll send v4 in a minute.
On 16/03/2016 06:12, "Ilya Maximets" <i.maxim...@samsung.com> wrote:
>On 16.03.2016 01:30, Daniele Di Proietto wrote:
>> This introduces in dpif-netdev and netdev-dpdk the first use for the
>> newly introduce reconfigure netdev call.
>>
>> When a request to change the number of queues comes, netdev-dpdk will
>> remember this and notify the upper layer via
>> netdev_request_reconfigure().
>>
>> The datapath, instead of periodically calling netdev_set_multiq(), can
>> detect this and call reconfigure().
>>
>> This mechanism can also be used to:
>> * Automatically match the number of rxq with the one provided by qemu
>> via the new_device callback.
>> * Provide a way to change the MTU of dpdk devices at runtime.
>>
>> Signed-off-by: Daniele Di Proietto <diproiet...@vmware.com>
[...]
>>
>> @@ -2637,8 +2577,79 @@ static const struct dpdk_qos_ops
>>egress_policer_ops = {
>> egress_policer_run
>> };
>>
>> -#define NETDEV_DPDK_CLASS(NAME, INIT, CONSTRUCT, DESTRUCT, MULTIQ,
>>SEND, \
>> - GET_CARRIER, GET_STATS, GET_FEATURES, GET_STATUS, RXQ_RECV)
>> \
>> +static int
>> +netdev_dpdk_reconfigure(struct netdev *netdev_)
>> +{
>> + struct netdev_dpdk *netdev = netdev_dpdk_cast(netdev_);
>> + int err = 0;
>> +
>> + ovs_mutex_lock(&dpdk_mutex);
>> + ovs_mutex_lock(&netdev->mutex);
>> + rte_eth_dev_stop(netdev->port_id);
>
>Checking must be performed prior to stopping of device.
You're right, thanks for spotting that.
I fixed it
>
>> +
>> + if (netdev_->n_txq == netdev->requested_n_txq
>> + && netdev_->n_rxq == netdev->requested_n_rxq) {
>> + /* Reconfiguration is unnecessary */
>> +
>> + goto out;
>> + }
>> +
>> + netdev_->n_txq = netdev->requested_n_txq;
>> + netdev_->n_rxq = netdev->requested_n_rxq;
>> +
>> + rte_free(netdev->tx_q);
>> + err = dpdk_eth_dev_init(netdev);
>> + netdev_dpdk_alloc_txq(netdev, netdev->real_n_txq);
>> +
>> + netdev->txq_needs_locking = netdev->real_n_txq != netdev->up.n_txq;
>> +
>> +out:
>> +
>> + ovs_mutex_unlock(&netdev->mutex);
>> + ovs_mutex_unlock(&dpdk_mutex);
>> +
>> + return err;
>> +}
>> +
>> +static int
>> +netdev_dpdk_vhost_user_reconfigure(struct netdev *netdev_)
>> +{
>> + struct netdev_dpdk *netdev = netdev_dpdk_cast(netdev_);
>> +
>> + ovs_mutex_lock(&dpdk_mutex);
>> + ovs_mutex_lock(&netdev->mutex);
>> +
>> + netdev->up.n_txq = netdev->requested_n_txq;
>> + netdev->up.n_rxq = netdev->requested_n_rxq;
>> +
>> + ovs_mutex_unlock(&netdev->mutex);
>> + ovs_mutex_unlock(&dpdk_mutex);
>> +
>> + return 0;
>> +}
>> +
>> +static int
>> +netdev_dpdk_vhost_cuse_reconfigure(struct netdev *netdev_)
>> +{
>> + struct netdev_dpdk *netdev = netdev_dpdk_cast(netdev_);
>> +
>> + ovs_mutex_lock(&dpdk_mutex);
>> + ovs_mutex_lock(&netdev->mutex);
>> +
>> + netdev->up.n_txq = netdev->requested_n_txq;
>> + netdev->real_n_txq = 1;
>> + netdev->up.n_rxq = 1;
>> + netdev->txq_needs_locking = netdev->real_n_txq != netdev->up.n_txq;
>> +
>> + ovs_mutex_unlock(&netdev->mutex);
>> + ovs_mutex_unlock(&dpdk_mutex);
>> +
>> + return 0;
>> +}
>> +
>> +#define NETDEV_DPDK_CLASS(NAME, INIT, CONSTRUCT, DESTRUCT, SEND, \
>> + GET_CARRIER, GET_STATS, GET_FEATURES, \
>> + GET_STATUS, RECONFIGURE, RXQ_RECV) \
>> { \
>> NAME, \
>> INIT, /* init */ \
>> @@ -2656,7 +2667,7 @@ static const struct dpdk_qos_ops
>>egress_policer_ops = {
>> NULL, /* push header */ \
>> NULL, /* pop header */ \
>> netdev_dpdk_get_numa_id, /* get_numa_id */ \
>> - MULTIQ, /* set_multiq */ \
>> + netdev_dpdk_set_multiq, \
>> \
>> SEND, /* send */ \
>> NULL, /* send_wait */ \
>> @@ -2696,7 +2707,7 @@ static const struct dpdk_qos_ops
>>egress_policer_ops = {
>> NULL, /* arp_lookup */ \
>> \
>> netdev_dpdk_update_flags, \
>> - NULL, /* reconfigure */ \
>> + RECONFIGURE, \
>> \
>> netdev_dpdk_rxq_alloc, \
>> netdev_dpdk_rxq_construct, \
>> @@ -2811,12 +2822,12 @@ static const struct netdev_class dpdk_class =
>> NULL,
>> netdev_dpdk_construct,
>> netdev_dpdk_destruct,
>> - netdev_dpdk_set_multiq,
>> netdev_dpdk_eth_send,
>> netdev_dpdk_get_carrier,
>> netdev_dpdk_get_stats,
>> netdev_dpdk_get_features,
>> netdev_dpdk_get_status,
>> + netdev_dpdk_reconfigure,
>> netdev_dpdk_rxq_recv);
>>
>> static const struct netdev_class dpdk_ring_class =
>> @@ -2825,12 +2836,12 @@ static const struct netdev_class
>>dpdk_ring_class =
>> NULL,
>> netdev_dpdk_ring_construct,
>> netdev_dpdk_destruct,
>> - netdev_dpdk_set_multiq,
>> netdev_dpdk_ring_send,
>> netdev_dpdk_get_carrier,
>> netdev_dpdk_get_stats,
>> netdev_dpdk_get_features,
>> netdev_dpdk_get_status,
>> + netdev_dpdk_reconfigure,
>> netdev_dpdk_rxq_recv);
>>
>> static const struct netdev_class OVS_UNUSED dpdk_vhost_cuse_class =
>> @@ -2839,12 +2850,12 @@ static const struct netdev_class OVS_UNUSED
>>dpdk_vhost_cuse_class =
>> dpdk_vhost_cuse_class_init,
>> netdev_dpdk_vhost_cuse_construct,
>> netdev_dpdk_vhost_destruct,
>> - netdev_dpdk_vhost_cuse_set_multiq,
>> netdev_dpdk_vhost_send,
>> netdev_dpdk_vhost_get_carrier,
>> netdev_dpdk_vhost_get_stats,
>> NULL,
>> NULL,
>> + netdev_dpdk_vhost_cuse_reconfigure,
>> netdev_dpdk_vhost_rxq_recv);
>>
>> static const struct netdev_class OVS_UNUSED dpdk_vhost_user_class =
>> @@ -2853,12 +2864,12 @@ static const struct netdev_class OVS_UNUSED
>>dpdk_vhost_user_class =
>> dpdk_vhost_user_class_init,
>> netdev_dpdk_vhost_user_construct,
>> netdev_dpdk_vhost_destruct,
>> - netdev_dpdk_vhost_set_multiq,
>> netdev_dpdk_vhost_send,
>> netdev_dpdk_vhost_get_carrier,
>> netdev_dpdk_vhost_get_stats,
>> NULL,
>> NULL,
>> + netdev_dpdk_vhost_user_reconfigure,
>> netdev_dpdk_vhost_rxq_recv);
>>
>> void
>> diff --git a/lib/netdev-provider.h b/lib/netdev-provider.h
>> index 0317910..ea40d61 100644
>> --- a/lib/netdev-provider.h
>> +++ b/lib/netdev-provider.h
>> @@ -67,8 +67,6 @@ struct netdev {
>> * modify them. */
>> int n_txq;
>> int n_rxq;
>> - /* Number of rx queues requested by user. */
>> - int requested_n_rxq;
>> int ref_cnt; /* Times this devices was
>>opened. */
>> struct shash_node *node; /* Pointer to element in
>>global map. */
>> struct ovs_list saved_flags_list; /* Contains "struct
>>netdev_saved_flags". */
>> @@ -295,13 +293,8 @@ struct netdev_class {
>> * such info, returns NETDEV_NUMA_UNSPEC. */
>> int (*get_numa_id)(const struct netdev *netdev);
>>
>> - /* Configures the number of tx queues and rx queues of 'netdev'.
>> - * Return 0 if successful, otherwise a positive errno value.
>> - *
>> - * 'n_rxq' specifies the maximum number of receive queues to
>>create.
>> - * The netdev provider might choose to create less (e.g. if the
>>hardware
>> - * supports only a smaller number). The actual number of queues
>>created
>> - * is stored in the 'netdev->n_rxq' field.
>> + /* Configures the number of tx queues of 'netdev'. Returns 0 if
>>successful,
>> + * otherwise a positive errno value.
>> *
>> * 'n_txq' specifies the exact number of transmission queues to
>>create.
>> * The caller will call netdev_send() concurrently from 'n_txq'
>>different
>> @@ -309,12 +302,8 @@ struct netdev_class {
>> * making sure that these concurrent calls do not create a race
>>condition
>> * by using multiple hw queues or locking.
>
>Comment above (a little corrupted by diff) needs to be rewritten because
>of
>undocumented relations between 'set_multiq' and 'reconfigure' calls
>described in
>my next comment.
You're right, I've added the following paragraph:
* The caller will call netdev_reconfigure() (if necessary) before using
* netdev_send() on any of the newly configured queues, giving the
provider
* a chance to adjust its settings.
>
>> *
>> - * On error, the tx queue and rx queue configuration is
>>indeterminant.
>> - * Caller should make decision on whether to restore the previous
>>or
>> - * the default configuration. Also, caller must make sure there
>>is no
>> - * other thread accessing the queues at the same time. */
>> - int (*set_multiq)(struct netdev *netdev, unsigned int n_txq,
>> - unsigned int n_rxq);
>> + * On error, the tx queue configuration is unchanged. */
>> + int (*set_multiq)(struct netdev *netdev, unsigned int n_txq);
>>
>> /* Sends buffers on 'netdev'.
>> * Returns 0 if successful (for every buffer), otherwise a
>>positive errno
>> diff --git a/lib/netdev.c b/lib/netdev.c
>> index 7cb7085..8c1d7a4 100644
>> --- a/lib/netdev.c
>> +++ b/lib/netdev.c
>> @@ -106,12 +106,6 @@ netdev_n_rxq(const struct netdev *netdev)
>> return netdev->n_rxq;
>> }
>>
>> -int
>> -netdev_requested_n_rxq(const struct netdev *netdev)
>> -{
>> - return netdev->requested_n_rxq;
>> -}
>> -
>> bool
>> netdev_is_pmd(const struct netdev *netdev)
>> {
>> @@ -384,7 +378,6 @@ netdev_open(const char *name, const char *type,
>>struct netdev **netdevp)
>> /* By default enable one tx and rx queue per netdev. */
>> netdev->n_txq = netdev->netdev_class->send ? 1 : 0;
>> netdev->n_rxq = netdev->netdev_class->rxq_alloc ? 1 :
>>0;
>> - netdev->requested_n_rxq = netdev->n_rxq;
>>
>> list_init(&netdev->saved_flags_list);
>>
>> @@ -685,37 +678,26 @@ netdev_rxq_drain(struct netdev_rxq *rx)
>> : 0);
>> }
>>
>> -/* Configures the number of tx queues and rx queues of 'netdev'.
>> - * Return 0 if successful, otherwise a positive errno value.
>> - *
>> - * 'n_rxq' specifies the maximum number of receive queues to create.
>> - * The netdev provider might choose to create less (e.g. if the
>>hardware
>> - * supports only a smaller number). The caller can check how many
>>have been
>> - * actually created by calling 'netdev_n_rxq()'
>> +/* Configures the number of tx queues of 'netdev'. Returns 0 if
>>successful,
>> + * otherwise a positive errno value.
>> *
>> * 'n_txq' specifies the exact number of transmission queues to create.
>> * If this function returns successfully, the caller can make 'n_txq'
>> * concurrent calls to netdev_send() (each one with a different 'qid'
>>in the
>> * range [0..'n_txq'-1]).
>
>This is not true now. The caller can make concurrent calls to
>netdev_send()
>only after successful netdev_reconfigure().
Good point, we have to document this interface change.
I've added this paragraph:
* The change might not effective immediately. The caller must check if a
* reconfiguration is required with netdev_is_reconf_required() and
eventually
* call netdev_reconfigure() before using the new queues.
>
>> *
>> - * On error, the tx queue and rx queue configuration is indeterminant.
>> - * Caller should make decision on whether to restore the previous or
>> - * the default configuration. Also, caller must make sure there is no
>> - * other thread accessing the queues at the same time. */
>> + * On error, the tx queue configuration is unchanged */
>> int
>> -netdev_set_multiq(struct netdev *netdev, unsigned int n_txq,
>> - unsigned int n_rxq)
>> +netdev_set_multiq(struct netdev *netdev, unsigned int n_txq)
>> {
>> int error;
>>
>> error = (netdev->netdev_class->set_multiq
>> - ? netdev->netdev_class->set_multiq(netdev,
>> - MAX(n_txq, 1),
>> - MAX(n_rxq, 1))
>> + ? netdev->netdev_class->set_multiq(netdev, MAX(n_txq, 1))
>> : EOPNOTSUPP);
>>
>> if (error && error != EOPNOTSUPP) {
>> - VLOG_DBG_RL(&rl, "failed to set tx/rx queue for network device
>>%s:"
>> + VLOG_DBG_RL(&rl, "failed to set tx queue for network device
>>%s:"
>> "%s", netdev_get_name(netdev),
>>ovs_strerror(error));
>> }
>>
>> diff --git a/lib/netdev.h b/lib/netdev.h
>> index c2a1d6c..bb3d297 100644
>> --- a/lib/netdev.h
>> +++ b/lib/netdev.h
>> @@ -142,7 +142,6 @@ bool netdev_is_reserved_name(const char *name);
>>
>> int netdev_n_txq(const struct netdev *netdev);
>> int netdev_n_rxq(const struct netdev *netdev);
>> -int netdev_requested_n_rxq(const struct netdev *netdev);
>> bool netdev_is_pmd(const struct netdev *netdev);
>>
>> /* Open and close. */
>> @@ -168,7 +167,7 @@ const char *netdev_get_type_from_name(const char *);
>> int netdev_get_mtu(const struct netdev *, int *mtup);
>> int netdev_set_mtu(const struct netdev *, int mtu);
>> int netdev_get_ifindex(const struct netdev *);
>> -int netdev_set_multiq(struct netdev *, unsigned int n_txq, unsigned
>>int n_rxq);
>> +int netdev_set_multiq(struct netdev *, unsigned int n_txq);
>>
>> /* Packet reception. */
>> int netdev_rxq_open(struct netdev *, struct netdev_rxq **, int id);
>>
_______________________________________________
dev mailing list
dev@openvswitch.org
http://openvswitch.org/mailman/listinfo/dev
