Currently tx_qid is equal to pmd->core_id. This leads to unexpected
behavior if pmd-cpu-mask different from '/(0*)(1|3|7)?(f*)/',
e.g. if core_ids are not sequential, or doesn't start from 0, or both.
Example:
starting 2 pmd threads with 1 port, 2 rxqs per port,
pmd-cpu-mask = 00000014 and let dev->real_n_txq = 2
It that case pmd_1->tx_qid = 2, pmd_2->tx_qid = 4 and
txq_needs_locking = true (if device hasn't ovs_numa_get_n_cores()+1
queues).
In that case, after truncating in netdev_dpdk_send__():
'qid = qid % dev->real_n_txq;'
pmd_1: qid = 2 % 2 = 0
pmd_2: qid = 4 % 2 = 0
So, both threads will call dpdk_queue_pkts() with same qid = 0.
This is unexpected behavior if there is 2 tx queues in device.
Queue #1 will not be used and both threads will lock queue #0
on each send.
Fix that by using sequential tx_qids.
Signed-off-by: Ilya Maximets <i.maxim...@samsung.com>
---
lib/dpif-netdev.c | 38 ++++++++++++++++++++++++++------------
1 file changed, 26 insertions(+), 12 deletions(-)
diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c
index f055c6e..f0b99d4 100644
--- a/lib/dpif-netdev.c
+++ b/lib/dpif-netdev.c
@@ -1278,6 +1278,13 @@ get_port_by_name(struct dp_netdev *dp,
}
static int
+get_n_pmd_threads(struct dp_netdev *dp)
+{
+ /* There is one non pmd thread in dp->poll_threads */
+ return cmap_count(&dp->poll_threads) - 1;
+}
+
+static int
get_n_pmd_threads_on_numa(struct dp_netdev *dp, int numa_id)
{
struct dp_netdev_pmd_thread *pmd;
@@ -2800,16 +2807,6 @@ dp_netdev_pmd_get_next(struct dp_netdev *dp, struct
cmap_position *pos)
return next;
}
-static int
-core_id_to_qid(unsigned core_id)
-{
- if (core_id != NON_PMD_CORE_ID) {
- return core_id;
- } else {
- return ovs_numa_get_n_cores();
- }
-}
-
/* Configures the 'pmd' based on the input argument. */
static void
dp_netdev_configure_pmd(struct dp_netdev_pmd_thread *pmd, struct dp_netdev *dp,
@@ -2818,10 +2815,11 @@ dp_netdev_configure_pmd(struct dp_netdev_pmd_thread
*pmd, struct dp_netdev *dp,
pmd->dp = dp;
pmd->index = index;
pmd->core_id = core_id;
- pmd->tx_qid = core_id_to_qid(core_id);
pmd->numa_id = numa_id;
pmd->poll_cnt = 0;
+ pmd->tx_qid = (core_id == NON_PMD_CORE_ID) ? ovs_numa_get_n_cores()
+ : get_n_pmd_threads(dp);
ovs_refcount_init(&pmd->ref_cnt);
latch_init(&pmd->exit_latch);
atomic_init(&pmd->change_seq, PMD_INITIAL_SEQ);
@@ -2895,17 +2893,33 @@ dp_netdev_destroy_all_pmds(struct dp_netdev *dp)
}
}
-/* Deletes all pmd threads on numa node 'numa_id'. */
+/* Deletes all pmd threads on numa node 'numa_id' and
+ * fixes tx_qids of other threads to keep them sequential. */
static void
dp_netdev_del_pmds_on_numa(struct dp_netdev *dp, int numa_id)
{
struct dp_netdev_pmd_thread *pmd;
+ int n_pmds_on_numa, n_pmds;
+ int *free_idx, k = 0;
+
+ n_pmds_on_numa = get_n_pmd_threads_on_numa(dp, numa_id);
+ free_idx = xmalloc(n_pmds_on_numa * sizeof *free_idx);
CMAP_FOR_EACH (pmd, node, &dp->poll_threads) {
if (pmd->numa_id == numa_id) {
+ free_idx[k++] = pmd->tx_qid;
dp_netdev_del_pmd(dp, pmd);
}
}
+
+ n_pmds = get_n_pmd_threads(dp);
+ CMAP_FOR_EACH (pmd, node, &dp->poll_threads) {
+ if (pmd->tx_qid >= n_pmds) {
+ pmd->tx_qid = free_idx[--k];
+ }
+ }
+
+ free(free_idx);
}
/* Returns PMD thread from this numa node with fewer rx queues to poll.
--
2.5.0
_______________________________________________
dev mailing list
dev@openvswitch.org
http://openvswitch.org/mailman/listinfo/dev
