HI Maxime, Jens
Saturday, June 23, 2018 11:09 AM, Maxime Coquelin:
> Subject: [dpdk-dev] [PATCH v4 1/2] testpmd: add forwarding mode to
> simulate a noisy neighbour
>
> From: Jens Freimann <jfreim...@redhat.com>
>
> This adds a new forwarding mode to testpmd to simulate more realistic
> behavior of a guest machine engaged in receiving and sending packets
> performing Virtual Network Function (VNF).
I really like the idea of adding application w/ more realistic behavior to the
DPDK tree in order to use them in the non-regression cycle.
I am just wondering if putting it as another forward engine in testpmd is the
right place for it.
>From one side it reuses testpmd command line options which is good.
On the other hand it tends to have a lot of options specific only for the VNF
forward use case. And I am quite sure that w/ the time goes by the number of
configuration options will increase.
I think this code needs to be as a dedicated example under examples/.
Moreover I would refine from directly associate it with VNF. VNF can be one
example but for sure there are more.
We can call the example noisy_neigbour or any other name you like.
One more comment below.
>
> The goal is to enable a simple way of measuring performance impact on
> cache and memory footprint utilization from various VNF co-located on the
> same host machine. For this it does:
>
> * Buffer packets in a FIFO:
>
> Create a fifo to buffer received packets. Once it flows over put those packets
> into the actual tx queue. The fifo is created per tx queue and its size can be
> set with the --buffersize-before-sending commandline parameter.
>
> A second commandline parameter is used to set a timeout in milliseconds
> after which the fifo is flushed.
>
> --noisy-buffersize-before-sending [packet numbers] Keep the mbuf in a
> FIFO and forward the over flooding packets from the FIFO. This queue is per
> TX-queue (after all other packet processing).
>
> --noisy-flush-timeout [delay]
> Flush the packet queue if no packets have been seen during [delay]. As long
> as packets are seen, the timer is reset.
>
> Add several options to simulate route lookups (memory reads) in tables that
> can be quite large, as well as route hit statistics update.
> These options simulates the while stack traversal and will trash the cache.
> Memory access is random.
>
> * simulate route lookups:
>
> Allocate a buffer and perform reads and writes on it as specified by
> commandline options:
>
> --noisy-memory-footprint [size]
> Size of the VNF internal memory (MB), in which the random read/write will
> be done, allocated by rte_malloc (hugepages).
>
> --noisy-nb-rnd-write [num]
> Number of random writes in memory per packet should be performed,
> simulating hit-flags update. 64 bits per write, all write in different cache
> lines.
>
> --noisy-nb-rnd-read [num]
> Number of random reads in memory per packet should be performed,
> simulating FIB/table lookups. 64 bits per read, all write in different cache
> lines.
>
> --noisy-nb-rnd-read-write [num]
> Number of random reads and writes in memory per packet should be
> performed, simulating stats update. 64 bits per read-write, all reads and
> writes in different cache lines.
>
> Signed-off-by: Jens Freimann <jfreim...@redhat.com>
> Signed-off-by: Maxime Coquelin <maxime.coque...@redhat.com>
> ---
> app/test-pmd/Makefile | 1 +
> app/test-pmd/meson.build | 1 +
> app/test-pmd/noisy_vnf.c | 204
> ++++++++++++++++++++++++++++++++++++++++++++++
> app/test-pmd/noisy_vnf.h | 41 ++++++++++ app/test-pmd/parameters.c
> | 54 ++++++++++++
> app/test-pmd/testpmd.c | 52 ++++++++++++
> app/test-pmd/testpmd.h | 17 ++++
> 7 files changed, 370 insertions(+)
> create mode 100644 app/test-pmd/noisy_vnf.c create mode 100644
> app/test-pmd/noisy_vnf.h
>
> diff --git a/app/test-pmd/Makefile b/app/test-pmd/Makefile index
> a5a827bbd..2a1030120 100644
> --- a/app/test-pmd/Makefile
> +++ b/app/test-pmd/Makefile
> @@ -32,6 +32,7 @@ SRCS-y += rxonly.c
> SRCS-y += txonly.c
> SRCS-y += csumonly.c
> SRCS-y += icmpecho.c
> +SRCS-y += noisy_vnf.c
> SRCS-$(CONFIG_RTE_LIBRTE_IEEE1588) += ieee1588fwd.c
> SRCS-$(CONFIG_RTE_LIBRTE_BPF) += bpf_cmd.c
>
> diff --git a/app/test-pmd/meson.build b/app/test-pmd/meson.build index
> 4c9fbbfda..b3abc6e24 100644
> --- a/app/test-pmd/meson.build
> +++ b/app/test-pmd/meson.build
> @@ -16,6 +16,7 @@ sources = files('cmdline.c',
> 'iofwd.c',
> 'macfwd.c',
> 'macswap.c',
> + 'noisy_vnf.c',
> 'parameters.c',
> 'rxonly.c',
> 'testpmd.c',
> diff --git a/app/test-pmd/noisy_vnf.c b/app/test-pmd/noisy_vnf.c new file
> mode 100644 index 000000000..ff205495f
> --- /dev/null
> +++ b/app/test-pmd/noisy_vnf.c
> @@ -0,0 +1,204 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2018 Red Hat Corp.
> + */
> +
> +#include <stdarg.h>
> +#include <stdio.h>
> +#include <string.h>
> +#include <errno.h>
> +#include <stdint.h>
> +#include <unistd.h>
> +#include <inttypes.h>
> +
> +#include <sys/queue.h>
> +#include <sys/stat.h>
> +
> +#include <rte_common.h>
> +#include <rte_log.h>
> +#include <rte_debug.h>
> +#include <rte_cycles.h>
> +#include <rte_memory.h>
> +#include <rte_launch.h>
> +#include <rte_eal.h>
> +#include <rte_per_lcore.h>
> +#include <rte_lcore.h>
> +#include <rte_memcpy.h>
> +#include <rte_mempool.h>
> +#include <rte_mbuf.h>
> +#include <rte_ethdev.h>
> +#include <rte_flow.h>
> +#include <rte_malloc.h>
> +
> +#include "testpmd.h"
> +#include "noisy_vnf.h"
> +
> +static inline void
> +do_write(char *vnf_mem)
> +{
> + uint64_t i = rte_rand();
> + uint64_t w = rte_rand();
> +
> + vnf_mem[i % ((noisy_vnf_memory_footprint * 1024 * 1024) /
> + RTE_CACHE_LINE_SIZE)] = w;
> +}
> +
> + static inline void
> +do_read(char *vnf_mem)
> +{
> + uint64_t i = rte_rand();
> + uint64_t r;
> +
> + r = vnf_mem[i % ((noisy_vnf_memory_footprint * 1024 * 1024) /
> + RTE_CACHE_LINE_SIZE)];
> + r++;
> +}
> +
> + static inline void
> +do_rw(char *vnf_mem)
> +{
> + do_read(vnf_mem);
> + do_write(vnf_mem);
> +}
> +
> +/*
> + * Simulate route lookups as defined by commandline parameters */
> + static void
> +sim_memory_lookups(struct noisy_config *ncf, uint16_t nb_pkts) {
> + uint16_t i, j;
> +
> + for (i = 0; i < nb_pkts; i++) {
> + for (j = 0; j < noisy_nb_rnd_write; j++)
> + do_write(ncf->vnf_mem);
> + for (j = 0; j < noisy_nb_rnd_read; j++)
> + do_read(ncf->vnf_mem);
> + for (j = 0; j < noisy_nb_rnd_read_write; j++)
> + do_rw(ncf->vnf_mem);
> + }
> +}
> +
> +/*
> + * Forwarding of packets in I/O mode.
> + * Forward packets "as-is".
> + * This is the fastest possible forwarding operation, as it does not
> +access
> + * to packets data.
> + */
> +static void
> +pkt_burst_noisy_vnf(struct fwd_stream *fs) {
> + struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
> + uint16_t nb_rx;
> + uint16_t nb_tx = 0;
> + uint32_t retry;
> + const uint64_t freq_khz = rte_get_timer_hz() / 1000;
> + struct noisy_config *ncf = &noisy_cfg[fs->tx_queue];
> + struct rte_mbuf *tmp_pkts[MAX_PKT_BURST];
> + uint16_t nb_enqd;
> + uint16_t nb_deqd = 0;
> + uint64_t delta_ms;
> + uint64_t now;
> +
> + /*
> + * Receive a burst of packets and forward them.
> + */
> + nb_rx = rte_eth_rx_burst(fs->rx_port, fs->rx_queue,
> + pkts_burst, nb_pkt_per_burst);
> + if (unlikely(nb_rx == 0))
> + return;
> + fs->rx_packets += nb_rx;
> +
> + if (noisy_bsize_before_send > 0) {
> + if (rte_ring_free_count(ncf->f) >= nb_rx) {
> + /* enqueue into fifo */
> + nb_enqd = fifo_put(ncf->f, pkts_burst, nb_rx);
> + if (nb_enqd < nb_rx)
> + nb_rx = nb_enqd;
Am not sure I understand this part.
The simulation of the memory lookups should happen for every packet **before**
deciding on the output queue. Isn't it more realistic to simulate the memory
access after the rx_burst?
> + } else {
> + /* fifo is full, dequeue first */
> + nb_deqd = fifo_get(ncf->f, tmp_pkts, nb_rx);
> + /* enqueue into fifo */
> + nb_enqd = fifo_put(ncf->f, pkts_burst, nb_deqd);
> + sim_memory_lookups(ncf, nb_rx);
> + if (nb_enqd < nb_rx)
> + nb_rx = nb_enqd;
> + if (nb_deqd > 0)
> + nb_tx = rte_eth_tx_burst(fs->tx_port,
> + fs->tx_queue, tmp_pkts,
> + nb_deqd);
> + }
> + } else {
> + sim_memory_lookups(ncf, nb_rx);
> + nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue,
> + pkts_burst, nb_rx);
> + }
> +
> + /*
> + * TX burst queue drain
> + */
> + if (ncf->prev_time == 0)
> + now = ncf->prev_time = rte_get_timer_cycles();
> + else
> + now = rte_get_timer_cycles();
> + delta_ms = (now - ncf->prev_time) / freq_khz;
> + if (unlikely(delta_ms >= noisy_flush_timer) && noisy_flush_timer > 0
> &&
> + (nb_tx == 0)) {
> + while (fifo_count(ncf->f) > 0) {
> + nb_deqd = fifo_get(ncf->f, tmp_pkts, nb_rx);
> + nb_tx = rte_eth_tx_burst(fs->tx_port, fs-
> >tx_queue,
> + tmp_pkts, nb_deqd);
> + if (rte_ring_empty(ncf->f))
> + break;
> + }
> + ncf->prev_time = now;
> + }
> + if (nb_tx < nb_rx && fs->retry_enabled)
> + *pkts_burst = *tmp_pkts;
> +
> + /*
> + * Retry if necessary
> + */
> + if (unlikely(nb_tx < nb_rx) && fs->retry_enabled) {
> + retry = 0;
> + while (nb_tx < nb_rx && retry++ < burst_tx_retry_num) {
> + rte_delay_us(burst_tx_delay_time);
> + sim_memory_lookups(ncf, nb_rx);
> + nb_tx += rte_eth_tx_burst(fs->tx_port, fs-
> >tx_queue,
> + &pkts_burst[nb_tx], nb_rx - nb_tx);
> + }
> + }
> + fs->tx_packets += nb_tx;
> + if (unlikely(nb_tx < nb_rx)) {
> + fs->fwd_dropped += (nb_rx - nb_tx);
> + do {
> + rte_pktmbuf_free(pkts_burst[nb_tx]);
> + } while (++nb_tx < nb_rx);
> + }
> +}
> +
> +struct fwd_engine noisy_vnf_engine = {
> + .fwd_mode_name = "noisy",
> + .port_fwd_begin = NULL,
> + .port_fwd_end = NULL,
> + .packet_fwd = pkt_burst_noisy_vnf,
> +};
> +
> +#define NOISY_STRSIZE 256
> +#define NOISY_RING "noisy_ring_%d:%d\n"
> +struct rte_ring *
> +noisy_init(uint32_t qi, uint32_t pi)
> +{
> + struct noisy_config *n = &noisy_cfg[qi];
> + char name[NOISY_STRSIZE];
> +
> + snprintf(name, NOISY_STRSIZE, NOISY_RING, pi, qi);
> + n->f = rte_ring_create(name, noisy_bsize_before_send,
> + rte_socket_id(), 0);
> + n->vnf_mem = (char *) rte_zmalloc("vnf sim memory",
> + noisy_vnf_memory_footprint * 1024 * 1024,
> + RTE_CACHE_LINE_SIZE);
> + if (n->vnf_mem == NULL)
> + printf("allocating vnf memory failed\n");
> +
> + return n->f;
> +}
> diff --git a/app/test-pmd/noisy_vnf.h b/app/test-pmd/noisy_vnf.h new file
> mode 100644 index 000000000..6c6df7a9e
> --- /dev/null
> +++ b/app/test-pmd/noisy_vnf.h
> @@ -0,0 +1,41 @@
> +#ifndef __NOISY_VNF_H
> +#define __NOISY_VNF_H
> +#include <stdint.h>
> +#include <rte_mbuf.h>
> +#include <rte_ring.h>
> +#include "testpmd.h"
> +
> +/**
> + * Add elements to fifo. Return number of written elements */ static
> +inline unsigned int fifo_put(struct rte_ring *r, struct rte_mbuf
> +**data, unsigned num) {
> +
> + return rte_ring_enqueue_burst(r, (void **)data, num, NULL); }
> +
> +/**
> + * Get elements from fifo. Return number of read elements */ static
> +inline unsigned int fifo_get(struct rte_ring *r, struct rte_mbuf
> +**data, unsigned num) {
> + return rte_ring_dequeue_burst(r, (void **) data, num, NULL); }
> +
> +static inline unsigned int
> +fifo_count(struct rte_ring *r)
> +{
> + return rte_ring_count(r);
> +}
> +
> +static inline int
> +fifo_full(struct rte_ring *r)
> +{
> + return rte_ring_full(r);
> +}
> +
> +struct rte_ring *noisy_init(uint32_t qi, uint32_t pi);
> +
> +#endif
> diff --git a/app/test-pmd/parameters.c b/app/test-pmd/parameters.c index
> 75807623c..e6d8b3ef9 100644
> --- a/app/test-pmd/parameters.c
> +++ b/app/test-pmd/parameters.c
> @@ -625,6 +625,12 @@ launch_args_parse(int argc, char** argv)
> { "vxlan-gpe-port", 1, 0, 0 },
> { "mlockall", 0, 0, 0 },
> { "no-mlockall", 0, 0, 0 },
> + { "noisy-buffersize-before-sending", 1, 0, 0 },
> + { "noisy-flush-timeout", 1, 0, 0 },
> + { "noisy-memory-footprint", 1, 0, 0 },
> + { "noisy-nb-rnd-write", 1, 0, 0 },
> + { "noisy-nb-rnd-read", 1, 0, 0 },
> + { "noisy-nb-rnd-read-write", 1, 0, 0 },
> { 0, 0, 0, 0 },
> };
>
> @@ -1145,6 +1151,54 @@ launch_args_parse(int argc, char** argv)
> do_mlockall = 1;
> if (!strcmp(lgopts[opt_idx].name, "no-mlockall"))
> do_mlockall = 0;
> + if (!strcmp(lgopts[opt_idx].name, "noisy-buffersize-
> before-sending")) {
> + n = atoi(optarg);
> + if (n > 0)
> + noisy_bsize_before_send =
> (uint16_t) n;
> + else
> + rte_exit(EXIT_FAILURE,
> + "noisy-buffersize-before-
> sending must be > 0\n");
> + }
> + if (!strcmp(lgopts[opt_idx].name, "noisy-flush-
> timeout")) {
> + n = atoi(optarg);
> + if (n >= 0)
> + noisy_flush_timer = (uint16_t) n;
> + else
> + rte_exit(EXIT_FAILURE,
> + "noisy-flush-timeout must
> be > 0\n");
> + }
> + if (!strcmp(lgopts[opt_idx].name, "noisy-memory-
> footprint")) {
> + n = atoi(optarg);
> + if (n > 0)
> + noisy_vnf_memory_footprint =
> (uint16_t) n;
> + else
> + rte_exit(EXIT_FAILURE,
> + "noisy-memory-footprint
> must be > 0\n");
> + }
> + if (!strcmp(lgopts[opt_idx].name, "noisy-nb-rnd-
> write")) {
> + n = atoi(optarg);
> + if (n > 0)
> + noisy_nb_rnd_write = (uint16_t) n;
> + else
> + rte_exit(EXIT_FAILURE,
> + "noisy-nb-rnd-write must be
> > 0\n");
> + }
> + if (!strcmp(lgopts[opt_idx].name, "noisy-nb-rnd-
> read")) {
> + n = atoi(optarg);
> + if (n > 0)
> + noisy_nb_rnd_read = (uint16_t) n;
> + else
> + rte_exit(EXIT_FAILURE,
> + "noisy-nb-rnd-read must be
> > 0\n");
> + }
> + if (!strcmp(lgopts[opt_idx].name, "noisy-nb-rnd-
> read-write")) {
> + n = atoi(optarg);
> + if (n > 0)
> + noisy_nb_rnd_read_write =
> (uint16_t) n;
> + else
> + rte_exit(EXIT_FAILURE,
> + "noisy-nb-rnd-read-write
> must be > 0\n");
> + }
> break;
> case 'h':
> usage(argv[0]);
> diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c index
> 24c199844..7f3dc6f1c 100644
> --- a/app/test-pmd/testpmd.c
> +++ b/app/test-pmd/testpmd.c
> @@ -61,6 +61,7 @@
> #include <rte_latencystats.h>
> #endif
>
> +#include "noisy_vnf.h"
> #include "testpmd.h"
>
> uint16_t verbose_level = 0; /**< Silent by default. */ @@ -155,6 +156,7 @@
> struct fwd_engine * fwd_engines[] = {
> &tx_only_engine,
> &csum_fwd_engine,
> &icmp_echo_engine,
> + &noisy_vnf_engine,
> #if defined RTE_LIBRTE_PMD_SOFTNIC && defined RTE_LIBRTE_SCHED
> &softnic_tm_engine,
> &softnic_tm_bypass_engine,
> @@ -251,6 +253,40 @@ int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
> */
> int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
>
> +/*
> + * Configurable value of buffered packed before sending.
> + */
> +uint16_t noisy_bsize_before_send;
> +
> +/*
> + * Configurable value of packet buffer timeout.
> + */
> +uint16_t noisy_flush_timer;
> +
> +/*
> + * Configurable value for size of VNF internal memory area
> + * used for simulating noisy neighbour behaviour */ uint64_t
> +noisy_vnf_memory_footprint;
> +
> +/*
> + * Configurable value of number of random writes done in
> + * VNF simulation memory area.
> + */
> +uint64_t noisy_nb_rnd_write;
> +
> +/*
> + * Configurable value of number of random reads done in
> + * VNF simulation memory area.
> + */
> +uint64_t noisy_nb_rnd_read;
> +
> +/*
> + * Configurable value of number of random reads/wirtes done in
> + * VNF simulation memory area.
> + */
> +uint64_t noisy_nb_rnd_read_write;
> +
> /*
> * Receive Side Scaling (RSS) configuration.
> */
> @@ -1641,12 +1677,26 @@ start_port(portid_t pid)
> return -1;
> }
> }
> + noisy_cfg = (struct noisy_config *) rte_zmalloc(
> + "testpmd noisy fifo and timers",
> + nb_txq * sizeof(struct noisy_config),
> + RTE_CACHE_LINE_SIZE);
> + if (noisy_cfg == NULL) {
> + rte_exit(EXIT_FAILURE,
> + "rte_zmalloc(%d) struct noisy_config)
> failed\n",
> + (int)(nb_txq *
> + sizeof(struct noisy_config)));
> + }
> if (port->need_reconfig_queues > 0) {
> port->need_reconfig_queues = 0;
> /* setup tx queues */
> for (qi = 0; qi < nb_txq; qi++) {
> port->tx_conf[qi].txq_flags =
> ETH_TXQ_FLAGS_IGNORE;
> + if (!noisy_init(qi, pi) &&
> + noisy_bsize_before_send >
> 0)
> + rte_exit(EXIT_FAILURE, "%s\n",
> + rte_strerror(rte_errno));
> if ((numa_support) &&
> (txring_numa[pi] !=
> NUMA_NO_CONFIG))
> diag = rte_eth_tx_queue_setup(pi,
> qi, @@ -1817,6 +1867,8 @@ stop_port(portid_t pid)
> RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
> printf("Port %d can not be set into stopped\n", pi);
> need_check_link_status = 1;
> +
> + rte_free(noisy_cfg);
> }
> if (need_check_link_status && !no_link_check)
> check_all_ports_link_status(RTE_PORT_ALL);
> diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h index
> f51cd9dd9..75fb18a25 100644
> --- a/app/test-pmd/testpmd.h
> +++ b/app/test-pmd/testpmd.h
> @@ -9,6 +9,7 @@
> #include <rte_bus_pci.h>
> #include <rte_gro.h>
> #include <rte_gso.h>
> +#include "noisy_vnf.h"
>
> #define RTE_PORT_ALL (~(portid_t)0x0)
>
> @@ -122,6 +123,14 @@ struct fwd_stream { #endif };
>
> +struct noisy_config {
> + struct rte_ring *f;
> + uint64_t prev_time;
> + char *vnf_mem;
> +};
> +struct noisy_config *noisy_cfg;
> +
> +
> /** Descriptor for a single flow. */
> struct port_flow {
> size_t size; /**< Allocated space including data[]. */ @@ -266,6
> +275,7 @@ extern struct fwd_engine rx_only_engine; extern struct
> fwd_engine tx_only_engine; extern struct fwd_engine csum_fwd_engine;
> extern struct fwd_engine icmp_echo_engine;
> +extern struct fwd_engine noisy_vnf_engine;
> #ifdef TM_MODE
> extern struct fwd_engine softnic_tm_engine; extern struct fwd_engine
> softnic_tm_bypass_engine; @@ -399,6 +409,13 @@ extern int8_t
> rx_drop_en; extern int16_t tx_free_thresh; extern int16_t tx_rs_thresh;
>
> +extern uint16_t noisy_bsize_before_send; extern uint16_t
> +noisy_flush_timer; extern uint64_t noisy_vnf_memory_footprint; extern
> +uint64_t noisy_nb_rnd_write; extern uint64_t noisy_nb_rnd_read; extern
> +uint64_t noisy_nb_rnd_read_write;
> +
> extern uint8_t dcb_config;
> extern uint8_t dcb_test;
>
> --
> 2.14.4
