>Add worker for handling vector events to perf tests, vector events could
>be generated by crypto adapter producer.
>
>Example:
> ./dpdk-test-eventdev -l 0-2 -a <EVENTDEV> -a <CRYPTODEV> -- \
> --prod_type_cryptodev --crypto_adptr_mode 1 --test=perf_queue \
> --stlist=a --wlcores 1 --plcores 2 --prod_enq_burst_sz 32 \
> --enable_vector --vector_tmo_ns 0 --nb_flows 2
>
>Signed-off-by: Volodymyr Fialko <vfia...@marvell.com>
Acked-by: Shijith Thotton <sthot...@marvell.com>
>---
> app/test-eventdev/test_perf_atq.c | 62 ++++++++++++++--
> app/test-eventdev/test_perf_common.c | 68 +++++++++++++++---
> app/test-eventdev/test_perf_common.h | 102
>++++++++++++++++++++++++++-
> app/test-eventdev/test_perf_queue.c | 63 +++++++++++++++--
> doc/guides/tools/testeventdev.rst | 12 ++--
> 5 files changed, 279 insertions(+), 28 deletions(-)
>
>diff --git a/app/test-eventdev/test_perf_atq.c b/app/test-
>eventdev/test_perf_atq.c
>index 9d30081117..4ac60cc38b 100644
>--- a/app/test-eventdev/test_perf_atq.c
>+++ b/app/test-eventdev/test_perf_atq.c
>@@ -24,14 +24,22 @@ atq_fwd_event(struct rte_event *const ev, uint8_t *const
>sched_type_list,
> ev->event_type = RTE_EVENT_TYPE_CPU;
> }
>
>+static __rte_always_inline void
>+atq_fwd_event_vector(struct rte_event *const ev, uint8_t *const
>sched_type_list,
>+ const uint8_t nb_stages)
>+{
>+ ev->sub_event_type++;
>+ ev->sched_type = sched_type_list[ev->sub_event_type % nb_stages];
>+ ev->op = RTE_EVENT_OP_FORWARD;
>+ ev->event_type = RTE_EVENT_TYPE_CPU_VECTOR;
>+}
>+
> static int
> perf_atq_worker(void *arg, const int enable_fwd_latency)
> {
>- struct perf_elt *pe = NULL;
> uint16_t enq = 0, deq = 0;
> struct rte_event ev;
> PERF_WORKER_INIT;
>- uint8_t stage;
>
> while (t->done == false) {
> deq = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
>@@ -79,9 +87,7 @@ perf_atq_worker_burst(void *arg, const int
>enable_fwd_latency)
> /* +1 to avoid prefetch out of array check */
> struct rte_event ev[BURST_SIZE + 1];
> uint16_t enq = 0, nb_rx = 0;
>- struct perf_elt *pe = NULL;
> PERF_WORKER_INIT;
>- uint8_t stage;
> uint16_t i;
>
> while (t->done == false) {
>@@ -134,6 +140,50 @@ perf_atq_worker_burst(void *arg, const int
>enable_fwd_latency)
> return 0;
> }
>
>+static int
>+perf_atq_worker_vector(void *arg, const int enable_fwd_latency)
>+{
>+ uint16_t enq = 0, deq = 0;
>+ struct rte_event ev;
>+ PERF_WORKER_INIT;
>+
>+ RTE_SET_USED(sz);
>+ RTE_SET_USED(cnt);
>+ RTE_SET_USED(prod_crypto_type);
>+
>+ while (t->done == false) {
>+ deq = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
>+
>+ if (!deq)
>+ continue;
>+
>+ if (ev.event_type == RTE_EVENT_TYPE_CRYPTODEV_VECTOR) {
>+ if (perf_handle_crypto_vector_ev(&ev, &pe,
>enable_fwd_latency))
>+ continue;
>+ }
>+
>+ stage = ev.sub_event_type % nb_stages;
>+ /* First q in pipeline, mark timestamp to compute fwd latency */
>+ if (enable_fwd_latency && !prod_timer_type && stage == 0)
>+ perf_mark_fwd_latency(pe);
>+
>+ /* Last stage in pipeline */
>+ if (unlikely(stage == laststage)) {
>+ perf_process_vector_last_stage(pool, t->ca_op_pool,
>&ev, w,
>+ enable_fwd_latency);
>+ } else {
>+ atq_fwd_event_vector(&ev, sched_type_list, nb_stages);
>+ do {
>+ enq = rte_event_enqueue_burst(dev, port, &ev,
>1);
>+ } while (!enq && !t->done);
>+ }
>+ }
>+
>+ perf_worker_cleanup(pool, dev, port, &ev, enq, deq);
>+
>+ return 0;
>+}
>+
> static int
> worker_wrapper(void *arg)
> {
>@@ -144,7 +194,9 @@ worker_wrapper(void *arg)
> const int fwd_latency = opt->fwd_latency;
>
> /* allow compiler to optimize */
>- if (!burst && !fwd_latency)
>+ if (opt->ena_vector && opt->prod_type ==
>EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR)
>+ return perf_atq_worker_vector(arg, fwd_latency);
>+ else if (!burst && !fwd_latency)
> return perf_atq_worker(arg, 0);
> else if (!burst && fwd_latency)
> return perf_atq_worker(arg, 1);
>diff --git a/app/test-eventdev/test_perf_common.c b/app/test-
>eventdev/test_perf_common.c
>index 140c0c2dc3..8d7e483c55 100644
>--- a/app/test-eventdev/test_perf_common.c
>+++ b/app/test-eventdev/test_perf_common.c
>@@ -827,10 +827,13 @@ perf_event_timer_adapter_setup(struct test_perf *t)
> static int
> perf_event_crypto_adapter_setup(struct test_perf *t, struct prod_data *p)
> {
>+ struct rte_event_crypto_adapter_queue_conf conf;
> struct evt_options *opt = t->opt;
> uint32_t cap;
> int ret;
>
>+ memset(&conf, 0, sizeof(conf));
>+
> ret = rte_event_crypto_adapter_caps_get(p->dev_id, p->ca.cdev_id,
>&cap);
> if (ret) {
> evt_err("Failed to get crypto adapter capabilities");
>@@ -849,19 +852,53 @@ perf_event_crypto_adapter_setup(struct test_perf *t,
>struct prod_data *p)
> return -ENOTSUP;
> }
>
>- if (cap &
>RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_QP_EV_BIND) {
>- struct rte_event_crypto_adapter_queue_conf conf;
>+ if (opt->ena_vector) {
>+ struct rte_event_crypto_adapter_vector_limits limits;
>+
>+ if (!(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_EVENT_VECTOR))
>{
>+ evt_err("Crypto adapter doesn't support event vector");
>+ return -EINVAL;
>+ }
>+
>+ ret = rte_event_crypto_adapter_vector_limits_get(p->dev_id, p-
>>ca.cdev_id, &limits);
>+ if (ret) {
>+ evt_err("Failed to get crypto adapter's vector limits");
>+ return ret;
>+ }
>
>- memset(&conf, 0, sizeof(conf));
>+ if (opt->vector_size < limits.min_sz || opt->vector_size >
>limits.max_sz) {
>+ evt_err("Vector size [%d] not within limits max[%d]
>min[%d]",
>+ opt->vector_size, limits.max_sz, limits.min_sz);
>+ return -EINVAL;
>+ }
>+
>+ if (limits.log2_sz && !rte_is_power_of_2(opt->vector_size)) {
>+ evt_err("Vector size [%d] not power of 2", opt-
>>vector_size);
>+ return -EINVAL;
>+ }
>+
>+ if (opt->vector_tmo_nsec > limits.max_timeout_ns ||
>+ opt->vector_tmo_nsec < limits.min_timeout_ns) {
>+ evt_err("Vector timeout [%" PRIu64 "] not within limits
>"
>+ "max[%" PRIu64 "] min[%" PRIu64 "]",
>+ opt->vector_tmo_nsec, limits.max_timeout_ns,
>limits.min_timeout_ns);
>+ return -EINVAL;
>+ }
>+
>+ conf.vector_mp = t->ca_vector_pool;
>+ conf.vector_sz = opt->vector_size;
>+ conf.vector_timeout_ns = opt->vector_tmo_nsec;
>+ conf.flags |= RTE_EVENT_CRYPTO_ADAPTER_EVENT_VECTOR;
>+ }
>+
>+ if (cap &
>RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_QP_EV_BIND) {
> conf.ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
> conf.ev.queue_id = p->queue_id;
>- ret = rte_event_crypto_adapter_queue_pair_add(
>- TEST_PERF_CA_ID, p->ca.cdev_id, p->ca.cdev_qp_id,
>&conf);
>- } else {
>- ret = rte_event_crypto_adapter_queue_pair_add(
>- TEST_PERF_CA_ID, p->ca.cdev_id, p->ca.cdev_qp_id,
>NULL);
> }
>
>+ ret = rte_event_crypto_adapter_queue_pair_add(
>+ TEST_PERF_CA_ID, p->ca.cdev_id, p->ca.cdev_qp_id, &conf);
>+
> return ret;
> }
>
>@@ -1411,6 +1448,19 @@ perf_cryptodev_setup(struct evt_test *test, struct
>evt_options *opt)
> goto err;
> }
>
>+ if (opt->ena_vector) {
>+ unsigned int nb_elem = (opt->pool_sz / opt->vector_size) * 2;
>+ nb_elem = RTE_MAX(512U, nb_elem);
>+ nb_elem += evt_nr_active_lcores(opt->wlcores) * 32;
>+ t->ca_vector_pool =
>rte_event_vector_pool_create("vector_pool", nb_elem, 32,
>+ opt->vector_size, opt->socket_id);
>+ if (t->ca_vector_pool == NULL) {
>+ evt_err("Failed to create event vector pool");
>+ ret = -ENOMEM;
>+ goto err;
>+ }
>+ }
>+
> /*
> * Calculate number of needed queue pairs, based on the amount of
> * available number of logical cores and crypto devices. For instance,
>@@ -1467,6 +1517,7 @@ perf_cryptodev_setup(struct evt_test *test, struct
>evt_options *opt)
> rte_mempool_free(t->ca_op_pool);
> rte_mempool_free(t->ca_sess_pool);
> rte_mempool_free(t->ca_asym_sess_pool);
>+ rte_mempool_free(t->ca_vector_pool);
>
> return ret;
> }
>@@ -1507,6 +1558,7 @@ perf_cryptodev_destroy(struct evt_test *test, struct
>evt_options *opt)
> rte_mempool_free(t->ca_op_pool);
> rte_mempool_free(t->ca_sess_pool);
> rte_mempool_free(t->ca_asym_sess_pool);
>+ rte_mempool_free(t->ca_vector_pool);
> }
>
> int
>diff --git a/app/test-eventdev/test_perf_common.h b/app/test-
>eventdev/test_perf_common.h
>index 503b6aa1db..faedd471c6 100644
>--- a/app/test-eventdev/test_perf_common.h
>+++ b/app/test-eventdev/test_perf_common.h
>@@ -71,6 +71,7 @@ struct test_perf {
> struct rte_mempool *ca_op_pool;
> struct rte_mempool *ca_sess_pool;
> struct rte_mempool *ca_asym_sess_pool;
>+ struct rte_mempool *ca_vector_pool;
> } __rte_cache_aligned;
>
> struct perf_elt {
>@@ -103,6 +104,8 @@ struct perf_elt {
> uint8_t cnt = 0;\
> void *bufs[16] __rte_cache_aligned;\
> int const sz = RTE_DIM(bufs);\
>+ uint8_t stage;\
>+ struct perf_elt *pe = NULL;\
> if (opt->verbose_level > 1)\
> printf("%s(): lcore %d dev_id %d port=%d\n", __func__,\
> rte_lcore_id(), dev, port)
>@@ -143,6 +146,64 @@ perf_handle_crypto_ev(struct rte_event *ev, struct
>perf_elt **pe, int enable_fwd
> return 0;
> }
>
>+static __rte_always_inline struct perf_elt *
>+perf_elt_from_vec_get(struct rte_event_vector *vec)
>+{
>+ /* Timestamp for vector event stored in first element */
>+ struct rte_crypto_op *cop = vec->ptrs[0];
>+ struct rte_mbuf *m;
>+
>+ if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
>+ m = cop->sym->m_dst == NULL ? cop->sym->m_src : cop->sym-
>>m_dst;
>+ return rte_pktmbuf_mtod(m, struct perf_elt *);
>+ } else {
>+ return RTE_PTR_ADD(cop->asym->modex.result.data, cop-
>>asym->modex.result.length);
>+ }
>+}
>+
>+static __rte_always_inline int
>+perf_handle_crypto_vector_ev(struct rte_event *ev, struct perf_elt **pe,
>+ const int enable_fwd_latency)
>+{
>+ struct rte_event_vector *vec = ev->vec;
>+ struct rte_crypto_op *cop;
>+ struct rte_mbuf *m;
>+ int i, n = 0;
>+ void *data;
>+
>+ for (i = 0; i < vec->nb_elem; i++) {
>+ cop = vec->ptrs[i];
>+ if (unlikely(cop->status != RTE_CRYPTO_OP_STATUS_SUCCESS)) {
>+ if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
>+ m = cop->sym->m_dst == NULL ? cop->sym-
>>m_src : cop->sym->m_dst;
>+ rte_pktmbuf_free(m);
>+ } else {
>+ data = cop->asym->modex.result.data;
>+
> rte_mempool_put(rte_mempool_from_obj(data), data);
>+ }
>+ rte_crypto_op_free(cop);
>+ continue;
>+ }
>+ vec->ptrs[n++] = cop;
>+ }
>+
>+ /* All cops failed, free the vector */
>+ if (n == 0) {
>+ rte_mempool_put(rte_mempool_from_obj(vec), vec);
>+ return -ENOENT;
>+ }
>+
>+ vec->nb_elem = n;
>+
>+ /* Forward latency not enabled - perf data will be not accessed */
>+ if (!enable_fwd_latency)
>+ return 0;
>+
>+ /* Get pointer to perf data */
>+ *pe = perf_elt_from_vec_get(vec);
>+
>+ return 0;
>+}
>
> static __rte_always_inline int
> perf_process_last_stage(struct rte_mempool *const pool, uint8_t
>prod_crypto_type,
>@@ -195,9 +256,8 @@ perf_process_last_stage_latency(struct rte_mempool
>*const pool, uint8_t prod_cry
> struct perf_elt *pe;
> void *to_free_in_bulk;
>
>- /* release fence here ensures event_prt is
>- * stored before updating the number of
>- * processed packets for worker lcores
>+ /* Release fence here ensures event_prt is stored before updating the
>number of processed
>+ * packets for worker lcores.
> */
> rte_atomic_thread_fence(__ATOMIC_RELEASE);
> w->processed_pkts++;
>@@ -237,6 +297,42 @@ perf_process_last_stage_latency(struct rte_mempool
>*const pool, uint8_t prod_cry
> return count;
> }
>
>+static __rte_always_inline void
>+perf_process_vector_last_stage(struct rte_mempool *const pool,
>+ struct rte_mempool *const ca_pool, struct rte_event *const ev,
>+ struct worker_data *const w, const bool enable_fwd_latency)
>+{
>+ struct rte_event_vector *vec = ev->vec;
>+ struct rte_crypto_op *cop;
>+ void *bufs[vec->nb_elem];
>+ struct perf_elt *pe;
>+ uint64_t latency;
>+ int i;
>+
>+ /* Release fence here ensures event_prt is stored before updating the
>number of processed
>+ * packets for worker lcores.
>+ */
>+ rte_atomic_thread_fence(__ATOMIC_RELEASE);
>+ w->processed_pkts += vec->nb_elem;
>+
>+ if (enable_fwd_latency) {
>+ pe = perf_elt_from_vec_get(vec);
>+ latency = rte_get_timer_cycles() - pe->timestamp;
>+ w->latency += latency;
>+ }
>+
>+ for (i = 0; i < vec->nb_elem; i++) {
>+ cop = vec->ptrs[i];
>+ if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC)
>+ bufs[i] = cop->sym->m_dst == NULL ? cop->sym->m_src :
>cop->sym->m_dst;
>+ else
>+ bufs[i] = cop->asym->modex.result.data;
>+ }
>+
>+ rte_mempool_put_bulk(pool, bufs, vec->nb_elem);
>+ rte_mempool_put_bulk(ca_pool, (void * const *)vec->ptrs, vec-
>>nb_elem);
>+ rte_mempool_put(rte_mempool_from_obj(vec), vec);
>+}
>
> static inline int
> perf_nb_event_ports(struct evt_options *opt)
>diff --git a/app/test-eventdev/test_perf_queue.c b/app/test-
>eventdev/test_perf_queue.c
>index 69ef0ebbac..2399cfb69b 100644
>--- a/app/test-eventdev/test_perf_queue.c
>+++ b/app/test-eventdev/test_perf_queue.c
>@@ -25,15 +25,22 @@ fwd_event(struct rte_event *const ev, uint8_t *const
>sched_type_list,
> ev->event_type = RTE_EVENT_TYPE_CPU;
> }
>
>+static __rte_always_inline void
>+fwd_event_vector(struct rte_event *const ev, uint8_t *const sched_type_list,
>+ const uint8_t nb_stages)
>+{
>+ ev->queue_id++;
>+ ev->sched_type = sched_type_list[ev->queue_id % nb_stages];
>+ ev->op = RTE_EVENT_OP_FORWARD;
>+ ev->event_type = RTE_EVENT_TYPE_CPU_VECTOR;
>+}
>+
> static int
> perf_queue_worker(void *arg, const int enable_fwd_latency)
> {
>- struct perf_elt *pe = NULL;
> uint16_t enq = 0, deq = 0;
> struct rte_event ev;
> PERF_WORKER_INIT;
>- uint8_t stage;
>-
>
> while (t->done == false) {
> deq = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
>@@ -82,9 +89,7 @@ perf_queue_worker_burst(void *arg, const int
>enable_fwd_latency)
> /* +1 to avoid prefetch out of array check */
> struct rte_event ev[BURST_SIZE + 1];
> uint16_t enq = 0, nb_rx = 0;
>- struct perf_elt *pe = NULL;
> PERF_WORKER_INIT;
>- uint8_t stage;
> uint16_t i;
>
> while (t->done == false) {
>@@ -137,6 +142,50 @@ perf_queue_worker_burst(void *arg, const int
>enable_fwd_latency)
> return 0;
> }
>
>+static int
>+perf_queue_worker_vector(void *arg, const int enable_fwd_latency)
>+{
>+ uint16_t enq = 0, deq = 0;
>+ struct rte_event ev;
>+ PERF_WORKER_INIT;
>+
>+ RTE_SET_USED(sz);
>+ RTE_SET_USED(cnt);
>+ RTE_SET_USED(prod_crypto_type);
>+
>+ while (t->done == false) {
>+ deq = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
>+
>+ if (!deq)
>+ continue;
>+
>+ if (ev.event_type == RTE_EVENT_TYPE_CRYPTODEV_VECTOR) {
>+ if (perf_handle_crypto_vector_ev(&ev, &pe,
>enable_fwd_latency))
>+ continue;
>+ }
>+
>+ stage = ev.queue_id % nb_stages;
>+ /* First q in pipeline, mark timestamp to compute fwd latency */
>+ if (enable_fwd_latency && !prod_timer_type && stage == 0)
>+ perf_mark_fwd_latency(pe);
>+
>+ /* Last stage in pipeline */
>+ if (unlikely(stage == laststage)) {
>+ perf_process_vector_last_stage(pool, t->ca_op_pool,
>&ev, w,
>+ enable_fwd_latency);
>+ } else {
>+ fwd_event_vector(&ev, sched_type_list, nb_stages);
>+ do {
>+ enq = rte_event_enqueue_burst(dev, port, &ev,
>1);
>+ } while (!enq && !t->done);
>+ }
>+ }
>+
>+ perf_worker_cleanup(pool, dev, port, &ev, enq, deq);
>+
>+ return 0;
>+}
>+
> static int
> worker_wrapper(void *arg)
> {
>@@ -147,7 +196,9 @@ worker_wrapper(void *arg)
> const int fwd_latency = opt->fwd_latency;
>
> /* allow compiler to optimize */
>- if (!burst && !fwd_latency)
>+ if (opt->ena_vector && opt->prod_type ==
>EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR)
>+ return perf_queue_worker_vector(arg, fwd_latency);
>+ else if (!burst && !fwd_latency)
> return perf_queue_worker(arg, 0);
> else if (!burst && fwd_latency)
> return perf_queue_worker(arg, 1);
>diff --git a/doc/guides/tools/testeventdev.rst
>b/doc/guides/tools/testeventdev.rst
>index cd278e8998..6f065b9752 100644
>--- a/doc/guides/tools/testeventdev.rst
>+++ b/doc/guides/tools/testeventdev.rst
>@@ -185,18 +185,18 @@ The following are the application command-line options:
>
> * ``--enable_vector``
>
>- Enable event vector for Rx/Tx adapters.
>- Only applicable for `pipeline_atq` and `pipeline_queue` tests.
>+ Enable event vector for Rx/Tx/crypto adapters.
>+ Only applicable for `pipeline_*` and `perf_*` tests.
>
> * ``--vector_size``
>
>- Vector size to configure for the Rx adapter.
>- Only applicable for `pipeline_atq` and `pipeline_queue` tests.
>+ Vector size to configure for the Rx/crypto adapter.
>+ Only applicable for `pipeline_*` and `perf_*` tests.
>
> * ``--vector_tmo_ns``
>
>- Vector timeout nanoseconds to be configured for the Rx adapter.
>- Only applicable for `pipeline_atq` and `pipeline_queue` tests.
>+ Vector timeout nanoseconds to be configured for the Rx/crypto adapter.
>+ Only applicable for `pipeline_*` and `perf_*` tests.
>
> * ``--per_port_pool``
>
>--
>2.25.1
