Add helper functions and macros to help drivers to
transition to new datapath interface.
Signed-off-by: Akhil Goyal <gak...@marvell.com>
---
lib/cryptodev/cryptodev_pmd.h | 246 ++++++++++++++++++++++++++++++++++
lib/cryptodev/rte_cryptodev.c | 40 +++++-
lib/cryptodev/version.map | 4 +
3 files changed, 289 insertions(+), 1 deletion(-)
diff --git a/lib/cryptodev/cryptodev_pmd.h b/lib/cryptodev/cryptodev_pmd.h
index eeaea13a23..d40e5cee94 100644
--- a/lib/cryptodev/cryptodev_pmd.h
+++ b/lib/cryptodev/cryptodev_pmd.h
@@ -70,6 +70,13 @@ struct cryptodev_driver {
const struct rte_driver *driver;
uint8_t id;
};
+/**
+ * @internal
+ * The pool of *rte_cryptodev* structures. The size of the pool
+ * is configured at compile-time in the <rte_cryptodev.c> file.
+ */
+extern struct rte_cryptodev rte_crypto_devices[];
+
/**
* Get the rte_cryptodev structure device pointer for the device. Assumes a
@@ -529,6 +536,245 @@ __rte_internal
void
rte_cryptodev_api_reset(struct rte_cryptodev_api *api);
+/**
+ * @internal
+ * Helper routine for cryptodev_dequeue_burst.
+ * Should be called as first thing on entrance to the PMD's
+ * rte_cryptodev_dequeue_burst implementation.
+ * Does necessary checks and returns pointer to cryptodev identifier.
+ *
+ * @param dev_id
+ * The device identifier of the crypto device.
+ * @param qp_id
+ * The index of the queue pair from which processed crypto ops will
+ * be dequeued.
+ *
+ * @return
+ * Pointer to device queue pair on success or NULL otherwise.
+ */
+__rte_internal
+static inline void *
+_rte_cryptodev_dequeue_prolog(uint8_t dev_id, uint8_t qp_id)
+{
+ struct rte_cryptodev *dev = &rte_cryptodevs[dev_id];
+
+ return dev->data->queue_pairs[qp_id];
+}
+
+/**
+ * @internal
+ * Helper routine for crypto driver dequeue API.
+ * Should be called at exit from PMD's rte_cryptodev_dequeue_burst
+ * implementation.
+ * Does necessary post-processing - invokes RX callbacks if any, tracing, etc.
+ *
+ * @param dev_id
+ * The device identifier of the Crypto device.
+ * @param qp_id
+ * The index of the queue pair from which to retrieve input crypto_ops.
+ * @param ops
+ * The address of an array of pointers to *rte_crypto_op* structures that
+ * have been retrieved from the device.
+ * @param nb_ops
+ * The number of ops that were retrieved from the device.
+ *
+ * @return
+ * The number of crypto ops effectively supplied to the *ops* array.
+ */
+__rte_internal
+static inline uint16_t
+_rte_cryptodev_dequeue_epilog(uint16_t dev_id, uint16_t qp_id,
+ struct rte_crypto_op **ops, uint16_t nb_ops)
+{
+#ifdef RTE_CRYPTO_CALLBACKS
+ struct rte_cryptodev *dev = &rte_cryptodevs[dev_id];
+
+ if (unlikely(dev->deq_cbs != NULL)) {
+ struct rte_cryptodev_cb_rcu *list;
+ struct rte_cryptodev_cb *cb;
+
+ /* __ATOMIC_RELEASE memory order was used when the
+ * call back was inserted into the list.
+ * Since there is a clear dependency between loading
+ * cb and cb->fn/cb->next, __ATOMIC_ACQUIRE memory order is
+ * not required.
+ */
+ list = &dev->deq_cbs[qp_id];
+ rte_rcu_qsbr_thread_online(list->qsbr, 0);
+ cb = __atomic_load_n(&list->next, __ATOMIC_RELAXED);
+
+ while (cb != NULL) {
+ nb_ops = cb->fn(dev_id, qp_id, ops, nb_ops,
+ cb->arg);
+ cb = cb->next;
+ };
+
+ rte_rcu_qsbr_thread_offline(list->qsbr, 0);
+ }
+#endif
+
+ return nb_ops;
+}
+#define _RTE_CRYPTO_DEQ_FUNC(fn) _rte_crypto_deq_##fn
+
+/**
+ * @internal
+ * Helper macro to create new API wrappers for existing PMD dequeue functions.
+ */
+#define _RTE_CRYPTO_DEQ_PROTO(fn) \
+ uint16_t _RTE_CRYPTO_DEQ_FUNC(fn)(uint8_t dev_id, uint8_t qp_id, \
+ struct rte_crypto_op **ops, uint16_t nb_ops)
+
+/**
+ * @internal
+ * Helper macro to create new API wrappers for existing PMD dequeue functions.
+ */
+#define _RTE_CRYPTO_DEQ_DEF(fn) \
+_RTE_CRYPTO_DEQ_PROTO(fn) \
+{ \
+ void *qp = _rte_cryptodev_dequeue_prolog(dev_id, qp_id); \
+ if (qp == NULL) \
+ return 0; \
+ nb_ops = fn(qp, ops, nb_ops); \
+ return _rte_cryptodev_dequeue_epilog(dev_id, qp_id, ops, nb_ops); \
+}
+
+/**
+ * @internal
+ * Helper routine for cryptodev_enqueue_burst.
+ * Should be called as first thing on entrance to the PMD's
+ * rte_cryptodev_enqueue_burst implementation.
+ * Does necessary checks and returns pointer to cryptodev queue pair.
+ *
+ * @param dev_id
+ * The device identifier of the crypto device.
+ * @param qp_id
+ * The index of the queue pair in which packets will be enqueued.
+ * @param ops
+ * The address of an array of pointers to *rte_crypto_op* structures that
+ * will be enqueued to the device.
+ * @param nb_ops
+ * The number of ops that will be sent to the device.
+ *
+ * @return
+ * Pointer to device queue pair on success or NULL otherwise.
+ */
+__rte_internal
+static inline void *
+_rte_cryptodev_enqueue_prolog(uint8_t dev_id, uint8_t qp_id,
+ struct rte_crypto_op **ops, uint16_t nb_ops)
+{
+ struct rte_cryptodev *dev = &rte_cryptodevs[dev_id];
+
+#ifdef RTE_CRYPTO_CALLBACKS
+ if (unlikely(dev->enq_cbs != NULL)) {
+ struct rte_cryptodev_cb_rcu *list;
+ struct rte_cryptodev_cb *cb;
+
+ /* __ATOMIC_RELEASE memory order was used when the
+ * call back was inserted into the list.
+ * Since there is a clear dependency between loading
+ * cb and cb->fn/cb->next, __ATOMIC_ACQUIRE memory order is
+ * not required.
+ */
+ list = &dev->enq_cbs[qp_id];
+ rte_rcu_qsbr_thread_online(list->qsbr, 0);
+ cb = __atomic_load_n(&list->next, __ATOMIC_RELAXED);
+
+ while (cb != NULL) {
+ nb_ops = cb->fn(dev_id, qp_id, ops, nb_ops,
+ cb->arg);
+ cb = cb->next;
+ };
+
+ rte_rcu_qsbr_thread_offline(list->qsbr, 0);
+ }
+#endif
+ return dev->data->queue_pairs[qp_id];
+}
+
+#define _RTE_CRYPTO_ENQ_FUNC(fn) _rte_crypto_enq_##fn
+
+/**
+ * @internal
+ * Helper macro to create new API wrappers for existing PMD enqueue functions.
+ */
+#define _RTE_CRYPTO_ENQ_PROTO(fn) \
+ uint16_t _RTE_CRYPTO_ENQ_FUNC(fn)(uint8_t dev_id, uint8_t qp_id, \
+ struct rte_crypto_op **ops, uint16_t nb_ops)
+
+/**
+ * @internal
+ * Helper macro to create new API wrappers for existing PMD enqueue functions.
+ */
+#define _RTE_CRYPTO_ENQ_DEF(fn) \
+_RTE_CRYPTO_ENQ_PROTO(fn) \
+{ \
+ void *qp = _rte_cryptodev_enqueue_prolog(dev_id, qp_id, ops, nb_ops); \
+ if (qp == NULL) \
+ return 0; \
+ return fn(qp, ops, nb_ops); \
+}
+
+/**
+ * @internal
+ * Helper routine to get enqueue burst function of a given device.
+ *
+ * @param dev_id
+ * The device identifier of the Crypto device.
+ *
+ * @return
+ * The function if valid else NULL
+ */
+__rte_internal
+rte_crypto_enqueue_burst_t
+rte_crypto_get_enq_burst_fn(uint8_t dev_id);
+
+/**
+ * @internal
+ * Helper routine to get dequeue burst function of a given device.
+ *
+ * @param dev_id
+ * The device identifier of the Crypto device.
+ *
+ * @return
+ * The function if valid else NULL
+ */
+__rte_internal
+rte_crypto_dequeue_burst_t
+rte_crypto_get_deq_burst_fn(uint8_t dev_id);
+
+/**
+ * @internal
+ * Helper routine to set enqueue burst function of a given device.
+ *
+ * @param dev_id
+ * The device identifier of the Crypto device.
+ *
+ * @return
+ * 0 Success.
+ * -EINVAL Failure if dev_id or fn are in-valid.
+ */
+__rte_internal
+int
+rte_crypto_set_enq_burst_fn(uint8_t dev_id, rte_crypto_enqueue_burst_t fn);
+
+/**
+ * @internal
+ * Helper routine to set dequeue burst function of a given device.
+ *
+ * @param dev_id
+ * The device identifier of the Crypto device.
+ *
+ * @return
+ * 0 Success.
+ * -EINVAL Failure if dev_id or fn are in-valid.
+ */
+__rte_internal
+int
+rte_crypto_set_deq_burst_fn(uint8_t dev_id, rte_crypto_dequeue_burst_t fn);
+
+
static inline void *
get_sym_session_private_data(const struct rte_cryptodev_sym_session *sess,
uint8_t driver_id) {
diff --git a/lib/cryptodev/rte_cryptodev.c b/lib/cryptodev/rte_cryptodev.c
index 26f8390668..4ab82d21d0 100644
--- a/lib/cryptodev/rte_cryptodev.c
+++ b/lib/cryptodev/rte_cryptodev.c
@@ -44,7 +44,7 @@
static uint8_t nb_drivers;
-static struct rte_cryptodev rte_crypto_devices[RTE_CRYPTO_MAX_DEVS];
+struct rte_cryptodev rte_crypto_devices[RTE_CRYPTO_MAX_DEVS];
struct rte_cryptodev *rte_cryptodevs = rte_crypto_devices;
@@ -1270,6 +1270,44 @@ rte_cryptodev_queue_pair_setup(uint8_t dev_id, uint16_t
queue_pair_id,
socket_id);
}
+rte_crypto_enqueue_burst_t
+rte_crypto_get_enq_burst_fn(uint8_t dev_id)
+{
+ if (dev_id >= RTE_CRYPTO_MAX_DEVS) {
+ rte_errno = EINVAL;
+ return NULL;
+ }
+ return rte_cryptodev_api[dev_id].enqueue_burst;
+}
+
+rte_crypto_dequeue_burst_t
+rte_crypto_get_deq_burst_fn(uint8_t dev_id)
+{
+ if (dev_id >= RTE_CRYPTO_MAX_DEVS) {
+ rte_errno = EINVAL;
+ return NULL;
+ }
+ return rte_cryptodev_api[dev_id].dequeue_burst;
+}
+
+int
+rte_crypto_set_enq_burst_fn(uint8_t dev_id, rte_crypto_enqueue_burst_t fn)
+{
+ if (dev_id >= RTE_CRYPTO_MAX_DEVS || fn == NULL)
+ return -EINVAL;
+ rte_cryptodev_api[dev_id].enqueue_burst = fn;
+ return 0;
+}
+
+int
+rte_crypto_set_deq_burst_fn(uint8_t dev_id, rte_crypto_dequeue_burst_t fn)
+{
+ if (dev_id >= RTE_CRYPTO_MAX_DEVS || fn == NULL)
+ return -EINVAL;
+ rte_cryptodev_api[dev_id].dequeue_burst = fn;
+ return 0;
+}
+
struct rte_cryptodev_cb *
rte_cryptodev_add_enq_callback(uint8_t dev_id,
uint16_t qp_id,
diff --git a/lib/cryptodev/version.map b/lib/cryptodev/version.map
index 050089ae55..b64384cc05 100644
--- a/lib/cryptodev/version.map
+++ b/lib/cryptodev/version.map
@@ -116,6 +116,10 @@ EXPERIMENTAL {
INTERNAL {
global:
+ rte_crypto_get_deq_burst_fn;
+ rte_crypto_get_enq_burst_fn;
+ rte_crypto_set_deq_burst_fn;
+ rte_crypto_set_enq_burst_fn;
rte_cryptodev_allocate_driver;
rte_cryptodev_api_reset;
rte_cryptodev_pmd_allocate;
--
2.25.1
