This defines the main crypto operation enqueue and dequeue handlers.
Signed-off-by: Andrew Boyer <andrew.bo...@amd.com>
---
doc/guides/cryptodevs/features/ionic.ini | 8 +
doc/guides/cryptodevs/ionic.rst | 12 +
drivers/crypto/ionic/ionic_crypto.h | 17 ++
drivers/crypto/ionic/ionic_crypto_caps.c | 30 +++
drivers/crypto/ionic/ionic_crypto_main.c | 222 ++++++++++++++++-
drivers/crypto/ionic/ionic_crypto_ops.c | 295 +++++++++++++++++++++++
6 files changed, 582 insertions(+), 2 deletions(-)
diff --git a/doc/guides/cryptodevs/features/ionic.ini
b/doc/guides/cryptodevs/features/ionic.ini
index 62b7e9e8f2..d3e00bd795 100644
--- a/doc/guides/cryptodevs/features/ionic.ini
+++ b/doc/guides/cryptodevs/features/ionic.ini
@@ -4,6 +4,12 @@
; Refer to default.ini for the full list of available PMD features.
;
[Features]
+Symmetric crypto = Y
+HW Accelerated = Y
+In Place SGL = Y
+OOP SGL In LB Out = Y
+OOP SGL In SGL Out = Y
+OOP LB In LB Out = Y
;
; Supported crypto algorithms of 'ionic' crypto driver.
@@ -19,6 +25,8 @@
; Supported AEAD algorithms of 'ionic' crypto driver.
;
[AEAD]
+AES GCM (128) = Y
+AES GCM (256) = Y
;
; Supported Asymmetric algorithms of the 'ionic' crypto driver.
diff --git a/doc/guides/cryptodevs/ionic.rst b/doc/guides/cryptodevs/ionic.rst
index b54ddd37a3..bc0f0ca6aa 100644
--- a/doc/guides/cryptodevs/ionic.rst
+++ b/doc/guides/cryptodevs/ionic.rst
@@ -25,9 +25,21 @@ Limitations
- Host-side access via PCI is not yet supported
- Multiprocess applications are not yet supported
+- Sessionless APIs are not yet supported
Runtime Configuration
---------------------
None
+Features
+--------
+
+The ionic crypto PMD has support for:
+
+Symmetric Crypto Algorithms
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+AEAD algorithms:
+
+* ``RTE_CRYPTO_AEAD_AES_GCM``
diff --git a/drivers/crypto/ionic/ionic_crypto.h
b/drivers/crypto/ionic/ionic_crypto.h
index f50c4b4291..e05b458926 100644
--- a/drivers/crypto/ionic/ionic_crypto.h
+++ b/drivers/crypto/ionic/ionic_crypto.h
@@ -27,6 +27,8 @@
#define IOCPT_MIN_RING_DESC 16
#define IOCPT_ADMINQ_LENGTH 16 /* must be a power of two */
+#define IOCPT_CRYPTOQ_WAIT 10 /* 1s */
+
extern int iocpt_logtype;
#define RTE_LOGTYPE_IOCPT iocpt_logtype
@@ -155,6 +157,14 @@ struct iocpt_admin_q {
uint16_t flags;
};
+struct iocpt_crypto_q {
+ /* cacheline0, cacheline1 */
+ IOCPT_COMMON_FIELDS;
+
+ /* cacheline2 */
+ uint16_t flags;
+};
+
#define IOCPT_S_F_INITED BIT(0)
struct iocpt_session_priv {
@@ -212,6 +222,7 @@ struct iocpt_dev {
rte_spinlock_t adminq_service_lock;
struct iocpt_admin_q *adminq;
+ struct iocpt_crypto_q **cryptoqs;
struct rte_bitmap *sess_bm; /* SET bit indicates index is free */
@@ -259,6 +270,8 @@ int iocpt_remove(struct rte_device *rte_dev);
void iocpt_configure(struct iocpt_dev *dev);
int iocpt_assign_ops(struct rte_cryptodev *cdev);
+int iocpt_start(struct iocpt_dev *dev);
+void iocpt_stop(struct iocpt_dev *dev);
void iocpt_deinit(struct iocpt_dev *dev);
int iocpt_dev_identify(struct iocpt_dev *dev);
@@ -268,6 +281,10 @@ void iocpt_dev_reset(struct iocpt_dev *dev);
int iocpt_adminq_post_wait(struct iocpt_dev *dev, struct iocpt_admin_ctx *ctx);
+int iocpt_cryptoq_alloc(struct iocpt_dev *dev, uint32_t socket_id,
+ uint32_t index, uint16_t ndescs);
+void iocpt_cryptoq_free(struct iocpt_crypto_q *cptq);
+
int iocpt_session_init(struct iocpt_session_priv *priv);
int iocpt_session_update(struct iocpt_session_priv *priv);
void iocpt_session_deinit(struct iocpt_session_priv *priv);
diff --git a/drivers/crypto/ionic/ionic_crypto_caps.c
b/drivers/crypto/ionic/ionic_crypto_caps.c
index c22681fabc..da5a69be3d 100644
--- a/drivers/crypto/ionic/ionic_crypto_caps.c
+++ b/drivers/crypto/ionic/ionic_crypto_caps.c
@@ -7,6 +7,36 @@
#include "ionic_crypto.h"
static const struct rte_cryptodev_capabilities iocpt_sym_caps[] = {
+ { /* AES GCM */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
+ {.aead = {
+ .algo = RTE_CRYPTO_AEAD_AES_GCM,
+ .block_size = 16,
+ .key_size = {
+ .min = 16,
+ .max = 32,
+ .increment = 16
+ },
+ .digest_size = {
+ .min = 16,
+ .max = 16,
+ .increment = 0
+ },
+ .aad_size = {
+ .min = 0,
+ .max = 1024,
+ .increment = 1
+ },
+ .iv_size = {
+ .min = 12,
+ .max = 12,
+ .increment = 0
+ }
+ }, }
+ }, }
+ },
RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
};
diff --git a/drivers/crypto/ionic/ionic_crypto_main.c
b/drivers/crypto/ionic/ionic_crypto_main.c
index 8e319b8830..179ef740ba 100644
--- a/drivers/crypto/ionic/ionic_crypto_main.c
+++ b/drivers/crypto/ionic/ionic_crypto_main.c
@@ -29,6 +29,15 @@ iocpt_cq_init(struct iocpt_cq *cq, uint16_t num_descs)
return 0;
}
+static void
+iocpt_cq_reset(struct iocpt_cq *cq)
+{
+ cq->tail_idx = 0;
+ cq->done_color = 1;
+
+ memset(cq->base, 0, sizeof(struct iocpt_nop_comp) * cq->num_descs);
+}
+
static void
iocpt_cq_map(struct iocpt_cq *cq, void *base, rte_iova_t base_pa)
{
@@ -89,6 +98,13 @@ iocpt_q_init(struct iocpt_queue *q, uint8_t type, uint32_t
index,
return 0;
}
+static void
+iocpt_q_reset(struct iocpt_queue *q)
+{
+ q->head_idx = 0;
+ q->tail_idx = 0;
+}
+
static void
iocpt_q_map(struct iocpt_queue *q, void *base, rte_iova_t base_pa)
{
@@ -348,12 +364,154 @@ iocpt_commonq_alloc(struct iocpt_dev *dev,
return err;
}
+int
+iocpt_cryptoq_alloc(struct iocpt_dev *dev, uint32_t socket_id, uint32_t index,
+ uint16_t num_descs)
+{
+ struct iocpt_crypto_q *cptq;
+ uint16_t flags = 0;
+ int err;
+
+ /* CryptoQ always supports scatter-gather */
+ flags |= IOCPT_Q_F_SG;
+
+ IOCPT_PRINT(DEBUG, "cptq %u num_descs %u num_segs %u",
+ index, num_descs, 1);
+
+ err = iocpt_commonq_alloc(dev,
+ IOCPT_QTYPE_CRYPTOQ,
+ sizeof(struct iocpt_crypto_q),
+ socket_id,
+ index,
+ "crypto",
+ flags,
+ num_descs,
+ 1,
+ sizeof(struct iocpt_crypto_desc),
+ sizeof(struct iocpt_crypto_comp),
+ sizeof(struct iocpt_crypto_sg_desc),
+ (struct iocpt_common_q **)&cptq);
+ if (err != 0)
+ return err;
+
+ cptq->flags = flags;
+
+ dev->cryptoqs[index] = cptq;
+
+ return 0;
+}
+
struct ionic_doorbell *
iocpt_db_map(struct iocpt_dev *dev, struct iocpt_queue *q)
{
return dev->db_pages + q->hw_type;
}
+static int
+iocpt_cryptoq_init(struct iocpt_crypto_q *cptq)
+{
+ struct iocpt_queue *q = &cptq->q;
+ struct iocpt_dev *dev = cptq->dev;
+ struct iocpt_cq *cq = &cptq->cq;
+ struct iocpt_admin_ctx ctx = {
+ .pending_work = true,
+ .cmd.q_init = {
+ .opcode = IOCPT_CMD_Q_INIT,
+ .type = IOCPT_QTYPE_CRYPTOQ,
+ .ver = dev->qtype_info[IOCPT_QTYPE_CRYPTOQ].version,
+ .index = rte_cpu_to_le_32(q->index),
+ .flags = rte_cpu_to_le_16(IOCPT_QINIT_F_ENA |
+ IOCPT_QINIT_F_SG),
+ .intr_index = rte_cpu_to_le_16(IONIC_INTR_NONE),
+ .ring_size = rte_log2_u32(q->num_descs),
+ .ring_base = rte_cpu_to_le_64(q->base_pa),
+ .cq_ring_base = rte_cpu_to_le_64(cq->base_pa),
+ .sg_ring_base = rte_cpu_to_le_64(q->sg_base_pa),
+ },
+ };
+ int err;
+
+ IOCPT_PRINT(DEBUG, "cptq_init.index %d", q->index);
+ IOCPT_PRINT(DEBUG, "cptq_init.ring_base %#jx", q->base_pa);
+ IOCPT_PRINT(DEBUG, "cptq_init.ring_size %d",
+ ctx.cmd.q_init.ring_size);
+ IOCPT_PRINT(DEBUG, "cptq_init.ver %u", ctx.cmd.q_init.ver);
+
+ iocpt_q_reset(q);
+ iocpt_cq_reset(cq);
+
+ err = iocpt_adminq_post_wait(dev, &ctx);
+ if (err != 0)
+ return err;
+
+ q->hw_type = ctx.comp.q_init.hw_type;
+ q->hw_index = rte_le_to_cpu_32(ctx.comp.q_init.hw_index);
+ q->db = iocpt_db_map(dev, q);
+
+ IOCPT_PRINT(DEBUG, "cptq->hw_type %d", q->hw_type);
+ IOCPT_PRINT(DEBUG, "cptq->hw_index %d", q->hw_index);
+ IOCPT_PRINT(DEBUG, "cptq->db %p", q->db);
+
+ cptq->flags |= IOCPT_Q_F_INITED;
+
+ return 0;
+}
+
+static void
+iocpt_cryptoq_deinit(struct iocpt_crypto_q *cptq)
+{
+ struct iocpt_dev *dev = cptq->dev;
+ struct iocpt_admin_ctx ctx = {
+ .pending_work = true,
+ .cmd.q_control = {
+ .opcode = IOCPT_CMD_Q_CONTROL,
+ .type = IOCPT_QTYPE_CRYPTOQ,
+ .index = rte_cpu_to_le_32(cptq->q.index),
+ .oper = IOCPT_Q_DISABLE,
+ },
+ };
+ unsigned long sleep_usec = 100UL * 1000;
+ uint32_t sleep_cnt, sleep_max = IOCPT_CRYPTOQ_WAIT;
+ int err;
+
+ for (sleep_cnt = 0; sleep_cnt < sleep_max; sleep_cnt++) {
+ ctx.pending_work = true;
+
+ err = iocpt_adminq_post_wait(dev, &ctx);
+ if (err != -EAGAIN)
+ break;
+
+ rte_delay_us_block(sleep_usec);
+ }
+
+ if (err != 0)
+ IOCPT_PRINT(ERR, "Deinit queue %u returned %d after %u ms",
+ cptq->q.index, err, sleep_cnt * 100);
+ else
+ IOCPT_PRINT(DEBUG, "Deinit queue %u returned %d after %u ms",
+ cptq->q.index, err, sleep_cnt * 100);
+
+ cptq->flags &= ~IOCPT_Q_F_INITED;
+}
+
+void
+iocpt_cryptoq_free(struct iocpt_crypto_q *cptq)
+{
+ if (cptq == NULL)
+ return;
+
+ if (cptq->base_z != NULL) {
+ rte_memzone_free(cptq->base_z);
+ cptq->base_z = NULL;
+ cptq->base = NULL;
+ cptq->base_pa = 0;
+ }
+
+ iocpt_q_free(&cptq->q);
+
+ rte_free(cptq);
+}
+
static int
iocpt_adminq_alloc(struct iocpt_dev *dev)
{
@@ -421,6 +579,14 @@ iocpt_alloc_objs(struct iocpt_dev *dev)
IOCPT_PRINT(DEBUG, "Crypto: %s", dev->name);
+ dev->cryptoqs = rte_calloc_socket("iocpt",
+ dev->max_qps, sizeof(*dev->cryptoqs),
+ RTE_CACHE_LINE_SIZE, dev->socket_id);
+ if (dev->cryptoqs == NULL) {
+ IOCPT_PRINT(ERR, "Cannot allocate tx queues array");
+ return -ENOMEM;
+ }
+
rte_spinlock_init(&dev->adminq_lock);
rte_spinlock_init(&dev->adminq_service_lock);
@@ -428,7 +594,7 @@ iocpt_alloc_objs(struct iocpt_dev *dev)
if (err != 0) {
IOCPT_PRINT(ERR, "Cannot allocate admin queue");
err = -ENOMEM;
- goto err_out;
+ goto err_free_cryptoqs;
}
dev->info_sz = RTE_ALIGN(sizeof(*dev->info), rte_mem_page_size());
@@ -473,7 +639,9 @@ iocpt_alloc_objs(struct iocpt_dev *dev)
err_free_adminq:
iocpt_adminq_free(dev->adminq);
dev->adminq = NULL;
-err_out:
+err_free_cryptoqs:
+ rte_free(dev->cryptoqs);
+ dev->cryptoqs = NULL;
return err;
}
@@ -502,6 +670,43 @@ iocpt_configure(struct iocpt_dev *dev)
RTE_SET_USED(dev);
}
+int
+iocpt_start(struct iocpt_dev *dev)
+{
+ uint32_t i;
+ int err;
+
+ IOCPT_PRINT(DEBUG, "Starting %u queues",
+ dev->crypto_dev->data->nb_queue_pairs);
+
+ for (i = 0; i < dev->crypto_dev->data->nb_queue_pairs; i++) {
+ err = iocpt_cryptoq_init(dev->cryptoqs[i]);
+ if (err != 0)
+ return err;
+ }
+
+ dev->state |= IOCPT_DEV_F_UP;
+
+ return 0;
+}
+
+void
+iocpt_stop(struct iocpt_dev *dev)
+{
+ uint32_t i;
+
+ IOCPT_PRINT_CALL();
+
+ dev->state &= ~IOCPT_DEV_F_UP;
+
+ for (i = 0; i < dev->crypto_dev->data->nb_queue_pairs; i++) {
+ struct iocpt_crypto_q *cptq = dev->cryptoqs[i];
+
+ if (cptq->flags & IOCPT_Q_F_INITED)
+ (void)iocpt_cryptoq_deinit(cptq);
+ }
+}
+
void
iocpt_deinit(struct iocpt_dev *dev)
{
@@ -518,8 +723,16 @@ iocpt_deinit(struct iocpt_dev *dev)
static void
iocpt_free_objs(struct iocpt_dev *dev)
{
+ void **queue_pairs = dev->crypto_dev->data->queue_pairs;
+ uint32_t i;
+
IOCPT_PRINT_CALL();
+ for (i = 0; i < dev->crypto_dev->data->nb_queue_pairs; i++) {
+ iocpt_cryptoq_free(queue_pairs[i]);
+ queue_pairs[i] = NULL;
+ }
+
if (dev->sess_bm != NULL) {
rte_bitmap_free(dev->sess_bm);
rte_free(dev->sess_bm);
@@ -531,6 +744,11 @@ iocpt_free_objs(struct iocpt_dev *dev)
dev->adminq = NULL;
}
+ if (dev->cryptoqs != NULL) {
+ rte_free(dev->cryptoqs);
+ dev->cryptoqs = NULL;
+ }
+
if (dev->info != NULL) {
rte_memzone_free(dev->info_z);
dev->info_z = NULL;
diff --git a/drivers/crypto/ionic/ionic_crypto_ops.c
b/drivers/crypto/ionic/ionic_crypto_ops.c
index e6b3402b63..28b099dea2 100644
--- a/drivers/crypto/ionic/ionic_crypto_ops.c
+++ b/drivers/crypto/ionic/ionic_crypto_ops.c
@@ -21,6 +21,22 @@ iocpt_op_config(struct rte_cryptodev *cdev,
return 0;
}
+static int
+iocpt_op_start(struct rte_cryptodev *cdev)
+{
+ struct iocpt_dev *dev = cdev->data->dev_private;
+
+ return iocpt_start(dev);
+}
+
+static void
+iocpt_op_stop(struct rte_cryptodev *cdev)
+{
+ struct iocpt_dev *dev = cdev->data->dev_private;
+
+ return iocpt_stop(dev);
+}
+
static int
iocpt_op_close(struct rte_cryptodev *cdev)
{
@@ -48,6 +64,53 @@ iocpt_op_info_get(struct rte_cryptodev *cdev, struct
rte_cryptodev_info *info)
info->min_mbuf_tailroom_req = 0;
}
+static int
+iocpt_op_queue_release(struct rte_cryptodev *cdev, uint16_t queue_id)
+{
+ struct iocpt_crypto_q *cptq = cdev->data->queue_pairs[queue_id];
+
+ IOCPT_PRINT(DEBUG, "queue_id %u", queue_id);
+
+ assert(!(cptq->flags & IOCPT_Q_F_INITED));
+
+ iocpt_cryptoq_free(cptq);
+
+ cdev->data->queue_pairs[queue_id] = NULL;
+
+ return 0;
+}
+
+static int
+iocpt_op_queue_setup(struct rte_cryptodev *cdev, uint16_t queue_id,
+ const struct rte_cryptodev_qp_conf *qp_conf,
+ int socket_id)
+{
+ struct iocpt_dev *dev = cdev->data->dev_private;
+ int err;
+
+ if (cdev->data->queue_pairs[queue_id] != NULL)
+ iocpt_op_queue_release(cdev, queue_id);
+
+ if (qp_conf->nb_descriptors < (1 << IOCPT_QSIZE_MIN_LG2) ||
+ qp_conf->nb_descriptors > (1 << IOCPT_QSIZE_MAX_LG2)) {
+ IOCPT_PRINT(ERR, "invalid nb_descriptors %u, use range %u..%u",
+ qp_conf->nb_descriptors,
+ 1 << IOCPT_QSIZE_MIN_LG2, 1 << IOCPT_QSIZE_MAX_LG2);
+ return -ERANGE;
+ }
+
+ IOCPT_PRINT(DEBUG, "queue_id %u", queue_id);
+
+ err = iocpt_cryptoq_alloc(dev, socket_id, queue_id,
+ qp_conf->nb_descriptors);
+ if (err != 0)
+ return err;
+
+ cdev->data->queue_pairs[queue_id] = dev->cryptoqs[queue_id];
+
+ return 0;
+}
+
static unsigned int
iocpt_op_get_session_size(struct rte_cryptodev *cdev __rte_unused)
{
@@ -167,11 +230,238 @@ iocpt_op_session_clear(struct rte_cryptodev *cdev
__rte_unused,
iocpt_session_clear(sess);
}
+static inline void
+iocpt_fill_sge(struct iocpt_crypto_sg_elem *arr, uint8_t idx,
+ uint64_t addr, uint16_t len)
+{
+ arr[idx].addr = rte_cpu_to_le_64(addr);
+ arr[idx].len = rte_cpu_to_le_16(len);
+}
+
+static __rte_always_inline int
+iocpt_enq_one_aead(struct iocpt_crypto_q *cptq,
+ struct iocpt_session_priv *priv, struct rte_crypto_op *op)
+{
+ struct rte_crypto_sym_op *sym_op = op->sym;
+ struct iocpt_queue *q = &cptq->q;
+ struct iocpt_crypto_desc *desc, *desc_base = q->base;
+ struct iocpt_crypto_sg_desc *sg_desc, *sg_desc_base = q->sg_base;
+ struct iocpt_crypto_sg_elem *src, *dst;
+ rte_iova_t aad_addr, digest_addr, iv_addr, seg_addr;
+ uint32_t data_len, data_offset, seg_len;
+ uint8_t nsge_src = 0, nsge_dst = 0, flags = 0;
+ struct rte_mbuf *m;
+
+ desc = &desc_base[q->head_idx];
+ sg_desc = &sg_desc_base[q->head_idx];
+ src = sg_desc->src_elems;
+ dst = sg_desc->dst_elems;
+
+ /* Fill the first SGE with the IV / Nonce */
+ iv_addr = rte_crypto_op_ctophys_offset(op, priv->iv_offset);
+ iocpt_fill_sge(src, nsge_src++, iv_addr, priv->iv_length);
+
+ /* Fill the second SGE with the AAD, if applicable */
+ if (priv->aad_length > 0) {
+ aad_addr = sym_op->aead.aad.phys_addr;
+ iocpt_fill_sge(src, nsge_src++, aad_addr, priv->aad_length);
+ flags |= IOCPT_DESC_F_AAD_VALID;
+ }
+
+ m = sym_op->m_src;
+ data_len = sym_op->aead.data.length;
+
+ /* Fast-forward through mbuf chain to account for data offset */
+ data_offset = sym_op->aead.data.offset;
+ while (m != NULL && data_offset >= m->data_len) {
+ data_offset -= m->data_len;
+ m = m->next;
+ }
+
+ /* Fill the next SGEs with the payload segments */
+ while (m != NULL && data_len > 0) {
+ seg_addr = rte_mbuf_data_iova(m) + data_offset;
+ seg_len = RTE_MIN(m->data_len - data_offset, data_len);
+ data_offset = 0;
+ data_len -= seg_len;
+
+ /* Use -1 to save room for digest */
+ if (nsge_src >= IOCPT_CRYPTO_MAX_SG_ELEMS - 1)
+ return -ERANGE;
+
+ iocpt_fill_sge(src, nsge_src++, seg_addr, seg_len);
+
+ m = m->next;
+ }
+
+ /* AEAD AES-GCM: digest == authentication tag */
+ digest_addr = sym_op->aead.digest.phys_addr;
+ iocpt_fill_sge(src, nsge_src++, digest_addr, priv->digest_length);
+
+ /* Process Out-Of-Place destination SGL */
+ if (sym_op->m_dst != NULL) {
+ /* Put the AAD here, too */
+ if (priv->aad_length > 0)
+ iocpt_fill_sge(dst, nsge_dst++,
+ sym_op->aead.aad.phys_addr, priv->aad_length);
+
+ m = sym_op->m_dst;
+ data_len = sym_op->aead.data.length;
+
+ /* Fast-forward through chain to account for data offset */
+ data_offset = sym_op->aead.data.offset;
+ while (m != NULL && data_offset >= m->data_len) {
+ data_offset -= m->data_len;
+ m = m->next;
+ }
+
+ /* Fill in the SGEs with the payload segments */
+ while (m != NULL && data_len > 0) {
+ seg_addr = rte_mbuf_data_iova(m) + data_offset;
+ seg_len = RTE_MIN(m->data_len - data_offset, data_len);
+ data_offset = 0;
+ data_len -= seg_len;
+
+ if (nsge_dst >= IOCPT_CRYPTO_MAX_SG_ELEMS)
+ return -ERANGE;
+
+ iocpt_fill_sge(dst, nsge_dst++, seg_addr, seg_len);
+
+ m = m->next;
+ }
+ }
+
+ desc->opcode = priv->op;
+ desc->flags = flags;
+ desc->num_src_dst_sgs = iocpt_encode_nsge_src_dst(nsge_src, nsge_dst);
+ desc->session_tag = rte_cpu_to_le_32(priv->index);
+
+ op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
+ q->info[q->head_idx] = op;
+ q->head_idx = Q_NEXT_TO_POST(q, 1);
+
+ return 0;
+}
+
+static uint16_t
+iocpt_enqueue_sym(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
+{
+ struct iocpt_crypto_q *cptq = qp;
+ struct rte_crypto_op *op;
+ struct iocpt_session_priv *priv;
+ uint16_t avail, count;
+ int err;
+
+ avail = iocpt_q_space_avail(&cptq->q);
+ if (unlikely(nb_ops > avail))
+ nb_ops = avail;
+
+ count = 0;
+ while (likely(count < nb_ops)) {
+ op = ops[count];
+
+ if (unlikely(op->sess_type != RTE_CRYPTO_OP_WITH_SESSION)) {
+ op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+ break;
+ }
+
+ priv = CRYPTODEV_GET_SYM_SESS_PRIV(op->sym->session);
+ if (unlikely(priv == NULL)) {
+ op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
+ break;
+ }
+
+ err = iocpt_enq_one_aead(cptq, priv, op);
+ if (unlikely(err != 0)) {
+ op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+ break;
+ }
+
+ count++;
+ }
+
+ if (likely(count > 0))
+ iocpt_q_flush(&cptq->q);
+
+ return count;
+}
+
+static uint16_t
+iocpt_dequeue_sym(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
+{
+ struct iocpt_crypto_q *cptq = qp;
+ struct iocpt_queue *q = &cptq->q;
+ struct iocpt_cq *cq = &cptq->cq;
+ struct rte_crypto_op *op;
+ struct iocpt_crypto_comp *cq_desc_base = cq->base;
+ volatile struct iocpt_crypto_comp *cq_desc;
+ uint16_t count = 0;
+
+ cq_desc = &cq_desc_base[cq->tail_idx];
+
+ /* First walk the CQ to update any completed op's status
+ * NB: These can arrive out of order!
+ */
+ while ((cq_desc->color & 0x1) == cq->done_color) {
+ cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1);
+ if (unlikely(cq->tail_idx == 0))
+ cq->done_color = !cq->done_color;
+
+ op = q->info[rte_le_to_cpu_16(cq_desc->comp_index)];
+
+ /* Process returned CQ descriptor status */
+ if (unlikely(cq_desc->status)) {
+ switch (cq_desc->status) {
+ case IOCPT_COMP_SYMM_AUTH_VERIFY_ERROR:
+ op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
+ break;
+ case IOCPT_COMP_INVAL_OPCODE_ERROR:
+ case IOCPT_COMP_UNSUPP_OPCODE_ERROR:
+ case IOCPT_COMP_SYMM_SRC_SG_ERROR:
+ case IOCPT_COMP_SYMM_DST_SG_ERROR:
+ case IOCPT_COMP_SYMM_SRC_DST_LEN_MISMATCH:
+ case IOCPT_COMP_SYMM_KEY_IDX_ERROR:
+ op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+ break;
+ default:
+ op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+ break;
+ }
+ } else
+ op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+
+ cq_desc = &cq_desc_base[cq->tail_idx];
+ }
+
+ /* Next walk the SQ to pop off completed ops in-order */
+ while (count < nb_ops) {
+ op = q->info[q->tail_idx];
+
+ /* No more completions */
+ if (op == NULL ||
+ op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
+ break;
+
+ ops[count] = op;
+ q->info[q->tail_idx] = NULL;
+
+ q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
+ count++;
+ }
+
+ return count;
+}
+
static struct rte_cryptodev_ops iocpt_ops = {
.dev_configure = iocpt_op_config,
+ .dev_start = iocpt_op_start,
+ .dev_stop = iocpt_op_stop,
.dev_close = iocpt_op_close,
.dev_infos_get = iocpt_op_info_get,
+ .queue_pair_setup = iocpt_op_queue_setup,
+ .queue_pair_release = iocpt_op_queue_release,
+
.sym_session_get_size = iocpt_op_get_session_size,
.sym_session_configure = iocpt_op_session_cfg,
.sym_session_clear = iocpt_op_session_clear,
@@ -185,5 +475,10 @@ iocpt_assign_ops(struct rte_cryptodev *cdev)
cdev->dev_ops = &iocpt_ops;
cdev->feature_flags = dev->features;
+ if (dev->features & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) {
+ cdev->enqueue_burst = iocpt_enqueue_sym;
+ cdev->dequeue_burst = iocpt_dequeue_sym;
+ }
+
return 0;
}
--
2.17.1
