This defines the adminq used for control path commands. The adminq is
faster and more flexible than the device command interface.
Signed-off-by: Andrew Boyer <andrew.bo...@amd.com>
---
drivers/crypto/ionic/ionic_crypto.h | 115 +++++++++
drivers/crypto/ionic/ionic_crypto_cmds.c | 302 +++++++++++++++++++++++
drivers/crypto/ionic/ionic_crypto_main.c | 295 +++++++++++++++++++++-
3 files changed, 711 insertions(+), 1 deletion(-)
diff --git a/drivers/crypto/ionic/ionic_crypto.h
b/drivers/crypto/ionic/ionic_crypto.h
index a7999a0f68..065e1bd826 100644
--- a/drivers/crypto/ionic/ionic_crypto.h
+++ b/drivers/crypto/ionic/ionic_crypto.h
@@ -22,6 +22,10 @@
/* Devargs */
/* NONE */
+#define IOCPT_MAX_RING_DESC 32768
+#define IOCPT_MIN_RING_DESC 16
+#define IOCPT_ADMINQ_LENGTH 16 /* must be a power of two */
+
extern int iocpt_logtype;
#define RTE_LOGTYPE_IOCPT iocpt_logtype
@@ -91,6 +95,63 @@ struct iocpt_qtype_info {
uint16_t sg_desc_stride;
};
+#define IOCPT_Q_F_INITED BIT(0)
+#define IOCPT_Q_F_DEFERRED BIT(1)
+#define IOCPT_Q_F_SG BIT(2)
+
+#define Q_NEXT_TO_POST(_q, _n) (((_q)->head_idx + (_n)) & ((_q)->size_mask))
+#define Q_NEXT_TO_SRVC(_q, _n) (((_q)->tail_idx + (_n)) & ((_q)->size_mask))
+
+#define IOCPT_INFO_SZ(_q) ((_q)->num_segs * sizeof(void *))
+#define IOCPT_INFO_IDX(_q, _i) ((_i) * (_q)->num_segs)
+#define IOCPT_INFO_PTR(_q, _i) (&(_q)->info[IOCPT_INFO_IDX((_q), _i)])
+
+struct iocpt_queue {
+ uint16_t num_descs;
+ uint16_t num_segs;
+ uint16_t head_idx;
+ uint16_t tail_idx;
+ uint16_t size_mask;
+ uint8_t type;
+ uint8_t hw_type;
+ void *base;
+ void *sg_base;
+ struct ionic_doorbell __iomem *db;
+ void **info;
+
+ uint32_t index;
+ uint32_t hw_index;
+ rte_iova_t base_pa;
+ rte_iova_t sg_base_pa;
+};
+
+struct iocpt_cq {
+ uint16_t tail_idx;
+ uint16_t num_descs;
+ uint16_t size_mask;
+ bool done_color;
+ void *base;
+ rte_iova_t base_pa;
+};
+
+#define IOCPT_COMMON_FIELDS \
+ struct iocpt_queue q; \
+ struct iocpt_cq cq; \
+ struct iocpt_dev *dev; \
+ const struct rte_memzone *base_z; \
+ void *base; \
+ rte_iova_t base_pa
+
+struct iocpt_common_q {
+ IOCPT_COMMON_FIELDS;
+};
+
+struct iocpt_admin_q {
+ IOCPT_COMMON_FIELDS;
+
+ uint16_t flags;
+};
+
#define IOCPT_DEV_F_INITED BIT(0)
#define IOCPT_DEV_F_UP BIT(1)
#define IOCPT_DEV_F_FW_RESET BIT(2)
@@ -118,6 +179,11 @@ struct iocpt_dev {
uint8_t driver_id;
uint8_t socket_id;
+ rte_spinlock_t adminq_lock;
+ rte_spinlock_t adminq_service_lock;
+
+ struct iocpt_admin_q *adminq;
+
uint64_t features;
uint32_t hw_features;
@@ -144,6 +210,17 @@ iocpt_setup_bars(struct iocpt_dev *dev)
return (*dev->intf->setup_bars)(dev);
}
+/** iocpt_admin_ctx - Admin command context.
+ * @pending_work: Flag that indicates a completion.
+ * @cmd: Admin command (64B) to be copied to the queue.
+ * @comp: Admin completion (16B) copied from the queue.
+ */
+struct iocpt_admin_ctx {
+ bool pending_work;
+ union iocpt_adminq_cmd cmd;
+ union iocpt_adminq_comp comp;
+};
+
int iocpt_probe(void *bus_dev, struct rte_device *rte_dev,
struct iocpt_dev_bars *bars, const struct iocpt_dev_intf *intf,
uint8_t driver_id, uint8_t socket_id);
@@ -154,8 +231,46 @@ void iocpt_deinit(struct iocpt_dev *dev);
int iocpt_dev_identify(struct iocpt_dev *dev);
int iocpt_dev_init(struct iocpt_dev *dev, rte_iova_t info_pa);
+int iocpt_dev_adminq_init(struct iocpt_dev *dev);
void iocpt_dev_reset(struct iocpt_dev *dev);
+int iocpt_adminq_post_wait(struct iocpt_dev *dev, struct iocpt_admin_ctx *ctx);
+
+struct ionic_doorbell __iomem *iocpt_db_map(struct iocpt_dev *dev,
+ struct iocpt_queue *q);
+
+typedef bool (*iocpt_cq_cb)(struct iocpt_cq *cq, uint16_t cq_desc_index,
+ void *cb_arg);
+uint32_t iocpt_cq_service(struct iocpt_cq *cq, uint32_t work_to_do,
+ iocpt_cq_cb cb, void *cb_arg);
+
+static inline uint16_t
+iocpt_q_space_avail(struct iocpt_queue *q)
+{
+ uint16_t avail = q->tail_idx;
+
+ if (q->head_idx >= avail)
+ avail += q->num_descs - q->head_idx - 1;
+ else
+ avail -= q->head_idx + 1;
+
+ return avail;
+}
+
+static inline void
+iocpt_q_flush(struct iocpt_queue *q)
+{
+ uint64_t val = IONIC_DBELL_QID(q->hw_index) | q->head_idx;
+
+#if defined(RTE_LIBRTE_IONIC_PMD_BARRIER_ERRATA)
+ /* On some devices the standard 'dmb' barrier is insufficient */
+ asm volatile("dsb st" : : : "memory");
+ rte_write64_relaxed(rte_cpu_to_le_64(val), q->db);
+#else
+ rte_write64(rte_cpu_to_le_64(val), q->db);
+#endif
+}
+
static inline bool
iocpt_is_embedded(void)
{
diff --git a/drivers/crypto/ionic/ionic_crypto_cmds.c
b/drivers/crypto/ionic/ionic_crypto_cmds.c
index 105005539b..8165a6a7c8 100644
--- a/drivers/crypto/ionic/ionic_crypto_cmds.c
+++ b/drivers/crypto/ionic/ionic_crypto_cmds.c
@@ -16,6 +16,55 @@ static const uint8_t iocpt_qtype_vers[IOCPT_QTYPE_MAX] = {
[IOCPT_QTYPE_CRYPTOQ] = 0, /* 0 = Base version */
};
+static const char *
+iocpt_error_to_str(enum iocpt_status_code code)
+{
+ switch (code) {
+ case IOCPT_RC_SUCCESS:
+ return "IOCPT_RC_SUCCESS";
+ case IOCPT_RC_EVERSION:
+ return "IOCPT_RC_EVERSION";
+ case IOCPT_RC_EOPCODE:
+ return "IOCPT_RC_EOPCODE";
+ case IOCPT_RC_EIO:
+ return "IOCPT_RC_EIO";
+ case IOCPT_RC_EPERM:
+ return "IOCPT_RC_EPERM";
+ case IOCPT_RC_EQID:
+ return "IOCPT_RC_EQID";
+ case IOCPT_RC_EQTYPE:
+ return "IOCPT_RC_EQTYPE";
+ case IOCPT_RC_ENOENT:
+ return "IOCPT_RC_ENOENT";
+ case IOCPT_RC_EINTR:
+ return "IOCPT_RC_EINTR";
+ case IOCPT_RC_EAGAIN:
+ return "IOCPT_RC_EAGAIN";
+ case IOCPT_RC_ENOMEM:
+ return "IOCPT_RC_ENOMEM";
+ case IOCPT_RC_EFAULT:
+ return "IOCPT_RC_EFAULT";
+ case IOCPT_RC_EBUSY:
+ return "IOCPT_RC_EBUSY";
+ case IOCPT_RC_EEXIST:
+ return "IOCPT_RC_EEXIST";
+ case IOCPT_RC_EINVAL:
+ return "IOCPT_RC_EINVAL";
+ case IOCPT_RC_ENOSPC:
+ return "IOCPT_RC_ENOSPC";
+ case IOCPT_RC_ERANGE:
+ return "IOCPT_RC_ERANGE";
+ case IOCPT_RC_BAD_ADDR:
+ return "IOCPT_RC_BAD_ADDR";
+ case IOCPT_RC_DEV_CMD:
+ return "IOCPT_RC_DEV_CMD";
+ case IOCPT_RC_ERROR:
+ return "IOCPT_RC_ERROR";
+ default:
+ return "IOCPT_RC_UNKNOWN";
+ }
+}
+
static const char *
iocpt_opcode_to_str(enum iocpt_cmd_opcode opcode)
{
@@ -97,6 +146,17 @@ iocpt_dev_cmd_wait(struct iocpt_dev *dev, unsigned long
max_wait)
return -ETIMEDOUT;
}
+static void
+iocpt_dev_cmd_comp(struct iocpt_dev *dev, void *mem)
+{
+ union iocpt_dev_cmd_comp *comp = mem;
+ uint32_t comp_size = RTE_DIM(comp->words);
+ uint32_t i;
+
+ for (i = 0; i < comp_size; i++)
+ comp->words[i] = ioread32(&dev->dev_cmd->comp.words[i]);
+}
+
static int
iocpt_dev_cmd_wait_check(struct iocpt_dev *dev, unsigned long max_wait)
{
@@ -175,6 +235,29 @@ iocpt_dev_cmd_queue_identify(struct iocpt_dev *dev,
iocpt_dev_cmd_go(dev, &cmd);
}
+static void
+iocpt_dev_cmd_adminq_init(struct iocpt_dev *dev)
+{
+ struct iocpt_queue *q = &dev->adminq->q;
+ struct iocpt_cq *cq = &dev->adminq->cq;
+
+ union iocpt_dev_cmd cmd = {
+ .q_init.opcode = IOCPT_CMD_Q_INIT,
+ .q_init.type = q->type,
+ .q_init.ver = dev->qtype_info[q->type].version,
+ .q_init.index = rte_cpu_to_le_32(q->index),
+ .q_init.flags = rte_cpu_to_le_16(IOCPT_QINIT_F_ENA),
+ .q_init.intr_index = rte_cpu_to_le_16(IONIC_INTR_NONE),
+ .q_init.ring_size = rte_log2_u32(q->num_descs),
+ .q_init.ring_base = rte_cpu_to_le_64(q->base_pa),
+ .q_init.cq_ring_base = rte_cpu_to_le_64(cq->base_pa),
+ };
+
+ IOCPT_PRINT(DEBUG, "adminq.q_init.ver %u", cmd.q_init.ver);
+
+ iocpt_dev_cmd_go(dev, &cmd);
+}
+
/* Dev_cmd consumers */
static void
@@ -346,3 +429,222 @@ iocpt_dev_reset(struct iocpt_dev *dev)
iocpt_dev_cmd_reset(dev);
(void)iocpt_dev_cmd_wait_check(dev, IONIC_DEVCMD_TIMEOUT);
}
+
+int
+iocpt_dev_adminq_init(struct iocpt_dev *dev)
+{
+ struct iocpt_queue *q = &dev->adminq->q;
+ struct iocpt_q_init_comp comp;
+ uint32_t retries = 5;
+ int err;
+
+retry_adminq_init:
+ iocpt_dev_cmd_adminq_init(dev);
+
+ err = iocpt_dev_cmd_wait_check(dev, IONIC_DEVCMD_TIMEOUT);
+ if (err == -EAGAIN && retries > 0) {
+ retries--;
+ rte_delay_us_block(IONIC_DEVCMD_RETRY_WAIT_US);
+ goto retry_adminq_init;
+ }
+ if (err != 0)
+ return err;
+
+ iocpt_dev_cmd_comp(dev, &comp);
+
+ q->hw_type = comp.hw_type;
+ q->hw_index = rte_le_to_cpu_32(comp.hw_index);
+ q->db = iocpt_db_map(dev, q);
+
+ IOCPT_PRINT(DEBUG, "adminq->hw_type %d", q->hw_type);
+ IOCPT_PRINT(DEBUG, "adminq->hw_index %d", q->hw_index);
+ IOCPT_PRINT(DEBUG, "adminq->db %p", q->db);
+
+ dev->adminq->flags |= IOCPT_Q_F_INITED;
+
+ return 0;
+}
+
+/* Admin_cmd interface */
+
+static bool
+iocpt_adminq_service(struct iocpt_cq *cq, uint16_t cq_desc_index,
+ void *cb_arg __rte_unused)
+{
+ struct iocpt_admin_comp *cq_desc_base = cq->base;
+ struct iocpt_admin_comp *cq_desc = &cq_desc_base[cq_desc_index];
+ struct iocpt_admin_q *adminq =
+ container_of(cq, struct iocpt_admin_q, cq);
+ struct iocpt_queue *q = &adminq->q;
+ struct iocpt_admin_ctx *ctx;
+ uint16_t curr_q_tail_idx;
+ uint16_t stop_index;
+ void **info;
+
+ if (!iocpt_color_match(cq_desc->color, cq->done_color))
+ return false;
+
+ stop_index = rte_le_to_cpu_16(cq_desc->comp_index);
+
+ do {
+ info = IOCPT_INFO_PTR(q, q->tail_idx);
+
+ ctx = info[0];
+ if (ctx != NULL) {
+ memcpy(&ctx->comp, cq_desc, sizeof(*cq_desc));
+ ctx->pending_work = false; /* done */
+ }
+
+ curr_q_tail_idx = q->tail_idx;
+ q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
+ } while (curr_q_tail_idx != stop_index);
+
+ return true;
+}
+
+/** iocpt_adminq_post - Post an admin command.
+ * @dev: Handle to dev.
+ * @cmd_ctx: Api admin command context.
+ *
+ * Return: zero or negative error status.
+ */
+static int
+iocpt_adminq_post(struct iocpt_dev *dev, struct iocpt_admin_ctx *ctx)
+{
+ struct iocpt_queue *q = &dev->adminq->q;
+ struct iocpt_admin_cmd *q_desc_base = q->base;
+ struct iocpt_admin_cmd *q_desc;
+ void **info;
+ int err = 0;
+
+ rte_spinlock_lock(&dev->adminq_lock);
+
+ if (iocpt_q_space_avail(q) < 1) {
+ err = -ENOSPC;
+ goto err_out;
+ }
+
+ q_desc = &q_desc_base[q->head_idx];
+
+ memcpy(q_desc, &ctx->cmd, sizeof(ctx->cmd));
+
+ info = IOCPT_INFO_PTR(q, q->head_idx);
+ info[0] = ctx;
+
+ q->head_idx = Q_NEXT_TO_POST(q, 1);
+
+ /* Ring doorbell */
+ iocpt_q_flush(q);
+
+err_out:
+ rte_spinlock_unlock(&dev->adminq_lock);
+
+ return err;
+}
+
+static int
+iocpt_adminq_wait_for_completion(struct iocpt_dev *dev,
+ struct iocpt_admin_ctx *ctx, unsigned long max_wait)
+{
+ struct iocpt_queue *q = &dev->adminq->q;
+ unsigned long step_usec = IONIC_DEVCMD_CHECK_PERIOD_US;
+ unsigned long step_deadline;
+ unsigned long max_wait_usec = max_wait * 1000000L;
+ unsigned long elapsed_usec = 0;
+ int budget = 8;
+ uint16_t idx;
+ void **info;
+
+ step_deadline = IONIC_ADMINQ_WDOG_MS * 1000 / step_usec;
+
+ while (ctx->pending_work && elapsed_usec < max_wait_usec) {
+ /*
+ * Locking here as adminq is served inline and could be
+ * called from multiple places
+ */
+ rte_spinlock_lock(&dev->adminq_service_lock);
+
+ iocpt_cq_service(&dev->adminq->cq, budget,
+ iocpt_adminq_service, NULL);
+
+ /*
+ * Ring the doorbell again if work is pending after step_usec.
+ */
+ if (ctx->pending_work && !step_deadline) {
+ step_deadline = IONIC_ADMINQ_WDOG_MS *
+ 1000 / step_usec;
+
+ rte_spinlock_lock(&dev->adminq_lock);
+ idx = Q_NEXT_TO_POST(q, -1);
+ info = IOCPT_INFO_PTR(q, idx);
+ if (info[0] == ctx)
+ iocpt_q_flush(q);
+ rte_spinlock_unlock(&dev->adminq_lock);
+ }
+
+ rte_spinlock_unlock(&dev->adminq_service_lock);
+
+ rte_delay_us_block(step_usec);
+ elapsed_usec += step_usec;
+ step_deadline--;
+ }
+
+ return (!ctx->pending_work);
+}
+
+static int
+iocpt_adminq_check_err(struct iocpt_admin_ctx *ctx, bool timeout)
+{
+ const char *name;
+ const char *status;
+
+ name = iocpt_opcode_to_str(ctx->cmd.cmd.opcode);
+
+ if (ctx->comp.comp.status == IOCPT_RC_EAGAIN) {
+ IOCPT_PRINT(DEBUG, "%s (%d) returned EAGAIN (%d)",
+ name, ctx->cmd.cmd.opcode,
+ ctx->comp.comp.status);
+ return -EAGAIN;
+ }
+ if (ctx->comp.comp.status != 0 || timeout) {
+ status = iocpt_error_to_str(ctx->comp.comp.status);
+ IOCPT_PRINT(ERR, "%s (%d) failed: %s (%d)",
+ name,
+ ctx->cmd.cmd.opcode,
+ timeout ? "TIMEOUT" : status,
+ timeout ? -1 : ctx->comp.comp.status);
+ return -EIO;
+ }
+
+ if (ctx->cmd.cmd.opcode != IOCPT_CMD_SESS_CONTROL) {
+ IOCPT_PRINT(DEBUG, "%s (%d) succeeded",
+ name, ctx->cmd.cmd.opcode);
+ }
+
+ return 0;
+}
+
+int
+iocpt_adminq_post_wait(struct iocpt_dev *dev, struct iocpt_admin_ctx *ctx)
+{
+ bool done;
+ int err;
+
+ if (ctx->cmd.cmd.opcode != IOCPT_CMD_SESS_CONTROL) {
+ IOCPT_PRINT(DEBUG, "Sending %s (%d) via the admin queue",
+ iocpt_opcode_to_str(ctx->cmd.cmd.opcode),
+ ctx->cmd.cmd.opcode);
+ }
+
+ err = iocpt_adminq_post(dev, ctx);
+ if (err != 0) {
+ IOCPT_PRINT(ERR, "Failure posting %d to the admin queue (%d)",
+ ctx->cmd.cmd.opcode, err);
+ return err;
+ }
+
+ done = iocpt_adminq_wait_for_completion(dev, ctx,
+ IONIC_DEVCMD_TIMEOUT);
+
+ return iocpt_adminq_check_err(ctx, !done /* timed out */);
+}
diff --git a/drivers/crypto/ionic/ionic_crypto_main.c
b/drivers/crypto/ionic/ionic_crypto_main.c
index 4f782f3fe4..8d4ee7f29f 100644
--- a/drivers/crypto/ionic/ionic_crypto_main.c
+++ b/drivers/crypto/ionic/ionic_crypto_main.c
@@ -10,6 +10,108 @@
#include "ionic_crypto.h"
+static int
+iocpt_cq_init(struct iocpt_cq *cq, uint16_t num_descs)
+{
+ if (!rte_is_power_of_2(num_descs) ||
+ num_descs < IOCPT_MIN_RING_DESC ||
+ num_descs > IOCPT_MAX_RING_DESC) {
+ IOCPT_PRINT(ERR, "%u descriptors (min: %u max: %u)",
+ num_descs, IOCPT_MIN_RING_DESC, IOCPT_MAX_RING_DESC);
+ return -EINVAL;
+ }
+
+ cq->num_descs = num_descs;
+ cq->size_mask = num_descs - 1;
+ cq->tail_idx = 0;
+ cq->done_color = 1;
+
+ return 0;
+}
+
+static void
+iocpt_cq_map(struct iocpt_cq *cq, void *base, rte_iova_t base_pa)
+{
+ cq->base = base;
+ cq->base_pa = base_pa;
+}
+
+uint32_t
+iocpt_cq_service(struct iocpt_cq *cq, uint32_t work_to_do,
+ iocpt_cq_cb cb, void *cb_arg)
+{
+ uint32_t work_done = 0;
+
+ if (work_to_do == 0)
+ return 0;
+
+ while (cb(cq, cq->tail_idx, cb_arg)) {
+ cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1);
+ if (cq->tail_idx == 0)
+ cq->done_color = !cq->done_color;
+
+ if (++work_done == work_to_do)
+ break;
+ }
+
+ return work_done;
+}
+
+static int
+iocpt_q_init(struct iocpt_queue *q, uint8_t type, uint32_t index,
+ uint16_t num_descs, uint16_t num_segs, uint32_t socket_id)
+{
+ uint32_t ring_size;
+
+ if (!rte_is_power_of_2(num_descs))
+ return -EINVAL;
+
+ ring_size = rte_log2_u32(num_descs);
+ if (ring_size < 2 || ring_size > 16)
+ return -EINVAL;
+
+ q->type = type;
+ q->index = index;
+ q->num_descs = num_descs;
+ q->num_segs = num_segs;
+ q->size_mask = num_descs - 1;
+ q->head_idx = 0;
+ q->tail_idx = 0;
+
+ q->info = rte_calloc_socket("iocpt",
+ num_descs * num_segs, sizeof(void *),
+ rte_mem_page_size(), socket_id);
+ if (q->info == NULL) {
+ IOCPT_PRINT(ERR, "Cannot allocate queue info");
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static void
+iocpt_q_map(struct iocpt_queue *q, void *base, rte_iova_t base_pa)
+{
+ q->base = base;
+ q->base_pa = base_pa;
+}
+
+static void
+iocpt_q_sg_map(struct iocpt_queue *q, void *base, rte_iova_t base_pa)
+{
+ q->sg_base = base;
+ q->sg_base_pa = base_pa;
+}
+
+static void
+iocpt_q_free(struct iocpt_queue *q)
+{
+ if (q->info != NULL) {
+ rte_free(q->info);
+ q->info = NULL;
+ }
+}
+
static const struct rte_memzone *
iocpt_dma_zone_reserve(const char *type_name, uint16_t qid, size_t size,
unsigned int align, int socket_id)
@@ -33,6 +135,173 @@ iocpt_dma_zone_reserve(const char *type_name, uint16_t
qid, size_t size,
RTE_MEMZONE_IOVA_CONTIG, align);
}
+static int
+iocpt_commonq_alloc(struct iocpt_dev *dev,
+ uint8_t type,
+ size_t struct_size,
+ uint32_t socket_id,
+ uint32_t index,
+ const char *type_name,
+ uint16_t flags,
+ uint16_t num_descs,
+ uint16_t num_segs,
+ uint16_t desc_size,
+ uint16_t cq_desc_size,
+ uint16_t sg_desc_size,
+ struct iocpt_common_q **comq)
+{
+ struct iocpt_common_q *new;
+ uint32_t q_size, cq_size, sg_size, total_size;
+ void *q_base, *cq_base, *sg_base;
+ rte_iova_t q_base_pa = 0;
+ rte_iova_t cq_base_pa = 0;
+ rte_iova_t sg_base_pa = 0;
+ size_t page_size = rte_mem_page_size();
+ int err;
+
+ *comq = NULL;
+
+ q_size = num_descs * desc_size;
+ cq_size = num_descs * cq_desc_size;
+ sg_size = num_descs * sg_desc_size;
+
+ /*
+ * Note: aligning q_size/cq_size is not enough due to cq_base address
+ * aligning as q_base could be not aligned to the page.
+ * Adding page_size.
+ */
+ total_size = RTE_ALIGN(q_size, page_size) +
+ RTE_ALIGN(cq_size, page_size) + page_size;
+ if (flags & IOCPT_Q_F_SG)
+ total_size += RTE_ALIGN(sg_size, page_size) + page_size;
+
+ new = rte_zmalloc_socket("iocpt", struct_size,
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (new == NULL) {
+ IOCPT_PRINT(ERR, "Cannot allocate queue structure");
+ return -ENOMEM;
+ }
+
+ new->dev = dev;
+
+ err = iocpt_q_init(&new->q, type, index, num_descs, num_segs,
+ socket_id);
+ if (err != 0) {
+ IOCPT_PRINT(ERR, "Queue initialization failed");
+ goto err_free_q;
+ }
+
+ err = iocpt_cq_init(&new->cq, num_descs);
+ if (err != 0) {
+ IOCPT_PRINT(ERR, "Completion queue initialization failed");
+ goto err_deinit_q;
+ }
+
+ new->base_z = iocpt_dma_zone_reserve(type_name, index, total_size,
+ IONIC_ALIGN, socket_id);
+ if (new->base_z == NULL) {
+ IOCPT_PRINT(ERR, "Cannot reserve queue DMA memory");
+ err = -ENOMEM;
+ goto err_deinit_cq;
+ }
+
+ new->base = new->base_z->addr;
+ new->base_pa = new->base_z->iova;
+
+ q_base = new->base;
+ q_base_pa = new->base_pa;
+ iocpt_q_map(&new->q, q_base, q_base_pa);
+
+ cq_base = (void *)RTE_ALIGN((uintptr_t)q_base + q_size, page_size);
+ cq_base_pa = RTE_ALIGN(q_base_pa + q_size, page_size);
+ iocpt_cq_map(&new->cq, cq_base, cq_base_pa);
+
+ if (flags & IOCPT_Q_F_SG) {
+ sg_base = (void *)RTE_ALIGN((uintptr_t)cq_base + cq_size,
+ page_size);
+ sg_base_pa = RTE_ALIGN(cq_base_pa + cq_size, page_size);
+ iocpt_q_sg_map(&new->q, sg_base, sg_base_pa);
+ }
+
+ IOCPT_PRINT(DEBUG, "q_base_pa %#jx cq_base_pa %#jx sg_base_pa %#jx",
+ q_base_pa, cq_base_pa, sg_base_pa);
+
+ *comq = new;
+
+ return 0;
+
+err_deinit_cq:
+err_deinit_q:
+ iocpt_q_free(&new->q);
+err_free_q:
+ rte_free(new);
+ return err;
+}
+
+struct ionic_doorbell *
+iocpt_db_map(struct iocpt_dev *dev, struct iocpt_queue *q)
+{
+ return dev->db_pages + q->hw_type;
+}
+
+static int
+iocpt_adminq_alloc(struct iocpt_dev *dev)
+{
+ struct iocpt_admin_q *aq;
+ uint16_t num_descs = IOCPT_ADMINQ_LENGTH;
+ uint16_t flags = 0;
+ int err;
+
+ err = iocpt_commonq_alloc(dev,
+ IOCPT_QTYPE_ADMINQ,
+ sizeof(struct iocpt_admin_q),
+ rte_socket_id(),
+ 0,
+ "admin",
+ flags,
+ num_descs,
+ 1,
+ sizeof(struct iocpt_admin_cmd),
+ sizeof(struct iocpt_admin_comp),
+ 0,
+ (struct iocpt_common_q **)&aq);
+ if (err != 0)
+ return err;
+
+ aq->flags = flags;
+
+ dev->adminq = aq;
+
+ return 0;
+}
+
+static int
+iocpt_adminq_init(struct iocpt_dev *dev)
+{
+ return iocpt_dev_adminq_init(dev);
+}
+
+static void
+iocpt_adminq_deinit(struct iocpt_dev *dev)
+{
+ dev->adminq->flags &= ~IOCPT_Q_F_INITED;
+}
+
+static void
+iocpt_adminq_free(struct iocpt_admin_q *aq)
+{
+ if (aq->base_z != NULL) {
+ rte_memzone_free(aq->base_z);
+ aq->base_z = NULL;
+ aq->base = NULL;
+ aq->base_pa = 0;
+ }
+
+ iocpt_q_free(&aq->q);
+
+ rte_free(aq);
+}
+
static int
iocpt_alloc_objs(struct iocpt_dev *dev)
{
@@ -40,13 +309,23 @@ iocpt_alloc_objs(struct iocpt_dev *dev)
IOCPT_PRINT(DEBUG, "Crypto: %s", dev->name);
+ rte_spinlock_init(&dev->adminq_lock);
+ rte_spinlock_init(&dev->adminq_service_lock);
+
+ err = iocpt_adminq_alloc(dev);
+ if (err != 0) {
+ IOCPT_PRINT(ERR, "Cannot allocate admin queue");
+ err = -ENOMEM;
+ goto err_out;
+ }
+
dev->info_sz = RTE_ALIGN(sizeof(*dev->info), rte_mem_page_size());
dev->info_z = iocpt_dma_zone_reserve("info", 0, dev->info_sz,
IONIC_ALIGN, dev->socket_id);
if (dev->info_z == NULL) {
IOCPT_PRINT(ERR, "Cannot allocate dev info memory");
err = -ENOMEM;
- goto err_out;
+ goto err_free_adminq;
}
dev->info = dev->info_z->addr;
@@ -54,6 +333,9 @@ iocpt_alloc_objs(struct iocpt_dev *dev)
return 0;
+err_free_adminq:
+ iocpt_adminq_free(dev->adminq);
+ dev->adminq = NULL;
err_out:
return err;
}
@@ -68,6 +350,10 @@ iocpt_init(struct iocpt_dev *dev)
if (err != 0)
return err;
+ err = iocpt_adminq_init(dev);
+ if (err != 0)
+ return err;
+
dev->state |= IOCPT_DEV_F_INITED;
return 0;
@@ -87,6 +373,8 @@ iocpt_deinit(struct iocpt_dev *dev)
if (!(dev->state & IOCPT_DEV_F_INITED))
return;
+ iocpt_adminq_deinit(dev);
+
dev->state &= ~IOCPT_DEV_F_INITED;
}
@@ -95,6 +383,11 @@ iocpt_free_objs(struct iocpt_dev *dev)
{
IOCPT_PRINT_CALL();
+ if (dev->adminq != NULL) {
+ iocpt_adminq_free(dev->adminq);
+ dev->adminq = NULL;
+ }
+
if (dev->info != NULL) {
rte_memzone_free(dev->info_z);
dev->info_z = NULL;
--
2.17.1
