From: Archana Muniganti <march...@marvell.com>
Added IPsec SG support in CN9K
Signed-off-by: Archana Muniganti <march...@marvell.com>
Signed-off-by: Tejasree Kondoj <ktejas...@marvell.com>
---
drivers/common/cnxk/roc_api.h | 1 +
drivers/common/cnxk/roc_cpt_sg.h | 37 +++
drivers/common/cnxk/roc_ie_on.h | 9 +-
drivers/common/cnxk/roc_se.h | 28 --
drivers/crypto/cnxk/cn9k_cryptodev_ops.c | 75 ++---
drivers/crypto/cnxk/cn9k_ipsec_la_ops.h | 171 ++++++++--
drivers/crypto/cnxk/cnxk_cryptodev_ops.c | 25 +-
drivers/crypto/cnxk/cnxk_cryptodev_ops.h | 19 ++
drivers/crypto/cnxk/cnxk_se.h | 387 +++--------------------
drivers/crypto/cnxk/cnxk_sg.h | 273 ++++++++++++++++
10 files changed, 594 insertions(+), 431 deletions(-)
create mode 100644 drivers/common/cnxk/roc_cpt_sg.h
create mode 100644 drivers/crypto/cnxk/cnxk_sg.h
diff --git a/drivers/common/cnxk/roc_api.h b/drivers/common/cnxk/roc_api.h
index 072f16d77d..14a11321e0 100644
--- a/drivers/common/cnxk/roc_api.h
+++ b/drivers/common/cnxk/roc_api.h
@@ -86,6 +86,7 @@
/* CPT microcode */
#include "roc_ae.h"
#include "roc_ae_fpm_tables.h"
+#include "roc_cpt_sg.h"
#include "roc_ie.h"
#include "roc_ie_on.h"
#include "roc_ie_ot.h"
diff --git a/drivers/common/cnxk/roc_cpt_sg.h b/drivers/common/cnxk/roc_cpt_sg.h
new file mode 100644
index 0000000000..8a97e1aa5b
--- /dev/null
+++ b/drivers/common/cnxk/roc_cpt_sg.h
@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2022 Marvell.
+ */
+
+#ifndef _ROC_CPT_SG_H_
+#define _ROC_CPT_SG_H_
+
+#define ROC_DMA_MODE_SG (1 << 7)
+
+#define ROC_MAX_SG_IN_OUT_CNT 32
+#define ROC_MAX_SG_CNT (ROC_MAX_SG_IN_OUT_CNT / 2)
+
+#define ROC_SG_LIST_HDR_SIZE (8u)
+#define ROC_SG_ENTRY_SIZE sizeof(struct roc_sglist_comp)
+
+struct roc_sglist_comp {
+ union {
+ uint64_t len;
+ struct {
+ uint16_t len[4];
+ } s;
+ } u;
+ uint64_t ptr[4];
+};
+
+struct roc_sg2list_comp {
+ union {
+ uint64_t len;
+ struct {
+ uint16_t len[3];
+ uint16_t valid_segs;
+ } s;
+ } u;
+ uint64_t ptr[3];
+};
+
+#endif /* _ROC_CPT_SG_H_ */
diff --git a/drivers/common/cnxk/roc_ie_on.h b/drivers/common/cnxk/roc_ie_on.h
index 057ff95362..9933ffa148 100644
--- a/drivers/common/cnxk/roc_ie_on.h
+++ b/drivers/common/cnxk/roc_ie_on.h
@@ -25,10 +25,11 @@ enum roc_ie_on_ucc_ipsec {
};
/* Helper macros */
-#define ROC_IE_ON_INB_RPTR_HDR 16
-#define ROC_IE_ON_MAX_IV_LEN 16
-#define ROC_IE_ON_PER_PKT_IV BIT(43)
-#define ROC_IE_ON_INPLACE_BIT BIT(6)
+#define ROC_IE_ON_OUTB_DPTR_HDR 16
+#define ROC_IE_ON_INB_RPTR_HDR 16
+#define ROC_IE_ON_MAX_IV_LEN 16
+#define ROC_IE_ON_PER_PKT_IV BIT(43)
+#define ROC_IE_ON_INPLACE_BIT BIT(6)
enum {
ROC_IE_ON_SA_ENC_NULL = 0,
diff --git a/drivers/common/cnxk/roc_se.h b/drivers/common/cnxk/roc_se.h
index c357c19c0b..a8f0f49479 100644
--- a/drivers/common/cnxk/roc_se.h
+++ b/drivers/common/cnxk/roc_se.h
@@ -27,13 +27,6 @@
#define ROC_SE_MAX_MAC_LEN 64
#define ROC_SE_OFF_CTRL_LEN 8
-#define ROC_SE_DMA_MODE (1 << 7)
-
-#define ROC_SE_MAX_SG_IN_OUT_CNT 32
-#define ROC_SE_MAX_SG_CNT (ROC_SE_MAX_SG_IN_OUT_CNT / 2)
-
-#define ROC_SE_SG_LIST_HDR_SIZE (8u)
-#define ROC_SE_SG_ENTRY_SIZE sizeof(struct roc_se_sglist_comp)
#define ROC_SE_ZS_EA 0x1
#define ROC_SE_ZS_IA 0x2
@@ -173,27 +166,6 @@ typedef enum {
ROC_SE_PDCP_MAC_LEN_128_BIT = 0x3
} roc_se_pdcp_mac_len_type;
-struct roc_se_sglist_comp {
- union {
- uint64_t len;
- struct {
- uint16_t len[4];
- } s;
- } u;
- uint64_t ptr[4];
-};
-
-struct roc_se_sg2list_comp {
- union {
- uint64_t len;
- struct {
- uint16_t len[3];
- uint16_t valid_segs;
- } s;
- } u;
- uint64_t ptr[3];
-};
-
struct roc_se_enc_context {
uint64_t iv_source : 1;
uint64_t aes_key : 2;
diff --git a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
index 04c004bc7a..cfe1e08dff 100644
--- a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
@@ -18,13 +18,11 @@
#include "cnxk_se.h"
static __rte_always_inline int __rte_hot
-cn9k_cpt_sec_inst_fill(struct rte_crypto_op *op,
- struct cpt_inflight_req *infl_req,
- struct cpt_inst_s *inst)
+cn9k_cpt_sec_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
+ struct cpt_inflight_req *infl_req, struct cpt_inst_s
*inst)
{
struct rte_crypto_sym_op *sym_op = op->sym;
struct cn9k_sec_session *sec_sess;
- int ret;
sec_sess = (struct cn9k_sec_session *)(op->sym->session);
@@ -33,22 +31,10 @@ cn9k_cpt_sec_inst_fill(struct rte_crypto_op *op,
return -ENOTSUP;
}
- if (unlikely(!rte_pktmbuf_is_contiguous(sym_op->m_src))) {
- plt_dp_err("Scatter Gather mode is not supported");
- return -ENOTSUP;
- }
-
if (sec_sess->is_outbound)
- ret = process_outb_sa(op, sec_sess, inst);
- else {
- infl_req->op_flags |= CPT_OP_FLAGS_IPSEC_DIR_INBOUND;
- process_inb_sa(op, sec_sess, inst);
- if (unlikely(sec_sess->replay_win_sz))
- infl_req->op_flags |= CPT_OP_FLAGS_IPSEC_INB_REPLAY;
- ret = 0;
- }
-
- return ret;
+ return process_outb_sa(&qp->meta_info, op, sec_sess, inst,
infl_req);
+ else
+ return process_inb_sa(&qp->meta_info, op, sec_sess, inst,
infl_req);
}
static inline struct cnxk_se_sess *
@@ -94,7 +80,7 @@ cn9k_cpt_inst_prep(struct cnxk_cpt_qp *qp, struct
rte_crypto_op *op,
ret = cpt_sym_inst_fill(qp, op, sess, infl_req, inst,
false);
inst->w7.u64 = sess->cpt_inst_w7;
} else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)
- ret = cn9k_cpt_sec_inst_fill(op, infl_req, inst);
+ ret = cn9k_cpt_sec_inst_fill(qp, op, infl_req, inst);
else {
sess = cn9k_cpt_sym_temp_sess_create(qp, op);
if (unlikely(sess == NULL)) {
@@ -522,45 +508,60 @@ cn9k_cpt_sec_post_process(struct rte_crypto_op *cop,
{
struct rte_crypto_sym_op *sym_op = cop->sym;
struct rte_mbuf *m = sym_op->m_src;
+ struct roc_ie_on_inb_hdr *hdr;
struct cn9k_sec_session *priv;
struct rte_ipv6_hdr *ip6;
struct rte_ipv4_hdr *ip;
uint16_t m_len = 0;
- char *data;
if (infl_req->op_flags & CPT_OP_FLAGS_IPSEC_DIR_INBOUND) {
- data = rte_pktmbuf_mtod(m, char *);
- if (unlikely(infl_req->op_flags &
- CPT_OP_FLAGS_IPSEC_INB_REPLAY)) {
+ hdr = (struct roc_ie_on_inb_hdr *)rte_pktmbuf_mtod(m, char *);
+
+ if (likely(m->next == NULL)) {
+ ip = PLT_PTR_ADD(hdr, ROC_IE_ON_INB_RPTR_HDR);
+ } else {
+ ip = (struct rte_ipv4_hdr *)hdr;
+ hdr = infl_req->mdata;
+ }
+
+ if (unlikely(infl_req->op_flags &
CPT_OP_FLAGS_IPSEC_INB_REPLAY)) {
int ret;
priv = (struct cn9k_sec_session *)(sym_op->session);
- ret = ipsec_antireplay_check(priv, priv->replay_win_sz,
- (struct roc_ie_on_inb_hdr
*)data);
+ ret = ipsec_antireplay_check(priv, priv->replay_win_sz,
hdr);
if (unlikely(ret)) {
cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
return;
}
}
- ip = (struct rte_ipv4_hdr *)(data + ROC_IE_ON_INB_RPTR_HDR);
-
- if (((ip->version_ihl & 0xf0) >> RTE_IPV4_IHL_MULTIPLIER) ==
- IPVERSION) {
+ if (((ip->version_ihl & 0xf0) >> RTE_IPV4_IHL_MULTIPLIER) ==
IPVERSION) {
m_len = rte_be_to_cpu_16(ip->total_length);
} else {
- PLT_ASSERT(((ip->version_ihl & 0xf0) >>
- RTE_IPV4_IHL_MULTIPLIER) == 6);
+ PLT_ASSERT(((ip->version_ihl & 0xf0) >>
RTE_IPV4_IHL_MULTIPLIER) == 6);
ip6 = (struct rte_ipv6_hdr *)ip;
- m_len = rte_be_to_cpu_16(ip6->payload_len) +
- sizeof(struct rte_ipv6_hdr);
+ m_len = rte_be_to_cpu_16(ip6->payload_len) +
sizeof(struct rte_ipv6_hdr);
}
- m->data_len = m_len;
- m->pkt_len = m_len;
- m->data_off += ROC_IE_ON_INB_RPTR_HDR;
+ if (likely(m->next == NULL)) {
+ m->data_len = m_len;
+ m->pkt_len = m_len;
+
+ m->data_off += ROC_IE_ON_INB_RPTR_HDR;
+ } else {
+ struct rte_mbuf *temp = m;
+ uint8_t m_len_s = m_len;
+
+ while (m_len_s - temp->data_len > 0) {
+ m_len_s -= temp->data_len;
+ temp = temp->next;
+ }
+
+ temp->data_len = m_len_s;
+ m->pkt_len = m_len;
+ }
}
}
diff --git a/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h
b/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h
index f1298017ce..3d9c851f10 100644
--- a/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h
+++ b/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h
@@ -10,7 +10,9 @@
#include <rte_security.h>
#include "cn9k_ipsec.h"
+#include "cnxk_cryptodev_ops.h"
#include "cnxk_security_ar.h"
+#include "cnxk_sg.h"
static __rte_always_inline int32_t
ipsec_po_out_rlen_get(struct cn9k_sec_session *sess, uint32_t plen, struct
rte_mbuf *m_src)
@@ -58,7 +60,9 @@ ipsec_po_out_rlen_get(struct cn9k_sec_session *sess, uint32_t
plen, struct rte_m
}
static __rte_always_inline int
-process_outb_sa(struct rte_crypto_op *cop, struct cn9k_sec_session *sess,
struct cpt_inst_s *inst)
+process_outb_sa(struct cpt_qp_meta_info *m_info, struct rte_crypto_op *cop,
+ struct cn9k_sec_session *sess, struct cpt_inst_s *inst,
+ struct cpt_inflight_req *infl_req)
{
const unsigned int hdr_len = sess->custom_hdr_len;
struct rte_crypto_sym_op *sym_op = cop->sym;
@@ -74,24 +78,90 @@ process_outb_sa(struct rte_crypto_op *cop, struct
cn9k_sec_session *sess, struct
rlen = ipsec_po_out_rlen_get(sess, pkt_len, m_src);
extend_tail = rlen - dlen;
- if (unlikely(extend_tail > rte_pktmbuf_tailroom(m_src))) {
- plt_dp_err("Not enough tail room (required: %d, available: %d)",
- extend_tail, rte_pktmbuf_tailroom(m_src));
- return -ENOMEM;
- }
+ pkt_len += extend_tail;
- if (unlikely(hdr_len > data_off)) {
- plt_dp_err("Not enough head room (required: %d, available: %d)",
- hdr_len, rte_pktmbuf_headroom(m_src));
- return -ENOMEM;
- }
+ if (likely(m_src->next == NULL)) {
+ if (unlikely(extend_tail > rte_pktmbuf_tailroom(m_src))) {
+ plt_dp_err("Not enough tail room (required: %d,
available: %d)",
+ extend_tail, rte_pktmbuf_tailroom(m_src));
+ return -ENOMEM;
+ }
- pkt_len += extend_tail;
+ if (unlikely(hdr_len > data_off)) {
+ plt_dp_err("Not enough head room (required: %d,
available: %d)", hdr_len,
+ rte_pktmbuf_headroom(m_src));
+ return -ENOMEM;
+ }
- m_src->data_len = pkt_len;
- m_src->pkt_len = pkt_len;
+ m_src->data_len = pkt_len;
+
+ hdr = PLT_PTR_ADD(m_src->buf_addr, data_off - hdr_len);
+
+ inst->dptr = PLT_U64_CAST(hdr);
+ inst->w4.u64 = sess->inst.w4 | dlen;
+ } else {
+ struct roc_sglist_comp *scatter_comp, *gather_comp;
+ uint32_t g_size_bytes, s_size_bytes;
+ struct rte_mbuf *last_seg;
+ uint8_t *in_buffer;
+ void *m_data;
+ int i;
+
+ last_seg = rte_pktmbuf_lastseg(m_src);
+
+ if (unlikely(extend_tail > rte_pktmbuf_tailroom(last_seg))) {
+ plt_dp_err("Not enough tail room (required: %d,
available: %d)",
+ extend_tail, rte_pktmbuf_tailroom(last_seg));
+ return -ENOMEM;
+ }
+
+ m_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool,
infl_req);
+ if (unlikely(m_data == NULL)) {
+ plt_dp_err("Error allocating meta buffer for request");
+ return -ENOMEM;
+ }
+
+ hdr = m_data;
+
+ m_data = (uint8_t *)m_data + hdr_len;
+ in_buffer = m_data;
+
+ ((uint16_t *)in_buffer)[0] = 0;
+ ((uint16_t *)in_buffer)[1] = 0;
+
+ /*
+ * Input Gather List
+ */
+ i = 0;
+ gather_comp = (struct roc_sglist_comp *)((uint8_t *)m_data + 8);
+
+ i = fill_sg_comp(gather_comp, i, (uint64_t)hdr, hdr_len);
+ i = fill_ipsec_sg_comp_from_pkt(gather_comp, i, m_src);
+ ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
+
+ g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
+
+ /*
+ * output Scatter List
+ */
+ last_seg->data_len += extend_tail;
+
+ i = 0;
+ scatter_comp = (struct roc_sglist_comp *)((uint8_t
*)gather_comp + g_size_bytes);
+
+ i = fill_sg_comp(scatter_comp, i, (uint64_t)hdr, hdr_len);
+ i = fill_ipsec_sg_comp_from_pkt(scatter_comp, i, m_src);
+ ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
- hdr = PLT_PTR_ADD(m_src->buf_addr, data_off - hdr_len);
+ s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
+
+ dlen = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;
+
+ inst->dptr = (uint64_t)in_buffer;
+
+ inst->w4.u64 = sess->inst.w4 | dlen;
+ inst->w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG;
+ }
#ifdef LA_IPSEC_DEBUG
if (sess->inst.w4 & ROC_IE_ON_PER_PKT_IV) {
@@ -101,6 +171,7 @@ process_outb_sa(struct rte_crypto_op *cop, struct
cn9k_sec_session *sess, struct
}
#endif
+ m_src->pkt_len = pkt_len;
esn = ++sess->esn;
/* Set ESN seq hi */
@@ -114,22 +185,80 @@ process_outb_sa(struct rte_crypto_op *cop, struct
cn9k_sec_session *sess, struct
hdr->ip_id = seq_lo;
/* Prepare CPT instruction */
- inst->w4.u64 = sess->inst.w4 | dlen;
- inst->dptr = PLT_U64_CAST(hdr);
inst->w7.u64 = sess->inst.w7;
return 0;
}
-static __rte_always_inline void
-process_inb_sa(struct rte_crypto_op *cop, struct cn9k_sec_session *sess,
struct cpt_inst_s *inst)
+static __rte_always_inline int
+process_inb_sa(struct cpt_qp_meta_info *m_info, struct rte_crypto_op *cop,
struct cn9k_sec_session *sess, struct cpt_inst_s *inst, struct cpt_inflight_req
*infl_req)
{
+ const unsigned int hdr_len = ROC_IE_ON_INB_RPTR_HDR;
struct rte_crypto_sym_op *sym_op = cop->sym;
struct rte_mbuf *m_src = sym_op->m_src;
+ struct roc_ie_on_inb_hdr *hdr;
+ uint32_t dlen;
+
+ infl_req->op_flags |= CPT_OP_FLAGS_IPSEC_DIR_INBOUND;
+ if (likely(m_src->next == NULL)) {
+ dlen = rte_pktmbuf_pkt_len(m_src);
+ inst->dptr = rte_pktmbuf_mtod(m_src, uint64_t);
+ inst->w4.u64 = sess->inst.w4 | dlen;
+ } else {
+ struct roc_sglist_comp *scatter_comp, *gather_comp;
+ uint32_t g_size_bytes, s_size_bytes;
+ uint8_t *in_buffer;
+ void *m_data;
+ int i;
+
+ m_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool,
infl_req);
+ if (unlikely(m_data == NULL)) {
+ plt_dp_err("Error allocating meta buffer for request");
+ return -ENOMEM;
+ }
+
+ hdr = m_data;
+
+ m_data = (uint8_t *)m_data + hdr_len;
+ in_buffer = m_data;
+
+ ((uint16_t *)in_buffer)[0] = 0;
+ ((uint16_t *)in_buffer)[1] = 0;
+
+ /*
+ * Input Gather List
+ */
+ i = 0;
+ gather_comp = (struct roc_sglist_comp *)((uint8_t *)m_data + 8);
+ i = fill_ipsec_sg_comp_from_pkt(gather_comp, i, m_src);
+ ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
+
+ g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
+
+ /*
+ * Output Scatter List
+ */
+ i = 0;
+ scatter_comp = (struct roc_sglist_comp *)((uint8_t
*)gather_comp + g_size_bytes);
+ i = fill_sg_comp(scatter_comp, i, (uint64_t)hdr, hdr_len);
+ i = fill_ipsec_sg_comp_from_pkt(scatter_comp, i, m_src);
+ ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
+
+ s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
+
+ dlen = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;
+
+ inst->dptr = (uint64_t)in_buffer;
+ inst->w4.u64 = sess->inst.w4 | dlen;
+ inst->w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG;
+ }
/* Prepare CPT instruction */
- inst->w4.u64 = sess->inst.w4 | rte_pktmbuf_pkt_len(m_src);
- inst->dptr = rte_pktmbuf_mtod(m_src, uint64_t);
inst->w7.u64 = sess->inst.w7;
+
+ if (unlikely(sess->replay_win_sz))
+ infl_req->op_flags |= CPT_OP_FLAGS_IPSEC_INB_REPLAY;
+
+ return 0;
}
#endif /* __CN9K_IPSEC_LA_OPS_H__ */
diff --git a/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
b/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
index a9c42205e6..eb2ed0d103 100644
--- a/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
@@ -30,10 +30,22 @@ cnxk_cpt_get_mlen(void)
/* For PDCP_CHAIN passthrough alignment */
len += 8;
len += ROC_SE_OFF_CTRL_LEN + ROC_CPT_AES_CBC_IV_LEN;
- len += RTE_ALIGN_CEIL(
- (ROC_SE_SG_LIST_HDR_SIZE +
- (RTE_ALIGN_CEIL(ROC_SE_MAX_SG_IN_OUT_CNT, 4) >> 2) *
ROC_SE_SG_ENTRY_SIZE),
- 8);
+ len += RTE_ALIGN_CEIL((ROC_SG_LIST_HDR_SIZE +
+ (RTE_ALIGN_CEIL(ROC_MAX_SG_IN_OUT_CNT, 4) >> 2)
* ROC_SG_ENTRY_SIZE),
+ 8);
+
+ return len;
+}
+
+static int
+cnxk_cpt_sec_get_mlen(void)
+{
+ uint32_t len;
+
+ len = ROC_IE_ON_OUTB_DPTR_HDR + ROC_IE_ON_MAX_IV_LEN;
+ len += RTE_ALIGN_CEIL((ROC_SG_LIST_HDR_SIZE +
+ (RTE_ALIGN_CEIL(ROC_MAX_SG_IN_OUT_CNT, 4) >> 2)
* ROC_SG_ENTRY_SIZE),
+ 8);
return len;
}
@@ -196,6 +208,11 @@ cnxk_cpt_metabuf_mempool_create(const struct rte_cryptodev
*dev,
mlen = cnxk_cpt_get_mlen();
}
+ if (dev->feature_flags & RTE_CRYPTODEV_FF_SECURITY) {
+ /* Get meta len for security operations */
+ mlen = cnxk_cpt_sec_get_mlen();
+ }
+
if (dev->feature_flags & RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO) {
/* Get meta len required for asymmetric operations */
diff --git a/drivers/crypto/cnxk/cnxk_cryptodev_ops.h
b/drivers/crypto/cnxk/cnxk_cryptodev_ops.h
index 13c90444d6..5153d334ba 100644
--- a/drivers/crypto/cnxk/cnxk_cryptodev_ops.h
+++ b/drivers/crypto/cnxk/cnxk_cryptodev_ops.h
@@ -158,4 +158,23 @@ pending_queue_free_cnt(uint64_t head, uint64_t tail, const
uint64_t mask)
return mask - pending_queue_infl_cnt(head, tail, mask);
}
+static __rte_always_inline void *
+alloc_op_meta(struct roc_se_buf_ptr *buf, int32_t len, struct rte_mempool
*cpt_meta_pool,
+ struct cpt_inflight_req *infl_req)
+{
+ uint8_t *mdata;
+
+ if (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0))
+ return NULL;
+
+ if (likely(buf)) {
+ buf->vaddr = mdata;
+ buf->size = len;
+ }
+
+ infl_req->mdata = mdata;
+ infl_req->op_flags |= CPT_OP_FLAGS_METABUF;
+
+ return mdata;
+}
#endif /* _CNXK_CRYPTODEV_OPS_H_ */
diff --git a/drivers/crypto/cnxk/cnxk_se.h b/drivers/crypto/cnxk/cnxk_se.h
index 2944d3c287..88049ac431 100644
--- a/drivers/crypto/cnxk/cnxk_se.h
+++ b/drivers/crypto/cnxk/cnxk_se.h
@@ -8,11 +8,12 @@
#include "cnxk_cryptodev.h"
#include "cnxk_cryptodev_ops.h"
+#include "cnxk_sg.h"
#define SRC_IOV_SIZE
\
- (sizeof(struct roc_se_iov_ptr) + (sizeof(struct roc_se_buf_ptr) *
ROC_SE_MAX_SG_CNT))
+ (sizeof(struct roc_se_iov_ptr) + (sizeof(struct roc_se_buf_ptr) *
ROC_MAX_SG_CNT))
#define DST_IOV_SIZE
\
- (sizeof(struct roc_se_iov_ptr) + (sizeof(struct roc_se_buf_ptr) *
ROC_SE_MAX_SG_CNT))
+ (sizeof(struct roc_se_iov_ptr) + (sizeof(struct roc_se_buf_ptr) *
ROC_MAX_SG_CNT))
enum cpt_dp_thread_type {
CPT_DP_THREAD_TYPE_FC_CHAIN = 0x1,
@@ -193,272 +194,14 @@ cpt_fc_salt_update(struct roc_se_ctx *se_ctx, uint8_t
*salt)
memcpy(fctx->enc.encr_iv, salt, 4);
}
-static __rte_always_inline uint32_t
-fill_sg_comp(struct roc_se_sglist_comp *list, uint32_t i, phys_addr_t dma_addr,
- uint32_t size)
-{
- struct roc_se_sglist_comp *to = &list[i >> 2];
-
- to->u.s.len[i % 4] = rte_cpu_to_be_16(size);
- to->ptr[i % 4] = rte_cpu_to_be_64(dma_addr);
- i++;
- return i;
-}
-
-static __rte_always_inline uint32_t
-fill_sg_comp_from_buf(struct roc_se_sglist_comp *list, uint32_t i,
- struct roc_se_buf_ptr *from)
-{
- struct roc_se_sglist_comp *to = &list[i >> 2];
-
- to->u.s.len[i % 4] = rte_cpu_to_be_16(from->size);
- to->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)from->vaddr);
- i++;
- return i;
-}
-
-static __rte_always_inline uint32_t
-fill_sg_comp_from_buf_min(struct roc_se_sglist_comp *list, uint32_t i,
- struct roc_se_buf_ptr *from, uint32_t *psize)
-{
- struct roc_se_sglist_comp *to = &list[i >> 2];
- uint32_t size = *psize;
- uint32_t e_len;
-
- e_len = (size > from->size) ? from->size : size;
- to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
- to->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)from->vaddr);
- *psize -= e_len;
- i++;
- return i;
-}
-
-/*
- * This fills the MC expected SGIO list
- * from IOV given by user.
- */
-static __rte_always_inline uint32_t
-fill_sg_comp_from_iov(struct roc_se_sglist_comp *list, uint32_t i,
- struct roc_se_iov_ptr *from, uint32_t from_offset,
- uint32_t *psize, struct roc_se_buf_ptr *extra_buf,
- uint32_t extra_offset)
-{
- int32_t j;
- uint32_t extra_len = extra_buf ? extra_buf->size : 0;
- uint32_t size = *psize;
-
- for (j = 0; j < from->buf_cnt; j++) {
- struct roc_se_sglist_comp *to = &list[i >> 2];
- uint32_t buf_sz = from->bufs[j].size;
- void *vaddr = from->bufs[j].vaddr;
- uint64_t e_vaddr;
- uint32_t e_len;
-
- if (unlikely(from_offset)) {
- if (from_offset >= buf_sz) {
- from_offset -= buf_sz;
- continue;
- }
- e_vaddr = (uint64_t)vaddr + from_offset;
- e_len = (size > (buf_sz - from_offset)) ?
- (buf_sz - from_offset) :
- size;
- from_offset = 0;
- } else {
- e_vaddr = (uint64_t)vaddr;
- e_len = (size > buf_sz) ? buf_sz : size;
- }
-
- to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
- to->ptr[i % 4] = rte_cpu_to_be_64(e_vaddr);
-
- if (extra_len && (e_len >= extra_offset)) {
- /* Break the data at given offset */
- uint32_t next_len = e_len - extra_offset;
- uint64_t next_vaddr = e_vaddr + extra_offset;
-
- if (!extra_offset) {
- i--;
- } else {
- e_len = extra_offset;
- size -= e_len;
- to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
- }
-
- extra_len = RTE_MIN(extra_len, size);
- /* Insert extra data ptr */
- if (extra_len) {
- i++;
- to = &list[i >> 2];
- to->u.s.len[i % 4] =
- rte_cpu_to_be_16(extra_len);
- to->ptr[i % 4] = rte_cpu_to_be_64(
- (uint64_t)extra_buf->vaddr);
- size -= extra_len;
- }
-
- next_len = RTE_MIN(next_len, size);
- /* insert the rest of the data */
- if (next_len) {
- i++;
- to = &list[i >> 2];
- to->u.s.len[i % 4] = rte_cpu_to_be_16(next_len);
- to->ptr[i % 4] = rte_cpu_to_be_64(next_vaddr);
- size -= next_len;
- }
- extra_len = 0;
-
- } else {
- size -= e_len;
- }
- if (extra_offset)
- extra_offset -= size;
- i++;
-
- if (unlikely(!size))
- break;
- }
-
- *psize = size;
- return (uint32_t)i;
-}
-
-static __rte_always_inline uint32_t
-fill_sg2_comp(struct roc_se_sg2list_comp *list, uint32_t i, phys_addr_t
dma_addr, uint32_t size)
-{
- struct roc_se_sg2list_comp *to = &list[i / 3];
-
- to->u.s.len[i % 3] = (size);
- to->ptr[i % 3] = (dma_addr);
- to->u.s.valid_segs = (i % 3) + 1;
- i++;
- return i;
-}
-
-static __rte_always_inline uint32_t
-fill_sg2_comp_from_buf(struct roc_se_sg2list_comp *list, uint32_t i, struct
roc_se_buf_ptr *from)
-{
- struct roc_se_sg2list_comp *to = &list[i / 3];
-
- to->u.s.len[i % 3] = (from->size);
- to->ptr[i % 3] = ((uint64_t)from->vaddr);
- to->u.s.valid_segs = (i % 3) + 1;
- i++;
- return i;
-}
-
-static __rte_always_inline uint32_t
-fill_sg2_comp_from_buf_min(struct roc_se_sg2list_comp *list, uint32_t i,
- struct roc_se_buf_ptr *from, uint32_t *psize)
-{
- struct roc_se_sg2list_comp *to = &list[i / 3];
- uint32_t size = *psize;
- uint32_t e_len;
-
- e_len = (size > from->size) ? from->size : size;
- to->u.s.len[i % 3] = (e_len);
- to->ptr[i % 3] = ((uint64_t)from->vaddr);
- to->u.s.valid_segs = (i % 3) + 1;
- *psize -= e_len;
- i++;
- return i;
-}
-
-static __rte_always_inline uint32_t
-fill_sg2_comp_from_iov(struct roc_se_sg2list_comp *list, uint32_t i, struct
roc_se_iov_ptr *from,
- uint32_t from_offset, uint32_t *psize, struct
roc_se_buf_ptr *extra_buf,
- uint32_t extra_offset)
-{
- int32_t j;
- uint32_t extra_len = extra_buf ? extra_buf->size : 0;
- uint32_t size = *psize;
-
- rte_prefetch2(psize);
-
- for (j = 0; j < from->buf_cnt; j++) {
- struct roc_se_sg2list_comp *to = &list[i / 3];
- uint32_t buf_sz = from->bufs[j].size;
- void *vaddr = from->bufs[j].vaddr;
- uint64_t e_vaddr;
- uint32_t e_len;
-
- if (unlikely(from_offset)) {
- if (from_offset >= buf_sz) {
- from_offset -= buf_sz;
- continue;
- }
- e_vaddr = (uint64_t)vaddr + from_offset;
- e_len = (size > (buf_sz - from_offset)) ? (buf_sz -
from_offset) : size;
- from_offset = 0;
- } else {
- e_vaddr = (uint64_t)vaddr;
- e_len = (size > buf_sz) ? buf_sz : size;
- }
-
- to->u.s.len[i % 3] = (e_len);
- to->ptr[i % 3] = (e_vaddr);
- to->u.s.valid_segs = (i % 3) + 1;
-
- if (extra_len && (e_len >= extra_offset)) {
- /* Break the data at given offset */
- uint32_t next_len = e_len - extra_offset;
- uint64_t next_vaddr = e_vaddr + extra_offset;
-
- if (!extra_offset) {
- i--;
- } else {
- e_len = extra_offset;
- size -= e_len;
- to->u.s.len[i % 3] = (e_len);
- }
-
- extra_len = RTE_MIN(extra_len, size);
- /* Insert extra data ptr */
- if (extra_len) {
- i++;
- to = &list[i / 3];
- to->u.s.len[i % 3] = (extra_len);
- to->ptr[i % 3] = ((uint64_t)extra_buf->vaddr);
- to->u.s.valid_segs = (i % 3) + 1;
- size -= extra_len;
- }
-
- next_len = RTE_MIN(next_len, size);
- /* insert the rest of the data */
- if (next_len) {
- i++;
- to = &list[i / 3];
- to->u.s.len[i % 3] = (next_len);
- to->ptr[i % 3] = (next_vaddr);
- to->u.s.valid_segs = (i % 3) + 1;
- size -= next_len;
- }
- extra_len = 0;
-
- } else {
- size -= e_len;
- }
- if (extra_offset)
- extra_offset -= size;
- i++;
-
- if (unlikely(!size))
- break;
- }
-
- *psize = size;
- return (uint32_t)i;
-}
-
static __rte_always_inline int
sg_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst,
uint64_t offset_ctrl,
uint8_t *iv_s, int iv_len, uint8_t pack_iv, uint8_t pdcp_alg_type,
int32_t inputlen,
int32_t outputlen, uint32_t passthrough_len, uint32_t req_flags,
int pdcp_flag,
int decrypt)
{
+ struct roc_sglist_comp *gather_comp, *scatter_comp;
void *m_vaddr = params->meta_buf.vaddr;
- struct roc_se_sglist_comp *gather_comp;
- struct roc_se_sglist_comp *scatter_comp;
struct roc_se_buf_ptr *aad_buf = NULL;
uint32_t mac_len = 0, aad_len = 0;
struct roc_se_ctx *se_ctx;
@@ -485,7 +228,7 @@ sg_inst_prep(struct roc_se_fc_params *params, struct
cpt_inst_s *inst, uint64_t
m_vaddr = (uint8_t *)m_vaddr + ROC_SE_OFF_CTRL_LEN +
RTE_ALIGN_CEIL(iv_len, 8);
- inst->w4.s.opcode_major |= (uint64_t)ROC_SE_DMA_MODE;
+ inst->w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG;
/* iv offset is 0 */
*offset_vaddr = offset_ctrl;
@@ -503,7 +246,7 @@ sg_inst_prep(struct roc_se_fc_params *params, struct
cpt_inst_s *inst, uint64_t
/* DPTR has SG list */
/* TODO Add error check if space will be sufficient */
- gather_comp = (struct roc_se_sglist_comp *)((uint8_t *)m_vaddr + 8);
+ gather_comp = (struct roc_sglist_comp *)((uint8_t *)m_vaddr + 8);
/*
* Input Gather List
@@ -562,13 +305,13 @@ sg_inst_prep(struct roc_se_fc_params *params, struct
cpt_inst_s *inst, uint64_t
((uint16_t *)in_buffer)[1] = 0;
((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
- g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp);
+ g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
/*
* Output Scatter List
*/
i = 0;
- scatter_comp = (struct roc_se_sglist_comp *)((uint8_t *)gather_comp +
g_size_bytes);
+ scatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp +
g_size_bytes);
if (zsk_flags == 0x1) {
/* IV in SLIST only for EEA3 & UEA2 or for F8 */
@@ -626,9 +369,9 @@ sg_inst_prep(struct roc_se_fc_params *params, struct
cpt_inst_s *inst, uint64_t
}
}
((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
- s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp);
+ s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
- size = g_size_bytes + s_size_bytes + ROC_SE_SG_LIST_HDR_SIZE;
+ size = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;
/* This is DPTR len in case of SG mode */
inst->w4.s.dlen = size;
@@ -643,18 +386,17 @@ sg2_inst_prep(struct roc_se_fc_params *params, struct
cpt_inst_s *inst, uint64_t
int32_t outputlen, uint32_t passthrough_len, uint32_t req_flags,
int pdcp_flag,
int decrypt)
{
+ struct roc_sg2list_comp *gather_comp, *scatter_comp;
void *m_vaddr = params->meta_buf.vaddr;
- uint32_t i, g_size_bytes;
- struct roc_se_sg2list_comp *gather_comp;
- struct roc_se_sg2list_comp *scatter_comp;
struct roc_se_buf_ptr *aad_buf = NULL;
+ uint32_t mac_len = 0, aad_len = 0;
+ union cpt_inst_w5 cpt_inst_w5;
+ union cpt_inst_w6 cpt_inst_w6;
struct roc_se_ctx *se_ctx;
+ uint32_t i, g_size_bytes;
uint64_t *offset_vaddr;
- uint32_t mac_len = 0, aad_len = 0;
int zsk_flags;
uint32_t size;
- union cpt_inst_w5 cpt_inst_w5;
- union cpt_inst_w6 cpt_inst_w6;
uint8_t *iv_d;
se_ctx = params->ctx;
@@ -672,7 +414,7 @@ sg2_inst_prep(struct roc_se_fc_params *params, struct
cpt_inst_s *inst, uint64_t
m_vaddr = (uint8_t *)m_vaddr + ROC_SE_OFF_CTRL_LEN +
RTE_ALIGN_CEIL(iv_len, 8);
- inst->w4.s.opcode_major |= (uint64_t)ROC_SE_DMA_MODE;
+ inst->w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG;
/* iv offset is 0 */
*offset_vaddr = offset_ctrl;
@@ -689,7 +431,7 @@ sg2_inst_prep(struct roc_se_fc_params *params, struct
cpt_inst_s *inst, uint64_t
/* DPTR has SG list */
/* TODO Add error check if space will be sufficient */
- gather_comp = (struct roc_se_sg2list_comp *)((uint8_t *)m_vaddr);
+ gather_comp = (struct roc_sg2list_comp *)((uint8_t *)m_vaddr);
/*
* Input Gather List
@@ -746,13 +488,13 @@ sg2_inst_prep(struct roc_se_fc_params *params, struct
cpt_inst_s *inst, uint64_t
cpt_inst_w5.s.gather_sz = ((i + 2) / 3);
- g_size_bytes = ((i + 2) / 3) * sizeof(struct roc_se_sg2list_comp);
+ g_size_bytes = ((i + 2) / 3) * sizeof(struct roc_sg2list_comp);
/*
* Output Scatter List
*/
i = 0;
- scatter_comp = (struct roc_se_sg2list_comp *)((uint8_t *)gather_comp +
g_size_bytes);
+ scatter_comp = (struct roc_sg2list_comp *)((uint8_t *)gather_comp +
g_size_bytes);
if (zsk_flags == 0x1) {
/* IV in SLIST only for EEA3 & UEA2 or for F8 */
@@ -829,16 +571,15 @@ static __rte_always_inline int
cpt_digest_gen_sg_ver1_prep(uint32_t flags, uint64_t d_lens, struct
roc_se_fc_params *params,
struct cpt_inst_s *inst)
{
+ struct roc_sglist_comp *gather_comp, *scatter_comp;
void *m_vaddr = params->meta_buf.vaddr;
- uint32_t size, i;
+ uint32_t g_size_bytes, s_size_bytes;
uint16_t data_len, mac_len, key_len;
+ union cpt_inst_w4 cpt_inst_w4;
roc_se_auth_type hash_type;
struct roc_se_ctx *ctx;
- struct roc_se_sglist_comp *gather_comp;
- struct roc_se_sglist_comp *scatter_comp;
uint8_t *in_buffer;
- uint32_t g_size_bytes, s_size_bytes;
- union cpt_inst_w4 cpt_inst_w4;
+ uint32_t size, i;
ctx = params->ctx;
@@ -851,13 +592,11 @@ cpt_digest_gen_sg_ver1_prep(uint32_t flags, uint64_t
d_lens, struct roc_se_fc_pa
cpt_inst_w4.s.opcode_minor = 0;
cpt_inst_w4.s.param2 = ((uint16_t)hash_type << 8) | mac_len;
if (ctx->hmac) {
- cpt_inst_w4.s.opcode_major =
- ROC_SE_MAJOR_OP_HMAC | ROC_SE_DMA_MODE;
+ cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_HMAC |
ROC_DMA_MODE_SG;
cpt_inst_w4.s.param1 = key_len;
cpt_inst_w4.s.dlen = data_len + RTE_ALIGN_CEIL(key_len, 8);
} else {
- cpt_inst_w4.s.opcode_major =
- ROC_SE_MAJOR_OP_HASH | ROC_SE_DMA_MODE;
+ cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_HASH |
ROC_DMA_MODE_SG;
cpt_inst_w4.s.param1 = 0;
cpt_inst_w4.s.dlen = data_len;
}
@@ -878,7 +617,7 @@ cpt_digest_gen_sg_ver1_prep(uint32_t flags, uint64_t
d_lens, struct roc_se_fc_pa
((uint16_t *)in_buffer)[1] = 0;
/* TODO Add error check if space will be sufficient */
- gather_comp = (struct roc_se_sglist_comp *)((uint8_t *)m_vaddr + 8);
+ gather_comp = (struct roc_sglist_comp *)((uint8_t *)m_vaddr + 8);
/*
* Input gather list
@@ -901,15 +640,14 @@ cpt_digest_gen_sg_ver1_prep(uint32_t flags, uint64_t
d_lens, struct roc_se_fc_pa
return -1;
}
((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
- g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp);
+ g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
/*
* Output Gather list
*/
i = 0;
- scatter_comp = (struct roc_se_sglist_comp *)((uint8_t *)gather_comp +
- g_size_bytes);
+ scatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp +
g_size_bytes);
if (flags & ROC_SE_VALID_MAC_BUF) {
if (unlikely(params->mac_buf.size < mac_len)) {
@@ -932,9 +670,9 @@ cpt_digest_gen_sg_ver1_prep(uint32_t flags, uint64_t
d_lens, struct roc_se_fc_pa
}
((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
- s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp);
+ s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
- size = g_size_bytes + s_size_bytes + ROC_SE_SG_LIST_HDR_SIZE;
+ size = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;
/* This is DPTR len in case of SG mode */
cpt_inst_w4.s.dlen = size;
@@ -949,17 +687,16 @@ static __rte_always_inline int
cpt_digest_gen_sg_ver2_prep(uint32_t flags, uint64_t d_lens, struct
roc_se_fc_params *params,
struct cpt_inst_s *inst)
{
+ struct roc_sg2list_comp *gather_comp, *scatter_comp;
void *m_vaddr = params->meta_buf.vaddr;
- uint32_t size, i;
uint16_t data_len, mac_len, key_len;
- roc_se_auth_type hash_type;
- struct roc_se_ctx *ctx;
- struct roc_se_sg2list_comp *gather_comp;
- struct roc_se_sg2list_comp *scatter_comp;
+ union cpt_inst_w4 cpt_inst_w4;
union cpt_inst_w5 cpt_inst_w5;
union cpt_inst_w6 cpt_inst_w6;
+ roc_se_auth_type hash_type;
+ struct roc_se_ctx *ctx;
uint32_t g_size_bytes;
- union cpt_inst_w4 cpt_inst_w4;
+ uint32_t size, i;
ctx = params->ctx;
@@ -993,7 +730,7 @@ cpt_digest_gen_sg_ver2_prep(uint32_t flags, uint64_t
d_lens, struct roc_se_fc_pa
/* DPTR has SG list */
/* TODO Add error check if space will be sufficient */
- gather_comp = (struct roc_se_sg2list_comp *)((uint8_t *)m_vaddr + 0);
+ gather_comp = (struct roc_sg2list_comp *)((uint8_t *)m_vaddr + 0);
/*
* Input gather list
@@ -1016,14 +753,14 @@ cpt_digest_gen_sg_ver2_prep(uint32_t flags, uint64_t
d_lens, struct roc_se_fc_pa
}
cpt_inst_w5.s.gather_sz = ((i + 2) / 3);
- g_size_bytes = ((i + 2) / 3) * sizeof(struct roc_se_sg2list_comp);
+ g_size_bytes = ((i + 2) / 3) * sizeof(struct roc_sg2list_comp);
/*
* Output Gather list
*/
i = 0;
- scatter_comp = (struct roc_se_sg2list_comp *)((uint8_t *)gather_comp +
g_size_bytes);
+ scatter_comp = (struct roc_sg2list_comp *)((uint8_t *)gather_comp +
g_size_bytes);
if (flags & ROC_SE_VALID_MAC_BUF) {
if (unlikely(params->mac_buf.size < mac_len)) {
@@ -1482,8 +1219,7 @@ cpt_pdcp_chain_alg_prep(uint32_t req_flags, uint64_t
d_offs, uint64_t d_lens,
pdcp_iv_copy(iv_d, auth_iv, pdcp_auth_alg, pack_iv);
} else {
-
- struct roc_se_sglist_comp *scatter_comp, *gather_comp;
+ struct roc_sglist_comp *scatter_comp, *gather_comp;
void *m_vaddr = params->meta_buf.vaddr;
uint32_t i, g_size_bytes, s_size_bytes;
uint8_t *in_buffer;
@@ -1494,7 +1230,7 @@ cpt_pdcp_chain_alg_prep(uint32_t req_flags, uint64_t
d_offs, uint64_t d_lens,
m_vaddr = PLT_PTR_ADD(m_vaddr, ROC_SE_OFF_CTRL_LEN +
RTE_ALIGN_CEIL(hdr_len, 8));
- cpt_inst_w4.s.opcode_major |= (uint64_t)ROC_SE_DMA_MODE;
+ cpt_inst_w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG;
/* DPTR has SG list */
in_buffer = m_vaddr;
@@ -1502,8 +1238,7 @@ cpt_pdcp_chain_alg_prep(uint32_t req_flags, uint64_t
d_offs, uint64_t d_lens,
((uint16_t *)in_buffer)[0] = 0;
((uint16_t *)in_buffer)[1] = 0;
- gather_comp =
- (struct roc_se_sglist_comp *)((uint8_t *)m_vaddr + 8);
+ gather_comp = (struct roc_sglist_comp *)((uint8_t *)m_vaddr +
8);
/* Input Gather List */
i = 0;
@@ -1536,15 +1271,14 @@ cpt_pdcp_chain_alg_prep(uint32_t req_flags, uint64_t
d_offs, uint64_t d_lens,
}
}
((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
- g_size_bytes =
- ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp);
+ g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
/*
* Output Scatter List
*/
i = 0;
- scatter_comp = (struct roc_se_sglist_comp *)((uint8_t
*)gather_comp + g_size_bytes);
+ scatter_comp = (struct roc_sglist_comp *)((uint8_t
*)gather_comp + g_size_bytes);
if ((hdr_len)) {
i = fill_sg_comp(scatter_comp, i,
@@ -1573,10 +1307,9 @@ cpt_pdcp_chain_alg_prep(uint32_t req_flags, uint64_t
d_offs, uint64_t d_lens,
}
((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
- s_size_bytes =
- ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp);
+ s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
- size = g_size_bytes + s_size_bytes + ROC_SE_SG_LIST_HDR_SIZE;
+ size = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;
/* This is DPTR len in case of SG mode */
cpt_inst_w4.s.dlen = size;
@@ -1800,7 +1533,7 @@ cpt_kasumi_enc_prep(uint32_t req_flags, uint64_t d_offs,
uint64_t d_lens,
}
}
- cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_KASUMI | ROC_SE_DMA_MODE;
+ cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_KASUMI | ROC_DMA_MODE_SG;
/* Indicate ECB/CBC, direction, CTX from CPTR, IV from DPTR */
cpt_inst_w4.s.opcode_minor =
@@ -1841,11 +1574,11 @@ cpt_kasumi_dec_prep(uint64_t d_offs, uint64_t d_lens,
struct roc_se_fc_params *p
flags = se_ctx->zsk_flags;
cpt_inst_w4.u64 = 0;
- cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_KASUMI | ROC_SE_DMA_MODE;
+ cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_KASUMI | ROC_DMA_MODE_SG;
/* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */
- cpt_inst_w4.s.opcode_minor = ((1 << 6) | (se_ctx->k_ecb << 5) |
- (dir << 4) | (0 << 3) | (flags & 0x7));
+ cpt_inst_w4.s.opcode_minor =
+ ((1 << 6) | (se_ctx->k_ecb << 5) | (dir << 4) | (0 << 3) |
(flags & 0x7));
/*
* GP op header, lengths are expected in bits.
@@ -2290,28 +2023,8 @@ fill_sess_gmac(struct rte_crypto_sym_xform *xform,
struct cnxk_se_sess *sess)
return 0;
}
-static __rte_always_inline void *
-alloc_op_meta(struct roc_se_buf_ptr *buf, int32_t len,
- struct rte_mempool *cpt_meta_pool,
- struct cpt_inflight_req *infl_req)
-{
- uint8_t *mdata;
-
- if (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0))
- return NULL;
-
- buf->vaddr = mdata;
- buf->size = len;
-
- infl_req->mdata = mdata;
- infl_req->op_flags |= CPT_OP_FLAGS_METABUF;
-
- return mdata;
-}
-
static __rte_always_inline uint32_t
-prepare_iov_from_pkt(struct rte_mbuf *pkt, struct roc_se_iov_ptr *iovec,
- uint32_t start_offset)
+prepare_iov_from_pkt(struct rte_mbuf *pkt, struct roc_se_iov_ptr *iovec,
uint32_t start_offset)
{
uint16_t index = 0;
void *seg_data = NULL;
diff --git a/drivers/crypto/cnxk/cnxk_sg.h b/drivers/crypto/cnxk/cnxk_sg.h
new file mode 100644
index 0000000000..1dfca261cf
--- /dev/null
+++ b/drivers/crypto/cnxk/cnxk_sg.h
@@ -0,0 +1,273 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2022 Marvell.
+ */
+
+#ifndef _CNXK_SG_H_
+#define _CNXK_SG_H_
+
+static __rte_always_inline uint32_t
+fill_sg_comp(struct roc_sglist_comp *list, uint32_t i, phys_addr_t dma_addr,
uint32_t size)
+{
+ struct roc_sglist_comp *to = &list[i >> 2];
+
+ to->u.s.len[i % 4] = rte_cpu_to_be_16(size);
+ to->ptr[i % 4] = rte_cpu_to_be_64(dma_addr);
+ return ++i;
+}
+
+static __rte_always_inline uint32_t
+fill_sg_comp_from_buf(struct roc_sglist_comp *list, uint32_t i, struct
roc_se_buf_ptr *from)
+{
+ struct roc_sglist_comp *to = &list[i >> 2];
+
+ to->u.s.len[i % 4] = rte_cpu_to_be_16(from->size);
+ to->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)from->vaddr);
+ return ++i;
+}
+
+static __rte_always_inline uint32_t
+fill_sg_comp_from_buf_min(struct roc_sglist_comp *list, uint32_t i, struct
roc_se_buf_ptr *from,
+ uint32_t *psize)
+{
+ struct roc_sglist_comp *to = &list[i >> 2];
+ uint32_t size = *psize;
+ uint32_t e_len;
+
+ e_len = RTE_MIN(from->size, size);
+ to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
+ to->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)from->vaddr);
+ *psize -= e_len;
+ return ++i;
+}
+
+/*
+ * This fills the MC expected SGIO list
+ * from IOV given by user.
+ */
+static __rte_always_inline uint32_t
+fill_sg_comp_from_iov(struct roc_sglist_comp *list, uint32_t i, struct
roc_se_iov_ptr *from,
+ uint32_t from_offset, uint32_t *psize, struct
roc_se_buf_ptr *extra_buf,
+ uint32_t extra_offset)
+{
+ uint32_t extra_len = extra_buf ? extra_buf->size : 0;
+ uint32_t size = *psize;
+ int32_t j;
+
+ for (j = 0; j < from->buf_cnt; j++) {
+ struct roc_sglist_comp *to = &list[i >> 2];
+ uint32_t buf_sz = from->bufs[j].size;
+ void *vaddr = from->bufs[j].vaddr;
+ uint64_t e_vaddr;
+ uint32_t e_len;
+
+ if (unlikely(from_offset)) {
+ if (from_offset >= buf_sz) {
+ from_offset -= buf_sz;
+ continue;
+ }
+ e_vaddr = (uint64_t)vaddr + from_offset;
+ e_len = RTE_MIN((buf_sz - from_offset), size);
+ from_offset = 0;
+ } else {
+ e_vaddr = (uint64_t)vaddr;
+ e_len = RTE_MIN(buf_sz, size);
+ }
+
+ to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
+ to->ptr[i % 4] = rte_cpu_to_be_64(e_vaddr);
+
+ if (extra_len && (e_len >= extra_offset)) {
+ /* Break the data at given offset */
+ uint32_t next_len = e_len - extra_offset;
+ uint64_t next_vaddr = e_vaddr + extra_offset;
+
+ if (!extra_offset) {
+ i--;
+ } else {
+ e_len = extra_offset;
+ size -= e_len;
+ to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
+ }
+
+ extra_len = RTE_MIN(extra_len, size);
+ /* Insert extra data ptr */
+ if (extra_len) {
+ i++;
+ to = &list[i >> 2];
+ to->u.s.len[i % 4] =
rte_cpu_to_be_16(extra_len);
+ to->ptr[i % 4] =
rte_cpu_to_be_64((uint64_t)extra_buf->vaddr);
+ size -= extra_len;
+ }
+
+ next_len = RTE_MIN(next_len, size);
+ /* insert the rest of the data */
+ if (next_len) {
+ i++;
+ to = &list[i >> 2];
+ to->u.s.len[i % 4] = rte_cpu_to_be_16(next_len);
+ to->ptr[i % 4] = rte_cpu_to_be_64(next_vaddr);
+ size -= next_len;
+ }
+ extra_len = 0;
+
+ } else {
+ size -= e_len;
+ }
+ if (extra_offset)
+ extra_offset -= size;
+ i++;
+
+ if (unlikely(!size))
+ break;
+ }
+
+ *psize = size;
+ return (uint32_t)i;
+}
+
+static __rte_always_inline uint32_t
+fill_ipsec_sg_comp_from_pkt(struct roc_sglist_comp *list, uint32_t i, struct
rte_mbuf *pkt)
+{
+ uint32_t buf_sz;
+ void *vaddr;
+
+ while (unlikely(pkt != NULL)) {
+ struct roc_sglist_comp *to = &list[i >> 2];
+ buf_sz = pkt->data_len;
+ vaddr = rte_pktmbuf_mtod(pkt, void *);
+
+ to->u.s.len[i % 4] = rte_cpu_to_be_16(buf_sz);
+ to->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)vaddr);
+
+ pkt = pkt->next;
+ i++;
+ }
+
+ return i;
+}
+
+static __rte_always_inline uint32_t
+fill_sg2_comp(struct roc_sg2list_comp *list, uint32_t i, phys_addr_t dma_addr,
uint32_t size)
+{
+ struct roc_sg2list_comp *to = &list[i / 3];
+
+ to->u.s.len[i % 3] = (size);
+ to->ptr[i % 3] = (dma_addr);
+ to->u.s.valid_segs = (i % 3) + 1;
+ return ++i;
+}
+
+static __rte_always_inline uint32_t
+fill_sg2_comp_from_buf(struct roc_sg2list_comp *list, uint32_t i, struct
roc_se_buf_ptr *from)
+{
+ struct roc_sg2list_comp *to = &list[i / 3];
+
+ to->u.s.len[i % 3] = (from->size);
+ to->ptr[i % 3] = ((uint64_t)from->vaddr);
+ to->u.s.valid_segs = (i % 3) + 1;
+ return ++i;
+}
+
+static __rte_always_inline uint32_t
+fill_sg2_comp_from_buf_min(struct roc_sg2list_comp *list, uint32_t i, struct
roc_se_buf_ptr *from,
+ uint32_t *psize)
+{
+ struct roc_sg2list_comp *to = &list[i / 3];
+ uint32_t size = *psize;
+ uint32_t e_len;
+
+ e_len = RTE_MIN(from->size, size);
+ to->u.s.len[i % 3] = (e_len);
+ to->ptr[i % 3] = ((uint64_t)from->vaddr);
+ to->u.s.valid_segs = (i % 3) + 1;
+ *psize -= e_len;
+ return ++i;
+}
+
+static __rte_always_inline uint32_t
+fill_sg2_comp_from_iov(struct roc_sg2list_comp *list, uint32_t i, struct
roc_se_iov_ptr *from,
+ uint32_t from_offset, uint32_t *psize, struct
roc_se_buf_ptr *extra_buf,
+ uint32_t extra_offset)
+{
+ uint32_t extra_len = extra_buf ? extra_buf->size : 0;
+ uint32_t size = *psize;
+ int32_t j;
+
+ rte_prefetch2(psize);
+
+ for (j = 0; j < from->buf_cnt; j++) {
+ struct roc_sg2list_comp *to = &list[i / 3];
+ uint32_t buf_sz = from->bufs[j].size;
+ void *vaddr = from->bufs[j].vaddr;
+ uint64_t e_vaddr;
+ uint32_t e_len;
+
+ if (unlikely(from_offset)) {
+ if (from_offset >= buf_sz) {
+ from_offset -= buf_sz;
+ continue;
+ }
+ e_vaddr = (uint64_t)vaddr + from_offset;
+ e_len = RTE_MIN((buf_sz - from_offset), size);
+ from_offset = 0;
+ } else {
+ e_vaddr = (uint64_t)vaddr;
+ e_len = RTE_MIN(buf_sz, size);
+ }
+
+ to->u.s.len[i % 3] = (e_len);
+ to->ptr[i % 3] = (e_vaddr);
+ to->u.s.valid_segs = (i % 3) + 1;
+
+ if (extra_len && (e_len >= extra_offset)) {
+ /* Break the data at given offset */
+ uint32_t next_len = e_len - extra_offset;
+ uint64_t next_vaddr = e_vaddr + extra_offset;
+
+ if (!extra_offset) {
+ i--;
+ } else {
+ e_len = extra_offset;
+ size -= e_len;
+ to->u.s.len[i % 3] = (e_len);
+ }
+
+ extra_len = RTE_MIN(extra_len, size);
+ /* Insert extra data ptr */
+ if (extra_len) {
+ i++;
+ to = &list[i / 3];
+ to->u.s.len[i % 3] = (extra_len);
+ to->ptr[i % 3] = ((uint64_t)extra_buf->vaddr);
+ to->u.s.valid_segs = (i % 3) + 1;
+ size -= extra_len;
+ }
+
+ next_len = RTE_MIN(next_len, size);
+ /* insert the rest of the data */
+ if (next_len) {
+ i++;
+ to = &list[i / 3];
+ to->u.s.len[i % 3] = (next_len);
+ to->ptr[i % 3] = (next_vaddr);
+ to->u.s.valid_segs = (i % 3) + 1;
+ size -= next_len;
+ }
+ extra_len = 0;
+
+ } else {
+ size -= e_len;
+ }
+ if (extra_offset)
+ extra_offset -= size;
+ i++;
+
+ if (unlikely(!size))
+ break;
+ }
+
+ *psize = size;
+ return (uint32_t)i;
+}
+
+#endif /*_CNXK_SG_H_ */
--
2.25.1
