From: Tomasz Jozwiak <tomaszx.jozw...@intel.com>
- removed unused args. from process_response function
- changed adf_modulo function
- changed qat_sym_process_response into static inline version
Signed-off-by: Tomasz Jozwiak <tomaszx.jozw...@intel.com>
Signed-off-by: Fiona Trahe <fiona.tr...@intel.com>
---
drivers/crypto/qat/qat_qp.c | 29 +++++----
drivers/crypto/qat/qat_qp.h | 10 +--
drivers/crypto/qat/qat_sym.c | 129 -------------------------------------
drivers/crypto/qat/qat_sym.h | 133 +++++++++++++++++++++++++++++++++++++--
drivers/crypto/qat/qat_sym_pmd.c | 1 -
5 files changed, 145 insertions(+), 157 deletions(-)
diff --git a/drivers/crypto/qat/qat_qp.c b/drivers/crypto/qat/qat_qp.c
index 569eace57..f9d3762d7 100644
--- a/drivers/crypto/qat/qat_qp.c
+++ b/drivers/crypto/qat/qat_qp.c
@@ -14,6 +14,7 @@
#include "qat_logs.h"
#include "qat_device.h"
#include "qat_qp.h"
+#include "qat_sym.h"
#include "adf_transport_access_macros.h"
@@ -238,7 +239,7 @@ int qat_qp_setup(struct qat_pci_device *qat_dev,
qp->qat_dev_gen = qat_dev->qat_dev_gen;
qp->build_request = qat_qp_conf->build_request;
- qp->process_response = qat_qp_conf->process_response;
+ qp->service_type = qat_qp_conf->hw->service_type;
qp->qat_dev = qat_dev;
PMD_DRV_LOG(DEBUG, "QP setup complete: id: %d, cookiepool: %s",
@@ -376,7 +377,7 @@ qat_queue_create(struct qat_pci_device *qat_dev, struct
qat_queue *queue,
queue->max_inflights = ADF_MAX_INFLIGHTS(queue->queue_size,
ADF_BYTES_TO_MSG_SIZE(desc_size));
- queue->modulo = ADF_RING_SIZE_MODULO(queue->queue_size);
+ queue->modulo_mask = (1 << ADF_RING_SIZE_MODULO(queue->queue_size)) - 1;
if (queue->max_inflights < 2) {
PMD_DRV_LOG(ERR, "Invalid num inflights");
@@ -401,11 +402,11 @@ qat_queue_create(struct qat_pci_device *qat_dev, struct
qat_queue *queue,
queue->hw_queue_number, queue_base);
PMD_DRV_LOG(DEBUG, "RING: Name:%s, size in CSR: %u, in bytes %u,"
- " nb msgs %u, msg_size %u, max_inflights %u modulo %u",
+ " nb msgs %u, msg_size %u, max_inflights %u modulo mask %u",
queue->memz_name,
queue->queue_size, queue_size_bytes,
qp_conf->nb_descriptors, desc_size,
- queue->max_inflights, queue->modulo);
+ queue->max_inflights, queue->modulo_mask);
return 0;
@@ -494,13 +495,9 @@ static void adf_configure_queues(struct qat_qp *qp)
queue->hw_queue_number, queue_config);
}
-
-static inline uint32_t adf_modulo(uint32_t data, uint32_t shift)
+static inline uint32_t adf_modulo(uint32_t data, uint32_t modulo_mask)
{
- uint32_t div = data >> shift;
- uint32_t mult = div << shift;
-
- return data - mult;
+ return data & modulo_mask;
}
static inline void
@@ -584,7 +581,7 @@ qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
goto kick_tail;
}
- tail = adf_modulo(tail + queue->msg_size, queue->modulo);
+ tail = adf_modulo(tail + queue->msg_size, queue->modulo_mask);
ops++;
nb_ops_sent++;
}
@@ -616,11 +613,13 @@ qat_dequeue_op_burst(void *qp, void **ops, uint16_t
nb_ops)
while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
resp_counter != nb_ops) {
- tmp_qp->process_response(ops, resp_msg,
- tmp_qp->op_cookies[head / rx_queue->msg_size],
- tmp_qp->qat_dev_gen);
+ if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
+ qat_sym_process_response(ops, resp_msg);
+ /* add qat_asym_process_response here */
+ /* add qat_comp_process_response here */
- head = adf_modulo(head + rx_queue->msg_size, rx_queue->modulo);
+ head = adf_modulo(head + rx_queue->msg_size,
+ rx_queue->modulo_mask);
resp_msg = (uint8_t *)rx_queue->base_addr + head;
ops++;
diff --git a/drivers/crypto/qat/qat_qp.h b/drivers/crypto/qat/qat_qp.h
index 6f07bd67c..59db945e7 100644
--- a/drivers/crypto/qat/qat_qp.h
+++ b/drivers/crypto/qat/qat_qp.h
@@ -21,11 +21,6 @@ typedef int (*build_request_t)(void *op,
enum qat_device_gen qat_dev_gen);
/**< Build a request from an op. */
-typedef int (*process_response_t)(void **ops,
- uint8_t *resp, void *op_cookie,
- enum qat_device_gen qat_dev_gen);
-/**< Process a response descriptor and return the associated op. */
-
/**
* Structure with data needed for creation of queue pair.
*/
@@ -46,7 +41,6 @@ struct qat_qp_config {
uint32_t cookie_size;
int socket_id;
build_request_t build_request;
- process_response_t process_response;
const char *service_str;
};
@@ -59,7 +53,7 @@ struct qat_queue {
rte_iova_t base_phys_addr; /* Queue physical address */
uint32_t head; /* Shadow copy of the head */
uint32_t tail; /* Shadow copy of the tail */
- uint32_t modulo;
+ uint32_t modulo_mask;
uint32_t msg_size;
uint16_t max_inflights;
uint32_t queue_size;
@@ -85,7 +79,7 @@ struct qat_qp {
uint32_t nb_descriptors;
enum qat_device_gen qat_dev_gen;
build_request_t build_request;
- process_response_t process_response;
+ enum qat_service_type service_type;
struct qat_pci_device *qat_dev;
/**< qat device this qp is on */
} __rte_cache_aligned;
diff --git a/drivers/crypto/qat/qat_sym.c b/drivers/crypto/qat/qat_sym.c
index 15244d113..887d4ebcd 100644
--- a/drivers/crypto/qat/qat_sym.c
+++ b/drivers/crypto/qat/qat_sym.c
@@ -12,44 +12,7 @@
#include <rte_byteorder.h>
#include "qat_logs.h"
-#include "qat_sym_session.h"
#include "qat_sym.h"
-#include "qat_sym_pmd.h"
-
-#define BYTE_LENGTH 8
-/* bpi is only used for partial blocks of DES and AES
- * so AES block len can be assumed as max len for iv, src and dst
- */
-#define BPI_MAX_ENCR_IV_LEN ICP_QAT_HW_AES_BLK_SZ
-
-/** Encrypt a single partial block
- * Depends on openssl libcrypto
- * Uses ECB+XOR to do CFB encryption, same result, more performant
- */
-static inline int
-bpi_cipher_encrypt(uint8_t *src, uint8_t *dst,
- uint8_t *iv, int ivlen, int srclen,
- void *bpi_ctx)
-{
- EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
- int encrypted_ivlen;
- uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
- uint8_t *encr = encrypted_iv;
-
- /* ECB method: encrypt the IV, then XOR this with plaintext */
- if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
- <= 0)
- goto cipher_encrypt_err;
-
- for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
- *dst = *src ^ *encr;
-
- return 0;
-
-cipher_encrypt_err:
- PMD_DRV_LOG(ERR, "libcrypto ECB cipher encrypt failed");
- return -EINVAL;
-}
/** Decrypt a single partial block
* Depends on openssl libcrypto
@@ -136,62 +99,6 @@ qat_bpicipher_preprocess(struct qat_sym_session *ctx,
return sym_op->cipher.data.length - last_block_len;
}
-static inline uint32_t
-qat_bpicipher_postprocess(struct qat_sym_session *ctx,
- struct rte_crypto_op *op)
-{
- int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
- struct rte_crypto_sym_op *sym_op = op->sym;
- uint8_t last_block_len = block_len > 0 ?
- sym_op->cipher.data.length % block_len : 0;
-
- if (last_block_len > 0 &&
- ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) {
-
- /* Encrypt last block */
- uint8_t *last_block, *dst, *iv;
- uint32_t last_block_offset;
-
- last_block_offset = sym_op->cipher.data.offset +
- sym_op->cipher.data.length - last_block_len;
- last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
- uint8_t *, last_block_offset);
-
- if (unlikely(sym_op->m_dst != NULL))
- /* out-of-place operation (OOP) */
- dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
- uint8_t *, last_block_offset);
- else
- dst = last_block;
-
- if (last_block_len < sym_op->cipher.data.length)
- /* use previous block ciphertext as IV */
- iv = dst - block_len;
- else
- /* runt block, i.e. less than one full block */
- iv = rte_crypto_op_ctod_offset(op, uint8_t *,
- ctx->cipher_iv.offset);
-
-#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
- rte_hexdump(stdout, "BPI: src before post-process:", last_block,
- last_block_len);
- if (sym_op->m_dst != NULL)
- rte_hexdump(stdout, "BPI: dst before post-process:",
- dst, last_block_len);
-#endif
- bpi_cipher_encrypt(last_block, dst, iv, block_len,
- last_block_len, ctx->bpi_ctx);
-#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
- rte_hexdump(stdout, "BPI: src after post-process:", last_block,
- last_block_len);
- if (sym_op->m_dst != NULL)
- rte_hexdump(stdout, "BPI: dst after post-process:", dst,
- last_block_len);
-#endif
- }
- return sym_op->cipher.data.length - last_block_len;
-}
-
static inline void
set_cipher_iv(uint16_t iv_length, uint16_t iv_offset,
struct icp_qat_fw_la_cipher_req_params *cipher_param,
@@ -659,39 +566,3 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,
#endif
return 0;
}
-
-int
-qat_sym_process_response(void **op, uint8_t *resp,
- __rte_unused void *op_cookie,
- __rte_unused enum qat_device_gen qat_dev_gen)
-{
-
- struct icp_qat_fw_comn_resp *resp_msg =
- (struct icp_qat_fw_comn_resp *)resp;
- struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t)
- (resp_msg->opaque_data);
-
-#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
- rte_hexdump(stdout, "qat_response:", (uint8_t *)resp_msg,
- sizeof(struct icp_qat_fw_comn_resp));
-#endif
-
- if (ICP_QAT_FW_COMN_STATUS_FLAG_OK !=
- ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
- resp_msg->comn_hdr.comn_status)) {
-
- rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
- } else {
- struct qat_sym_session *sess = (struct qat_sym_session *)
- get_session_private_data(
- rx_op->sym->session,
- cryptodev_qat_driver_id);
-
- if (sess->bpi_ctx)
- qat_bpicipher_postprocess(sess, rx_op);
- rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
- }
- *op = (void *)rx_op;
-
- return 0;
-}
diff --git a/drivers/crypto/qat/qat_sym.h b/drivers/crypto/qat/qat_sym.h
index bdb672be4..ddd0f473f 100644
--- a/drivers/crypto/qat/qat_sym.h
+++ b/drivers/crypto/qat/qat_sym.h
@@ -6,8 +6,19 @@
#define _QAT_SYM_H_
#include <rte_cryptodev_pmd.h>
+#include <rte_hexdump.h>
+
+#include <openssl/evp.h>
#include "qat_common.h"
+#include "qat_sym_session.h"
+#include "qat_sym_pmd.h"
+
+#define BYTE_LENGTH 8
+/* bpi is only used for partial blocks of DES and AES
+ * so AES block len can be assumed as max len for iv, src and dst
+ */
+#define BPI_MAX_ENCR_IV_LEN ICP_QAT_HW_AES_BLK_SZ
struct qat_sym_session;
@@ -21,9 +32,123 @@ struct qat_sym_op_cookie {
int
qat_sym_build_request(void *in_op, uint8_t *out_msg,
void *op_cookie, enum qat_device_gen qat_dev_gen);
-int
-qat_sym_process_response(void **op, uint8_t *resp,
- __rte_unused void *op_cookie,
- __rte_unused enum qat_device_gen qat_dev_gen);
+
+/** Encrypt a single partial block
+ * Depends on openssl libcrypto
+ * Uses ECB+XOR to do CFB encryption, same result, more performant
+ */
+static inline int
+bpi_cipher_encrypt(uint8_t *src, uint8_t *dst,
+ uint8_t *iv, int ivlen, int srclen,
+ void *bpi_ctx)
+{
+ EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
+ int encrypted_ivlen;
+ uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
+ uint8_t *encr = encrypted_iv;
+
+ /* ECB method: encrypt the IV, then XOR this with plaintext */
+ if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
+ <= 0)
+ goto cipher_encrypt_err;
+
+ for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
+ *dst = *src ^ *encr;
+
+ return 0;
+
+cipher_encrypt_err:
+ PMD_DRV_LOG(ERR, "libcrypto ECB cipher encrypt failed");
+ return -EINVAL;
+}
+
+static inline uint32_t
+qat_bpicipher_postprocess(struct qat_sym_session *ctx,
+ struct rte_crypto_op *op)
+{
+ int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
+ struct rte_crypto_sym_op *sym_op = op->sym;
+ uint8_t last_block_len = block_len > 0 ?
+ sym_op->cipher.data.length % block_len : 0;
+
+ if (last_block_len > 0 &&
+ ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) {
+
+ /* Encrypt last block */
+ uint8_t *last_block, *dst, *iv;
+ uint32_t last_block_offset;
+
+ last_block_offset = sym_op->cipher.data.offset +
+ sym_op->cipher.data.length - last_block_len;
+ last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
+ uint8_t *, last_block_offset);
+
+ if (unlikely(sym_op->m_dst != NULL))
+ /* out-of-place operation (OOP) */
+ dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
+ uint8_t *, last_block_offset);
+ else
+ dst = last_block;
+
+ if (last_block_len < sym_op->cipher.data.length)
+ /* use previous block ciphertext as IV */
+ iv = dst - block_len;
+ else
+ /* runt block, i.e. less than one full block */
+ iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+ ctx->cipher_iv.offset);
+
+#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
+ rte_hexdump(stdout, "BPI: src before post-process:", last_block,
+ last_block_len);
+ if (sym_op->m_dst != NULL)
+ rte_hexdump(stdout, "BPI: dst before post-process:",
+ dst, last_block_len);
+#endif
+ bpi_cipher_encrypt(last_block, dst, iv, block_len,
+ last_block_len, ctx->bpi_ctx);
+#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
+ rte_hexdump(stdout, "BPI: src after post-process:", last_block,
+ last_block_len);
+ if (sym_op->m_dst != NULL)
+ rte_hexdump(stdout, "BPI: dst after post-process:", dst,
+ last_block_len);
+#endif
+ }
+ return sym_op->cipher.data.length - last_block_len;
+}
+
+static inline void
+qat_sym_process_response(void **op, uint8_t *resp)
+{
+
+ struct icp_qat_fw_comn_resp *resp_msg =
+ (struct icp_qat_fw_comn_resp *)resp;
+ struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t)
+ (resp_msg->opaque_data);
+
+#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
+ rte_hexdump(stdout, "qat_response:", (uint8_t *)resp_msg,
+ sizeof(struct icp_qat_fw_comn_resp));
+#endif
+
+ if (ICP_QAT_FW_COMN_STATUS_FLAG_OK !=
+ ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
+ resp_msg->comn_hdr.comn_status)) {
+
+ rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
+ } else {
+ struct qat_sym_session *sess = (struct qat_sym_session *)
+ get_session_private_data(
+ rx_op->sym->session,
+ cryptodev_qat_driver_id);
+
+
+ if (sess->bpi_ctx)
+ qat_bpicipher_postprocess(sess, rx_op);
+ rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+ }
+ *op = (void *)rx_op;
+}
#endif /* _QAT_SYM_H_ */
diff --git a/drivers/crypto/qat/qat_sym_pmd.c b/drivers/crypto/qat/qat_sym_pmd.c
index 28e579b77..6b39b32f8 100644
--- a/drivers/crypto/qat/qat_sym_pmd.c
+++ b/drivers/crypto/qat/qat_sym_pmd.c
@@ -160,7 +160,6 @@ static int qat_sym_qp_setup(struct rte_cryptodev *dev,
uint16_t qp_id,
qat_qp_conf.hw = qp_hw_data;
qat_qp_conf.build_request = qat_sym_build_request;
- qat_qp_conf.process_response = qat_sym_process_response;
qat_qp_conf.cookie_size = sizeof(struct qat_sym_op_cookie);
qat_qp_conf.nb_descriptors = qp_conf->nb_descriptors;
qat_qp_conf.socket_id = socket_id;
--
2.13.6
