> Subject: [EXT] [PATCH v3 2/2] crypto/qat: add cipher-crc offload support
>
Update title as
crypto/qat: support cipher-crc offload
> This patch adds support to the QAT symmetric crypto PMD for combined
> cipher-crc offload feature, primarily for DOCSIS, on gen2/gen3/gen4
> QAT devices.
>
> A new parameter called qat_sym_cipher_crc_enable has been
> added to the PMD, which can be set on process start as follows:
A new devarg called ....
>
> -a <qat pci bdf>,qat_sym_cipher_crc_enable=1
>
> When enabled, a capability check for the combined cipher-crc offload
> feature is triggered to the QAT firmware during queue pair
> initialization. If supported by the firmware, any subsequent runtime
> DOCSIS cipher-crc requests handled by the QAT PMD are offloaded to the
> QAT device by setting up the content descriptor and request
> accordingly.
>
> If the combined DOCSIS cipher-crc feature is not supported by the
> firmware, the CRC continues to be calculated within the PMD, with just
> the cipher portion of the request being offloaded to the QAT device.
>
> Signed-off-by: Kevin O'Sullivan <kevin.osulli...@intel.com>
> Signed-off-by: David Coyle <david.co...@intel.com>
> ---
> v3: updated the file qat.rst with details of new configuration
> ---
> doc/guides/cryptodevs/qat.rst | 23 +++
> drivers/common/qat/qat_device.c | 12 +-
> drivers/common/qat/qat_device.h | 3 +-
> drivers/common/qat/qat_qp.c | 157 +++++++++++++++
> drivers/common/qat/qat_qp.h | 5 +
> drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c | 2 +-
> drivers/crypto/qat/dev/qat_crypto_pmd_gens.h | 24 ++-
> drivers/crypto/qat/dev/qat_sym_pmd_gen1.c | 4 +
> drivers/crypto/qat/qat_crypto.c | 22 ++-
> drivers/crypto/qat/qat_crypto.h | 1 +
> drivers/crypto/qat/qat_sym.c | 4 +
> drivers/crypto/qat/qat_sym.h | 7 +-
> drivers/crypto/qat/qat_sym_session.c | 196 ++++++++++++++++++-
> drivers/crypto/qat/qat_sym_session.h | 21 +-
> 14 files changed, 465 insertions(+), 16 deletions(-)
>
> diff --git a/doc/guides/cryptodevs/qat.rst b/doc/guides/cryptodevs/qat.rst
> index ef754106a8..32e0d8a562 100644
> --- a/doc/guides/cryptodevs/qat.rst
> +++ b/doc/guides/cryptodevs/qat.rst
> @@ -294,6 +294,29 @@ by comma. When the same parameter is used more
> than once first occurrence of the
> is used.
> Maximum threshold that can be set is 32.
>
> +
> +Running QAT PMD with Cipher-CRC offload feature
> +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
> +
> +Support has been added to the QAT symmetric crypto PMD for combined
> Cipher-CRC offload,
> +primarily for the Crypto-CRC DOCSIS security protocol, on GEN2/GEN3/GEN4
> QAT devices.
> +
> +The following parameter enables a Cipher-CRC offload capability check to
> determine
> +if the feature is supported on the QAT device.
> +
> +- qat_sym_cipher_crc_enable
Use the word devarg to make it uniform across DPDK.
> +
> +When enabled, a capability check for the combined Cipher-CRC offload feature
> is triggered
> +to the QAT firmware during queue pair initialization. If supported by the
> firmware,
> +any subsequent runtime Crypto-CRC DOCSIS security protocol requests handled
> by the QAT PMD
> +are offloaded to the QAT device by setting up the content descriptor and
> request accordingly.
> +If not supported, the CRC is calculated by the QAT PMD using the NET CRC API.
> +
> +To use this feature the user must set the parameter on process start as a
> device
> additional parameter::
> +
> + -a 03:01.1,qat_sym_cipher_crc_enable=1
> +
> +
> Running QAT PMD with Intel IPSEC MB library for symmetric precomputes
> function
>
> ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
> ~~~~~~~~~~~~~
>
> diff --git a/drivers/common/qat/qat_device.c
> b/drivers/common/qat/qat_device.c
> index 8bce2ac073..308c59c39f 100644
> --- a/drivers/common/qat/qat_device.c
> +++ b/drivers/common/qat/qat_device.c
> @@ -149,7 +149,16 @@ qat_dev_parse_cmd(const char *str, struct
> qat_dev_cmd_param
> } else {
> memcpy(value_str, arg2, iter);
> value = strtol(value_str, NULL, 10);
> - if (value > MAX_QP_THRESHOLD_SIZE) {
> + if (strcmp(param,
> + SYM_CIPHER_CRC_ENABLE_NAME) ==
> 0) {
> + if (value < 0 || value > 1) {
> + QAT_LOG(DEBUG, "The value
> for"
> + " qat_sym_cipher_crc_enable"
> + " should be set to 0 or 1,"
> + " setting to 0");
Do not split printable strings across multiple lines even if it cross max limit.
Fix this across the patch.
Moreover max limit is also increased from 80 -> 100
> + value = 0;
> + }
> + } else if (value > MAX_QP_THRESHOLD_SIZE) {
> QAT_LOG(DEBUG, "Exceeded max size
> of"
> " threshold, setting to %d",
> MAX_QP_THRESHOLD_SIZE);
> @@ -369,6 +378,7 @@ static int qat_pci_probe(struct rte_pci_driver *pci_drv
> __rte_unused,
> { SYM_ENQ_THRESHOLD_NAME, 0 },
> { ASYM_ENQ_THRESHOLD_NAME, 0 },
> { COMP_ENQ_THRESHOLD_NAME, 0 },
> + { SYM_CIPHER_CRC_ENABLE_NAME, 0 },
> [QAT_CMD_SLICE_MAP_POS] = {
> QAT_CMD_SLICE_MAP, 0},
> { NULL, 0 },
> };
> diff --git a/drivers/common/qat/qat_device.h
> b/drivers/common/qat/qat_device.h
> index bc3da04238..4188474dde 100644
> --- a/drivers/common/qat/qat_device.h
> +++ b/drivers/common/qat/qat_device.h
> @@ -21,8 +21,9 @@
> #define SYM_ENQ_THRESHOLD_NAME "qat_sym_enq_threshold"
> #define ASYM_ENQ_THRESHOLD_NAME "qat_asym_enq_threshold"
> #define COMP_ENQ_THRESHOLD_NAME "qat_comp_enq_threshold"
> +#define SYM_CIPHER_CRC_ENABLE_NAME "qat_sym_cipher_crc_enable"
> #define QAT_CMD_SLICE_MAP "qat_cmd_slice_disable"
> -#define QAT_CMD_SLICE_MAP_POS 4
> +#define QAT_CMD_SLICE_MAP_POS 5
> #define MAX_QP_THRESHOLD_SIZE 32
>
> /**
> diff --git a/drivers/common/qat/qat_qp.c b/drivers/common/qat/qat_qp.c
> index 9cbd19a481..1ce89c265f 100644
> --- a/drivers/common/qat/qat_qp.c
> +++ b/drivers/common/qat/qat_qp.c
> @@ -11,6 +11,9 @@
> #include <bus_pci_driver.h>
> #include <rte_atomic.h>
> #include <rte_prefetch.h>
> +#ifdef RTE_LIB_SECURITY
> +#include <rte_ether.h>
> +#endif
>
> #include "qat_logs.h"
> #include "qat_device.h"
> @@ -957,6 +960,160 @@ qat_cq_get_fw_version(struct qat_qp *qp)
> return -EINVAL;
> }
>
> +#ifdef BUILD_QAT_SYM
Where is this defined? Even no documentation about when to enable/disable it.
> +/* Sends an LA bulk req message to determine if a QAT device supports Cipher-
> CRC
> + * offload. This assumes that there are no inflight messages, i.e. assumes
> + * there's space on the qp, one message is sent and only one response
> + * collected. The status bit of the response and returned data are checked.
> + * Returns:
> + * 1 if status bit indicates success and returned data matches expected
> + * data (i.e. Cipher-CRC supported)
> + * 0 if status bit indicates error or returned data does not match
> expected
> + * data (i.e. Cipher-CRC not supported)
> + * Negative error code in case of error
> + */
> +int
> +qat_cq_get_fw_cipher_crc_cap(struct qat_qp *qp)
> +{
> + struct qat_queue *queue = &(qp->tx_q);
> + uint8_t *base_addr = (uint8_t *)queue->base_addr;
> + struct icp_qat_fw_la_bulk_req cipher_crc_cap_msg = {{0}};
> + struct icp_qat_fw_comn_resp response = {{0}};
> + struct icp_qat_fw_la_cipher_req_params *cipher_param;
> + struct icp_qat_fw_la_auth_req_params *auth_param;
> + struct qat_sym_session *session;
> + phys_addr_t phy_src_addr;
> + uint64_t *src_data_addr;
> + int ret;
> + uint8_t cipher_offset = 18;
> + uint8_t crc_offset = 6;
> + uint8_t ciphertext[34] = {
> + /* Outer protocol header */
> + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
> + /* Ethernet frame */
> + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x06, 0x05,
> + 0x04, 0x03, 0x02, 0x01, 0xD6, 0xE2, 0x70, 0x5C,
> + 0xE6, 0x4D, 0xCC, 0x8C, 0x47, 0xB7, 0x09, 0xD6,
> + /* CRC */
> + 0x54, 0x85, 0xF8, 0x32
> + };
> + uint8_t plaintext[34] = {
> + /* Outer protocol header */
> + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
> + /* Ethernet frame */
> + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x06, 0x05,
> + 0x04, 0x03, 0x02, 0x01, 0x08, 0x00, 0xAA, 0xAA,
> + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA,
> + /* CRC */
> + 0xFF, 0xFF, 0xFF, 0xFF
> + };
> + uint8_t key[16] = {
> + 0x00, 0x00, 0x00, 0x00, 0xAA, 0xBB, 0xCC, 0xDD,
> + 0xEE, 0xFF, 0x00, 0x11, 0x22, 0x33, 0x44, 0x55
> + };
> + uint8_t iv[16] = {
> + 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11,
> + 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11
> + };
Is it not better to define them as macros?
> +
> + session = rte_zmalloc(NULL, sizeof(struct qat_sym_session), 0);
> + if (session == NULL)
> + return -EINVAL;
> +
> + /* Verify the session physical address is known */
> + rte_iova_t session_paddr = rte_mem_virt2iova(session);
> + if (session_paddr == 0 || session_paddr == RTE_BAD_IOVA) {
> + QAT_LOG(ERR, "Session physical address unknown.");
> + return -EINVAL;
> + }
> +
> + /* Prepare the LA bulk request */
> + ret = qat_cipher_crc_cap_msg_sess_prepare(session,
> + session_paddr,
> + key,
> + sizeof(key),
> + qp->qat_dev_gen);
> + if (ret < 0) {
> + rte_free(session);
> + /* Returning 0 here to allow qp setup to continue, but
> + * indicate that Cipher-CRC offload is not supported on the
> + * device
> + */
> + return 0;
> + }
> +
> + cipher_crc_cap_msg = session->fw_req;
> +
> + src_data_addr = rte_zmalloc(NULL, sizeof(plaintext), 0);
> + if (src_data_addr == NULL) {
> + rte_free(session);
> + return -EINVAL;
> + }
> +
> + rte_memcpy(src_data_addr, plaintext, sizeof(plaintext));
> +
> + phy_src_addr = rte_mem_virt2iova(src_data_addr);
> + if (phy_src_addr == 0 || phy_src_addr == RTE_BAD_IOVA) {
> + QAT_LOG(ERR, "Source physical address unknown.");
> + return -EINVAL;
> + }
> +
> + cipher_crc_cap_msg.comn_mid.src_data_addr = phy_src_addr;
> + cipher_crc_cap_msg.comn_mid.src_length = sizeof(plaintext);
> + cipher_crc_cap_msg.comn_mid.dest_data_addr = phy_src_addr;
> + cipher_crc_cap_msg.comn_mid.dst_length = sizeof(plaintext);
> +
> + cipher_param = (void *)&cipher_crc_cap_msg.serv_specif_rqpars;
> + auth_param = (void *)((uint8_t *)cipher_param +
> + ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
> +
> + rte_memcpy(cipher_param->u.cipher_IV_array, iv, sizeof(iv));
> +
> + cipher_param->cipher_offset = cipher_offset;
> + cipher_param->cipher_length = sizeof(plaintext) - cipher_offset;
> + auth_param->auth_off = crc_offset;
> + auth_param->auth_len = sizeof(plaintext) -
> + crc_offset -
> + RTE_ETHER_CRC_LEN;
> +
> + ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(
> + cipher_crc_cap_msg.comn_hdr.serv_specif_flags,
> + ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
> +
> +#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
> + QAT_DP_HEXDUMP_LOG(DEBUG, "LA Bulk request",
> &cipher_crc_cap_msg,
> + sizeof(cipher_crc_cap_msg));
> +#endif
> +
> + /* Send the cipher_crc_cap_msg request */
> + memcpy(base_addr + queue->tail,
> + &cipher_crc_cap_msg,
> + sizeof(cipher_crc_cap_msg));
> + queue->tail = adf_modulo(queue->tail + queue->msg_size,
> + queue->modulo_mask);
> + txq_write_tail(qp->qat_dev_gen, qp, queue);
> +
> + /* Check for response and verify data is same as ciphertext */
> + if (qat_cq_dequeue_response(qp, &response)) {
> +#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
> + QAT_DP_HEXDUMP_LOG(DEBUG, "LA response:", &response,
> + sizeof(response));
> +#endif
> +
> + if (memcmp(src_data_addr, ciphertext, sizeof(ciphertext)) != 0)
> + ret = 0; /* Cipher-CRC offload not supported */
> + else
> + ret = 1;
> + } else {
> + ret = -EINVAL;
> + }
> +
> + rte_free(src_data_addr);
> + rte_free(session);
> + return ret;
> +}
> +#endif
> +
> __rte_weak int
> qat_comp_process_response(void **op __rte_unused, uint8_t *resp
> __rte_unused,
> void *op_cookie __rte_unused,
> diff --git a/drivers/common/qat/qat_qp.h b/drivers/common/qat/qat_qp.h
> index 66f00943a5..d19fc387e4 100644
> --- a/drivers/common/qat/qat_qp.h
> +++ b/drivers/common/qat/qat_qp.h
> @@ -153,6 +153,11 @@ qat_qp_get_hw_data(struct qat_pci_device *qat_dev,
> int
> qat_cq_get_fw_version(struct qat_qp *qp);
>
> +#ifdef BUILD_QAT_SYM
> +int
> +qat_cq_get_fw_cipher_crc_cap(struct qat_qp *qp);
> +#endif
> +
> /* Needed for weak function*/
> int
> qat_comp_process_response(void **op __rte_unused, uint8_t *resp
> __rte_unused,
> diff --git a/drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c
> b/drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c
> index 60ca0fc0d2..1f3e2b1d99 100644
> --- a/drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c
> +++ b/drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c
> @@ -163,7 +163,7 @@ qat_sym_crypto_qp_setup_gen2(struct rte_cryptodev
> *dev, uint16_t qp_id,
> QAT_LOG(DEBUG, "unknown QAT firmware version");
>
> /* set capabilities based on the fw version */
> - qat_sym_private->internal_capabilities = QAT_SYM_CAP_VALID |
> + qat_sym_private->internal_capabilities |= QAT_SYM_CAP_VALID |
> ((ret >= MIXED_CRYPTO_MIN_FW_VER) ?
> QAT_SYM_CAP_MIXED_CRYPTO : 0);
> return 0;
> diff --git a/drivers/crypto/qat/dev/qat_crypto_pmd_gens.h
> b/drivers/crypto/qat/dev/qat_crypto_pmd_gens.h
> index 524c291340..70942906ea 100644
> --- a/drivers/crypto/qat/dev/qat_crypto_pmd_gens.h
> +++ b/drivers/crypto/qat/dev/qat_crypto_pmd_gens.h
> @@ -399,8 +399,13 @@ qat_sym_convert_op_to_vec_chain(struct
> rte_crypto_op *op,
> cipher_ofs = op->sym->cipher.data.offset >> 3;
> break;
> case 0:
> - cipher_len = op->sym->cipher.data.length;
> - cipher_ofs = op->sym->cipher.data.offset;
> + if (ctx->bpi_ctx) {
> + cipher_len = qat_bpicipher_preprocess(ctx, op);
> + cipher_ofs = op->sym->cipher.data.offset;
> + } else {
> + cipher_len = op->sym->cipher.data.length;
> + cipher_ofs = op->sym->cipher.data.offset;
> + }
> break;
> default:
> QAT_DP_LOG(ERR,
> @@ -428,8 +433,10 @@ qat_sym_convert_op_to_vec_chain(struct
> rte_crypto_op *op,
>
> max_len = RTE_MAX(cipher_ofs + cipher_len, auth_ofs + auth_len);
>
> - /* digest in buffer check. Needed only for wireless algos */
> - if (ret == 1) {
> + /* digest in buffer check. Needed only for wireless algos
> + * or combined cipher-crc operations
> + */
> + if (ret == 1 || ctx->bpi_ctx) {
> /* Handle digest-encrypted cases, i.e.
> * auth-gen-then-cipher-encrypt and
> * cipher-decrypt-then-auth-verify
> @@ -456,8 +463,9 @@ qat_sym_convert_op_to_vec_chain(struct
> rte_crypto_op *op,
> auth_len;
>
> /* Then check if digest-encrypted conditions are met */
> - if ((auth_ofs + auth_len < cipher_ofs + cipher_len) &&
> - (digest->iova == auth_end_iova))
> + if (((auth_ofs + auth_len < cipher_ofs + cipher_len) &&
> + (digest->iova == auth_end_iova)) ||
> + ctx->bpi_ctx)
> max_len = RTE_MAX(max_len, auth_ofs + auth_len +
> ctx->digest_length);
> }
> @@ -691,9 +699,9 @@ enqueue_one_chain_job_gen1(struct qat_sym_session
> *ctx,
> auth_param->auth_len;
>
> /* Then check if digest-encrypted conditions are met */
> - if ((auth_param->auth_off + auth_param->auth_len <
> + if (((auth_param->auth_off + auth_param->auth_len <
> cipher_param->cipher_offset + cipher_param->cipher_length)
> &&
> - (digest->iova == auth_iova_end)) {
> + (digest->iova == auth_iova_end)) || ctx->bpi_ctx) {
> /* Handle partial digest encryption */
> if (cipher_param->cipher_offset + cipher_param->cipher_length
> <
> auth_param->auth_off + auth_param->auth_len +
> diff --git a/drivers/crypto/qat/dev/qat_sym_pmd_gen1.c
> b/drivers/crypto/qat/dev/qat_sym_pmd_gen1.c
> index 91d5cfa71d..590eaa0057 100644
> --- a/drivers/crypto/qat/dev/qat_sym_pmd_gen1.c
> +++ b/drivers/crypto/qat/dev/qat_sym_pmd_gen1.c
> @@ -1205,6 +1205,10 @@ qat_sym_crypto_set_session_gen1(void *cryptodev
> __rte_unused, void *session)
> } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) {
> /* do_auth = 0; do_cipher = 1; */
> build_request = qat_sym_build_op_cipher_gen1;
> + } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_CRC) {
> + /* do_auth = 1; do_cipher = 1; */
> + build_request = qat_sym_build_op_chain_gen1;
> + handle_mixed = 1;
> }
>
> if (build_request)
> diff --git a/drivers/crypto/qat/qat_crypto.c b/drivers/crypto/qat/qat_crypto.c
> index 84c26a8062..861679373b 100644
> --- a/drivers/crypto/qat/qat_crypto.c
> +++ b/drivers/crypto/qat/qat_crypto.c
> @@ -172,5 +172,25 @@ qat_cryptodev_qp_setup(struct rte_cryptodev *dev,
> uint16_t qp_id,
> qat_asym_init_op_cookie(qp->op_cookies[i]);
> }
>
> - return ret;
> + if (qat_private->cipher_crc_offload_enable) {
> + ret = qat_cq_get_fw_cipher_crc_cap(qp);
> + if (ret < 0) {
> + qat_cryptodev_qp_release(dev, qp_id);
> + return ret;
> + }
> +
> + if (ret != 0)
> + QAT_LOG(DEBUG, "Cipher CRC supported on QAT
> device");
> + else
> + QAT_LOG(DEBUG, "Cipher CRC not supported on QAT
> device");
> +
> + /* Only send the cipher crc offload capability message once */
> + qat_private->cipher_crc_offload_enable = 0;
> + /* Set cipher crc offload indicator */
> + if (ret)
> + qat_private->internal_capabilities |=
> + QAT_SYM_CAP_CIPHER_CRC;
> + }
> +
> + return 0;
> }
> diff --git a/drivers/crypto/qat/qat_crypto.h b/drivers/crypto/qat/qat_crypto.h
> index 6fe1326c51..e20f16236e 100644
> --- a/drivers/crypto/qat/qat_crypto.h
> +++ b/drivers/crypto/qat/qat_crypto.h
> @@ -36,6 +36,7 @@ struct qat_cryptodev_private {
> /* Shared memzone for storing capabilities */
> uint16_t min_enq_burst_threshold;
> uint32_t internal_capabilities; /* see flags QAT_SYM_CAP_xxx */
> + bool cipher_crc_offload_enable;
> enum qat_service_type service_type;
> };
>
> diff --git a/drivers/crypto/qat/qat_sym.c b/drivers/crypto/qat/qat_sym.c
> index 08e92191a3..345c845325 100644
> --- a/drivers/crypto/qat/qat_sym.c
> +++ b/drivers/crypto/qat/qat_sym.c
> @@ -279,6 +279,10 @@ qat_sym_dev_create(struct qat_pci_device
> *qat_pci_dev,
> if (!strcmp(qat_dev_cmd_param[i].name,
> SYM_ENQ_THRESHOLD_NAME))
> internals->min_enq_burst_threshold =
> qat_dev_cmd_param[i].val;
> + if (!strcmp(qat_dev_cmd_param[i].name,
> + SYM_CIPHER_CRC_ENABLE_NAME))
> + internals->cipher_crc_offload_enable =
> + qat_dev_cmd_param[i].val;
> if (!strcmp(qat_dev_cmd_param[i].name, QAT_IPSEC_MB_LIB))
> qat_ipsec_mb_lib = qat_dev_cmd_param[i].val;
> if (!strcmp(qat_dev_cmd_param[i].name,
> QAT_CMD_SLICE_MAP))
> diff --git a/drivers/crypto/qat/qat_sym.h b/drivers/crypto/qat/qat_sym.h
> index 9a4251e08b..3d841d0eba 100644
> --- a/drivers/crypto/qat/qat_sym.h
> +++ b/drivers/crypto/qat/qat_sym.h
> @@ -32,6 +32,7 @@
>
> /* Internal capabilities */
> #define QAT_SYM_CAP_MIXED_CRYPTO (1 << 0)
> +#define QAT_SYM_CAP_CIPHER_CRC (1 << 1)
> #define QAT_SYM_CAP_VALID (1 << 31)
>
> /**
> @@ -282,7 +283,8 @@ qat_sym_preprocess_requests(void **ops, uint16_t
> nb_ops)
> if (ctx == NULL || ctx->bpi_ctx == NULL)
> continue;
>
> - qat_crc_generate(ctx, op);
> + if (ctx->qat_cmd !=
> ICP_QAT_FW_LA_CMD_CIPHER_CRC)
> + qat_crc_generate(ctx, op);
> }
> }
> }
> @@ -330,7 +332,8 @@ qat_sym_process_response(void **op, uint8_t *resp,
> void *op_cookie,
> if (sess->bpi_ctx) {
> qat_bpicipher_postprocess(sess, rx_op);
> #ifdef RTE_LIB_SECURITY
> - if (is_docsis_sec)
> + if (is_docsis_sec && sess->qat_cmd !=
> +
> ICP_QAT_FW_LA_CMD_CIPHER_CRC)
> qat_crc_verify(sess, rx_op);
> #endif
> }
> diff --git a/drivers/crypto/qat/qat_sym_session.c
> b/drivers/crypto/qat/qat_sym_session.c
> index 6ad6c7ee3a..c0217654c1 100644
> --- a/drivers/crypto/qat/qat_sym_session.c
> +++ b/drivers/crypto/qat/qat_sym_session.c
> @@ -27,6 +27,7 @@
> #include <rte_crypto_sym.h>
> #ifdef RTE_LIB_SECURITY
> #include <rte_security_driver.h>
> +#include <rte_ether.h>
> #endif
>
> #include "qat_logs.h"
> @@ -68,6 +69,13 @@ static void ossl_legacy_provider_unload(void)
>
> extern int qat_ipsec_mb_lib;
>
> +#define ETH_CRC32_POLYNOMIAL 0x04c11db7
> +#define ETH_CRC32_INIT_VAL 0xffffffff
> +#define ETH_CRC32_XOR_OUT 0xffffffff
> +#define ETH_CRC32_POLYNOMIAL_BE RTE_BE32(ETH_CRC32_POLYNOMIAL)
> +#define ETH_CRC32_INIT_VAL_BE RTE_BE32(ETH_CRC32_INIT_VAL)
> +#define ETH_CRC32_XOR_OUT_BE RTE_BE32(ETH_CRC32_XOR_OUT)
> +
> /* SHA1 - 20 bytes - Initialiser state can be found in FIPS stds 180-2 */
> static const uint8_t sha1InitialState[] = {
> 0x67, 0x45, 0x23, 0x01, 0xef, 0xcd, 0xab, 0x89, 0x98, 0xba,
> @@ -115,6 +123,10 @@ qat_sym_cd_cipher_set(struct qat_sym_session *cd,
> const uint8_t *enckey,
> uint32_t enckeylen);
>
> +static int
> +qat_sym_cd_crc_set(struct qat_sym_session *cdesc,
> + enum qat_device_gen qat_dev_gen);
> +
> static int
> qat_sym_cd_auth_set(struct qat_sym_session *cdesc,
> const uint8_t *authkey,
> @@ -122,6 +134,7 @@ qat_sym_cd_auth_set(struct qat_sym_session *cdesc,
> uint32_t aad_length,
> uint32_t digestsize,
> unsigned int operation);
> +
> static void
> qat_sym_session_init_common_hdr(struct qat_sym_session *session);
>
> @@ -630,6 +643,7 @@ qat_sym_session_set_parameters(struct rte_cryptodev
> *dev,
> case ICP_QAT_FW_LA_CMD_MGF1:
> case ICP_QAT_FW_LA_CMD_AUTH_PRE_COMP:
> case ICP_QAT_FW_LA_CMD_CIPHER_PRE_COMP:
> + case ICP_QAT_FW_LA_CMD_CIPHER_CRC:
> case ICP_QAT_FW_LA_CMD_DELIMITER:
> QAT_LOG(ERR, "Unsupported Service %u",
> session->qat_cmd);
> @@ -645,6 +659,45 @@ qat_sym_session_set_parameters(struct rte_cryptodev
> *dev,
> (void *)session);
> }
>
> +int
> +qat_cipher_crc_cap_msg_sess_prepare(struct qat_sym_session *session,
> + rte_iova_t session_paddr,
> + const uint8_t *cipherkey,
> + uint32_t cipherkeylen,
> + enum qat_device_gen qat_dev_gen)
> +{
> + int ret;
> +
> + /* Set content descriptor physical address */
> + session->cd_paddr = session_paddr +
> + offsetof(struct qat_sym_session, cd);
> +
> + /* Set up some pre-requisite variables */
> + session->qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_NONE;
> + session->is_ucs = 0;
> + session->qat_cmd = ICP_QAT_FW_LA_CMD_CIPHER_CRC;
> + session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
> + session->qat_cipher_alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
> + session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
> + session->is_auth = 1;
> + session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_NULL;
> + session->auth_mode = ICP_QAT_HW_AUTH_MODE0;
> + session->auth_op = ICP_QAT_HW_AUTH_GENERATE;
> + session->digest_length = RTE_ETHER_CRC_LEN;
> +
> + ret = qat_sym_cd_cipher_set(session, cipherkey, cipherkeylen);
> + if (ret < 0)
> + return -EINVAL;
> +
> + ret = qat_sym_cd_crc_set(session, qat_dev_gen);
> + if (ret < 0)
> + return -EINVAL;
> +
> + qat_sym_session_finalize(session);
> +
> + return 0;
> +}
> +
> static int
> qat_sym_session_handle_single_pass(struct qat_sym_session *session,
> const struct rte_crypto_aead_xform *aead_xform)
> @@ -697,7 +750,7 @@ qat_sym_session_configure_auth(struct rte_cryptodev
> *dev,
> switch (auth_xform->algo) {
> case RTE_CRYPTO_AUTH_SM3:
> session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SM3;
> - session->auth_mode = ICP_QAT_HW_AUTH_MODE0;
> + session->auth_mode = ICP_QAT_HW_AUTH_MODE2;
> break;
> case RTE_CRYPTO_AUTH_SHA1:
> session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA1;
> @@ -1866,6 +1919,9 @@ int qat_sym_cd_cipher_set(struct qat_sym_session
> *cdesc,
> ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl,
> ICP_QAT_FW_SLICE_DRAM_WR);
> cdesc->cd_cur_ptr = (uint8_t *)&cdesc->cd;
> + } else if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_CRC) {
> + cd_pars->u.s.content_desc_addr = cdesc->cd_paddr;
> + cdesc->cd_cur_ptr = (uint8_t *)&cdesc->cd;
> } else if (cdesc->qat_cmd != ICP_QAT_FW_LA_CMD_HASH_CIPHER) {
> QAT_LOG(ERR, "Invalid param, must be a cipher command.");
> return -EFAULT;
> @@ -2641,6 +2697,135 @@ qat_sec_session_check_docsis(struct
> rte_security_session_conf *conf)
> return -EINVAL;
> }
>
> +static int
> +qat_sym_cd_crc_set(struct qat_sym_session *cdesc,
> + enum qat_device_gen qat_dev_gen)
> +{
> + struct icp_qat_hw_gen2_crc_cd *crc_cd_gen2;
> + struct icp_qat_hw_gen3_crc_cd *crc_cd_gen3;
> + struct icp_qat_hw_gen4_crc_cd *crc_cd_gen4;
> + struct icp_qat_fw_la_bulk_req *req_tmpl = &cdesc->fw_req;
> + struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl-
> >cd_pars;
> + void *ptr = &req_tmpl->cd_ctrl;
> + struct icp_qat_fw_auth_cd_ctrl_hdr *crc_cd_ctrl = ptr;
> + struct icp_qat_fw_la_auth_req_params *crc_param =
> + (struct icp_qat_fw_la_auth_req_params *)
> + ((char *)&req_tmpl->serv_specif_rqpars +
> +
> ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
> + struct icp_qat_fw_ucs_slice_cipher_config crc_cfg;
> + uint16_t crc_cfg_offset, cd_size;
> +
> + crc_cfg_offset = cdesc->cd_cur_ptr - ((uint8_t *)&cdesc->cd);
> +
> + switch (qat_dev_gen) {
> + case QAT_GEN2:
> + crc_cd_gen2 =
> + (struct icp_qat_hw_gen2_crc_cd *)cdesc->cd_cur_ptr;
> + crc_cd_gen2->flags = 0;
> + crc_cd_gen2->initial_crc = 0;
> + memset(&crc_cd_gen2->reserved1,
> + 0,
> + sizeof(crc_cd_gen2->reserved1));
> + memset(&crc_cd_gen2->reserved2,
> + 0,
> + sizeof(crc_cd_gen2->reserved2));
> + cdesc->cd_cur_ptr += sizeof(struct icp_qat_hw_gen2_crc_cd);
> + break;
> + case QAT_GEN3:
> + crc_cd_gen3 =
> + (struct icp_qat_hw_gen3_crc_cd *)cdesc->cd_cur_ptr;
> + crc_cd_gen3->flags =
> ICP_QAT_HW_GEN3_CRC_FLAGS_BUILD(1, 1);
> + crc_cd_gen3->polynomial = ETH_CRC32_POLYNOMIAL;
> + crc_cd_gen3->initial_crc = ETH_CRC32_INIT_VAL;
> + crc_cd_gen3->xor_val = ETH_CRC32_XOR_OUT;
> + memset(&crc_cd_gen3->reserved1,
> + 0,
> + sizeof(crc_cd_gen3->reserved1));
> + memset(&crc_cd_gen3->reserved2,
> + 0,
> + sizeof(crc_cd_gen3->reserved2));
> + crc_cd_gen3->reserved3 = 0;
> + cdesc->cd_cur_ptr += sizeof(struct icp_qat_hw_gen3_crc_cd);
> + break;
> + case QAT_GEN4:
> + crc_cfg.mode = ICP_QAT_HW_CIPHER_ECB_MODE;
> + crc_cfg.algo = ICP_QAT_HW_CIPHER_ALGO_NULL;
> + crc_cfg.hash_cmp_val = 0;
> + crc_cfg.dir = ICP_QAT_HW_CIPHER_ENCRYPT;
> + crc_cfg.associated_data_len_in_bytes = 0;
> + crc_cfg.crc_reflect_out =
> +
> ICP_QAT_HW_CIPHER_UCS_REFLECT_OUT_ENABLED;
> + crc_cfg.crc_reflect_in =
> +
> ICP_QAT_HW_CIPHER_UCS_REFLECT_IN_ENABLED;
> + crc_cfg.crc_encoding = ICP_QAT_HW_CIPHER_UCS_CRC32;
> +
> + crc_cd_gen4 =
> + (struct icp_qat_hw_gen4_crc_cd *)cdesc->cd_cur_ptr;
> + crc_cd_gen4->ucs_config[0] =
> +
> ICP_QAT_HW_UCS_CIPHER_GEN4_BUILD_CONFIG_LOWER(crc_cfg);
> + crc_cd_gen4->ucs_config[1] =
> +
> ICP_QAT_HW_UCS_CIPHER_GEN4_BUILD_CONFIG_UPPER(crc_cfg);
> + crc_cd_gen4->polynomial = ETH_CRC32_POLYNOMIAL_BE;
> + crc_cd_gen4->initial_crc = ETH_CRC32_INIT_VAL_BE;
> + crc_cd_gen4->xor_val = ETH_CRC32_XOR_OUT_BE;
> + crc_cd_gen4->reserved1 = 0;
> + crc_cd_gen4->reserved2 = 0;
> + crc_cd_gen4->reserved3 = 0;
> + cdesc->cd_cur_ptr += sizeof(struct icp_qat_hw_gen4_crc_cd);
> + break;
> + default:
> + return -EINVAL;
> + }
> +
> + crc_cd_ctrl->hash_cfg_offset = crc_cfg_offset >> 3;
> + crc_cd_ctrl->hash_flags =
> ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
> + crc_cd_ctrl->inner_res_sz = cdesc->digest_length;
> + crc_cd_ctrl->final_sz = cdesc->digest_length;
> + crc_cd_ctrl->inner_state1_sz = 0;
> + crc_cd_ctrl->inner_state2_sz = 0;
> + crc_cd_ctrl->inner_state2_offset = 0;
> + crc_cd_ctrl->outer_prefix_sz = 0;
> + crc_cd_ctrl->outer_config_offset = 0;
> + crc_cd_ctrl->outer_state1_sz = 0;
> + crc_cd_ctrl->outer_res_sz = 0;
> + crc_cd_ctrl->outer_prefix_offset = 0;
> +
> + crc_param->auth_res_sz = cdesc->digest_length;
> + crc_param->u2.aad_sz = 0;
> + crc_param->hash_state_sz = 0;
> +
> + cd_size = cdesc->cd_cur_ptr - (uint8_t *)&cdesc->cd;
> + cd_pars->u.s.content_desc_addr = cdesc->cd_paddr;
> + cd_pars->u.s.content_desc_params_sz = RTE_ALIGN_CEIL(cd_size, 8) >>
> 3;
> +
> + return 0;
> +}
> +
> +static int
> +qat_sym_session_configure_crc(struct rte_cryptodev *dev,
> + const struct rte_crypto_sym_xform *cipher_xform,
> + struct qat_sym_session *session)
> +{
> + struct qat_cryptodev_private *internals = dev->data->dev_private;
> + enum qat_device_gen qat_dev_gen = internals->qat_dev-
> >qat_dev_gen;
> + int ret;
> +
> + session->is_auth = 1;
> + session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_NULL;
> + session->auth_mode = ICP_QAT_HW_AUTH_MODE0;
> + session->auth_op = cipher_xform->cipher.op ==
> + RTE_CRYPTO_CIPHER_OP_ENCRYPT ?
> + ICP_QAT_HW_AUTH_GENERATE :
> + ICP_QAT_HW_AUTH_VERIFY;
> + session->digest_length = RTE_ETHER_CRC_LEN;
> +
> + ret = qat_sym_cd_crc_set(session, qat_dev_gen);
> + if (ret < 0)
> + return ret;
> +
> + return 0;
> +}
> +
> static int
> qat_sec_session_set_docsis_parameters(struct rte_cryptodev *dev,
> struct rte_security_session_conf *conf, void *session_private,
> @@ -2681,12 +2866,21 @@ qat_sec_session_set_docsis_parameters(struct
> rte_cryptodev *dev,
> if (qat_cmd_id != ICP_QAT_FW_LA_CMD_CIPHER) {
> QAT_LOG(ERR, "Unsupported xform chain requested");
> return -ENOTSUP;
> + } else if (internals->internal_capabilities
> + & QAT_SYM_CAP_CIPHER_CRC) {
> + qat_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_CRC;
> }
> session->qat_cmd = (enum icp_qat_fw_la_cmd_id)qat_cmd_id;
>
> ret = qat_sym_session_configure_cipher(dev, xform, session);
> if (ret < 0)
> return ret;
> +
> + if (qat_cmd_id == ICP_QAT_FW_LA_CMD_CIPHER_CRC) {
> + ret = qat_sym_session_configure_crc(dev, xform, session);
> + if (ret < 0)
> + return ret;
> + }
> qat_sym_session_finalize(session);
>
> return qat_sym_gen_dev_ops[qat_dev_gen].set_session((void *)cdev,
> diff --git a/drivers/crypto/qat/qat_sym_session.h
> b/drivers/crypto/qat/qat_sym_session.h
> index 6322d7e3bc..9b5d11ac88 100644
> --- a/drivers/crypto/qat/qat_sym_session.h
> +++ b/drivers/crypto/qat/qat_sym_session.h
> @@ -46,6 +46,12 @@
> ICP_QAT_HW_CIPHER_KEY_CONVERT,
> \
> ICP_QAT_HW_CIPHER_DECRYPT)
>
> +#define ICP_QAT_HW_GEN3_CRC_FLAGS_BUILD(ref_in, ref_out) \
> + (((ref_in & QAT_GEN3_COMP_REFLECT_IN_MASK) << \
> + QAT_GEN3_COMP_REFLECT_IN_BITPOS) | \
> + ((ref_out & QAT_GEN3_COMP_REFLECT_OUT_MASK) << \
> + QAT_GEN3_COMP_REFLECT_OUT_BITPOS))
> +
> #define QAT_AES_CMAC_CONST_RB 0x87
>
> #define QAT_CRYPTO_SLICE_SPC 1
> @@ -76,7 +82,12 @@ typedef int (*qat_sym_build_request_t)(void *in_op,
> struct qat_sym_session *ctx,
> /* Common content descriptor */
> struct qat_sym_cd {
> struct icp_qat_hw_cipher_algo_blk cipher;
> - struct icp_qat_hw_auth_algo_blk hash;
> + union {
> + struct icp_qat_hw_auth_algo_blk hash;
> + struct icp_qat_hw_gen2_crc_cd crc_gen2;
> + struct icp_qat_hw_gen3_crc_cd crc_gen3;
> + struct icp_qat_hw_gen4_crc_cd crc_gen4;
> + };
> } __rte_packed __rte_cache_aligned;
>
> struct qat_sym_session {
> @@ -152,10 +163,18 @@ qat_sym_session_clear(struct rte_cryptodev *dev,
> unsigned int
> qat_sym_session_get_private_size(struct rte_cryptodev *dev);
>
> +int
> +qat_cipher_crc_cap_msg_sess_prepare(struct qat_sym_session *session,
> + rte_iova_t session_paddr,
> + const uint8_t *cipherkey,
> + uint32_t cipherkeylen,
> + enum qat_device_gen qat_dev_gen);
> +
> void
> qat_sym_sesssion_init_common_hdr(struct qat_sym_session *session,
> struct icp_qat_fw_comn_req_hdr
> *header,
> enum qat_sym_proto_flag
> proto_flags);
> +
> int
> qat_sym_validate_aes_key(int key_len, enum icp_qat_hw_cipher_algo *alg);
> int
> --
> 2.34.1
>
> --------------------------------------------------------------
> Intel Research and Development Ireland Limited
> Registered in Ireland
> Registered Office: Collinstown Industrial Park, Leixlip, County Kildare
> Registered Number: 308263
>
>
> This e-mail and any attachments may contain confidential material for the sole
> use of the intended recipient(s). Any review or distribution by others is
> strictly prohibited. If you are not the intended recipient, please contact the
> sender and delete all copies.
