This patch adds AES-GCM algorithm support to AESNI-MB PMD.
Signed-off-by: Fan Zhang <roy.fan.zh...@intel.com>
---
drivers/crypto/aesni_mb/aesni_mb_ops.h | 28 +++-
drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c | 159 +++++++++++++++------
drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c | 30 ++++
drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h | 3 +
4 files changed, 172 insertions(+), 48 deletions(-)
diff --git a/drivers/crypto/aesni_mb/aesni_mb_ops.h
b/drivers/crypto/aesni_mb/aesni_mb_ops.h
index 5a1cba6cb..7257e40aa 100644
--- a/drivers/crypto/aesni_mb/aesni_mb_ops.h
+++ b/drivers/crypto/aesni_mb/aesni_mb_ops.h
@@ -39,6 +39,8 @@ typedef void (*aes_cmac_sub_key_gen_t)
(const void *exp_key, void *k2, void *k3);
typedef void (*aes_cmac_keyexp_t)
(const void *key, void *keyexp);
+typedef void (*aes_gcm_keyexp_t)
+ (const void *key, struct gcm_key_data *keyexp);
/** Multi-buffer library function pointer table */
struct aesni_mb_op_fns {
@@ -86,6 +88,12 @@ struct aesni_mb_op_fns {
/**< AES CMAC subkey expansions */
aes_cmac_keyexp_t aes_cmac_expkey;
/**< AES CMAC key expansions */
+ aes_gcm_keyexp_t aes_gcm_128;
+ /**< AES GCM 128 key expansions */
+ aes_gcm_keyexp_t aes_gcm_192;
+ /**< AES GCM 192 key expansions */
+ aes_gcm_keyexp_t aes_gcm_256;
+ /**< AES GCM 256 key expansions */
} keyexp;
/**< Key expansion functions */
} aux;
@@ -130,7 +138,10 @@ static const struct aesni_mb_op_fns job_ops[] = {
aes_keyexp_256_sse,
aes_xcbc_expand_key_sse,
aes_cmac_subkey_gen_sse,
- aes_keyexp_128_enc_sse
+ aes_keyexp_128_enc_sse,
+ aes_gcm_pre_128_sse,
+ aes_gcm_pre_192_sse,
+ aes_gcm_pre_256_sse
}
}
},
@@ -157,7 +168,10 @@ static const struct aesni_mb_op_fns job_ops[] = {
aes_keyexp_256_avx,
aes_xcbc_expand_key_avx,
aes_cmac_subkey_gen_avx,
- aes_keyexp_128_enc_avx
+ aes_keyexp_128_enc_avx,
+ aes_gcm_pre_128_avx_gen2,
+ aes_gcm_pre_192_avx_gen2,
+ aes_gcm_pre_256_avx_gen2
}
}
},
@@ -184,7 +198,10 @@ static const struct aesni_mb_op_fns job_ops[] = {
aes_keyexp_256_avx2,
aes_xcbc_expand_key_avx2,
aes_cmac_subkey_gen_avx2,
- aes_keyexp_128_enc_avx2
+ aes_keyexp_128_enc_avx2,
+ aes_gcm_pre_128_avx_gen4,
+ aes_gcm_pre_192_avx_gen4,
+ aes_gcm_pre_256_avx_gen4
}
}
},
@@ -211,7 +228,10 @@ static const struct aesni_mb_op_fns job_ops[] = {
aes_keyexp_256_avx512,
aes_xcbc_expand_key_avx512,
aes_cmac_subkey_gen_avx512,
- aes_keyexp_128_enc_avx512
+ aes_keyexp_128_enc_avx512,
+ aes_gcm_pre_128_avx_gen4,
+ aes_gcm_pre_192_avx_gen4,
+ aes_gcm_pre_256_avx_gen4
}
}
}
diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
index 93dc7a443..fe04ae48c 100644
--- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
+++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
@@ -83,7 +83,8 @@ aesni_mb_get_chain_order(const struct rte_crypto_sym_xform
*xform)
}
if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
- if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) {
+ if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM ||
+ xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) {
if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
return AESNI_MB_OP_AEAD_CIPHER_HASH;
else
@@ -330,7 +331,10 @@ aesni_mb_set_session_aead_parameters(const struct
aesni_mb_op_fns *mb_ops,
struct aesni_mb_session *sess,
const struct rte_crypto_sym_xform *xform)
{
- aes_keyexp_t aes_keyexp_fn;
+ union {
+ aes_keyexp_t aes_keyexp_fn;
+ aes_gcm_keyexp_t aes_gcm_keyexp_fn;
+ } keyexp;
switch (xform->aead.op) {
case RTE_CRYPTO_AEAD_OP_ENCRYPT:
@@ -350,7 +354,53 @@ aesni_mb_set_session_aead_parameters(const struct
aesni_mb_op_fns *mb_ops,
case RTE_CRYPTO_AEAD_AES_CCM:
sess->cipher.mode = CCM;
sess->auth.algo = AES_CCM;
+
+ /* Check key length and choose key expansion function for AES */
+ switch (xform->aead.key.length) {
+ case AES_128_BYTES:
+ sess->cipher.key_length_in_bytes = AES_128_BYTES;
+ keyexp.aes_keyexp_fn = mb_ops->aux.keyexp.aes128;
+ break;
+ default:
+ AESNI_MB_LOG(ERR, "Invalid cipher key length");
+ return -EINVAL;
+ }
+
+ /* Expanded cipher keys */
+ (*keyexp.aes_keyexp_fn)(xform->aead.key.data,
+ sess->cipher.expanded_aes_keys.encode,
+ sess->cipher.expanded_aes_keys.decode);
+ break;
+
+ case RTE_CRYPTO_AEAD_AES_GCM:
+ sess->cipher.mode = GCM;
+ sess->auth.algo = AES_GMAC;
+
+ switch (xform->aead.key.length) {
+ case AES_128_BYTES:
+ sess->cipher.key_length_in_bytes = AES_128_BYTES;
+ keyexp.aes_gcm_keyexp_fn =
+ mb_ops->aux.keyexp.aes_gcm_128;
+ break;
+ case AES_192_BYTES:
+ sess->cipher.key_length_in_bytes = AES_192_BYTES;
+ keyexp.aes_gcm_keyexp_fn =
+ mb_ops->aux.keyexp.aes_gcm_192;
+ break;
+ case AES_256_BYTES:
+ sess->cipher.key_length_in_bytes = AES_256_BYTES;
+ keyexp.aes_gcm_keyexp_fn =
+ mb_ops->aux.keyexp.aes_gcm_256;
+ break;
+ default:
+ AESNI_MB_LOG(ERR, "Invalid cipher key length");
+ return -EINVAL;
+ }
+
+ (keyexp.aes_gcm_keyexp_fn)(xform->aead.key.data,
+ &sess->cipher.gcm_key);
break;
+
default:
AESNI_MB_LOG(ERR, "Unsupported aead mode parameter");
return -ENOTSUP;
@@ -360,23 +410,6 @@ aesni_mb_set_session_aead_parameters(const struct
aesni_mb_op_fns *mb_ops,
sess->iv.offset = xform->aead.iv.offset;
sess->iv.length = xform->aead.iv.length;
- /* Check key length and choose key expansion function for AES */
-
- switch (xform->aead.key.length) {
- case AES_128_BYTES:
- sess->cipher.key_length_in_bytes = AES_128_BYTES;
- aes_keyexp_fn = mb_ops->aux.keyexp.aes128;
- break;
- default:
- AESNI_MB_LOG(ERR, "Invalid cipher key length");
- return -EINVAL;
- }
-
- /* Expanded cipher keys */
- (*aes_keyexp_fn)(xform->aead.key.data,
- sess->cipher.expanded_aes_keys.encode,
- sess->cipher.expanded_aes_keys.decode);
-
return 0;
}
@@ -573,38 +606,62 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp
*qp,
job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
- if (job->cipher_mode == DES3) {
- job->aes_enc_key_expanded =
- session->cipher.exp_3des_keys.ks_ptr;
- job->aes_dec_key_expanded =
- session->cipher.exp_3des_keys.ks_ptr;
- } else {
- job->aes_enc_key_expanded =
- session->cipher.expanded_aes_keys.encode;
- job->aes_dec_key_expanded =
- session->cipher.expanded_aes_keys.decode;
- }
-
-
-
-
/* Set authentication parameters */
job->hash_alg = session->auth.algo;
- if (job->hash_alg == AES_XCBC) {
+
+ switch (job->hash_alg) {
+ case AES_XCBC:
job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
job->u.XCBC._k2 = session->auth.xcbc.k2;
job->u.XCBC._k3 = session->auth.xcbc.k3;
- } else if (job->hash_alg == AES_CCM) {
+
+ job->aes_enc_key_expanded =
+ session->cipher.expanded_aes_keys.encode;
+ job->aes_dec_key_expanded =
+ session->cipher.expanded_aes_keys.decode;
+ break;
+
+ case AES_CCM:
job->u.CCM.aad = op->sym->aead.aad.data + 18;
job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
- } else if (job->hash_alg == AES_CMAC) {
+ job->aes_enc_key_expanded =
+ session->cipher.expanded_aes_keys.encode;
+ job->aes_dec_key_expanded =
+ session->cipher.expanded_aes_keys.decode;
+ break;
+
+ case AES_CMAC:
job->u.CMAC._key_expanded = session->auth.cmac.expkey;
job->u.CMAC._skey1 = session->auth.cmac.skey1;
job->u.CMAC._skey2 = session->auth.cmac.skey2;
+ job->aes_enc_key_expanded =
+ session->cipher.expanded_aes_keys.encode;
+ job->aes_dec_key_expanded =
+ session->cipher.expanded_aes_keys.decode;
+ break;
- } else {
+ case AES_GMAC:
+ job->u.GCM.aad = op->sym->aead.aad.data;
+ job->u.GCM.aad_len_in_bytes = session->aead.aad_len;
+ job->aes_enc_key_expanded = &session->cipher.gcm_key;
+ job->aes_dec_key_expanded = &session->cipher.gcm_key;
+ break;
+
+ default:
job->u.HMAC._hashed_auth_key_xor_ipad =
session->auth.pads.inner;
job->u.HMAC._hashed_auth_key_xor_opad =
session->auth.pads.outer;
+
+ if (job->cipher_mode == DES3) {
+ job->aes_enc_key_expanded =
+ session->cipher.exp_3des_keys.ks_ptr;
+ job->aes_dec_key_expanded =
+ session->cipher.exp_3des_keys.ks_ptr;
+ } else {
+ job->aes_enc_key_expanded =
+ session->cipher.expanded_aes_keys.encode;
+ job->aes_dec_key_expanded =
+ session->cipher.expanded_aes_keys.decode;
+ }
}
/* Mutable crypto operation parameters */
@@ -625,7 +682,7 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
rte_pktmbuf_data_len(op->sym->m_src));
} else {
m_dst = m_src;
- if (job->hash_alg == AES_CCM)
+ if (job->hash_alg == AES_CCM || job->hash_alg == AES_GMAC)
m_offset = op->sym->aead.data.offset;
else
m_offset = op->sym->cipher.data.offset;
@@ -637,7 +694,7 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
job->auth_tag_output = qp->temp_digests[*digest_idx];
*digest_idx = (*digest_idx + 1) % MAX_JOBS;
} else {
- if (job->hash_alg == AES_CCM)
+ if (job->hash_alg == AES_CCM || job->hash_alg == AES_GMAC)
job->auth_tag_output = op->sym->aead.digest.data;
else
job->auth_tag_output = op->sym->auth.digest.data;
@@ -647,7 +704,8 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
* Multi-buffer library current only support returning a truncated
* digest length as specified in the relevant IPsec RFCs
*/
- if (job->hash_alg != AES_CCM && job->hash_alg != AES_CMAC)
+ if (job->hash_alg != AES_CCM && job->hash_alg != AES_CMAC &&
+ job->hash_alg != AES_GMAC)
job->auth_tag_output_len_in_bytes =
get_truncated_digest_byte_length(job->hash_alg);
else
@@ -662,7 +720,8 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, m_offset);
- if (job->hash_alg == AES_CCM) {
+ switch (job->hash_alg) {
+ case AES_CCM:
job->cipher_start_src_offset_in_bytes =
op->sym->aead.data.offset;
job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;
@@ -671,7 +730,19 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp
*qp,
job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
session->iv.offset + 1);
- } else {
+ break;
+
+ case AES_GMAC:
+ job->cipher_start_src_offset_in_bytes =
+ op->sym->aead.data.offset;
+ job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;
+ job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;
+ job->msg_len_to_hash_in_bytes = job->msg_len_to_cipher_in_bytes;
+ job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+ session->iv.offset);
+ break;
+
+ default:
job->cipher_start_src_offset_in_bytes =
op->sym->cipher.data.offset;
job->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length;
@@ -693,7 +764,7 @@ static inline void
verify_digest(struct aesni_mb_qp *qp __rte_unused, JOB_AES_HMAC *job,
struct rte_crypto_op *op) {
/* Verify digest if required */
- if (job->hash_alg == AES_CCM) {
+ if (job->hash_alg == AES_CCM || job->hash_alg == AES_GMAC) {
if (memcmp(job->auth_tag_output, op->sym->aead.digest.data,
job->auth_tag_output_len_in_bytes) != 0)
op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c
b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c
index ab26e5ae4..5a726bee3 100644
--- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c
+++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c
@@ -330,6 +330,36 @@ static const struct rte_cryptodev_capabilities
aesni_mb_pmd_capabilities[] = {
}, }
}, }
},
+ { /* AES GCM */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
+ {.aead = {
+ .algo = RTE_CRYPTO_AEAD_AES_GCM,
+ .block_size = 16,
+ .key_size = {
+ .min = 16,
+ .max = 32,
+ .increment = 8
+ },
+ .digest_size = {
+ .min = 8,
+ .max = 16,
+ .increment = 4
+ },
+ .aad_size = {
+ .min = 0,
+ .max = 65535,
+ .increment = 1
+ },
+ .iv_size = {
+ .min = 12,
+ .max = 12,
+ .increment = 0
+ }
+ }, }
+ }, }
+ },
RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
};
diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h
b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h
index 70e9d18e5..24a943f2a 100644
--- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h
+++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h
@@ -91,6 +91,7 @@ static const unsigned auth_digest_byte_lengths[] = {
[SHA_512] = 64,
[AES_XCBC] = 16,
[AES_CMAC] = 16,
+ [AES_GMAC] = 12,
[NULL_HASH] = 0
};
@@ -180,6 +181,8 @@ struct aesni_mb_session {
const void *ks_ptr[3];
uint64_t key[3][16];
} exp_3des_keys;
+
+ struct gcm_key_data gcm_key;
};
/**< Expanded AES keys - Allocating space to
* contain the maximum expanded key size which
--
2.13.6
