The branch main has been updated by mhorne:
URL:
https://cgit.FreeBSD.org/src/commit/?id=4979620ece984ffb10c27c2db7d0e253eb84b2ba
commit 4979620ece984ffb10c27c2db7d0e253eb84b2ba
Author: Mitchell Horne <mho...@freebsd.org>
AuthorDate: 2021-01-07 19:30:40 +0000
Commit: Mitchell Horne <mho...@freebsd.org>
CommitDate: 2021-01-07 19:35:20 +0000
armv8crypto: add AES-XTS support
A straightforward(ish) port from aesni(4). This implementation does not
perform loop unrolling on the input blocks, so this is left as a future
performance improvement.
Submitted by: Greg V <greg AT unrelenting.technology>
Looks good: jhb, jmg
Tested by: mhorne
Differential Revision: https://reviews.freebsd.org/D21017
---
sys/crypto/armv8/armv8_crypto.c | 51 +++++++++++++++-----
sys/crypto/armv8/armv8_crypto.h | 6 +++
sys/crypto/armv8/armv8_crypto_wrap.c | 92 ++++++++++++++++++++++++++++++++++++
3 files changed, 138 insertions(+), 11 deletions(-)
diff --git a/sys/crypto/armv8/armv8_crypto.c b/sys/crypto/armv8/armv8_crypto.c
index 43dafd7b365c..90d096b53b1a 100644
--- a/sys/crypto/armv8/armv8_crypto.c
+++ b/sys/crypto/armv8/armv8_crypto.c
@@ -114,7 +114,7 @@ armv8_crypto_probe(device_t dev)
break;
}
- device_set_desc_copy(dev, "AES-CBC");
+ device_set_desc_copy(dev, "AES-CBC,AES-XTS");
/* TODO: Check more fields as we support more features */
@@ -204,6 +204,17 @@ armv8_crypto_probesession(device_t dev,
return (EINVAL);
}
break;
+ case CRYPTO_AES_XTS:
+ if (csp->csp_ivlen != AES_XTS_IV_LEN)
+ return (EINVAL);
+ switch (csp->csp_cipher_klen * 8) {
+ case 256:
+ case 512:
+ break;
+ default:
+ return (EINVAL);
+ }
+ break;
default:
return (EINVAL);
}
@@ -211,16 +222,19 @@ armv8_crypto_probesession(device_t dev,
default:
return (EINVAL);
}
- return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
+ return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
}
static void
armv8_crypto_cipher_setup(struct armv8_crypto_session *ses,
- const struct crypto_session_params *csp)
+ const struct crypto_session_params *csp, const uint8_t *key, int keylen)
{
int i;
- switch (csp->csp_cipher_klen * 8) {
+ if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
+ keylen /= 2;
+
+ switch (keylen * 8) {
case 128:
ses->rounds = AES128_ROUNDS;
break;
@@ -231,16 +245,19 @@ armv8_crypto_cipher_setup(struct armv8_crypto_session
*ses,
ses->rounds = AES256_ROUNDS;
break;
default:
- panic("invalid CBC key length");
+ panic("invalid AES key length");
}
- rijndaelKeySetupEnc(ses->enc_schedule, csp->csp_cipher_key,
- csp->csp_cipher_klen * 8);
- rijndaelKeySetupDec(ses->dec_schedule, csp->csp_cipher_key,
- csp->csp_cipher_klen * 8);
+ rijndaelKeySetupEnc(ses->enc_schedule, key, keylen * 8);
+ rijndaelKeySetupDec(ses->dec_schedule, key, keylen * 8);
+ if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
+ rijndaelKeySetupEnc(ses->xts_schedule, key + keylen, keylen *
8);
+
for (i = 0; i < nitems(ses->enc_schedule); i++) {
ses->enc_schedule[i] = bswap32(ses->enc_schedule[i]);
ses->dec_schedule[i] = bswap32(ses->dec_schedule[i]);
+ if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
+ ses->xts_schedule[i] = bswap32(ses->xts_schedule[i]);
}
}
@@ -259,7 +276,8 @@ armv8_crypto_newsession(device_t dev, crypto_session_t cses,
}
ses = crypto_get_driver_session(cses);
- armv8_crypto_cipher_setup(ses, csp);
+ armv8_crypto_cipher_setup(ses, csp, csp->csp_cipher_key,
+ csp->csp_cipher_klen);
rw_wunlock(&sc->lock);
return (0);
}
@@ -333,7 +351,8 @@ armv8_crypto_cipher_process(struct armv8_crypto_session
*ses,
}
if (crp->crp_cipher_key != NULL) {
- panic("armv8: new cipher key");
+ armv8_crypto_cipher_setup(ses, csp, crp->crp_cipher_key,
+ csp->csp_cipher_klen);
}
crypto_read_iv(crp, iv);
@@ -348,6 +367,16 @@ armv8_crypto_cipher_process(struct armv8_crypto_session
*ses,
armv8_aes_decrypt_cbc(ses->rounds, ses->dec_schedule,
crp->crp_payload_length, buf, iv);
break;
+ case CRYPTO_AES_XTS:
+ if (encflag)
+ armv8_aes_encrypt_xts(ses->rounds, ses->enc_schedule,
+ ses->xts_schedule, crp->crp_payload_length, buf,
+ buf, iv);
+ else
+ armv8_aes_decrypt_xts(ses->rounds, ses->dec_schedule,
+ ses->xts_schedule, crp->crp_payload_length, buf,
+ buf, iv);
+ break;
}
if (allocated)
diff --git a/sys/crypto/armv8/armv8_crypto.h b/sys/crypto/armv8/armv8_crypto.h
index 01d31f7cec78..2d0be163b072 100644
--- a/sys/crypto/armv8/armv8_crypto.h
+++ b/sys/crypto/armv8/armv8_crypto.h
@@ -40,6 +40,7 @@
struct armv8_crypto_session {
uint32_t enc_schedule[AES_SCHED_LEN/4];
uint32_t dec_schedule[AES_SCHED_LEN/4];
+ uint32_t xts_schedule[AES_SCHED_LEN/4];
int algo;
int rounds;
};
@@ -49,4 +50,9 @@ void armv8_aes_encrypt_cbc(int, const void *, size_t, const
uint8_t *,
void armv8_aes_decrypt_cbc(int, const void *, size_t, uint8_t *,
const uint8_t[static AES_BLOCK_LEN]);
+void armv8_aes_encrypt_xts(int, const void *, const void *, size_t,
+ const uint8_t *, uint8_t *, const uint8_t[AES_BLOCK_LEN]);
+void armv8_aes_decrypt_xts(int, const void *, const void *, size_t,
+ const uint8_t *, uint8_t *, const uint8_t[AES_BLOCK_LEN]);
+
#endif /* _ARMV8_CRYPTO_H_ */
diff --git a/sys/crypto/armv8/armv8_crypto_wrap.c
b/sys/crypto/armv8/armv8_crypto_wrap.c
index 643b2be2cb12..83af3fad40ef 100644
--- a/sys/crypto/armv8/armv8_crypto_wrap.c
+++ b/sys/crypto/armv8/armv8_crypto_wrap.c
@@ -126,3 +126,95 @@ armv8_aes_decrypt_cbc(int rounds, const void
*key_schedule, size_t len,
buf += AES_BLOCK_LEN;
}
}
+
+#define AES_XTS_BLOCKSIZE 16
+#define AES_XTS_IVSIZE 8
+#define AES_XTS_ALPHA 0x87 /* GF(2^128) generator
polynomial */
+
+static inline int32x4_t
+xts_crank_lfsr(int32x4_t inp)
+{
+ const int32x4_t alphamask = {AES_XTS_ALPHA, 1, 1, 1};
+ int32x4_t xtweak, ret;
+
+ /* set up xor mask */
+ xtweak = vextq_s32(inp, inp, 3);
+ xtweak = vshrq_n_s32(xtweak, 31);
+ xtweak &= alphamask;
+
+ /* next term */
+ ret = vshlq_n_s32(inp, 1);
+ ret ^= xtweak;
+
+ return ret;
+}
+
+static void
+armv8_aes_crypt_xts_block(int rounds, const uint8x16_t *key_schedule,
+ uint8x16_t *tweak, const uint8_t *from, uint8_t *to, int do_encrypt)
+{
+ uint8x16_t block;
+
+ block = vld1q_u8(from) ^ *tweak;
+
+ if (do_encrypt)
+ block = armv8_aes_enc(rounds - 1, key_schedule, block);
+ else
+ block = armv8_aes_dec(rounds - 1, key_schedule, block);
+
+ vst1q_u8(to, block ^ *tweak);
+
+ *tweak =
vreinterpretq_u8_s32(xts_crank_lfsr(vreinterpretq_s32_u8(*tweak)));
+}
+
+static void
+armv8_aes_crypt_xts(int rounds, const uint8x16_t *data_schedule,
+ const uint8x16_t *tweak_schedule, size_t len, const uint8_t *from,
+ uint8_t *to, const uint8_t iv[static AES_BLOCK_LEN], int do_encrypt)
+{
+ uint8x16_t tweakreg;
+ uint8_t tweak[AES_XTS_BLOCKSIZE] __aligned(16);
+ size_t i, cnt;
+
+ /*
+ * Prepare tweak as E_k2(IV). IV is specified as LE representation
+ * of a 64-bit block number which we allow to be passed in directly.
+ */
+#if BYTE_ORDER == LITTLE_ENDIAN
+ bcopy(iv, tweak, AES_XTS_IVSIZE);
+ /* Last 64 bits of IV are always zero. */
+ bzero(tweak + AES_XTS_IVSIZE, AES_XTS_IVSIZE);
+#else
+#error Only LITTLE_ENDIAN architectures are supported.
+#endif
+ tweakreg = vld1q_u8(tweak);
+ tweakreg = armv8_aes_enc(rounds - 1, tweak_schedule, tweakreg);
+
+ cnt = len / AES_XTS_BLOCKSIZE;
+ for (i = 0; i < cnt; i++) {
+ armv8_aes_crypt_xts_block(rounds, data_schedule, &tweakreg,
+ from, to, do_encrypt);
+ from += AES_XTS_BLOCKSIZE;
+ to += AES_XTS_BLOCKSIZE;
+ }
+}
+
+void
+armv8_aes_encrypt_xts(int rounds, const void *data_schedule,
+ const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to,
+ const uint8_t iv[static AES_BLOCK_LEN])
+{
+
+ armv8_aes_crypt_xts(rounds, data_schedule, tweak_schedule, len, from,
to,
+ iv, 1);
+}
+
+void
+armv8_aes_decrypt_xts(int rounds, const void *data_schedule,
+ const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to,
+ const uint8_t iv[static AES_BLOCK_LEN])
+{
+
+ armv8_aes_crypt_xts(rounds, data_schedule, tweak_schedule, len, from,
to,
+ iv, 0);
+}
_______________________________________________
dev-commits-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/dev-commits-src-all
To unsubscribe, send any mail to "dev-commits-src-all-unsubscr...@freebsd.org"
