The branch main has been updated by jhb:
URL:
https://cgit.FreeBSD.org/src/commit/?id=a221a8f4a0deb6619448e9d2df696625d454ab3d
commit a221a8f4a0deb6619448e9d2df696625d454ab3d
Author: John Baldwin <j...@freebsd.org>
AuthorDate: 2022-01-11 22:20:32 +0000
Commit: John Baldwin <j...@freebsd.org>
CommitDate: 2022-01-11 22:20:32 +0000
cryptosoft: Use multi-block encrypt/decrypt for AES-GCM.
Reviewed by: markj
Sponsored by: The FreeBSD Foundation
Differential Revision: https://reviews.freebsd.org/D33756
---
sys/opencrypto/cryptosoft.c | 130 +++++++++++++++++++++++++++++++-------------
1 file changed, 91 insertions(+), 39 deletions(-)
diff --git a/sys/opencrypto/cryptosoft.c b/sys/opencrypto/cryptosoft.c
index 864873944fd4..36081589cc65 100644
--- a/sys/opencrypto/cryptosoft.c
+++ b/sys/opencrypto/cryptosoft.c
@@ -411,12 +411,12 @@ swcr_gcm(const struct swcr_session *ses, struct cryptop
*crp)
struct crypto_buffer_cursor cc_in, cc_out;
const u_char *inblk;
u_char *outblk;
+ size_t inlen, outlen, todo;
const struct swcr_auth *swa;
const struct swcr_encdec *swe;
const struct enc_xform *exf;
void *ctx;
uint32_t *blkp;
- size_t len;
int blksz, error, ivlen, r, resid;
swa = &ses->swcr_auth;
@@ -441,30 +441,30 @@ swcr_gcm(const struct swcr_session *ses, struct cryptop
*crp)
/* Supply MAC with AAD */
if (crp->crp_aad != NULL) {
- len = rounddown(crp->crp_aad_length, blksz);
- if (len != 0)
- exf->update(ctx, crp->crp_aad, len);
- if (crp->crp_aad_length != len) {
+ inlen = rounddown2(crp->crp_aad_length, blksz);
+ if (inlen != 0)
+ exf->update(ctx, crp->crp_aad, inlen);
+ if (crp->crp_aad_length != inlen) {
memset(blk, 0, blksz);
- memcpy(blk, (char *)crp->crp_aad + len,
- crp->crp_aad_length - len);
+ memcpy(blk, (char *)crp->crp_aad + inlen,
+ crp->crp_aad_length - inlen);
exf->update(ctx, blk, blksz);
}
} else {
crypto_cursor_init(&cc_in, &crp->crp_buf);
crypto_cursor_advance(&cc_in, crp->crp_aad_start);
for (resid = crp->crp_aad_length; resid >= blksz;
- resid -= len) {
- inblk = crypto_cursor_segment(&cc_in, &len);
- if (len >= blksz) {
- len = rounddown(MIN(len, resid), blksz);
- crypto_cursor_advance(&cc_in, len);
+ resid -= inlen) {
+ inblk = crypto_cursor_segment(&cc_in, &inlen);
+ if (inlen >= blksz) {
+ inlen = rounddown2(MIN(inlen, resid), blksz);
+ crypto_cursor_advance(&cc_in, inlen);
} else {
- len = blksz;
- crypto_cursor_copydata(&cc_in, len, blk);
+ inlen = blksz;
+ crypto_cursor_copydata(&cc_in, inlen, blk);
inblk = blk;
}
- exf->update(ctx, inblk, len);
+ exf->update(ctx, inblk, inlen);
}
if (resid > 0) {
memset(blk, 0, blksz);
@@ -476,31 +476,57 @@ swcr_gcm(const struct swcr_session *ses, struct cryptop
*crp)
/* Do encryption with MAC */
crypto_cursor_init(&cc_in, &crp->crp_buf);
crypto_cursor_advance(&cc_in, crp->crp_payload_start);
+ inblk = crypto_cursor_segment(&cc_in, &inlen);
if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
crypto_cursor_init(&cc_out, &crp->crp_obuf);
crypto_cursor_advance(&cc_out, crp->crp_payload_output_start);
} else
cc_out = cc_in;
- for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) {
- inblk = crypto_cursor_segment(&cc_in, &len);
- if (len < blksz) {
+ outblk = crypto_cursor_segment(&cc_out, &outlen);
+
+ for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) {
+ if (inlen < blksz) {
crypto_cursor_copydata(&cc_in, blksz, blk);
inblk = blk;
- } else {
- crypto_cursor_advance(&cc_in, blksz);
+ inlen = blksz;
}
+
if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
- outblk = crypto_cursor_segment(&cc_out, &len);
- if (len < blksz)
+ if (outlen < blksz) {
outblk = blk;
- exf->encrypt(ctx, inblk, outblk);
- exf->update(ctx, outblk, blksz);
- if (outblk == blk)
+ outlen = blksz;
+ }
+
+ todo = rounddown2(MIN(resid, MIN(inlen, outlen)),
+ blksz);
+
+ exf->encrypt_multi(ctx, inblk, outblk, todo);
+ exf->update(ctx, outblk, todo);
+
+ if (outblk == blk) {
crypto_cursor_copyback(&cc_out, blksz, blk);
- else
- crypto_cursor_advance(&cc_out, blksz);
+ outblk = crypto_cursor_segment(&cc_out,
&outlen);
+ } else {
+ crypto_cursor_advance(&cc_out, todo);
+ outlen -= todo;
+ outblk += todo;
+ if (outlen == 0)
+ outblk = crypto_cursor_segment(&cc_out,
+ &outlen);
+ }
} else {
- exf->update(ctx, inblk, blksz);
+ todo = rounddown2(MIN(resid, inlen), blksz);
+ exf->update(ctx, inblk, todo);
+ }
+
+ if (inblk == blk) {
+ inblk = crypto_cursor_segment(&cc_in, &inlen);
+ } else {
+ crypto_cursor_advance(&cc_in, todo);
+ inlen -= todo;
+ inblk += todo;
+ if (inlen == 0)
+ inblk = crypto_cursor_segment(&cc_in, &inlen);
}
}
if (resid > 0) {
@@ -537,22 +563,48 @@ swcr_gcm(const struct swcr_session *ses, struct cryptop
*crp)
/* tag matches, decrypt data */
crypto_cursor_init(&cc_in, &crp->crp_buf);
crypto_cursor_advance(&cc_in, crp->crp_payload_start);
+ inblk = crypto_cursor_segment(&cc_in, &inlen);
+
for (resid = crp->crp_payload_length; resid > blksz;
- resid -= blksz) {
- inblk = crypto_cursor_segment(&cc_in, &len);
- if (len < blksz) {
+ resid -= todo) {
+ if (inlen < blksz) {
crypto_cursor_copydata(&cc_in, blksz, blk);
inblk = blk;
- } else
- crypto_cursor_advance(&cc_in, blksz);
- outblk = crypto_cursor_segment(&cc_out, &len);
- if (len < blksz)
+ inlen = blksz;
+ }
+ if (outlen < blksz) {
outblk = blk;
- exf->decrypt(ctx, inblk, outblk);
- if (outblk == blk)
+ outlen = blksz;
+ }
+
+ todo = rounddown2(MIN(resid, MIN(inlen, outlen)),
+ blksz);
+
+ exf->decrypt_multi(ctx, inblk, outblk, todo);
+
+ if (inblk == blk) {
+ inblk = crypto_cursor_segment(&cc_in, &inlen);
+ } else {
+ crypto_cursor_advance(&cc_in, todo);
+ inlen -= todo;
+ inblk += todo;
+ if (inlen == 0)
+ inblk = crypto_cursor_segment(&cc_in,
+ &inlen);
+ }
+
+ if (outblk == blk) {
crypto_cursor_copyback(&cc_out, blksz, blk);
- else
- crypto_cursor_advance(&cc_out, blksz);
+ outblk = crypto_cursor_segment(&cc_out,
+ &outlen);
+ } else {
+ crypto_cursor_advance(&cc_out, todo);
+ outlen -= todo;
+ outblk += todo;
+ if (outlen == 0)
+ outblk = crypto_cursor_segment(&cc_out,
+ &outlen);
+ }
}
if (resid > 0) {
crypto_cursor_copydata(&cc_in, resid, blk);
