The branch stable/13 has been updated by jhb: URL: https://cgit.FreeBSD.org/src/commit/?id=6ca6d4edc234cee9e3c85b058d8b0adda0a2026f
commit 6ca6d4edc234cee9e3c85b058d8b0adda0a2026f Author: John Baldwin <[email protected]> AuthorDate: 2021-03-03 23:17:43 +0000 Commit: John Baldwin <[email protected]> CommitDate: 2021-10-21 15:51:24 +0000 ossl: Add ChaCha20 cipher support. Sponsored by: Netflix Differential Revision: https://reviews.freebsd.org/D28756 (cherry picked from commit 92aecd1e6fac47ffc893f628c1fe289568bb19cb) --- share/man/man4/ossl.4 | 2 + sys/conf/files | 1 + sys/conf/files.amd64 | 1 + sys/conf/files.arm64 | 2 + sys/conf/files.i386 | 1 + sys/crypto/openssl/ossl.c | 82 ++++++++++++++++----- sys/crypto/openssl/ossl.h | 5 ++ sys/crypto/openssl/ossl_chacha.h | 42 +++++++++++ sys/crypto/openssl/ossl_chacha20.c | 141 +++++++++++++++++++++++++++++++++++++ sys/modules/ossl/Makefile | 4 ++ 10 files changed, 262 insertions(+), 19 deletions(-) diff --git a/share/man/man4/ossl.4 b/share/man/man4/ossl.4 index 9c0d7f897d53..2aa4b69eda31 100644 --- a/share/man/man4/ossl.4 +++ b/share/man/man4/ossl.4 @@ -74,6 +74,8 @@ driver includes support for the following algorithms: .Pp .Bl -bullet -compact .It +ChaCha20 +.It Poly1305 .It SHA1 diff --git a/sys/conf/files b/sys/conf/files index e680382210cd..a5f23ca09774 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -741,6 +741,7 @@ crypto/chacha20/chacha-sw.c optional crypto | ipsec | ipsec_support crypto/des/des_ecb.c optional netsmb crypto/des/des_setkey.c optional netsmb crypto/openssl/ossl.c optional ossl +crypto/openssl/ossl_chacha20.c optional ossl crypto/openssl/ossl_poly1305.c optional ossl crypto/openssl/ossl_sha1.c optional ossl crypto/openssl/ossl_sha256.c optional ossl diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64 index f6271d0e0e92..bf2213eec68c 100644 --- a/sys/conf/files.amd64 +++ b/sys/conf/files.amd64 @@ -139,6 +139,7 @@ cddl/dev/dtrace/amd64/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/amd64/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" crypto/aesni/aeskeys_amd64.S optional aesni crypto/des/des_enc.c optional netsmb +crypto/openssl/amd64/chacha-x86_64.S optional ossl crypto/openssl/amd64/poly1305-x86_64.S optional ossl crypto/openssl/amd64/sha1-x86_64.S optional ossl crypto/openssl/amd64/sha256-x86_64.S optional ossl diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64 index d6c9586f58f7..84fd0c7858ff 100644 --- a/sys/conf/files.arm64 +++ b/sys/conf/files.arm64 @@ -127,6 +127,8 @@ ghashv8-armx.o optional armv8crypto \ crypto/des/des_enc.c optional netsmb crypto/openssl/ossl_aarch64.c optional ossl +crypto/openssl/aarch64/chacha-armv8.S optional ossl \ + compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${PROF} ${.IMPSRC}" crypto/openssl/aarch64/poly1305-armv8.S optional ossl \ compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${PROF} ${.IMPSRC}" crypto/openssl/aarch64/sha1-armv8.S optional ossl \ diff --git a/sys/conf/files.i386 b/sys/conf/files.i386 index 5238e2bab40a..7beb3f21292c 100644 --- a/sys/conf/files.i386 +++ b/sys/conf/files.i386 @@ -77,6 +77,7 @@ compat/linux/linux.c optional compat_linux compat/ndis/winx32_wrap.S optional ndisapi pci crypto/aesni/aeskeys_i386.S optional aesni crypto/des/arch/i386/des_enc.S optional netsmb +crypto/openssl/i386/chacha-x86.S optional ossl crypto/openssl/i386/poly1305-x86.S optional ossl crypto/openssl/i386/sha1-586.S optional ossl crypto/openssl/i386/sha256-586.S optional ossl diff --git a/sys/crypto/openssl/ossl.c b/sys/crypto/openssl/ossl.c index 229729c27c21..0c863429939c 100644 --- a/sys/crypto/openssl/ossl.c +++ b/sys/crypto/openssl/ossl.c @@ -48,6 +48,7 @@ __FBSDID("$FreeBSD$"); #include <opencrypto/xform_auth.h> #include <crypto/openssl/ossl.h> +#include <crypto/openssl/ossl_chacha.h> #include "cryptodev_if.h" @@ -154,6 +155,16 @@ ossl_probesession(device_t dev, const struct crypto_session_params *csp) if (ossl_lookup_hash(csp) == NULL) return (EINVAL); break; + case CSP_MODE_CIPHER: + switch (csp->csp_cipher_alg) { + case CRYPTO_CHACHA20: + if (csp->csp_cipher_klen != CHACHA_KEY_SIZE) + return (EINVAL); + break; + default: + return (EINVAL); + } + break; default: return (EINVAL); } @@ -161,15 +172,12 @@ ossl_probesession(device_t dev, const struct crypto_session_params *csp) return (CRYPTODEV_PROBE_ACCEL_SOFTWARE); } -static int -ossl_newsession(device_t dev, crypto_session_t cses, +static void +ossl_newsession_hash(struct ossl_session *s, const struct crypto_session_params *csp) { - struct ossl_session *s; struct auth_hash *axf; - s = crypto_get_driver_session(cses); - axf = ossl_lookup_hash(csp); s->hash.axf = axf; if (csp->csp_auth_mlen == 0) @@ -195,31 +203,35 @@ ossl_newsession(device_t dev, crypto_session_t cses, fpu_kern_leave(curthread, NULL); } } +} + +static int +ossl_newsession(device_t dev, crypto_session_t cses, + const struct crypto_session_params *csp) +{ + struct ossl_session *s; + + s = crypto_get_driver_session(cses); + switch (csp->csp_mode) { + case CSP_MODE_DIGEST: + ossl_newsession_hash(s, csp); + break; + } + return (0); } static int -ossl_process(device_t dev, struct cryptop *crp, int hint) +ossl_process_hash(struct ossl_session *s, struct cryptop *crp, + const struct crypto_session_params *csp) { struct ossl_hash_context ctx; char digest[HASH_MAX_LEN]; - const struct crypto_session_params *csp; - struct ossl_session *s; struct auth_hash *axf; int error; - bool fpu_entered; - s = crypto_get_driver_session(crp->crp_session); - csp = crypto_get_params(crp->crp_session); axf = s->hash.axf; - if (is_fpu_kern_thread(0)) { - fpu_entered = false; - } else { - fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX); - fpu_entered = true; - } - if (crp->crp_auth_key == NULL) { ctx = s->hash.ictx; } else { @@ -273,13 +285,45 @@ ossl_process(device_t dev, struct cryptop *crp, int hint) explicit_bzero(digest, sizeof(digest)); out: + explicit_bzero(&ctx, sizeof(ctx)); + return (error); +} + +static int +ossl_process(device_t dev, struct cryptop *crp, int hint) +{ + const struct crypto_session_params *csp; + struct ossl_session *s; + int error; + bool fpu_entered; + + s = crypto_get_driver_session(crp->crp_session); + csp = crypto_get_params(crp->crp_session); + + if (is_fpu_kern_thread(0)) { + fpu_entered = false; + } else { + fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX); + fpu_entered = true; + } + + switch (csp->csp_mode) { + case CSP_MODE_DIGEST: + error = ossl_process_hash(s, crp, csp); + break; + case CSP_MODE_CIPHER: + error = ossl_chacha20(crp, csp); + break; + default: + __assert_unreachable(); + } + if (fpu_entered) fpu_kern_leave(curthread, NULL); crp->crp_etype = error; crypto_done(crp); - explicit_bzero(&ctx, sizeof(ctx)); return (0); } diff --git a/sys/crypto/openssl/ossl.h b/sys/crypto/openssl/ossl.h index 55022b10f377..b7c681d0fb1d 100644 --- a/sys/crypto/openssl/ossl.h +++ b/sys/crypto/openssl/ossl.h @@ -34,6 +34,11 @@ /* Compatibility shims. */ #define OPENSSL_cleanse explicit_bzero +struct cryptop; +struct crypto_session_params; + +int ossl_chacha20(struct cryptop *crp, + const struct crypto_session_params *csp); void ossl_cpuid(void); /* Needs to be big enough to hold any hash context. */ diff --git a/sys/crypto/openssl/ossl_chacha.h b/sys/crypto/openssl/ossl_chacha.h new file mode 100644 index 000000000000..781899c6bdf6 --- /dev/null +++ b/sys/crypto/openssl/ossl_chacha.h @@ -0,0 +1,42 @@ +/* + * Copyright 2015-2018 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the OpenSSL license (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +/* From include/crypto/chacha.h */ + +#ifndef OSSL_CRYPTO_CHACHA_H +#define OSSL_CRYPTO_CHACHA_H + +/* + * ChaCha20_ctr32 encrypts |len| bytes from |inp| with the given key and + * nonce and writes the result to |out|, which may be equal to |inp|. + * The |key| is not 32 bytes of verbatim key material though, but the + * said material collected into 8 32-bit elements array in host byte + * order. Same approach applies to nonce: the |counter| argument is + * pointer to concatenated nonce and counter values collected into 4 + * 32-bit elements. This, passing crypto material collected into 32-bit + * elements as opposite to passing verbatim byte vectors, is chosen for + * efficiency in multi-call scenarios. + */ +void ChaCha20_ctr32(unsigned char *out, const unsigned char *inp, + size_t len, const unsigned int key[8], + const unsigned int counter[4]); +/* + * You can notice that there is no key setup procedure. Because it's + * as trivial as collecting bytes into 32-bit elements, it's reckoned + * that below macro is sufficient. + */ +#define CHACHA_U8TOU32(p) ( \ + ((unsigned int)(p)[0]) | ((unsigned int)(p)[1]<<8) | \ + ((unsigned int)(p)[2]<<16) | ((unsigned int)(p)[3]<<24) ) + +#define CHACHA_KEY_SIZE 32 +#define CHACHA_CTR_SIZE 16 +#define CHACHA_BLK_SIZE 64 + +#endif diff --git a/sys/crypto/openssl/ossl_chacha20.c b/sys/crypto/openssl/ossl_chacha20.c new file mode 100644 index 000000000000..70a0a5718dbd --- /dev/null +++ b/sys/crypto/openssl/ossl_chacha20.c @@ -0,0 +1,141 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2020 Netflix, Inc + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer, + * without modification. + * 2. Redistributions in binary form must reproduce at minimum a disclaimer + * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any + * redistribution must be conditioned upon including a substantially + * similar Disclaimer requirement for further binary redistribution. + * + * NO WARRANTY + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY + * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, + * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGES. + */ + +#include <sys/types.h> +#include <sys/endian.h> +#include <sys/malloc.h> +#include <sys/time.h> + +#include <opencrypto/cryptodev.h> + +#include <crypto/openssl/ossl.h> +#include <crypto/openssl/ossl_chacha.h> + +int +ossl_chacha20(struct cryptop *crp, const struct crypto_session_params *csp) +{ + _Alignas(8) unsigned int key[CHACHA_KEY_SIZE / 4]; + unsigned int counter[CHACHA_CTR_SIZE / 4]; + unsigned char block[CHACHA_BLK_SIZE]; + struct crypto_buffer_cursor cc_in, cc_out; + const unsigned char *in, *inseg, *cipher_key; + unsigned char *out, *outseg; + size_t resid, todo, inlen, outlen; + uint32_t next_counter; + u_int i; + + if (crp->crp_cipher_key != NULL) + cipher_key = crp->crp_cipher_key; + else + cipher_key = csp->csp_cipher_key; + for (i = 0; i < nitems(key); i++) + key[i] = CHACHA_U8TOU32(cipher_key + i * 4); + crypto_read_iv(crp, counter); + for (i = 0; i < nitems(counter); i++) + counter[i] = le32toh(counter[i]); + + resid = crp->crp_payload_length; + crypto_cursor_init(&cc_in, &crp->crp_buf); + crypto_cursor_advance(&cc_in, crp->crp_payload_start); + inseg = crypto_cursor_segbase(&cc_in); + inlen = crypto_cursor_seglen(&cc_in); + if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { + crypto_cursor_init(&cc_out, &crp->crp_obuf); + crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); + } else + cc_out = cc_in; + outseg = crypto_cursor_segbase(&cc_out); + outlen = crypto_cursor_seglen(&cc_out); + while (resid >= CHACHA_BLK_SIZE) { + if (inlen < CHACHA_BLK_SIZE) { + crypto_cursor_copydata(&cc_in, CHACHA_BLK_SIZE, block); + in = block; + inlen = CHACHA_BLK_SIZE; + } else + in = inseg; + if (outlen < CHACHA_BLK_SIZE) { + out = block; + outlen = CHACHA_BLK_SIZE; + } else + out = outseg; + + /* Figure out how many blocks we can encrypt/decrypt at once. */ + todo = rounddown(MIN(inlen, outlen), CHACHA_BLK_SIZE); + +#ifdef __LP64__ + /* ChaCha20_ctr32() assumes length is <= 4GB. */ + todo = (uint32_t)todo; +#endif + + /* Truncate if the 32-bit counter would roll over. */ + next_counter = counter[0] + todo / CHACHA_BLK_SIZE; + if (next_counter < counter[0]) { + todo -= next_counter * CHACHA_BLK_SIZE; + next_counter = 0; + } + + ChaCha20_ctr32(out, in, todo, key, counter); + + counter[0] = next_counter; + if (counter[0] == 0) + counter[1]++; + + if (out == block) { + crypto_cursor_copyback(&cc_out, CHACHA_BLK_SIZE, block); + outseg = crypto_cursor_segbase(&cc_out); + outlen = crypto_cursor_seglen(&cc_out); + } else { + crypto_cursor_advance(&cc_out, todo); + outseg += todo; + outlen -= todo; + } + if (in == block) { + inseg = crypto_cursor_segbase(&cc_in); + inlen = crypto_cursor_seglen(&cc_in); + } else { + crypto_cursor_advance(&cc_in, todo); + inseg += todo; + inlen -= todo; + } + resid -= todo; + } + + if (resid > 0) { + memset(block, 0, sizeof(block)); + crypto_cursor_copydata(&cc_in, resid, block); + ChaCha20_ctr32(block, block, CHACHA_BLK_SIZE, key, counter); + crypto_cursor_copyback(&cc_out, resid, block); + } + + explicit_bzero(block, sizeof(block)); + explicit_bzero(counter, sizeof(counter)); + explicit_bzero(key, sizeof(key)); + return (0); +} diff --git a/sys/modules/ossl/Makefile b/sys/modules/ossl/Makefile index 2ddebefebd1a..dfd82dcf6e1f 100644 --- a/sys/modules/ossl/Makefile +++ b/sys/modules/ossl/Makefile @@ -8,6 +8,7 @@ SRCS= bus_if.h \ cryptodev_if.h \ device_if.h \ ossl.c \ + ossl_chacha20.c \ ossl_poly1305.c \ ossl_sha1.c \ ossl_sha256.c \ @@ -15,6 +16,7 @@ SRCS= bus_if.h \ ${SRCS.${MACHINE_CPUARCH}} SRCS.aarch64= \ + chacha-armv8.S \ poly1305-armv8.S \ sha1-armv8.S \ sha256-armv8.S \ @@ -22,6 +24,7 @@ SRCS.aarch64= \ ossl_aarch64.c SRCS.amd64= \ + chacha-x86_64.S \ poly1305-x86_64.S \ sha1-x86_64.S \ sha256-x86_64.S \ @@ -29,6 +32,7 @@ SRCS.amd64= \ ossl_x86.c SRCS.i386= \ + chacha-x86.S \ poly1305-x86.S \ sha1-586.S \ sha256-586.S \
