Author: kib
Date: Sun Nov 21 13:20:14 2010
New Revision: 215633
URL: http://svn.freebsd.org/changeset/base/215633
Log:
Bring aesni(4) driver from HEAD.
MFC r210409:
Crypto(4) driver for AESNI.
MFC r213064 (by pjd):
Simplify code a bit.
MFC r213066 (by pjd):
Add support for CRD_F_KEY_EXPLICIT flag.
MFC r213069 (by pjd):
Add support for CRYPTO_AES_XTS.
MFC r213166 (by pjd):
Fix two copy&paste bugs.
MFC r213797 (by dim):
Change two missed instances of 'retq' in aeskeys_i386.S to 'retl'.
MFC r215427:
Only save FPU context when not executing in the context of the crypto
thread.
Added:
stable/8/sys/crypto/aesni/
- copied from r210409, head/sys/crypto/aesni/
stable/8/sys/modules/aesni/
- copied from r210409, head/sys/modules/aesni/
Modified:
stable/8/sys/conf/files.amd64
stable/8/sys/conf/files.i386
stable/8/sys/crypto/aesni/aeskeys_i386.S
stable/8/sys/crypto/aesni/aesni.c
stable/8/sys/crypto/aesni/aesni.h
stable/8/sys/crypto/aesni/aesni_wrap.c
stable/8/sys/modules/Makefile
stable/8/sys/modules/aesni/Makefile
Directory Properties:
stable/8/sys/ (props changed)
stable/8/sys/amd64/include/xen/ (props changed)
stable/8/sys/cddl/contrib/opensolaris/ (props changed)
stable/8/sys/contrib/dev/acpica/ (props changed)
stable/8/sys/contrib/pf/ (props changed)
stable/8/sys/dev/xen/xenpci/ (props changed)
Modified: stable/8/sys/conf/files.amd64
==============================================================================
--- stable/8/sys/conf/files.amd64 Sun Nov 21 13:11:50 2010
(r215632)
+++ stable/8/sys/conf/files.amd64 Sun Nov 21 13:20:14 2010
(r215633)
@@ -136,6 +136,10 @@ amd64/amd64/uma_machdep.c standard
amd64/amd64/vm_machdep.c standard
amd64/pci/pci_bus.c optional pci
amd64/pci/pci_cfgreg.c optional pci
+crypto/aesni/aesencdec_amd64.S optional aesni
+crypto/aesni/aeskeys_amd64.S optional aesni
+crypto/aesni/aesni.c optional aesni
+crypto/aesni/aesni_wrap.c optional aesni
crypto/blowfish/bf_enc.c optional crypto | ipsec
crypto/des/des_enc.c optional crypto | ipsec | netsmb
crypto/via/padlock.c optional padlock
Modified: stable/8/sys/conf/files.i386
==============================================================================
--- stable/8/sys/conf/files.i386 Sun Nov 21 13:11:50 2010
(r215632)
+++ stable/8/sys/conf/files.i386 Sun Nov 21 13:20:14 2010
(r215633)
@@ -112,6 +112,10 @@ bf_enc.o optional crypto | ipsec \
dependency "$S/crypto/blowfish/arch/i386/bf_enc.S
$S/crypto/blowfish/arch/i386/bf_enc_586.S
$S/crypto/blowfish/arch/i386/bf_enc_686.S" \
compile-with "${CC} -c -I$S/crypto/blowfish/arch/i386 ${ASM_CFLAGS}
${WERROR} ${.IMPSRC}" \
no-implicit-rule
+crypto/aesni/aesencdec_i386.S optional aesni
+crypto/aesni/aeskeys_i386.S optional aesni
+crypto/aesni/aesni.c optional aesni
+crypto/aesni/aesni_wrap.c optional aesni
crypto/des/arch/i386/des_enc.S optional crypto | ipsec | netsmb
crypto/via/padlock.c optional padlock
crypto/via/padlock_cipher.c optional padlock
Modified: stable/8/sys/crypto/aesni/aeskeys_i386.S
==============================================================================
--- head/sys/crypto/aesni/aeskeys_i386.S Fri Jul 23 11:00:46 2010
(r210409)
+++ stable/8/sys/crypto/aesni/aeskeys_i386.S Sun Nov 21 13:20:14 2010
(r215633)
@@ -52,7 +52,7 @@ _key_expansion_256a:
pxor %xmm1,%xmm0
movaps %xmm0,(%edx)
addl $0x10,%edx
- retq
+ retl
.cfi_endproc
END(_key_expansion_128)
@@ -76,7 +76,7 @@ ENTRY(_key_expansion_192a)
shufps $0b01001110,%xmm2,%xmm1
movaps %xmm1,0x10(%edx)
addl $0x20,%edx
- retq
+ retl
.cfi_endproc
END(_key_expansion_192a)
Modified: stable/8/sys/crypto/aesni/aesni.c
==============================================================================
--- head/sys/crypto/aesni/aesni.c Fri Jul 23 11:00:46 2010
(r210409)
+++ stable/8/sys/crypto/aesni/aesni.c Sun Nov 21 13:20:14 2010
(r215633)
@@ -69,14 +69,12 @@ aesni_identify(driver_t *drv, device_t p
static int
aesni_probe(device_t dev)
{
- char capp[32];
if ((cpu_feature2 & CPUID2_AESNI) == 0) {
device_printf(dev, "No AESNI support.\n");
return (EINVAL);
}
- strlcpy(capp, "AES-CBC", sizeof(capp));
- device_set_desc_copy(dev, capp);
+ device_set_desc_copy(dev, "AES-CBC,AES-XTS");
return (0);
}
@@ -96,6 +94,7 @@ aesni_attach(device_t dev)
rw_init(&sc->lock, "aesni_lock");
crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0);
+ crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0);
return (0);
}
@@ -142,6 +141,7 @@ aesni_newsession(device_t dev, uint32_t
for (; cri != NULL; cri = cri->cri_next) {
switch (cri->cri_alg) {
case CRYPTO_AES_CBC:
+ case CRYPTO_AES_XTS:
if (encini != NULL)
return (EINVAL);
encini = cri;
@@ -174,6 +174,7 @@ aesni_newsession(device_t dev, uint32_t
ses->used = 1;
TAILQ_INSERT_TAIL(&sc->sessions, ses, next);
rw_wunlock(&sc->lock);
+ ses->algo = encini->cri_alg;
error = aesni_cipher_setup(ses, encini);
if (error != 0) {
@@ -245,6 +246,7 @@ aesni_process(device_t dev, struct crypt
for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
switch (crd->crd_alg) {
case CRYPTO_AES_CBC:
+ case CRYPTO_AES_XTS:
if (enccrd != NULL) {
error = EINVAL;
goto out;
Modified: stable/8/sys/crypto/aesni/aesni.h
==============================================================================
--- head/sys/crypto/aesni/aesni.h Fri Jul 23 11:00:46 2010
(r210409)
+++ stable/8/sys/crypto/aesni/aesni.h Sun Nov 21 13:20:14 2010
(r215633)
@@ -55,7 +55,9 @@
struct aesni_session {
uint8_t enc_schedule[AES_SCHED_LEN] __aligned(16);
uint8_t dec_schedule[AES_SCHED_LEN] __aligned(16);
+ uint8_t xts_schedule[AES_SCHED_LEN] __aligned(16);
uint8_t iv[AES_BLOCK_LEN];
+ int algo;
int rounds;
/* uint8_t *ses_ictx; */
/* uint8_t *ses_octx; */
Modified: stable/8/sys/crypto/aesni/aesni_wrap.c
==============================================================================
--- head/sys/crypto/aesni/aesni_wrap.c Fri Jul 23 11:00:46 2010
(r210409)
+++ stable/8/sys/crypto/aesni/aesni_wrap.c Sun Nov 21 13:20:14 2010
(r215633)
@@ -1,5 +1,6 @@
/*-
* Copyright (c) 2010 Konstantin Belousov <k...@freebsd.org>
+ * Copyright (c) 2010 Pawel Jakub Dawidek <p...@freebsd.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -105,35 +106,161 @@ aesni_decrypt_ecb(int rounds, const void
}
}
-int
-aesni_cipher_setup(struct aesni_session *ses, struct cryptoini *encini)
+#define AES_XTS_BLOCKSIZE 16
+#define AES_XTS_IVSIZE 8
+#define AES_XTS_ALPHA 0x87 /* GF(2^128) generator
polynomial */
+
+static void
+aesni_crypt_xts_block(int rounds, const void *key_schedule, uint8_t *tweak,
+ const uint8_t *from, uint8_t *to, int do_encrypt)
{
- struct thread *td;
- int error;
+ uint8_t block[AES_XTS_BLOCKSIZE];
+ u_int i, carry_in, carry_out;
- switch (encini->cri_klen) {
- case 128:
- ses->rounds = AES128_ROUNDS;
- break;
- case 192:
- ses->rounds = AES192_ROUNDS;
+ for (i = 0; i < AES_XTS_BLOCKSIZE; i++)
+ block[i] = from[i] ^ tweak[i];
+
+ if (do_encrypt)
+ aesni_enc(rounds - 1, key_schedule, block, to, NULL);
+ else
+ aesni_dec(rounds - 1, key_schedule, block, to, NULL);
+
+ for (i = 0; i < AES_XTS_BLOCKSIZE; i++)
+ to[i] ^= tweak[i];
+
+ /* Exponentiate tweak. */
+ carry_in = 0;
+ for (i = 0; i < AES_XTS_BLOCKSIZE; i++) {
+ carry_out = tweak[i] & 0x80;
+ tweak[i] = (tweak[i] << 1) | (carry_in ? 1 : 0);
+ carry_in = carry_out;
+ }
+ if (carry_in)
+ tweak[0] ^= AES_XTS_ALPHA;
+ bzero(block, sizeof(block));
+}
+
+static void
+aesni_crypt_xts(int rounds, const void *data_schedule,
+ const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to,
+ const uint8_t iv[AES_BLOCK_LEN], int do_encrypt)
+{
+ uint8_t tweak[AES_XTS_BLOCKSIZE];
+ uint64_t blocknum;
+ size_t i;
+
+ /*
+ * Prepare tweak as E_k2(IV). IV is specified as LE representation
+ * of a 64-bit block number which we allow to be passed in directly.
+ */
+ bcopy(iv, &blocknum, AES_XTS_IVSIZE);
+ for (i = 0; i < AES_XTS_IVSIZE; i++) {
+ tweak[i] = blocknum & 0xff;
+ blocknum >>= 8;
+ }
+ /* Last 64 bits of IV are always zero. */
+ bzero(tweak + AES_XTS_IVSIZE, AES_XTS_IVSIZE);
+ aesni_enc(rounds - 1, tweak_schedule, tweak, tweak, NULL);
+
+ len /= AES_XTS_BLOCKSIZE;
+ for (i = 0; i < len; i++) {
+ aesni_crypt_xts_block(rounds, data_schedule, tweak, from, to,
+ do_encrypt);
+ from += AES_XTS_BLOCKSIZE;
+ to += AES_XTS_BLOCKSIZE;
+ }
+
+ bzero(tweak, sizeof(tweak));
+}
+
+static void
+aesni_encrypt_xts(int rounds, const void *data_schedule,
+ const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to,
+ const uint8_t iv[AES_BLOCK_LEN])
+{
+
+ aesni_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to,
+ iv, 1);
+}
+
+static void
+aesni_decrypt_xts(int rounds, const void *data_schedule,
+ const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to,
+ const uint8_t iv[AES_BLOCK_LEN])
+{
+
+ aesni_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to,
+ iv, 0);
+}
+
+static int
+aesni_cipher_setup_common(struct aesni_session *ses, const uint8_t *key,
+ int keylen)
+{
+
+ switch (ses->algo) {
+ case CRYPTO_AES_CBC:
+ switch (keylen) {
+ case 128:
+ ses->rounds = AES128_ROUNDS;
+ break;
+ case 192:
+ ses->rounds = AES192_ROUNDS;
+ break;
+ case 256:
+ ses->rounds = AES256_ROUNDS;
+ break;
+ default:
+ return (EINVAL);
+ }
break;
- case 256:
- ses->rounds = AES256_ROUNDS;
+ case CRYPTO_AES_XTS:
+ switch (keylen) {
+ case 256:
+ ses->rounds = AES128_ROUNDS;
+ break;
+ case 512:
+ ses->rounds = AES256_ROUNDS;
+ break;
+ default:
+ return (EINVAL);
+ }
break;
default:
return (EINVAL);
}
+ aesni_set_enckey(key, ses->enc_schedule, ses->rounds);
+ aesni_set_deckey(ses->enc_schedule, ses->dec_schedule, ses->rounds);
+ if (ses->algo == CRYPTO_AES_CBC)
+ arc4rand(ses->iv, sizeof(ses->iv), 0);
+ else /* if (ses->algo == CRYPTO_AES_XTS) */ {
+ aesni_set_enckey(key + keylen / 16, ses->xts_schedule,
+ ses->rounds);
+ }
+
+ return (0);
+}
+
+int
+aesni_cipher_setup(struct aesni_session *ses, struct cryptoini *encini)
+{
+ struct thread *td;
+ int error, saved_ctx;
+
td = curthread;
- error = fpu_kern_enter(td, &ses->fpu_ctx, FPU_KERN_NORMAL);
+ if (!is_fpu_kern_thread(0)) {
+ error = fpu_kern_enter(td, &ses->fpu_ctx, FPU_KERN_NORMAL);
+ saved_ctx = 1;
+ } else {
+ error = 0;
+ saved_ctx = 0;
+ }
if (error == 0) {
- aesni_set_enckey(encini->cri_key, ses->enc_schedule,
- ses->rounds);
- aesni_set_deckey(ses->enc_schedule, ses->dec_schedule,
- ses->rounds);
- arc4rand(ses->iv, sizeof(ses->iv), 0);
- fpu_kern_leave(td, &ses->fpu_ctx);
+ error = aesni_cipher_setup_common(ses, encini->cri_key,
+ encini->cri_klen);
+ if (saved_ctx)
+ fpu_kern_leave(td, &ses->fpu_ctx);
}
return (error);
}
@@ -144,39 +271,61 @@ aesni_cipher_process(struct aesni_sessio
{
struct thread *td;
uint8_t *buf;
- int error, allocated;
+ int error, allocated, saved_ctx;
buf = aesni_cipher_alloc(enccrd, crp, &allocated);
- if (buf == NULL) {
- error = ENOMEM;
- goto out;
- }
+ if (buf == NULL)
+ return (ENOMEM);
td = curthread;
- error = fpu_kern_enter(td, &ses->fpu_ctx, FPU_KERN_NORMAL);
- if (error != 0)
- goto out1;
+ if (!is_fpu_kern_thread(0)) {
+ error = fpu_kern_enter(td, &ses->fpu_ctx, FPU_KERN_NORMAL);
+ if (error != 0)
+ goto out;
+ saved_ctx = 1;
+ } else {
+ saved_ctx = 0;
+ error = 0;
+ }
+
+ if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) {
+ error = aesni_cipher_setup_common(ses, enccrd->crd_key,
+ enccrd->crd_klen);
+ if (error != 0)
+ goto out;
+ }
if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) {
if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
bcopy(enccrd->crd_iv, ses->iv, AES_BLOCK_LEN);
-
if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0)
crypto_copyback(crp->crp_flags, crp->crp_buf,
enccrd->crd_inject, AES_BLOCK_LEN, ses->iv);
-
- aesni_encrypt_cbc(ses->rounds, ses->enc_schedule,
- enccrd->crd_len, buf, buf, ses->iv);
+ if (ses->algo == CRYPTO_AES_CBC) {
+ aesni_encrypt_cbc(ses->rounds, ses->enc_schedule,
+ enccrd->crd_len, buf, buf, ses->iv);
+ } else /* if (ses->algo == CRYPTO_AES_XTS) */ {
+ aesni_encrypt_xts(ses->rounds, ses->enc_schedule,
+ ses->xts_schedule, enccrd->crd_len, buf, buf,
+ ses->iv);
+ }
} else {
if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
bcopy(enccrd->crd_iv, ses->iv, AES_BLOCK_LEN);
else
crypto_copydata(crp->crp_flags, crp->crp_buf,
enccrd->crd_inject, AES_BLOCK_LEN, ses->iv);
- aesni_decrypt_cbc(ses->rounds, ses->dec_schedule,
- enccrd->crd_len, buf, ses->iv);
+ if (ses->algo == CRYPTO_AES_CBC) {
+ aesni_decrypt_cbc(ses->rounds, ses->dec_schedule,
+ enccrd->crd_len, buf, ses->iv);
+ } else /* if (ses->algo == CRYPTO_AES_XTS) */ {
+ aesni_decrypt_xts(ses->rounds, ses->dec_schedule,
+ ses->xts_schedule, enccrd->crd_len, buf, buf,
+ ses->iv);
+ }
}
- fpu_kern_leave(td, &ses->fpu_ctx);
+ if (saved_ctx)
+ fpu_kern_leave(td, &ses->fpu_ctx);
if (allocated)
crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
enccrd->crd_len, buf);
@@ -184,11 +333,10 @@ aesni_cipher_process(struct aesni_sessio
crypto_copydata(crp->crp_flags, crp->crp_buf,
enccrd->crd_skip + enccrd->crd_len - AES_BLOCK_LEN,
AES_BLOCK_LEN, ses->iv);
- out1:
+ out:
if (allocated) {
bzero(buf, enccrd->crd_len);
free(buf, M_AESNI);
}
- out:
return (error);
}
Modified: stable/8/sys/modules/Makefile
==============================================================================
--- stable/8/sys/modules/Makefile Sun Nov 21 13:11:50 2010
(r215632)
+++ stable/8/sys/modules/Makefile Sun Nov 21 13:20:14 2010
(r215633)
@@ -10,6 +10,7 @@ SUBDIR= ${_3dfx} \
accf_http \
${_acpi} \
ae \
+ ${_aesni} \
age \
${_agp} \
aha \
@@ -433,6 +434,9 @@ _zfs= zfs
.if ${MACHINE} == "i386"
_aac= aac
_acpi= acpi
+.if ${MK_CRYPT} != "no" || defined(ALL_MODULES)
+_aesni= aesni
+.endif
_ahb= ahb
_amdsbwd= amdsbwd
_amdtemp= amdtemp
@@ -489,6 +493,9 @@ _snc= snc
.if ${MACHINE_ARCH} == "amd64"
_aac= aac
_acpi= acpi
+.if ${MK_CRYPT} != "no" || defined(ALL_MODULES)
+_aesni= aesni
+.endif
_agp= agp
_an= an
_amdsbwd= amdsbwd
Modified: stable/8/sys/modules/aesni/Makefile
==============================================================================
--- head/sys/modules/aesni/Makefile Fri Jul 23 11:00:46 2010
(r210409)
+++ stable/8/sys/modules/aesni/Makefile Sun Nov 21 13:20:14 2010
(r215633)
@@ -4,7 +4,7 @@
KMOD= aesni
SRCS= aesni.c aesni_wrap.c
-SRCS+= aesencdec_$(MACHINE_CPUARCH).S aeskeys_$(MACHINE_CPUARCH).S
+SRCS+= aesencdec_$(MACHINE_ARCH).S aeskeys_$(MACHINE_ARCH).S
SRCS+= device_if.h bus_if.h opt_bus.h cryptodev_if.h
.include <bsd.kmod.mk>
_______________________________________________
svn-src-all@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
