tag 694091 + patch
tag 693460 + patch
thanks
Hi,
attached you can find a patch against bcrypt which makes it support
large files (> 2 GiB) and files which exceed the amount of available
RAM.
I tested this with a 128 MiB RAM VM:
root@squeezevm:/tmp# uname -a
Linux squeezevm 2.6.32-5-amd64 #1 SMP Sun Sep 23 10:07:46 UTC 2012 x86_64
GNU/Linux
root@squeezevm:/tmp# free -m
total used free shared buffers cached
Mem: 118 62 56 0 25 17
-/+ buffers/cache: 19 99
Swap: 382 0 382
root@squeezevm:/tmp# dd if=/dev/zero of=bigfile bs=10M count=20
20+0 records in
20+0 records out
209715200 bytes (210 MB) copied, 0.558276 s, 376 MB/s
root@squeezevm:/tmp# ./bcrypt -c bigfile
Encryption key:
Again:
root@squeezevm:/tmp# ls -hltr
total 201M
-rwxr-xr-x 1 root root 28K Nov 25 06:03 bcrypt
-rw-r--r-- 1 root root 201M Nov 25 06:04 bigfile.bfe
root@squeezevm:/tmp# free -m
total used free shared buffers cached
Mem: 118 75 43 0 21 34
-/+ buffers/cache: 20 98
Swap: 382 0 382
Also, here is a large file on an i686 (32-bit) machine:
root@stability /raid $ ls -l bigfile
-rw-r--r-- 1 root root 2621440000 2012-11-25 12:19 bigfile
root@stability /raid $ /home/michael/bcrypt-1.1/bcrypt -c bigfile
Encryption key:
Again:
/home/michael/bcrypt-1.1/bcrypt -c bigfile 188,96s user 1286,90s system 93%
cpu 26:11,79 total
root@stability /raid $ ls -l bigfile.bfe
-rw-r--r-- 1 root root 2621440066 2012-11-25 12:45 bigfile.bfe
root@stability /raid $ uname -a
Linux stability 2.6.30.1 #1 SMP Fri Jul 17 23:19:30 CEST 2009 i686
GNU/Linux
I have verified that the tool still works correctly by using a script.
The script runs 1000 iterations of each test, that
is, it tests that the new version can still read files created by the
old version, and it tests that the old version can still read files
created by the new version. You can find the script in the pull
request linked below.
I also tested that files created on big-endian machines can be
decrypted on little-endian machines and vice-versa.
I have also submitted this patch to the new upstream location at
https://github.com/casta/bcrypt/pull/1
--
Best regards,
Michael
diff -ur bcrypt-1.1.O/functions.h bcrypt-1.1/functions.h
--- bcrypt-1.1.O/functions.h 2002-09-12 11:09:38.000000000 +0200
+++ bcrypt-1.1/functions.h 2012-11-25 12:12:29.451956678 +0100
@@ -11,10 +11,8 @@
#include "blowfish.h"
/* from wrapbf.c */
-uLong BFEncrypt(char **input, char *key, uLong sz,
- BCoptions *options);
-uLong BFDecrypt(char **input, char *key, char *key2,
- uLong sz, BCoptions *options);
+off_t BFEncrypt(int infd, int outfd, char *key, BCoptions *options);
+off_t BFDecrypt(int infd, int outfd, char *key, char *key2, BCoptions *options);
/* from keys.c */
char * getkey(int type);
diff -ur bcrypt-1.1.O/main.c bcrypt-1.1/main.c
--- bcrypt-1.1.O/main.c 2002-09-12 11:50:12.000000000 +0200
+++ bcrypt-1.1/main.c 2012-11-25 12:12:41.835771398 +0100
@@ -6,6 +6,10 @@
* for details
* ====================================================================
*/
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
#include "includes.h"
#include "defines.h"
@@ -156,8 +160,8 @@
}
int main(int argc, char *argv[]) {
- uLong sz = 0;
- char *input = NULL, *key = NULL, *key2 = NULL;
+ off_t sz = 0;
+ char *key = NULL, *key2 = NULL;
char *infile = NULL, *outfile = NULL;
struct stat statbuf;
BCoptions options;
@@ -174,42 +178,35 @@
mutateKey(&key, &key2);
}
- sz = readfile(infile, &input, options.type, key, statbuf);
-
- if ((options.type == DECRYPT) && (testEndian(input)))
- swapCompressed(&input, sz);
+ int infd = open(infile, O_RDONLY);
+ if (infd == -1) {
+ perror("open(infile)");
+ continue;
+ }
- if ((options.compression == 1) && (options.type == ENCRYPT)) {
- options.origsize = sz;
- sz = docompress(&input, sz);
+ int outfd = open(outfile, O_CREAT | O_TRUNC | O_RDWR, 0644);
+ if (outfd == -1) {
+ perror("open(infile)");
+ continue;
}
if (options.type == ENCRYPT) {
- sz = attachKey(&input, key, sz);
- sz = padInput(&input, sz);
- }
+ options.origsize = statbuf.st_size;
- if (options.type == ENCRYPT)
- sz = BFEncrypt(&input, key, sz, &options);
- else if (options.type == DECRYPT)
- if ((sz = BFDecrypt(&input, key, key2, sz, &options)) == 0) {
+ sz = BFEncrypt(infd, outfd, key, &options);
+ } else {
+ if ((sz = BFDecrypt(infd, outfd, key, key2, &options)) == 0) {
fprintf(stderr, "Invalid encryption key for file: %s\n", infile);
exit(1);
}
+ ftruncate(outfd, sz);
+ }
- if ((options.compression == 1) && (options.type == DECRYPT))
- sz = douncompress(&input, sz, options);
-
- writefile(outfile, input, sz, options, statbuf);
-
- if ((input = realloc(input, sizeof(char *))) == NULL)
- memerror();
+ close(infd);
+ close(outfd);
if (options.remove == 1)
deletefile(infile, options, key, statbuf);
-
- if (input != NULL)
- free(input);
}
if(!sz)
diff -ur bcrypt-1.1.O/Makefile bcrypt-1.1/Makefile
--- bcrypt-1.1.O/Makefile 2012-11-25 12:31:43.000000000 +0100
+++ bcrypt-1.1/Makefile 2012-11-25 12:07:31.035957620 +0100
@@ -1,6 +1,6 @@
DEFAULTS = Makefile includes.h blowfish.h functions.h config.h
CC = gcc
-CFLAGS = -O2 -Wall
+CFLAGS = -O2 -Wall -D_FILE_OFFSET_BITS=64
COMPILE = ${CC} ${CFLAGS}
OBJS = main.o blowfish.o rwfile.o keys.o wrapbf.o endian.o wrapzl.o
LDFLAGS = -L/usr/local/lib -lz
diff -ur bcrypt-1.1.O/wrapbf.c bcrypt-1.1/wrapbf.c
--- bcrypt-1.1.O/wrapbf.c 2002-09-13 08:55:04.000000000 +0200
+++ bcrypt-1.1/wrapbf.c 2012-11-25 12:12:27.531985410 +0100
@@ -11,107 +11,244 @@
#include "defines.h"
#include "functions.h"
-uLong BFEncrypt(char **input, char *key, uLong sz, BCoptions *options) {
- uInt32 L, R;
- uLong i;
+#include <stdint.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+/* 256 KiB, as recommended by http://www.zlib.net/zlib_how.html */
+#define ZCHUNK (256 * 1024)
+
+off_t BFEncrypt(int infd, int outfd, char *key, BCoptions *options) {
BLOWFISH_CTX ctx;
- int j;
+ const int block_size = (sizeof(uint32_t) * 2);
unsigned char *myEndian = NULL;
- j = sizeof(uInt32);
+ uLong sz = 0;
+
+ /* zlib variables */
+ int ret, flush;
+ unsigned have;
+ z_stream strm;
+ uint8_t in[ZCHUNK];
+ uint8_t out[ZCHUNK];
+
+ strm.zalloc = Z_NULL;
+ strm.zfree = Z_NULL;
+ strm.opaque = Z_NULL;
+ ret = deflateInit(&strm, Z_DEFAULT_COMPRESSION);
+ if (ret != Z_OK)
+ exit(2);
+
getEndian(&myEndian);
- memmove(*input+2, *input, sz);
+ write(outfd, myEndian, sizeof(uint8_t));
+ write(outfd, &options->compression, sizeof(uint8_t));
+
+ Blowfish_Init (&ctx, (unsigned char*)key, MAXKEYBYTES);
- memcpy(*input, myEndian, 1);
- memcpy(*input+1, &options->compression, 1);
+ int n;
+ /* 2 * ZCHUNK for the overhead when compressing */
+ uint8_t rest[2 * ZCHUNK];
+ int restidx = 0;
+ uint32_t *rwalk = (uint32_t*)rest;
+ do {
+ if ((n = read(infd, in, ZCHUNK)) < 0) {
+ exit(3);
+ }
+ strm.avail_in = n;
+ strm.next_in = in;
+ flush = ((n == 0) ? Z_FINISH : Z_NO_FLUSH);
+ if (options->compression) {
+ do {
+ strm.avail_out = ZCHUNK;
+ strm.next_out = out;
+ ret = deflate(&strm, flush);
+
+ have = ZCHUNK - strm.avail_out;
+ memcpy(rest + restidx, out, have);
+ restidx += have;
+ sz += have;
+ } while (strm.avail_out == 0);
+ } else {
+ memcpy(rest + restidx, in, n);
+ restidx += n;
+ sz += n;
+ }
- sz += 2; /* add room for endian and compress flags */
+ /* Encrypt as many blocks (each block_size bytes big) as we can */
+ int i;
+ /* i adresses sizeof(uint32_t) = 4 byte blocks,
+ * but we use (restidx / block_size) loop up to full blocks */
+ for (i = 0; i < 2 * (restidx / block_size); i += 2) {
+ Blowfish_Encrypt(&ctx, &rwalk[i], &rwalk[i + 1]);
+ write(outfd, &rwalk[i], block_size);
+ }
+ if (i > 0) {
+ int remaining = restidx - (i * sizeof(uint32_t));
+ memmove(rest, &rwalk[i], remaining);
+ restidx = remaining;
+ }
+ } while (flush != Z_FINISH);
- Blowfish_Init (&ctx, key, MAXKEYBYTES);
+ /* Now append the key (MAXKEYBYTES bytes) */
+ memcpy(rest + restidx, key, MAXKEYBYTES);
+ restidx += MAXKEYBYTES;
+ sz += MAXKEYBYTES;
+
+ /* Pad the rest with 0-bytes */
+ int r;
+ if (sz >= block_size)
+ r = getremain(sz, block_size);
+ else
+ r = block_size - sz;
- for (i = 2; i < sz; i+=(j*2)) { /* start just after tags */
- memcpy(&L, *input+i, j);
- memcpy(&R, *input+i+j, j);
- Blowfish_Encrypt(&ctx, &L, &R);
- memcpy(*input+i, &L, j);
- memcpy(*input+i+j, &R, j);
+ memset(rest + restidx, '\0', r);
+ restidx += r;
+ sz += r;
+
+ /* Encrypt the rest */
+ int i;
+ for (i = 0; i < (restidx / sizeof(uint32_t)); i += 2) {
+ Blowfish_Encrypt(&ctx, &rwalk[i], &rwalk[i+1]);
+ write(outfd, rwalk + i, block_size);
}
- if (options->compression == 1) {
- if ((*input = realloc(*input, sz + j + 1)) == NULL)
- memerror();
-
- memset(*input+sz, 0, j + 1);
- memcpy(*input+sz, &options->origsize, j);
- sz += j; /* make room for the original size */
- }
+ if (options->compression)
+ write(outfd, &options->origsize, sizeof(uint32_t));
free(myEndian);
return(sz);
+
}
-uLong BFDecrypt(char **input, char *key, char *key2, uLong sz,
- BCoptions *options) {
- uInt32 L, R;
- uLong i;
+off_t BFDecrypt(int infd, int outfd, char *key, char *key2, BCoptions *options) {
BLOWFISH_CTX ctx;
- int j, swap = 0;
- unsigned char *myEndian = NULL;
+ const int block_size = (sizeof(uint32_t) * 2);
char *mykey = NULL;
+ int swap = 0;
+
+ /* zlib variables */
+ int ret;
+ unsigned have;
+ z_stream strm;
+ uint8_t in[ZCHUNK];
+ uint8_t out[ZCHUNK];
+
+ strm.zalloc = Z_NULL;
+ strm.zfree = Z_NULL;
+ strm.opaque = Z_NULL;
+ strm.avail_in = 0;
+ strm.next_in = Z_NULL;
+ ret = inflateInit(&strm);
+ if (ret != Z_OK)
+ exit(2);
- j = sizeof(uInt32);
- if ((mykey = malloc(MAXKEYBYTES + 1)) == NULL)
+ if ((mykey = calloc(MAXKEYBYTES + 1, 1)) == NULL)
memerror();
- memset(mykey, 0, MAXKEYBYTES + 1);
+ uint8_t endianness, compressed;
+ read(infd, &endianness, sizeof(uint8_t));
+ read(infd, &compressed, sizeof(uint8_t));
- if ((swap = testEndian(*input)) == 1)
+ if ((swap = testEndian(&endianness)) == 1)
memcpy(mykey, key2, MAXKEYBYTES);
else
memcpy(mykey, key, MAXKEYBYTES);
- memcpy(&options->compression, *input+1, 1);
-
- if (options->compression == 1) {
- memcpy(&options->origsize, *input+(sz - j), j);
- sz -= j; /* dump the size tag */
- }
-
- sz -= 2; /* now dump endian and compress flags */
-
- Blowfish_Init (&ctx, mykey, MAXKEYBYTES);
+ Blowfish_Init (&ctx, (unsigned char*)mykey, MAXKEYBYTES);
- for (i = 0; i < sz; i+=(j*2)) {
- memcpy(&L, *input+i+2, j);
- memcpy(&R, *input+i+j+2, j);
-
- if (swap == 1) {
- L = swapEndian(L);
- R = swapEndian(R);
+ int n;
+ /* 2 * ZCHUNK for the overhead when compressing */
+ uint8_t rest[2 * ZCHUNK];
+ int restidx = 0;
+ uint32_t *inwalk = (uint32_t*)in;
+ do {
+ if ((n = read(infd, in, ZCHUNK)) < 0) {
+ exit(3);
}
- Blowfish_Decrypt(&ctx, &L, &R);
+ /* Decrypt as many blocks (each block_size bytes big) as we can */
+ int i;
+ /* i adresses sizeof(uint32_t) = 4 byte blocks,
+ * but we use (restidx / block_size) loop up to full blocks */
+ for (i = 0; i < 2 * (n / block_size); i += 2) {
+ if (swap) {
+ inwalk[i] = swapEndian(inwalk[i]);
+ inwalk[i + 1] = swapEndian(inwalk[i + 1]);
+ }
+ Blowfish_Decrypt(&ctx, &inwalk[i], &inwalk[i + 1]);
+ if (swap) {
+ inwalk[i] = swapEndian(inwalk[i]);
+ inwalk[i + 1] = swapEndian(inwalk[i + 1]);
+ }
+ }
- if (swap == 1) {
- L = swapEndian(L);
- R = swapEndian(R);
+ strm.avail_in = n;
+ strm.next_in = in;
+ if (compressed) {
+ do {
+ strm.avail_out = ZCHUNK;
+ strm.next_out = out;
+ ret = inflate(&strm, Z_NO_FLUSH);
+ switch (ret) {
+ case Z_NEED_DICT:
+ ret = Z_DATA_ERROR;
+ /* fall-through */
+ case Z_DATA_ERROR:
+ case Z_MEM_ERROR:
+ exit(10);
+ }
+
+ have = ZCHUNK - strm.avail_out;
+ memcpy(rest + restidx, out, have);
+ write(outfd, out, have);
+ } while (strm.avail_out == 0);
+
+ /* If zlib is done and we still have data in our buffer, that extra data
+ * needs to be written out to file. It contains the key + padding +
+ * original filesize. */
+ if (strm.avail_in > sizeof(uint32_t)) {
+ /* We don't write the original file size since it is not stored for
+ * uncompressed files either. Actually, we don't need it at all and
+ * just support it for compatibility with earlier versions. */
+ write(outfd, strm.next_in, strm.avail_in - sizeof(uint32_t));
+ }
+ } else {
+ /* Abort the loop on EOF when the file is not compressed */
+ if (n == 0)
+ ret = Z_STREAM_END;
+
+ /* XXX: If we would buffer blocks as large as the maximum amount of
+ * padding + key can be, we could almost reliably detect the end of file
+ * and handle stdin to stdout decryptions correctly. */
+ write(outfd, in, n);
}
+ } while (ret != Z_STREAM_END);
- memcpy(*input+i, &L, j);
- memcpy(*input+i+j, &R, j);
+ /* Strip off the padding */
+ uint8_t lastbyte;
+ off_t seek_offset = -1, cur_offset;
+ do {
+ cur_offset = lseek(outfd, seek_offset, SEEK_CUR);
+ if (cur_offset == (off_t)-1) {
+ exit(8);
+ }
+ read(outfd, &lastbyte, sizeof(uint8_t));
+ seek_offset = -2;
+ } while (lastbyte == '\0');
+
+ cur_offset = lseek(outfd, -MAXKEYBYTES, SEEK_CUR);
+ if (cur_offset == (off_t)-1) {
+ exit(9);
}
- while (memcmp(*input+(sz-1), "\0", 1) == 0) /* strip excess nulls */
- sz--; /* from decrypted files */
+ /* Verify that the stored key matches our key */
+ uint8_t savedkey[MAXKEYBYTES];
+ read(outfd, savedkey, MAXKEYBYTES);
- sz -= MAXKEYBYTES;
+ if (memcmp(savedkey, mykey, MAXKEYBYTES) != 0)
+ return 0;
- if (memcmp(*input+sz, mykey, MAXKEYBYTES) != 0)
- return(0);
-
- free(mykey);
- free(myEndian);
- return(sz);
+ return cur_offset;
}
