Author: lkajan-guest Date: 2012-06-27 09:52:43 +0000 (Wed, 27 Jun 2012) New Revision: 11483
Added: trunk/packages/ffindex/trunk/debian/patches/40000000_bug trunk/packages/ffindex/trunk/debian/patches/hardening trunk/packages/ffindex/trunk/debian/patches/num_max_entries_bug Modified: trunk/packages/ffindex/trunk/debian/changelog trunk/packages/ffindex/trunk/debian/patches/Makefile trunk/packages/ffindex/trunk/debian/patches/series trunk/packages/ffindex/trunk/debian/rules Log: fixed 40000000 bug; index bug; added hardening flags Modified: trunk/packages/ffindex/trunk/debian/changelog =================================================================== --- trunk/packages/ffindex/trunk/debian/changelog 2012-06-27 06:01:53 UTC (rev 11482) +++ trunk/packages/ffindex/trunk/debian/changelog 2012-06-27 09:52:43 UTC (rev 11483) @@ -1,6 +1,7 @@ ffindex (0.9.6.1-1) unstable; urgency=low * New upstream release. + * Fixed many bugs. -- Laszlo Kajan <[email protected]> Tue, 26 Jun 2012 11:43:41 +0200 Added: trunk/packages/ffindex/trunk/debian/patches/40000000_bug =================================================================== --- trunk/packages/ffindex/trunk/debian/patches/40000000_bug (rev 0) +++ trunk/packages/ffindex/trunk/debian/patches/40000000_bug 2012-06-27 09:52:43 UTC (rev 11483) @@ -0,0 +1,104 @@ +From: Laszlo Kajan <[email protected]> +Subject: 40000000 bug + The default value 40000000 for FFINDEX_MAX_INDEX_ENTRIES_DEFAULT is too high for a machine with 2G (or less) RAM. + Bug fixed by introducing an environment variable FFINDEX_MAX_INDEX_ENTRIES to control the value. +Forwarded: yes + +--- a/src/ffindex.c ++++ b/src/ffindex.c +@@ -220,7 +220,7 @@ + ffindex_index_t* ffindex_index_parse(FILE *index_file, size_t num_max_entries) + { + if(num_max_entries == 0) +- num_max_entries = FFINDEX_MAX_INDEX_ENTRIES_DEFAULT; ++ num_max_entries = ffindex_max_index_entries(); + size_t nbytes = sizeof(ffindex_index_t) + (sizeof(ffindex_entry_t) * num_max_entries); + ffindex_index_t *index = (ffindex_index_t *)malloc(nbytes); + if(index == NULL) +--- a/src/ffindex.h ++++ b/src/ffindex.h +@@ -18,6 +18,8 @@ + #include <stdio.h> + + #define FFINDEX_VERSION 0.961 ++// lkajan: with ~72 bytes per entry, the below makes a default allocation over 2G. malloc in ffindex.c:225 fails on a machine with 'only' 2G of RAM. ++// lkajan: let the default remain 40000000, and let the actual value be controlled by the env var FFINDEX_MAX_INDEX_ENTRIES wherever FFINDEX_MAX_INDEX_ENTRIES_DEFAULT is used. + #define FFINDEX_MAX_INDEX_ENTRIES_DEFAULT 40000000 + #define FFINDEX_MAX_ENTRY_NAME_LENTH 56 + +@@ -41,6 +43,8 @@ + ffindex_entry_t entries[]; /* This array is as big as the excess memory allocated for this struct. */ + } ffindex_index_t; + ++inline size_t ffindex_max_index_entries(){ char *FMIE = getenv("FFINDEX_MAX_INDEX_ENTRIES"); if( FMIE ) { return atol( FMIE ); } else { return FFINDEX_MAX_INDEX_ENTRIES_DEFAULT; } } ++ + int ffindex_insert_memory(FILE *data_file, FILE *index_file, size_t *offset, char *from_start, size_t from_length, char *name); + + int ffindex_insert_file(FILE *data_file, FILE *index_file, size_t *offset, const char *path, char *name); +Binary files a/src/ffindex_apply and b/src/ffindex_apply differ +--- a/src/ffindex_apply.c ++++ b/src/ffindex_apply.c +@@ -34,7 +34,9 @@ + if(argn < 4) + { + fprintf(stderr, "USAGE: ffindex_apply DATA_FILENAME INDEX_FILENAME PROGRAM [PROGRAM_ARGS]*\n" +- "\nDesigned and implemented by Andy Hauser <[email protected]>.\n", ++ "ENVIRONMENT\n" ++ "\tFFINDEX_MAX_INDEX_ENTRIES - allocate memory for this number of entries\n" ++ "\nDesigned and implemented by Andy Hauser <[email protected]>.\n" + ); + return -1; + } +--- a/src/ffindex_build.c ++++ b/src/ffindex_build.c +@@ -39,6 +39,8 @@ + "\t\t-f can be specified up to %d times\n" + "\t-s\tsort index file\n" + "\t-v\tprint version and other info then exit\n" ++ "ENVIRONMENT\n" ++ "\tFFINDEX_MAX_INDEX_ENTRIES - allocate memory for this number of entries\n" + "\nDesigned and implemented by Andreas W. Hauser <[email protected]>.\n", MAX_FILENAME_LIST_FILES); + } + +--- a/src/ffindex_from_fasta.c ++++ b/src/ffindex_from_fasta.c +@@ -33,6 +33,8 @@ + { + fprintf(stderr, "USAGE: ffindex_from_fasta -v | [-s] data_filename index_filename fasta_filename\n" + "\t-s\tsort index file\n" ++ "ENVIRONMENT\n" ++ "\tFFINDEX_MAX_INDEX_ENTRIES - allocate memory for this number of entries\n" + "\nDesigned and implemented by Andreas W. Hauser <[email protected]>.\n"); + } + +--- a/src/ffindex_get.c ++++ b/src/ffindex_get.c +@@ -27,6 +27,8 @@ + if(argn < 3) + { + fprintf(stderr, "USAGE: ffindex_get data_filename index_filename filename(s)\n" ++ "ENVIRONMENT\n" ++ "\tFFINDEX_MAX_INDEX_ENTRIES - allocate memory for this number of entries\n" + "\nDesigned and implemented by Andreas W. Hauser <[email protected]>.\n"); + return -1; + } +--- a/src/ffindex_modify.c ++++ b/src/ffindex_modify.c +@@ -37,6 +37,8 @@ + "\t-t\tuse tree (default, will be removed soon)\n" + "\t-u\tunlink entry (remove from index only)\n" + "\t-v\tprint version and other info then exit\n" ++ "ENVIRONMENT\n" ++ "\tFFINDEX_MAX_INDEX_ENTRIES - allocate memory for this number of entries\n" + "\nDesigned and implemented by Andreas W. Hauser <[email protected]>.\n", + MAX_FILENAME_LIST_FILES); + } +@@ -125,7 +127,7 @@ + } + else + { +- char** sorted_names_to_unlink = malloc(FFINDEX_MAX_INDEX_ENTRIES_DEFAULT * sizeof(char *)); ++ char** sorted_names_to_unlink = malloc(ffindex_max_index_entries() * sizeof(char *)); + if(sorted_names_to_unlink == NULL) + fferror_print(__FILE__, __LINE__, __func__, "malloc failed"); + /* For each list_file unlink all entries */ Modified: trunk/packages/ffindex/trunk/debian/patches/Makefile =================================================================== --- trunk/packages/ffindex/trunk/debian/patches/Makefile 2012-06-27 06:01:53 UTC (rev 11482) +++ trunk/packages/ffindex/trunk/debian/patches/Makefile 2012-06-27 09:52:43 UTC (rev 11483) @@ -1,6 +1,12 @@ -* make Makefile DESTDIR-aware -* deinstall -> uninstall -* fix issues where the path in help pages is taken from the build-time location of the executables +From: Laszlo Kajan <[email protected]> +Subject: Makefile fixes + * make Makefile DESTDIR-aware + * other changes to allow help2man to work better. + * deinstall -> uninstall + . + Upstream is aware but not interested in DESTDIR. +Forwarded: yes + --- a/Makefile +++ b/Makefile @@ -6,15 +6,8 @@ @@ -24,7 +30,7 @@ + $(MAKE) -C src -f $(MFILE) $@ --- a/src/Makefile +++ b/src/Makefile -@@ -1,15 +1,19 @@ +@@ -1,15 +1,16 @@ -INSTALL_DIR?=.. -libdir=`([ -d /usr/lib64 ] && echo lib64) || echo lib` +PREFIX?=/usr @@ -36,9 +42,6 @@ +CC=gcc CFLAGS+=-std=c99 -O2 -g -fPIC -Wall -pedantic -LDFLAGS=-lffindex -L. -+CFLAGS+=-g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Werror=format-security -+CPPFLAGS+=-D_FORTIFY_SOURCE=2 -+LDFLAGS+=-Wl,-z,relro +SOCURRENT=0 +SOVERSION=$(SOCURRENT).1.0 ifdef FFINDEX_STATIC @@ -50,7 +53,7 @@ endif #-Wl,--version-script=libffindex.map -@@ -25,33 +29,34 @@ +@@ -25,33 +26,34 @@ ffindex_apply: $(FFINDEX_OBJECTS) ffindex_apply.o ffindex.h ffindex_apply_mpi: $(FFINDEX_OBJECTS) ffindex_apply_mpi.o ffindex.h @@ -95,7 +98,7 @@ rm -f /tmp/test.data /tmp/test.ffindex LD_LIBRARY_PATH=".:$(LD_LIBRARY_PATH)" ./ffindex_build -s /tmp/test.data /tmp/test.ffindex ../test/data ../test/data2 LD_LIBRARY_PATH=".:$(LD_LIBRARY_PATH)" ./ffindex_get /tmp/test.data /tmp/test.ffindex a b foo | tee /tmp/test.out -@@ -71,9 +76,11 @@ +@@ -71,9 +73,11 @@ LD_LIBRARY_PATH=".:$(LD_LIBRARY_PATH)" zsh -c "time ./ffindex_get ../test_big/nr20_11Jan10_a3m_db ../test_big/nr20_11Jan10_a3m_db.ffindex BABBAMIBA.a3m KARDUZEBA.a3m HARPAQIBA.a3m WOQQATEBA.a3m BEBRAQIBA.a3m GIRSAKEBA.a3m NAQTIFIBA.a3m BIWCIQABA.a3m > /dev/null" clean: @@ -108,7 +111,7 @@ install: mkdir -p $(INSTALL_DIR)/bin mkdir -p $(INSTALL_DIR)/$(libdir) -@@ -83,9 +90,10 @@ +@@ -83,9 +87,10 @@ install ffindex_modify $(INSTALL_DIR)/bin/ffindex_modify install ffindex_from_fasta $(INSTALL_DIR)/bin/ffindex_from_fasta install ffindex_apply $(INSTALL_DIR)/bin/ffindex_apply @@ -191,34 +194,31 @@ return -1; } char *data_filename = argv[1]; ---- a/src/ffindex.h -+++ b/src/ffindex.h -@@ -18,7 +18,8 @@ - #include <stdio.h> - - #define FFINDEX_VERSION 0.961 --#define FFINDEX_MAX_INDEX_ENTRIES_DEFAULT 40000000 -+// lkajan: with 72 bytes per entry, the below makes a default allocation over 2G. malloc in ffindex.c:225 fails with hardening flags. Let's give a value for approx. 256MB. -+#define FFINDEX_MAX_INDEX_ENTRIES_DEFAULT 3728270 - #define FFINDEX_MAX_ENTRY_NAME_LENTH 56 - - enum ffindex_type { PLAIN_FILE, SORTED_FILE, SORTED_ARRAY, TREE }; ---- a/src/ffindex.c -+++ b/src/ffindex.c -@@ -223,13 +223,14 @@ - num_max_entries = FFINDEX_MAX_INDEX_ENTRIES_DEFAULT; - size_t nbytes = sizeof(ffindex_index_t) + (sizeof(ffindex_entry_t) * num_max_entries); - ffindex_index_t *index = (ffindex_index_t *)malloc(nbytes); -- index->num_max_entries = num_max_entries; - if(index == NULL) +--- a/src/ffindex_apply.c ++++ b/src/ffindex_apply.c +@@ -33,9 +33,9 @@ + { + if(argn < 4) { -- fferror_print(__FILE__, __LINE__, __func__, "malloc failed"); -+ char buf[2048]; snprintf( buf, 2048, "malloc failed to allocate %ld bytes", nbytes ); -+ fferror_print(__FILE__, __LINE__, __func__, buf ); - return NULL; +- fprintf(stderr, "USAGE: %s DATA_FILENAME INDEX_FILENAME PROGRAM [PROGRAM_ARGS]*\n" ++ fprintf(stderr, "USAGE: ffindex_apply DATA_FILENAME INDEX_FILENAME PROGRAM [PROGRAM_ARGS]*\n" + "\nDesigned and implemented by Andy Hauser <[email protected]>.\n", +- argv[0]); ++ ); + return -1; } + char *data_filename = argv[1]; +--- a/src/ffindex_from_fasta.c ++++ b/src/ffindex_from_fasta.c +@@ -31,9 +31,9 @@ -+ index->num_max_entries = num_max_entries; - index->file = index_file; - index->index_data = ffindex_mmap_data(index_file, &(index->index_data_size)); - if(index->index_data_size == 0) + void usage(char *program_name) + { +- fprintf(stderr, "USAGE: %s -v | [-s] data_filename index_filename fasta_filename\n" ++ fprintf(stderr, "USAGE: ffindex_from_fasta -v | [-s] data_filename index_filename fasta_filename\n" + "\t-s\tsort index file\n" +- "\nDesigned and implemented by Andreas W. Hauser <[email protected]>.\n", program_name); ++ "\nDesigned and implemented by Andreas W. Hauser <[email protected]>.\n"); + } + + int main(int argn, char **argv) Added: trunk/packages/ffindex/trunk/debian/patches/hardening =================================================================== --- trunk/packages/ffindex/trunk/debian/patches/hardening (rev 0) +++ trunk/packages/ffindex/trunk/debian/patches/hardening 2012-06-27 09:52:43 UTC (rev 11483) @@ -0,0 +1,18 @@ +From: Laszlo Kajan <[email protected]> +Subject: hardening flags +Forwarded: yes + +--- a/src/Makefile ++++ b/src/Makefile +@@ -15,6 +15,11 @@ + + #-Wl,--version-script=libffindex.map + ++# hardening flags ++CFLAGS+=-g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Werror=format-security ++CPPFLAGS+=-D_FORTIFY_SOURCE=2 ++LDFLAGS+=-Wl,-z,relro ++ + ifdef HAVE_MPI + all: ffindex_apply_mpi ffindex_apply ffindex_from_fasta ffindex_get ffindex_build ffindex_modify libffindex.a libffindex.so + else Added: trunk/packages/ffindex/trunk/debian/patches/num_max_entries_bug =================================================================== --- trunk/packages/ffindex/trunk/debian/patches/num_max_entries_bug (rev 0) +++ trunk/packages/ffindex/trunk/debian/patches/num_max_entries_bug 2012-06-27 09:52:43 UTC (rev 11483) @@ -0,0 +1,23 @@ +From: Laszlo Kajan <[email protected]> +Subject: index pointer used before checking for NULL +Forwarded: yes + +--- a/src/ffindex.c ++++ b/src/ffindex.c +@@ -223,13 +223,14 @@ + num_max_entries = FFINDEX_MAX_INDEX_ENTRIES_DEFAULT; + size_t nbytes = sizeof(ffindex_index_t) + (sizeof(ffindex_entry_t) * num_max_entries); + ffindex_index_t *index = (ffindex_index_t *)malloc(nbytes); +- index->num_max_entries = num_max_entries; + if(index == NULL) + { +- fferror_print(__FILE__, __LINE__, __func__, "malloc failed"); ++ char buf[2048]; snprintf( buf, 2048, "malloc failed to allocate %ld bytes", nbytes ); ++ fferror_print(__FILE__, __LINE__, __func__, buf ); + return NULL; + } + ++ index->num_max_entries = num_max_entries; + index->file = index_file; + index->index_data = ffindex_mmap_data(index_file, &(index->index_data_size)); + if(index->index_data_size == 0) Modified: trunk/packages/ffindex/trunk/debian/patches/series =================================================================== --- trunk/packages/ffindex/trunk/debian/patches/series 2012-06-27 06:01:53 UTC (rev 11482) +++ trunk/packages/ffindex/trunk/debian/patches/series 2012-06-27 09:52:43 UTC (rev 11483) @@ -1 +1,4 @@ Makefile +hardening +num_max_entries_bug +40000000_bug Modified: trunk/packages/ffindex/trunk/debian/rules =================================================================== --- trunk/packages/ffindex/trunk/debian/rules 2012-06-27 06:01:53 UTC (rev 11482) +++ trunk/packages/ffindex/trunk/debian/rules 2012-06-27 09:52:43 UTC (rev 11483) @@ -3,6 +3,10 @@ # Uncomment this to turn on verbose mode. export DH_VERBOSE=1 +# [Laszlo Kajan <[email protected]>] +# ffindex has an overly large default for FFINDEX_MAX_INDEX_ENTRIES (40000000). In order to require less RAM for testing the package (make check), a more modest value is set below. +export FFINDEX_MAX_INDEX_ENTRIES=20000 + %: dh $@ --parallel _______________________________________________ debian-med-commit mailing list [email protected] http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/debian-med-commit
