changeset: 6923:42aa8b19da95 user: Kevin McCarthy <ke...@8t8.us> date: Sat Feb 04 12:53:38 2017 -0800 link: http://dev.mutt.org/hg/mutt/rev/42aa8b19da95
Add LMDB backend support for header cache. (see #3691) Based on the original from JP Mens: https://gist.github.com/jpmens/15969d9d678a3d450e4e The following performance patch was manually applied on top of the original patch: https://github.com/neomutt/neomutt/commit/7e5380cd4c40d119ff83b2cf5f51f2cdb8a95ab3 A variant of this patch was added to handle larger mailboxes: https://github.com/neomutt/neomutt/commit/6d337642e701b1dde4b5d0812e01c85f41ba65ca Thanks to all the developers and contributors to this patch, and to Fabian Groffen for bundling and posting this to mutt-dev. changeset: 6924:fca7e504ab6a user: Kevin McCarthy <ke...@8t8.us> date: Sat Feb 04 12:53:52 2017 -0800 link: http://dev.mutt.org/hg/mutt/rev/fca7e504ab6a Fixes to the LMDB header cache. (closes #3691) Use mdb_txn_abort() to free up readonly/reset transactions, and avoid leaking memory. Fix hcache_delete() key generation - looks like this was an incorrect copy/paste from the bdb code. Use dprint, not fprintf, for debugging messages. Remove strange blending of enum and bitfield logic for the txn_mode state. Remove some duplicate code from store/fetch raw. diffs (458 lines): diff -r 142a87f0c855 -r fca7e504ab6a configure.ac --- a/configure.ac Tue Jan 31 15:02:21 2017 -0800 +++ b/configure.ac Sat Feb 04 12:53:52 2017 -0800 @@ -881,6 +881,7 @@ AC_ARG_WITH(qdbm, AS_HELP_STRING([--without-qdbm],[Don't use qdbm even if it is available])) AC_ARG_WITH(gdbm, AS_HELP_STRING([--without-gdbm],[Don't use gdbm even if it is available])) AC_ARG_WITH(bdb, AS_HELP_STRING([--with-bdb@<:@=DIR@:>@],[Use BerkeleyDB4 if gdbm is not available])) +AC_ARG_WITH(lmdb, AS_HELP_STRING([--with-lmdb@<:@=DIR@:>@],[Use LMDB if gdbm is not available])) db_found=no if test x$enable_hcache = xyes @@ -925,6 +926,15 @@ db_requested=bdb fi fi + if test -n "$with_lmdb" && test "$with_lmdb" != "no" + then + if test "$db_requested" != "auto" + then + AC_MSG_ERROR([more than one header cache engine requested.]) + else + db_requested=lmdb + fi + fi dnl -- Tokyo Cabinet -- if test "$with_tokyocabinet" != "no" \ @@ -1012,7 +1022,8 @@ dnl -- BDB -- ac_bdb_prefix="$with_bdb" - if test x$ac_bdb_prefix != xno && test $db_found = no + if test x$with_bdb != xno && test $db_found = no \ + && test "$db_requested" = auto -o "$db_requested" = bdb then if test x$ac_bdb_prefix = xyes || test x$ac_bdb_prefix = x then @@ -1068,6 +1079,34 @@ fi fi + dnl -- LMDB -- + if test x$with_lmdb != xno && test $db_found = no \ + && test "$db_requested" = auto -o "$db_requested" = lmdb + then + if test "$with_lmdb" != "yes" + then + CPPFLAGS="$CPPFLAGS -I$with_lmdb/include" + LDFLAGS="$LDFLAGS -L$with_lmdb/lib" + fi + saved_LIBS="$LIBS" + LIBS="$LIBS -llmdb" + AC_CACHE_CHECK(for mdb_env_create, ac_cv_mdbenvcreate,[ + ac_cv_mdbenvcreate=no + AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <lmdb.h>]], [[mdb_env_create(0);]])],[ac_cv_mdbenvcreate=yes],[]) + ]) + LIBS="$saved_LIBS" + if test "$ac_cv_mdbenvcreate" = yes + then + AC_DEFINE(HAVE_LMDB, 1, [LMDB Support]) + MUTTLIBS="$MUTTLIBS -llmdb" + db_found=lmdb + fi + if test "$db_requested" != auto && test "$db_found" != "$db_requested" + then + AC_MSG_ERROR([LMDB could not be used. Check config.log for details.]) + fi + fi + if test $db_found = no then AC_MSG_ERROR([You need Tokyo Cabinet, QDBM, GDBM or Berkeley DB4 for hcache]) diff -r 142a87f0c855 -r fca7e504ab6a hcache.c --- a/hcache.c Tue Jan 31 15:02:21 2017 -0800 +++ b/hcache.c Sat Feb 04 12:53:52 2017 -0800 @@ -32,6 +32,12 @@ #include <gdbm.h> #elif HAVE_DB4 #include <db.h> +#elif HAVE_LMDB +/* This is the maximum size of the database file (2GiB) which is + * mmap(2)'ed into memory. This limit should be good for ~800,000 + * emails. */ +#define LMDB_DB_SIZE 2147483648 +#include <lmdb.h> #endif #include <errno.h> @@ -83,6 +89,78 @@ static void mutt_hcache_dbt_init(DBT * dbt, void *data, size_t len); static void mutt_hcache_dbt_empty_init(DBT * dbt); +#elif HAVE_LMDB +enum mdb_txn_mode +{ + txn_uninitialized = 0, + txn_read, + txn_write +}; +struct header_cache +{ + MDB_env *env; + MDB_txn *txn; + MDB_dbi db; + char *folder; + unsigned int crc; + enum mdb_txn_mode txn_mode; +}; + +static int mdb_get_r_txn(header_cache_t *h) +{ + int rc; + + if (h->txn) + { + if (h->txn_mode == txn_read || h->txn_mode == txn_write) + return MDB_SUCCESS; + + if ((rc = mdb_txn_renew (h->txn)) != MDB_SUCCESS) + { + h->txn = NULL; + dprint (2, (debugfile, "mdb_get_r_txn: mdb_txn_renew: %s\n", + mdb_strerror (rc))); + return rc; + } + h->txn_mode = txn_read; + return rc; + } + + if ((rc = mdb_txn_begin (h->env, NULL, MDB_RDONLY, &h->txn)) != MDB_SUCCESS) + { + h->txn = NULL; + dprint (2, (debugfile, "mdb_get_r_txn: mdb_txn_begin: %s\n", + mdb_strerror (rc))); + return rc; + } + h->txn_mode = txn_read; + return rc; +} + +static int mdb_get_w_txn(header_cache_t *h) +{ + int rc; + + if (h->txn) + { + if (h->txn_mode == txn_write) + return MDB_SUCCESS; + + /* Free up the memory for readonly or reset transactions */ + mdb_txn_abort (h->txn); + } + + if ((rc = mdb_txn_begin (h->env, NULL, 0, &h->txn)) != MDB_SUCCESS) + { + h->txn = NULL; + dprint (2, (debugfile, "mdb_get_w_txn: mdb_txn_begin %s\n", + mdb_strerror (rc))); + return rc; + } + + h->txn_mode = txn_write; + return rc; +} #endif typedef union @@ -732,11 +810,14 @@ #elif HAVE_DB4 DBT key; DBT data; +#elif HAVE_LMDB + MDB_val key; + MDB_val data; #endif - + if (!h) return NULL; - + #ifdef HAVE_DB4 if (filename[0] == '/') filename++; @@ -744,19 +825,20 @@ mutt_hcache_dbt_init(&key, (void *) filename, keylen(filename)); mutt_hcache_dbt_empty_init(&data); data.flags = DB_DBT_MALLOC; - + h->db->get(h->db, NULL, &key, &data, 0); - + return data.data; -#else +#endif + strncpy(path, h->folder, sizeof (path)); safe_strcat(path, sizeof (path), filename); ksize = strlen (h->folder) + keylen (path + strlen (h->folder)); -#endif + #ifdef HAVE_QDBM data = vlget(h->db, path, ksize, NULL); - + return data; #elif HAVE_TC data = tcbdbget(h->db, path, ksize, &sp); @@ -765,10 +847,21 @@ #elif HAVE_GDBM key.dptr = path; key.dsize = ksize; - + data = gdbm_fetch(h->db, key); - + return data.dptr; +#elif HAVE_LMDB + key.mv_data = path; + key.mv_size = ksize; + if ((mdb_get_r_txn (h) != MDB_SUCCESS) || + (mdb_get (h->txn, h->db, &key, &data) != MDB_SUCCESS)) + return NULL; + + /* Unlike other dbs, LMDB claims ownership of the returned data */ + char *d = safe_malloc (data.mv_size); + memcpy (d, data.mv_data, data.mv_size); + return d; #endif } @@ -813,30 +906,35 @@ #elif HAVE_DB4 DBT key; DBT databuf; +#elif HAVE_LMDB + MDB_val key; + MDB_val databuf; + int rc; #endif - + if (!h) return -1; #if HAVE_DB4 if (filename[0] == '/') filename++; - + mutt_hcache_dbt_init(&key, (void *) filename, keylen(filename)); - + mutt_hcache_dbt_empty_init(&databuf); databuf.flags = DB_DBT_USERMEM; databuf.data = data; databuf.size = dlen; databuf.ulen = dlen; - + return h->db->put(h->db, NULL, &key, &databuf, 0); -#else +#endif + strncpy(path, h->folder, sizeof (path)); safe_strcat(path, sizeof (path), filename); ksize = strlen(h->folder) + keylen(path + strlen(h->folder)); -#endif + #if HAVE_QDBM return vlput(h->db, path, ksize, data, dlen, VL_DOVER); #elif HAVE_TC @@ -844,11 +942,28 @@ #elif HAVE_GDBM key.dptr = path; key.dsize = ksize; - + databuf.dsize = dlen; databuf.dptr = data; - + return gdbm_store(h->db, key, databuf, GDBM_REPLACE); +#elif HAVE_LMDB + key.mv_data = path; + key.mv_size = ksize; + databuf.mv_data = data; + databuf.mv_size = dlen; + if ((rc = mdb_get_w_txn (h)) != MDB_SUCCESS) + return rc; + + if ((rc = mdb_put (h->txn, h->db, &key, &databuf, 0)) != MDB_SUCCESS) + { + dprint (2, (debugfile, "mutt_hcache_store_raw: mdb_put: %s\n", + mdb_strerror(rc))); + mdb_txn_abort (h->txn); + h->txn_mode = txn_uninitialized; + h->txn = NULL; + } + return rc; #endif } @@ -1134,6 +1249,117 @@ mutt_hcache_dbt_init(&key, (void *) filename, keylen(filename)); return h->db->del(h->db, NULL, &key, 0); } +#elif HAVE_LMDB + +static int +hcache_open_lmdb (struct header_cache* h, const char* path) +{ + int rc; + + h->txn = NULL; + + if ((rc = mdb_env_create(&h->env)) != MDB_SUCCESS) + { + dprint (2, (debugfile, "hcache_open_lmdb: mdb_env_create: %s\n", + mdb_strerror(rc))); + return -1; + } + + mdb_env_set_mapsize(h->env, LMDB_DB_SIZE); + + if ((rc = mdb_env_open(h->env, path, MDB_NOSUBDIR, 0644)) != MDB_SUCCESS) + { + dprint (2, (debugfile, "hcache_open_lmdb: mdb_env_open: %s\n", + mdb_strerror(rc))); + goto fail_env; + } + + if ((rc = mdb_get_r_txn(h)) != MDB_SUCCESS) + goto fail_env; + + if ((rc = mdb_dbi_open(h->txn, NULL, MDB_CREATE, &h->db)) != MDB_SUCCESS) + { + dprint (2, (debugfile, "hcache_open_lmdb: mdb_dbi_open: %s\n", + mdb_strerror(rc))); + goto fail_dbi; + } + + mdb_txn_reset(h->txn); + h->txn_mode = txn_uninitialized; + return 0; + +fail_dbi: + mdb_txn_abort(h->txn); + h->txn_mode = txn_uninitialized; + h->txn = NULL; + +fail_env: + mdb_env_close(h->env); + return -1; +} + +void +mutt_hcache_close(header_cache_t *h) +{ + int rc; + + if (!h) + return; + + if (h->txn) + { + if (h->txn_mode == txn_write) + { + if ((rc = mdb_txn_commit (h->txn)) != MDB_SUCCESS) + { + dprint (2, (debugfile, "mutt_hcache_close: mdb_txn_commit: %s\n", + mdb_strerror (rc))); + } + } + else + mdb_txn_abort (h->txn); + h->txn_mode = txn_uninitialized; + h->txn = NULL; + } + + mdb_env_close(h->env); + FREE (&h->folder); + FREE (&h); +} + +int +mutt_hcache_delete(header_cache_t *h, const char *filename, + size_t(*keylen) (const char *fn)) +{ + char path[_POSIX_PATH_MAX]; + int ksize; + MDB_val key; + int rc; + + if (!h) + return -1; + + strncpy(path, h->folder, sizeof (path)); + safe_strcat(path, sizeof (path), filename); + ksize = strlen (h->folder) + keylen (path + strlen (h->folder)); + + key.mv_data = path; + key.mv_size = ksize; + if (mdb_get_w_txn(h) != MDB_SUCCESS) + return -1; + rc = mdb_del(h->txn, h->db, &key, NULL); + if (rc != MDB_SUCCESS && rc != MDB_NOTFOUND) + { + dprint (2, (debugfile, "mutt_hcache_delete: mdb_del: %s\n", + mdb_strerror (rc))); + mdb_txn_abort(h->txn); + h->txn_mode = txn_uninitialized; + h->txn = NULL; + return -1; + } + + return 0; +} #endif header_cache_t * @@ -1151,6 +1377,8 @@ hcache_open = hcache_open_gdbm; #elif HAVE_DB4 hcache_open = hcache_open_db4; +#elif HAVE_LMDB + hcache_open = hcache_open_lmdb; #endif /* Calculate the current hcache version from dynamic configuration */ @@ -1188,7 +1416,11 @@ hcachever = digest.intval; } +#if HAVE_LMDB + h->db = 0; +#else h->db = NULL; +#endif h->folder = get_foldername(folder); h->crc = hcachever; @@ -1223,6 +1455,11 @@ { return DB_VERSION_STRING; } +#elif HAVE_LMDB +const char *mutt_hcache_backend (void) +{ + return "lmdb " MDB_VERSION_STRING; +} #elif HAVE_GDBM const char *mutt_hcache_backend (void) {
