On 27-01-2017 09:58:52 +0100, Fabian Groffen wrote: > > Two minor issues: > > - You may want to set LMDB_DB_SIZE to a larger value, see > > https://github.com/neomutt/neomutt/issues/267 > > Ok, I'll add that. > > > - There's a missing free(&ctx) in hcache_lmdb_open() when > > mdb_env_create() fails. > > I'll add it, thanks.
I looked at this but can't see a problem in the code. I think you have the NeoMutt version in mind because you reference ctx? Revised patch attached. Thanks, Fabian -- Fabian Groffen Gentoo on a different level
# HG changeset patch # Parent 17f834d977488a5d9b59a93a98d5a33a8ea10fee Add LMDB backend support for header cache Pietro Cerutti <g...@gahr.ch> Derived on the original from JP Mens: https://gist.github.com/jpmens/15969d9d678a3d450e4e The following performance patch was manually applied on top of the original patch: https://github.com/neomutt/neomutt/commit/7e5380cd4c40d119ff83b2cf5f51f2cdb8a95ab3 A variant of this patch was added to handle larger mailboxes: https://github.com/neomutt/neomutt/commit/6d337642e701b1dde4b5d0812e01c85f41ba65ca Related ticked: https://dev.mutt.org/trac/ticket/3691 diff -r 17f834d97748 configure.ac --- a/configure.ac Fri Jan 27 10:05:23 2017 +0100 +++ b/configure.ac Fri Jan 27 10:15:17 2017 +0100 @@ -915,6 +915,7 @@ AC_ARG_WITH(tokyocabinet, AS_HELP_STRING AC_ARG_WITH(qdbm, AS_HELP_STRING([--without-qdbm],[Don't use qdbm even if it is available])) AC_ARG_WITH(gdbm, AS_HELP_STRING([--without-gdbm],[Don't use gdbm even if it is available])) AC_ARG_WITH(bdb, AS_HELP_STRING([--with-bdb@<:@=DIR@:>@],[Use BerkeleyDB4 if gdbm is not available])) +AC_ARG_WITH(lmdb, AS_HELP_STRING([--with-lmdb@<:@=DIR@:>@],[Use LMDB if gdbm is not available])) db_found=no if test x$enable_hcache = xyes @@ -959,6 +960,15 @@ then db_requested=bdb fi fi + if test -n "$with_lmdb" && test "$with_lmdb" != "no" + then + if test "$db_requested" != "auto" + then + AC_MSG_ERROR([more than one header cache engine requested.]) + else + db_requested=lmdb + fi + fi dnl -- Tokyo Cabinet -- if test "$with_tokyocabinet" != "no" \ @@ -1046,7 +1056,8 @@ then dnl -- BDB -- ac_bdb_prefix="$with_bdb" - if test x$ac_bdb_prefix != xno && test $db_found = no + if test x$with_bdb != xno && test $db_found = no \ + && test "$db_requested" = auto -o "$db_requested" = bdb then if test x$ac_bdb_prefix = xyes || test x$ac_bdb_prefix = x then @@ -1102,6 +1113,34 @@ then fi fi + dnl -- LMDB -- + if test x$with_lmdb != xno && test $db_found = no \ + && test "$db_requested" = auto -o "$db_requested" = lmdb + then + if test "$with_lmdb" != "yes" + then + CPPFLAGS="$CPPFLAGS -I$with_lmdb/include" + LDFLAGS="$LDFLAGS -L$with_lmdb/lib" + fi + saved_LIBS="$LIBS" + LIBS="$LIBS -llmdb" + AC_CACHE_CHECK(for mdb_env_create, ac_cv_mdbenvcreate,[ + ac_cv_mdbenvcreate=no + AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <lmdb.h>]], [[mdb_env_create(0);]])],[ac_cv_mdbenvcreate=yes],[]) + ]) + LIBS="$saved_LIBS" + if test "$ac_cv_mdbenvcreate" = yes + then + AC_DEFINE(HAVE_LMDB, 1, [LMDB Support]) + MUTTLIBS="$MUTTLIBS -llmdb" + db_found=lmdb + fi + if test "$db_requested" != auto && test "$db_found" != "$db_requested" + then + AC_MSG_ERROR([LMDB could not be used. Check config.log for details.]) + fi + fi + if test $db_found = no then AC_MSG_ERROR([You need Tokyo Cabinet, QDBM, GDBM or Berkeley DB4 for hcache]) diff -r 17f834d97748 hcache.c --- a/hcache.c Fri Jan 27 10:05:23 2017 +0100 +++ b/hcache.c Fri Jan 27 10:15:17 2017 +0100 @@ -32,6 +32,12 @@ #include <gdbm.h> #elif HAVE_DB4 #include <db.h> +#elif HAVE_LMDB +/* This is the maximum size of the database file (2GiB) which is + * mmap(2)'ed into memory. This limit should be good for ~800,000 + * emails. */ +#define LMDB_DB_SIZE (2 * 1024 * 1024 * 1024) +#include <lmdb.h> #endif #include <errno.h> @@ -83,6 +89,61 @@ struct header_cache static void mutt_hcache_dbt_init(DBT * dbt, void *data, size_t len); static void mutt_hcache_dbt_empty_init(DBT * dbt); +#elif HAVE_LMDB +enum mdb_txn_mode +{ + txn_uninitialized = 0, + txn_read = 1 << 0, + txn_write = 1 << 1 +}; +struct header_cache +{ + MDB_env *env; + MDB_txn *txn; + MDB_dbi db; + char *folder; + unsigned int crc; + enum mdb_txn_mode txn_mode; +}; + +static int mdb_get_r_txn(header_cache_t *h) +{ + int rc; + + if (h->txn && (h->txn_mode & (txn_read | txn_write)) > 0) + return MDB_SUCCESS; + + if (h->txn) + rc = mdb_txn_renew(h->txn); + else + rc = mdb_txn_begin(h->env, NULL, MDB_RDONLY, &h->txn); + + if (rc == MDB_SUCCESS) + h->txn_mode = txn_read; + + return rc; +} + +static int mdb_get_w_txn(header_cache_t *h) +{ + int rc; + + if (h->txn && (h->txn_mode == txn_write)) + return MDB_SUCCESS; + + if (h->txn) + { + if (h->txn_mode == txn_read) + mdb_txn_reset(h->txn); + h->txn = NULL; + } + + rc = mdb_txn_begin(h->env, NULL, 0, &h->txn); + if (rc == MDB_SUCCESS) + h->txn_mode = txn_write; + + return rc; +} #endif typedef union @@ -744,6 +805,11 @@ mutt_hcache_fetch_raw (header_cache_t *h #elif HAVE_DB4 DBT key; DBT data; +#elif HAVE_LMDB + MDB_val key; + MDB_val data; + size_t folderlen; + int rc; #endif if (!h) @@ -760,6 +826,37 @@ mutt_hcache_fetch_raw (header_cache_t *h h->db->get(h->db, NULL, &key, &data, 0); return data.data; +#elif HAVE_LMDB + strncpy(path, h->folder, sizeof (path)); + safe_strcat(path, sizeof (path), filename); + + folderlen = strlen(h->folder); + ksize = folderlen + keylen(path + folderlen); + key.mv_data = (char *)path; + key.mv_size = ksize; + data.mv_data = NULL; + data.mv_size = 0; + rc = mdb_get_r_txn(h); + if (rc != MDB_SUCCESS) { + h->txn = NULL; + fprintf(stderr, "txn_renew: %s\n", mdb_strerror(rc)); + return NULL; + } + rc = mdb_get(h->txn, h->db, &key, &data); + if (rc == MDB_NOTFOUND) { + return NULL; + } + if (rc != MDB_SUCCESS) { + fprintf(stderr, "mdb_get: %s\n", mdb_strerror(rc)); + return NULL; + } + /* Caller frees the data we return, so I MUST make a copy of it */ + + char *d = safe_malloc(data.mv_size); + memcpy(d, data.mv_data, data.mv_size); + + return d; + #else strncpy(path, h->folder, sizeof (path)); safe_strcat(path, sizeof (path), filename); @@ -825,6 +922,11 @@ mutt_hcache_store_raw (header_cache_t* h #elif HAVE_DB4 DBT key; DBT databuf; +#elif HAVE_LMDB + MDB_val key; + MDB_val databuf; + size_t folderlen; + int rc; #endif if (!h) @@ -843,6 +945,30 @@ mutt_hcache_store_raw (header_cache_t* h databuf.ulen = dlen; return h->db->put(h->db, NULL, &key, &databuf, 0); +#elif HAVE_LMDB + folderlen = strlen(h->folder); + strncpy(path, h->folder, sizeof (path)); + safe_strcat(path, sizeof (path), filename); + ksize = folderlen + keylen(path + folderlen); + + key.mv_data = (char *)path; + key.mv_size = ksize; + databuf.mv_data = data; + databuf.mv_size = dlen; + rc = mdb_get_w_txn(h); + if (rc != MDB_SUCCESS) { + fprintf(stderr, "txn_begin: %s\n", mdb_strerror(rc)); + return rc; + } + rc = mdb_put(h->txn, h->db, &key, &databuf, 0); + if (rc != MDB_SUCCESS) { + fprintf(stderr, "mdb_put: %s\n", mdb_strerror(rc)); + mdb_txn_abort(h->txn); + h->txn_mode = txn_uninitialized; + h->txn = NULL; + return rc; + } + return rc; #else strncpy(path, h->folder, sizeof (path)); safe_strcat(path, sizeof (path), filename); @@ -1146,6 +1272,106 @@ mutt_hcache_delete(header_cache_t *h, co mutt_hcache_dbt_init(&key, (void *) filename, keylen(filename)); return h->db->del(h->db, NULL, &key, 0); } +#elif HAVE_LMDB + +static int +hcache_open_lmdb (struct header_cache* h, const char* path) +{ + int rc; + + h->txn = NULL; + + rc = mdb_env_create(&h->env); + if (rc != MDB_SUCCESS) { + fprintf(stderr, "hcache_open_lmdb: mdb_env_create: %s", mdb_strerror(rc)); + return -1; + } + + mdb_env_set_mapsize(h->env, LMDB_DB_SIZE); + + rc = mdb_env_open(h->env, path, MDB_NOSUBDIR, 0644); + if (rc != MDB_SUCCESS) { + fprintf(stderr, "hcache_open_lmdb: mdb_env_open: %s", mdb_strerror(rc)); + goto fail_env; + } + + rc = mdb_get_r_txn(h); + if (rc != MDB_SUCCESS) { + fprintf(stderr, "hcache_open_lmdb: mdb_txn_begin: %s", mdb_strerror(rc)); + goto fail_env; + } + + rc = mdb_dbi_open(h->txn, NULL, MDB_CREATE, &h->db); + if (rc != MDB_SUCCESS) { + fprintf(stderr, "hcache_open_lmdb: mdb_dbi_open: %s", mdb_strerror(rc)); + goto fail_dbi; + } + + mdb_txn_reset(h->txn); + h->txn_mode = txn_uninitialized; + return 0; + +fail_dbi: + mdb_txn_abort(h->txn); + h->txn_mode = txn_uninitialized; + h->txn = NULL; + +fail_env: + mdb_env_close(h->env); + return -1; +} + +void +mutt_hcache_close(header_cache_t *h) +{ + if (!h) + return; + + if (h->txn && h->txn_mode == txn_write) + { + mdb_txn_commit(h->txn); + h->txn_mode = txn_uninitialized; + h->txn = NULL; + } + + mdb_env_close(h->env); + FREE (&h->folder); + FREE (&h); +} + +int +mutt_hcache_delete(header_cache_t *h, const char *filename, + size_t(*keylen) (const char *fn)) +{ + MDB_val key; + int rc; + + if (!h) + return -1; + + if (filename[0] == '/') + filename++; + + key.mv_data = (char *)filename; + key.mv_size = strlen(filename); + rc = mdb_get_w_txn(h); + if (rc != MDB_SUCCESS) { + fprintf(stderr, "txn_begin: %s\n", mdb_strerror(rc)); + return rc; + } + rc = mdb_del(h->txn, h->db, &key, NULL); + if (rc != MDB_SUCCESS) { + if (rc != MDB_NOTFOUND) { + fprintf(stderr, "mdb_del: %s\n", mdb_strerror(rc)); + mdb_txn_abort(h->txn); + h->txn_mode = txn_uninitialized; + h->txn = NULL; + } + return rc; + } + + return rc; +} #endif header_cache_t * @@ -1163,6 +1389,8 @@ mutt_hcache_open(const char *path, const hcache_open = hcache_open_gdbm; #elif HAVE_DB4 hcache_open = hcache_open_db4; +#elif HAVE_LMDB + hcache_open = hcache_open_lmdb; #endif /* Calculate the current hcache version from dynamic configuration */ @@ -1200,7 +1428,11 @@ mutt_hcache_open(const char *path, const hcachever = digest.intval; } +#if HAVE_LMDB + h->db = 0; +#else h->db = NULL; +#endif h->folder = get_foldername(folder); h->crc = hcachever; @@ -1235,6 +1467,11 @@ const char *mutt_hcache_backend (void) { return DB_VERSION_STRING; } +#elif HAVE_LMDB +const char *mutt_hcache_backend (void) +{ + return "lmdb " MDB_VERSION_STRING; +} #elif HAVE_GDBM const char *mutt_hcache_backend (void) {
signature.asc
Description: Digital signature
