#3691: Add LMDB support
---------------------------+----------------------
Reporter: rsc | Owner: mutt-dev
Type: enhancement | Status: new
Priority: minor | Milestone:
Component: header cache | Version:
Resolution: | Keywords:
---------------------------+----------------------
Comment (by gahr2):
I have an alternative approach, which is, let's keep the last (read /
write) transaction open and reuse it until a transaction for a different
mode (write / read) is required. At that point, reset it if it was a read
transaction and commit it if it was a write transaction. I noticed a
relevant speed up with this patch.
{{{#!diff
diff -r f1f1af650910 configure.ac
--- a/configure.ac Tue May 24 12:08:46 2016 -0700
+++ b/configure.ac Mon Sep 19 14:18:01 2016 +0000
@@ -855,6 +855,7 @@
AC_ARG_WITH(qdbm, AS_HELP_STRING([--without-qdbm],[Don't use qdbm even if
it is available]))
AC_ARG_WITH(gdbm, AS_HELP_STRING([--without-gdbm],[Don't use gdbm even if
it is available]))
AC_ARG_WITH(bdb, AS_HELP_STRING([--with-bdb@<:@=DIR@:>@],[Use BerkeleyDB4
if gdbm is not available]))
+AC_ARG_WITH(lmdb, AS_HELP_STRING([--with-lmdb@<:@=DIR@:>@],[Use LMDB if
gdbm is not available]))
db_found=no
if test x$enable_hcache = xyes
@@ -899,6 +900,15 @@
db_requested=bdb
fi
fi
+ if test -n "$with_lmdb" && test "$with_lmdb" != "no"
+ then
+ if test "$db_requested" != "auto"
+ then
+ AC_MSG_ERROR([more than one header cache engine requested.])
+ else
+ db_requested=lmdb
+ fi
+ fi
dnl -- Tokyo Cabinet --
if test "$with_tokyocabinet" != "no" \
@@ -1042,6 +1052,34 @@
fi
fi
+ dnl -- LMDB --
+ if test x$with_lmdb != xno && test $db_found = no \
+ && test "$db_requested" = auto -o "$db_requested" = lmdb
+ then
+ if test "$with_lmdb" != "yes"
+ then
+ CPPFLAGS="$CPPFLAGS -I$with_lmdb/include"
+ LDFLAGS="$LDFLAGS -L$with_lmdb/lib"
+ fi
+ saved_LIBS="$LIBS"
+ LIBS="$LIBS -llmdb"
+ AC_CACHE_CHECK(for mdb_env_create, ac_cv_mdbenvcreate,[
+ ac_cv_mdbenvcreate=no
+ AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <lmdb.h>]],
[[mdb_env_create(0);]])],[ac_cv_mdbenvcreate=yes],[])
+ ])
+ LIBS="$saved_LIBS"
+ if test "$ac_cv_mdbenvcreate" = yes
+ then
+ AC_DEFINE(HAVE_LMDB, 1, [LMDB Support])
+ MUTTLIBS="$MUTTLIBS -llmdb"
+ db_found=lmdb
+ fi
+ if test "$db_requested" != auto && test "$db_found" !=
"$db_requested"
+ then
+ AC_MSG_ERROR([LMDB could not be used. Check config.log for
details.])
+ fi
+ fi
+
if test $db_found = no
then
AC_MSG_ERROR([You need Tokyo Cabinet, QDBM, GDBM or Berkeley DB4
for hcache])
diff -r f1f1af650910 hcache.c
--- a/hcache.c Tue May 24 12:08:46 2016 -0700
+++ b/hcache.c Mon Sep 19 14:18:01 2016 +0000
@@ -32,6 +32,9 @@
#include <gdbm.h>
#elif HAVE_DB4
#include <db.h>
+#elif HAVE_LMDB
+#define LMDB_DB_SIZE (1024 * 1024 * 1024)
+#include <lmdb.h>
#endif
#include <errno.h>
@@ -83,6 +86,52 @@
static void mutt_hcache_dbt_init(DBT * dbt, void *data, size_t len);
static void mutt_hcache_dbt_empty_init(DBT * dbt);
+#elif HAVE_LMDB
+struct header_cache
+{
+ MDB_env *env;
+ MDB_txn *txn;
+ MDB_dbi db;
+ char *folder;
+ unsigned int crc;
+ int last_mode; // 0: read, 1: write
+};
+
+static int mdb_get_r_txn(header_cache_t *h)
+{
+ if (h->txn && h->last_mode == 1)
+ {
+ mdb_txn_commit(h->txn);
+ h->txn = NULL;
+ }
+ h->last_mode = 0;
+ if (!h->txn)
+ {
+ return mdb_txn_begin(h->env, NULL, MDB_RDONLY, &h->txn);
+ }
+ else
+ {
+ return mdb_txn_renew(h->txn);
+ }
+}
+
+static int mdb_get_w_txn(header_cache_t *h)
+{
+ if (h->txn && h->last_mode == 0)
+ {
+ mdb_txn_reset(h->txn);
+ h->txn = NULL;
+ }
+ h->last_mode = 1;
+ if (h->txn)
+ {
+ return MDB_SUCCESS;
+ }
+ else
+ {
+ return mdb_txn_begin(h->env, NULL, 0, &h->txn);
+ }
+}
#endif
typedef union
@@ -732,6 +793,11 @@
#elif HAVE_DB4
DBT key;
DBT data;
+#elif HAVE_LMDB
+ MDB_val key;
+ MDB_val data;
+ size_t folderlen;
+ int rc;
#endif
if (!h)
@@ -748,6 +814,43 @@
h->db->get(h->db, NULL, &key, &data, 0);
return data.data;
+#elif HAVE_LMDB
+ strncpy(path, h->folder, sizeof (path));
+ safe_strcat(path, sizeof (path), filename);
+
+ folderlen = strlen(h->folder);
+ ksize = folderlen + keylen(path + folderlen);
+ key.mv_data = (char *)path;
+ key.mv_size = ksize;
+ data.mv_data = NULL;
+ data.mv_size = 0;
+ rc = mdb_get_r_txn(h);
+ if (rc != MDB_SUCCESS)
+ {
+ h->txn = NULL;
+ fprintf(stderr, "txn_renew: %s\n", mdb_strerror(rc));
+ return NULL;
+ }
+ rc = mdb_get(h->txn, h->db, &key, &data);
+ if (rc == MDB_NOTFOUND)
+ {
+ mdb_txn_reset(h->txn);
+ return NULL;
+ }
+ if (rc != MDB_SUCCESS)
+ {
+ fprintf(stderr, "mdb_get: %s\n", mdb_strerror(rc));
+ mdb_txn_reset(h->txn);
+ return NULL;
+ }
+ /* Caller frees the data we return, so I MUST make a copy of it */
+
+ char *d = safe_malloc(data.mv_size);
+ memcpy(d, data.mv_data, data.mv_size);
+ mdb_txn_reset(h->txn);
+
+ return d;
+
#else
strncpy(path, h->folder, sizeof (path));
safe_strcat(path, sizeof (path), filename);
@@ -813,6 +916,11 @@
#elif HAVE_DB4
DBT key;
DBT databuf;
+#elif HAVE_LMDB
+ MDB_val key;
+ MDB_val databuf;
+ size_t folderlen;
+ int rc;
#endif
if (!h)
@@ -831,6 +939,30 @@
databuf.ulen = dlen;
return h->db->put(h->db, NULL, &key, &databuf, 0);
+#elif HAVE_LMDB
+ folderlen = strlen(h->folder);
+ strncpy(path, h->folder, sizeof (path));
+ safe_strcat(path, sizeof (path), filename);
+ ksize = folderlen + keylen(path + folderlen);
+
+ key.mv_data = (char *)path;
+ key.mv_size = ksize;
+ databuf.mv_data = data;
+ databuf.mv_size = dlen;
+ rc = mdb_get_w_txn(h);
+ if (rc != MDB_SUCCESS)
+ {
+ fprintf(stderr, "txn_begin: %s\n", mdb_strerror(rc));
+ return rc;
+ }
+ rc = mdb_put(h->txn, h->db, &key, &databuf, 0);
+ if (rc != MDB_SUCCESS)
+ {
+ fprintf(stderr, "mdb_put: %s\n", mdb_strerror(rc));
+ mdb_txn_abort(h->txn);
+ return rc;
+ }
+ return rc;
#else
strncpy(path, h->folder, sizeof (path));
safe_strcat(path, sizeof (path), filename);
@@ -1134,6 +1266,108 @@
mutt_hcache_dbt_init(&key, (void *) filename, keylen(filename));
return h->db->del(h->db, NULL, &key, 0);
}
+#elif HAVE_LMDB
+
+static int
+hcache_open_lmdb (struct header_cache* h, const char* path)
+{
+ int rc;
+
+ h->txn = NULL;
+
+ rc = mdb_env_create(&h->env);
+ if (rc != MDB_SUCCESS)
+ {
+ fprintf(stderr, "hcache_open_lmdb: mdb_env_create: %s",
mdb_strerror(rc));
+ return -1;
+ }
+
+ mdb_env_set_mapsize(h->env, LMDB_DB_SIZE);
+
+ rc = mdb_env_open(h->env, path, MDB_NOSUBDIR, 0644);
+ if (rc != MDB_SUCCESS)
+ {
+ fprintf(stderr, "hcache_open_lmdb: mdb_env_open: %s",
mdb_strerror(rc));
+ goto fail_env;
+ }
+
+ rc = mdb_get_r_txn(h);
+ if (rc != MDB_SUCCESS)
+ {
+ fprintf(stderr, "hcache_open_lmdb: mdb_txn_begin: %s",
mdb_strerror(rc));
+ goto fail_env;
+ }
+
+ rc = mdb_dbi_open(h->txn, NULL, MDB_CREATE, &h->db);
+ if (rc != MDB_SUCCESS)
+ {
+ fprintf(stderr, "hcache_open_lmdb: mdb_dbi_open: %s",
mdb_strerror(rc));
+ goto fail_dbi;
+ }
+
+ mdb_txn_reset(h->txn);
+ return 0;
+
+fail_dbi:
+ mdb_txn_abort(h->txn);
+ h->txn = NULL;
+
+fail_env:
+ mdb_env_close(h->env);
+ return -1;
+}
+
+void
+mutt_hcache_close(header_cache_t *h)
+{
+ if (!h)
+ return;
+
+ if (h->txn && h->last_mode == 1)
+ {
+ mdb_txn_commit(h->txn);
+ }
+
+ mdb_env_close(h->env);
+ FREE (&h->folder);
+ FREE (&h);
+}
+
+int
+mutt_hcache_delete(header_cache_t *h, const char *filename,
+ size_t(*keylen) (const char *fn))
+{
+ MDB_val key;
+ MDB_txn *txn;
+ int rc;
+
+ if (!h)
+ return -1;
+
+ if (filename[0] == '/')
+ filename++;
+
+ key.mv_data = (char *)filename;
+ key.mv_size = strlen(filename);
+ rc = mdb_get_w_txn(h);
+ if (rc != MDB_SUCCESS)
+ {
+ fprintf(stderr, "txn_begin: %s\n", mdb_strerror(rc));
+ return rc;
+ }
+ rc = mdb_del(txn, h->db, &key, NULL);
+ if (rc != MDB_SUCCESS)
+ {
+ if (rc != MDB_NOTFOUND)
+ {
+ fprintf(stderr, "mdb_del: %s\n", mdb_strerror(rc));
+ }
+ mdb_txn_abort(txn);
+ return rc;
+ }
+
+ return rc;
+}
#endif
header_cache_t *
@@ -1151,6 +1385,8 @@
hcache_open = hcache_open_gdbm;
#elif HAVE_DB4
hcache_open = hcache_open_db4;
+#elif HAVE_LMDB
+ hcache_open = hcache_open_lmdb;
#endif
/* Calculate the current hcache version from dynamic configuration */
@@ -1188,7 +1424,11 @@
hcachever = digest.intval;
}
+#if HAVE_LMDB
+ h->db = 0;
+#else
h->db = NULL;
+#endif
h->folder = get_foldername(folder);
h->crc = hcachever;
@@ -1223,6 +1463,11 @@
{
return DB_VERSION_STRING;
}
+#elif HAVE_LMDB
+const char *mutt_hcache_backend (void)
+{
+ return "lmdb " MDB_VERSION_STRING;
+}
#elif HAVE_GDBM
const char *mutt_hcache_backend (void)
{
}}}
--
Ticket URL: <https://dev.mutt.org/trac/ticket/3691#comment:8>
Mutt <http://www.mutt.org/>
The Mutt mail user agent
