changeset: 6925:52481ceb6c6e user: Kevin McCarthy <ke...@8t8.us> date: Tue Feb 07 19:36:37 2017 -0800 link: http://dev.mutt.org/hg/mutt/rev/52481ceb6c6e
Add Kyoto Cabinet support to the header cache. Retain the defaults as they are, although we might switch to Kyoto Cabinet for the next major release. changeset: 6926:09bb4a62ceb1 user: Kevin McCarthy <ke...@8t8.us> date: Tue Feb 07 19:36:43 2017 -0800 link: http://dev.mutt.org/hg/mutt/rev/09bb4a62ceb1 Create function to free header cache data. Kyoto Cabinet documents that data from it should be freed via kcfree(). LMDB claims ownership of the data returned, so convert its free operation to be a noop and remove the malloc from its fetch function. diffs (390 lines): diff -r fca7e504ab6a -r 09bb4a62ceb1 configure.ac --- a/configure.ac Sat Feb 04 12:53:52 2017 -0800 +++ b/configure.ac Tue Feb 07 19:36:43 2017 -0800 @@ -882,6 +882,7 @@ AC_ARG_WITH(gdbm, AS_HELP_STRING([--without-gdbm],[Don't use gdbm even if it is available])) AC_ARG_WITH(bdb, AS_HELP_STRING([--with-bdb@<:@=DIR@:>@],[Use BerkeleyDB4 if gdbm is not available])) AC_ARG_WITH(lmdb, AS_HELP_STRING([--with-lmdb@<:@=DIR@:>@],[Use LMDB if gdbm is not available])) +AC_ARG_WITH(kyotocabinet, AS_HELP_STRING([--with-kyotocabinet@<:@=DIR@:>@],[Use kyotocabinet if gdbm is not available])) db_found=no if test x$enable_hcache = xyes @@ -935,6 +936,15 @@ db_requested=lmdb fi fi + if test -n "$with_kyotocabinet" && test "$with_kyotocabinet" != "no" + then + if test "$db_requested" != "auto" + then + AC_MSG_ERROR([more than one header cache engine requested.]) + else + db_requested=kc + fi + fi dnl -- Tokyo Cabinet -- if test "$with_tokyocabinet" != "no" \ @@ -1107,9 +1117,32 @@ fi fi + dnl -- Kyoto Cabinet -- + if test x$with_kyotocabinet != xno && test $db_found = no \ + && test "$db_requested" = auto -o "$db_requested" = kc + then + if test -n "$with_kyotocabinet" && test "$with_kyotocabinet" != "yes" + then + CPPFLAGS="$CPPFLAGS -I$with_kyotocabinet/include" + LDFLAGS="$LDFLAGS -L$with_kyotocabinet/lib" + fi + + AC_CHECK_HEADER(kclangc.h, + AC_CHECK_LIB(kyotocabinet, kcdbopen, + [MUTTLIBS="$MUTTLIBS -lkyotocabinet" + AC_DEFINE(HAVE_KC, 1, [Kyoto Cabinet Support]) + db_found=kc], + [CPPFLAGS="$OLDCPPFLAGS" + LDFLAGS="$OLDLDFLAGS"])) + if test "$db_requested" != auto && test "$db_found" != "$db_requested" + then + AC_MSG_ERROR([Kyoto Cabinet could not be used. Check config.log for details.]) + fi + fi + if test $db_found = no then - AC_MSG_ERROR([You need Tokyo Cabinet, QDBM, GDBM or Berkeley DB4 for hcache]) + AC_MSG_ERROR([You need Tokyo Cabinet, Kyoto Cabinet, QDBM, GDBM, LMDB or Berkeley DB4 for hcache]) fi fi dnl -- end cache -- diff -r fca7e504ab6a -r 09bb4a62ceb1 doc/manual.xml.head --- a/doc/manual.xml.head Sat Feb 04 12:53:52 2017 -0800 +++ b/doc/manual.xml.head Tue Feb 07 19:36:43 2017 -0800 @@ -8118,7 +8118,7 @@ Header caching can be enabled via the configure script and the <emphasis>--enable-hcache</emphasis> option. It's not turned on by default because external database libraries are required: one of -tokyocabinet, qdbm, gdbm or bdb must be present. +tokyocabinet, kyotocabinet, lmdb, qdbm, gdbm or bdb must be present. </para> <para> diff -r fca7e504ab6a -r 09bb4a62ceb1 hcache.c --- a/hcache.c Sat Feb 04 12:53:52 2017 -0800 +++ b/hcache.c Tue Feb 07 19:36:43 2017 -0800 @@ -28,6 +28,8 @@ #include <villa.h> #elif HAVE_TC #include <tcbdb.h> +#elif HAVE_KC +#include <kclangc.h> #elif HAVE_GDBM #include <gdbm.h> #elif HAVE_DB4 @@ -69,6 +71,13 @@ char *folder; unsigned int crc; }; +#elif HAVE_KC +struct header_cache +{ + KCDB *db; + char *folder; + unsigned int crc; +}; #elif HAVE_GDBM struct header_cache { @@ -784,7 +793,7 @@ if (!data || !crc_matches(data, h->crc)) { - FREE(&data); + mutt_hcache_free (&data); return NULL; } @@ -804,6 +813,9 @@ #elif HAVE_TC void *data; int sp; +#elif HAVE_KC + void *data; + size_t sp; #elif HAVE_GDBM datum key; datum data; @@ -844,6 +856,10 @@ data = tcbdbget(h->db, path, ksize, &sp); return data; +#elif HAVE_KC + data = kcdbget(h->db, path, ksize, &sp); + + return data; #elif HAVE_GDBM key.dptr = path; key.dsize = ksize; @@ -858,10 +874,9 @@ (mdb_get (h->txn, h->db, &key, &data) != MDB_SUCCESS)) return NULL; - /* Unlike other dbs, LMDB claims ownership of the returned data */ - char *d = safe_malloc (data.mv_size); - memcpy (d, data.mv_data, data.mv_size); - return d; + /* LMDB claims ownership of the returned data, so this will not be + * freed in mutt_hcache_free(). */ + return data.mv_data; #endif } @@ -939,6 +954,8 @@ return vlput(h->db, path, ksize, data, dlen, VL_DOVER); #elif HAVE_TC return tcbdbput(h->db, path, ksize, data, dlen); +#elif HAVE_KC + return kcdbset(h->db, path, ksize, data, dlen); #elif HAVE_GDBM key.dptr = path; key.dsize = ksize; @@ -1091,6 +1108,67 @@ return tcbdbout(h->db, path, ksize); } +#elif HAVE_KC +static int +hcache_open_kc (struct header_cache* h, const char* path) +{ + char fullpath[_POSIX_PATH_MAX]; + + /* Kyoto cabinet options are discussed at + * http://fallabs.com/kyotocabinet/spex.html + * - rcomp is by default lex, so there is no need to specify it. + * - opts=l enables linear collision chaining as opposed to using a binary tree. + * this isn't suggested unless you are tuning the number of buckets. + * - opts=c enables compression + */ + snprintf (fullpath, sizeof(fullpath), "%s#type=kct%s", path, + option(OPTHCACHECOMPRESS) ? "#opts=c" : ""); + h->db = kcdbnew(); + if (!h->db) + return -1; + if (kcdbopen(h->db, fullpath, KCOWRITER | KCOCREATE)) + return 0; + else + { + dprint (2, (debugfile, "kcdbopen failed for %s: %s (ecode %d)\n", fullpath, + kcdbemsg (h->db), kcdbecode (h->db))); + kcdbdel(h->db); + return -1; + } +} + +void +mutt_hcache_close(header_cache_t *h) +{ + if (!h) + return; + + if (!kcdbclose(h->db)) + dprint (2, (debugfile, "kcdbclose failed for %s: %s (ecode %d)\n", h->folder, + kcdbemsg (h->db), kcdbecode (h->db))); + kcdbdel(h->db); + FREE(&h->folder); + FREE(&h); +} + +int +mutt_hcache_delete(header_cache_t *h, const char *filename, + size_t(*keylen) (const char *fn)) +{ + char path[_POSIX_PATH_MAX]; + int ksize; + + if (!h) + return -1; + + strncpy(path, h->folder, sizeof (path)); + safe_strcat(path, sizeof (path), filename); + + ksize = strlen(h->folder) + keylen(path + strlen(h->folder)); + + return kcdbremove(h->db, path, ksize); +} + #elif HAVE_GDBM static int hcache_open_gdbm (struct header_cache* h, const char* path) @@ -1373,6 +1451,8 @@ hcache_open = hcache_open_qdbm; #elif HAVE_TC hcache_open= hcache_open_tc; +#elif HAVE_KC + hcache_open= hcache_open_kc; #elif HAVE_GDBM hcache_open = hcache_open_gdbm; #elif HAVE_DB4 @@ -1450,6 +1530,21 @@ } } +void mutt_hcache_free (void **data) +{ + if (!data || !*data) + return; + +#if HAVE_KC + kcfree (*data); + *data = NULL; +#elif HAVE_LMDB + /* LMDB owns the data returned. It should not be freed */ +#else + FREE (data); /* __FREE_CHECKED__ */ +#endif +} + #if HAVE_DB4 const char *mutt_hcache_backend (void) { @@ -1475,4 +1570,11 @@ { return "tokyocabinet " _TC_VERSION; } +#elif HAVE_KC +const char *mutt_hcache_backend (void) +{ + static char backend[SHORT_STRING]; + snprintf(backend, sizeof(backend), "kyotocabinet %s", KCVERSION); + return backend; +} #endif diff -r fca7e504ab6a -r 09bb4a62ceb1 hcache.h --- a/hcache.h Sat Feb 04 12:53:52 2017 -0800 +++ b/hcache.h Tue Feb 07 19:36:43 2017 -0800 @@ -33,6 +33,7 @@ void *mutt_hcache_fetch(header_cache_t *h, const char *filename, size_t (*keylen)(const char *fn)); void *mutt_hcache_fetch_raw (header_cache_t *h, const char *filename, size_t (*keylen)(const char *fn)); +void mutt_hcache_free (void **data); typedef enum { MUTT_GENERATE_UIDVALIDITY = 1 /* use gettimeofday() as value */ diff -r fca7e504ab6a -r 09bb4a62ceb1 imap/imap.c --- a/imap/imap.c Sat Feb 04 12:53:52 2017 -0800 +++ b/imap/imap.c Tue Feb 07 19:36:43 2017 -0800 @@ -1685,13 +1685,13 @@ { uidvalidity = mutt_hcache_fetch_raw (hc, "/UIDVALIDITY", imap_hcache_keylen); uidnext = mutt_hcache_fetch_raw (hc, "/UIDNEXT", imap_hcache_keylen); - mutt_hcache_close (hc); if (uidvalidity) { if (!status) { - FREE (&uidvalidity); - FREE (&uidnext); + mutt_hcache_free ((void **)&uidvalidity); + mutt_hcache_free ((void **)&uidnext); + mutt_hcache_close (hc); return imap_mboxcache_get (idata, mbox, 1); } status->uidvalidity = *uidvalidity; @@ -1699,8 +1699,9 @@ dprint (3, (debugfile, "mboxcache: hcache uidvalidity %d, uidnext %d\n", status->uidvalidity, status->uidnext)); } - FREE (&uidvalidity); - FREE (&uidnext); + mutt_hcache_free ((void **)&uidvalidity); + mutt_hcache_free ((void **)&uidnext); + mutt_hcache_close (hc); } #endif diff -r fca7e504ab6a -r 09bb4a62ceb1 imap/message.c --- a/imap/message.c Sat Feb 04 12:53:52 2017 -0800 +++ b/imap/message.c Tue Feb 07 19:36:43 2017 -0800 @@ -146,11 +146,11 @@ if (puidnext) { uidnext = *puidnext; - FREE (&puidnext); + mutt_hcache_free ((void **)&puidnext); } if (uid_validity && uidnext && *uid_validity == idata->uid_validity) evalhc = 1; - FREE (&uid_validity); + mutt_hcache_free ((void **)&uid_validity); } if (evalhc) { diff -r fca7e504ab6a -r 09bb4a62ceb1 imap/util.c --- a/imap/util.c Sat Feb 04 12:53:52 2017 -0800 +++ b/imap/util.c Tue Feb 07 19:36:43 2017 -0800 @@ -130,7 +130,7 @@ h = mutt_hcache_restore ((unsigned char*)uv, NULL); else dprint (3, (debugfile, "hcache uidvalidity mismatch: %u", *uv)); - FREE (&uv); + mutt_hcache_free ((void **)&uv); } return h; diff -r fca7e504ab6a -r 09bb4a62ceb1 init.h --- a/init.h Sat Feb 04 12:53:52 2017 -0800 +++ b/init.h Tue Feb 07 19:36:43 2017 -0800 @@ -984,12 +984,12 @@ ** Header caching can greatly improve speed when opening POP, IMAP ** MH or Maildir folders, see ``$caching'' for details. */ -#if defined(HAVE_QDBM) || defined(HAVE_TC) +#if defined(HAVE_QDBM) || defined(HAVE_TC) || defined(HAVE_KC) { "header_cache_compress", DT_BOOL, R_NONE, OPTHCACHECOMPRESS, 1 }, /* ** .pp - ** When mutt is compiled with qdbm or tokyocabinet as header cache backend, - ** this option determines whether the database will be compressed. + ** When mutt is compiled with qdbm, tokyocabinet, or kyotocabinet as header + ** cache backend, this option determines whether the database will be compressed. ** Compression results in database files roughly being one fifth ** of the usual diskspace, but the decompression can result in a ** slower opening of cached folder(s) which in general is still diff -r fca7e504ab6a -r 09bb4a62ceb1 mh.c --- a/mh.c Sat Feb 04 12:53:52 2017 -0800 +++ b/mh.c Tue Feb 07 19:36:43 2017 -0800 @@ -1195,7 +1195,7 @@ mutt_free_header (&p->h); #if USE_HCACHE } - FREE (&data); + mutt_hcache_free (&data); #endif last = p; } diff -r fca7e504ab6a -r 09bb4a62ceb1 mutt.h --- a/mutt.h Sat Feb 04 12:53:52 2017 -0800 +++ b/mutt.h Tue Feb 07 19:36:43 2017 -0800 @@ -360,7 +360,7 @@ OPTFORWQUOTE, #ifdef USE_HCACHE OPTHCACHEVERIFY, -#if defined(HAVE_QDBM) || defined(HAVE_TC) +#if defined(HAVE_QDBM) || defined(HAVE_TC) || defined(HAVE_KC) OPTHCACHECOMPRESS, #endif /* HAVE_QDBM */ #endif diff -r fca7e504ab6a -r 09bb4a62ceb1 pop.c --- a/pop.c Sat Feb 04 12:53:52 2017 -0800 +++ b/pop.c Tue Feb 07 19:36:43 2017 -0800 @@ -336,7 +336,7 @@ mutt_hcache_store (hc, ctx->hdrs[i]->data, ctx->hdrs[i], 0, strlen, MUTT_GENERATE_UIDVALIDITY); } - FREE(&data); + mutt_hcache_free (&data); #endif /*
