Changeset: 45b9f37c934b for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=45b9f37c934b
Modified Files:
clients/Tests/exports.stable.out
gdk/gdk.h
gdk/gdk_align.c
gdk/gdk_bat.c
gdk/gdk_bbp.c
gdk/gdk_mosaic.c
gdk/gdk_mosaic_templates.h
monetdb5/modules/mosaic/mosaic.c
monetdb5/modules/mosaic/mosaic.h
monetdb5/modules/mosaic/mosaic_dictionary_templates.h
monetdb5/modules/mosaic/mosaic_utility.h
sql/backends/monet5/sql_cat.c
sql/test/mosaic/Tests/analysis.stable.out
sql/test/mosaic/Tests/compression.stable.out
sql/test/mosaic/Tests/compressionRLE.stable.out
sql/test/mosaic/Tests/compressionRLE2.stable.out
sql/test/mosaic/Tests/join.stable.out
sql/test/mosaic/Tests/layout_delta.stable.out
sql/test/mosaic/Tests/layout_dict.stable.out
sql/test/mosaic/Tests/layout_frame.stable.out
sql/test/mosaic/Tests/layout_linear.stable.out
sql/test/mosaic/Tests/layout_prefix.stable.out
sql/test/mosaic/Tests/layout_raw.stable.out
sql/test/mosaic/Tests/mix.stable.out
sql/test/mosaic/Tests/session_exit.stable.out
sql/test/mosaic/Tests/xqueries.stable.out
sql/test/mosaic/Tests/xqueries_delta.stable.out
sql/test/mosaic/Tests/xqueries_rle.stable.out
Branch: mosaic
Log Message:
Repair mosaic heap persistence management.
diffs (truncated from 1789 to 300 lines):
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -338,7 +338,11 @@ void IMPSdestroy(BAT *b);
lng IMPSimprintsize(BAT *b);
const char *LEVEL_STR[];
LOG_LEVEL LVL_PER_COMPONENT[];
+Heap *MOScreate_mosaic_heap(BAT *b, BUN cap);
+Heap *MOScreate_vmosaic_heap(BAT *b, BUN cap);
void MOSdestroy(BAT *b);
+void MOSfree(BAT *bn);
+void MOSpersist(BAT *b);
void MOSsetLock(BAT *b);
void MOSunsetLock(BAT *b);
void MOSvirtualize(BAT *bn);
@@ -1552,9 +1556,9 @@ str MKEYconstbulk_rotate_xor_hash(bat *r
str MKEYhash(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
str MKEYrotate(lng *ret, const lng *v, const int *nbits);
str MKEYrotate_xor_hash(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
-str MOSAnalysis(BAT *b, BAT *btech, BAT *blayout, BAT *output, BAT *factor,
BAT *compress, BAT *decompress, str compressions);
+str MOSAnalysis(BAT *b, BAT *btech, BAT *blayout, BAT *output, BAT *factor,
BAT *compress, BAT *decompress, str compressions, str common_compressions);
str MOScompress(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str MOScompressInternal(BAT *bsrc, const char *compressions);
+str MOScompressInternal(BAT *bsrc, const char *compressions, bool persist);
str MOScreateCappedInfo(MOStask *task);
void MOScreateframeDictionary(MOStask *task);
void MOScreatevar(MOStask *task);
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -1622,8 +1622,12 @@ bunfastappVAR(BAT *b, const void *v)
/* support routines for the mosaic approach */
#define MOSAIC_VERSION 20140808
gdk_export gdk_return BATmosaic(BAT *b, BUN cap);
+gdk_export Heap* MOScreate_mosaic_heap(BAT *b, BUN cap);
+gdk_export Heap* MOScreate_vmosaic_heap(BAT *b, BUN cap);
gdk_export int BATcheckmosaic(BAT *b);
gdk_export void MOSdestroy(BAT *b);
+gdk_export void MOSfree(BAT *bn);
+gdk_export void MOSpersist(BAT *b);
gdk_export void MOSsetLock(BAT* b);
gdk_export void MOSunsetLock(BAT* b);
gdk_export void MOSvirtualize(BAT *bn);
@@ -1951,14 +1955,10 @@ gdk_export void VIEWbounds(BAT *b, BAT *
#define isVIEW(x) \
(assert((x)->batCacheid > 0), \
((x)->theap.parentid || \
- ((x)->tvheap && (x)->tvheap->parentid != (x)->batCacheid) || \
- ((x)->tmosaic && (x)->tmosaic->parentid != (x)->batCacheid)))
+ ((x)->tvheap && (x)->tvheap->parentid != (x)->batCacheid)))
#define VIEWtparent(x) ((x)->theap.parentid)
#define VIEWvtparent(x) ((x)->tvheap == NULL || (x)->tvheap->parentid
== (x)->batCacheid ? 0 : (x)->tvheap->parentid)
-// TODO check if this part of the check "(x)->tmosaic->parentid ==
(x)->batCacheid" is necessary.
-#define VIEWmosaictparent(x) ((x)->tmosaic == NULL || (x)->tmosaic->parentid
== (x)->batCacheid ? 0 : (x)->tmosaic->parentid)
-#define VIEWvmosaictparent(x) ((x)->tvmosaic == NULL ||
(x)->tvmosaic->parentid == (x)->batCacheid ? 0 : (x)->tvmosaic->parentid)
/*
* @+ BAT Iterators
diff --git a/gdk/gdk_align.c b/gdk/gdk_align.c
--- a/gdk/gdk_align.c
+++ b/gdk/gdk_align.c
@@ -236,10 +236,8 @@ VIEWunlink(BAT *b)
if (b) {
bat tp = VIEWtparent(b);
bat vtp = VIEWvtparent(b);
- bat mtp = VIEWmosaictparent(b);
BAT *tpb = NULL;
BAT *vtpb = NULL;
- BAT *mtpb = NULL;
assert(b->batCacheid > 0);
if (tp)
@@ -248,8 +246,6 @@ VIEWunlink(BAT *b)
vtp = tp;
if (vtp)
vtpb = BBP_cache(vtp);
- if (mtp)
- mtpb = BBP_cache(mtp);
if (tpb == NULL && vtpb == NULL)
return;
@@ -266,10 +262,6 @@ VIEWunlink(BAT *b)
/* unlink imprints shared with parent */
if (tpb && b->timprints && b->timprints == tpb->timprints)
b->timprints = NULL;
-
- /* unlink mosaic shared with parent */
- if (mtpb && b->tmosaic && b->tmosaic == mtpb->tmosaic)
- b->tmosaic = NULL;
}
}
diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c
--- a/gdk/gdk_bat.c
+++ b/gdk/gdk_bat.c
@@ -553,7 +553,7 @@ BATfree(BAT *b)
HASHfree(b);
IMPSfree(b);
OIDXfree(b);
- MOSdestroy(b);
+ MOSfree(b);
if (b->ttype)
HEAPfree(&b->theap, false);
else
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -1616,7 +1616,6 @@ BBPexit(void)
* to locks) do it */
bat tp = VIEWtparent(b);
bat vtp = VIEWvtparent(b);
- bat mosaictp =
VIEWmosaictparent(b);
if (tp) {
BBP_desc(tp)->batSharecnt--;
--BBP_lrefs(tp);
@@ -1625,10 +1624,6 @@ BBPexit(void)
BBP_desc(vtp)->batSharecnt--;
--BBP_lrefs(vtp);
}
- if (mosaictp) {
-
BBP_desc(mosaictp)->batSharecnt--;
- --BBP_lrefs(mosaictp);
- }
VIEWdestroy(b);
} else {
BATfree(b);
@@ -2494,7 +2489,7 @@ static inline int
incref(bat i, bool logical, bool lock)
{
int refs;
- bat tp, tvp, tmp;
+ bat tp, tvp;
BAT *b, *pb = NULL, *pvb = NULL, *pmb = NULL;
bool load = false;
@@ -2520,16 +2515,6 @@ incref(bat i, bool logical, bool lock)
BBPunfix(pb->batCacheid);
return 0;
}
- } // there could be a parent because of a shared mosaic
- if (b->tmosaic && b->tmosaic->parentid != i) {
- pmb = BATdescriptor(b->tmosaic->parentid);
- if (pmb == NULL) {
- if (pb)
- BBPunfix(pb->batCacheid);
- if (pvb)
- BBPunfix(pvb->batCacheid);
- return 0;
- }
}
}
@@ -2563,9 +2548,8 @@ incref(bat i, bool logical, bool lock)
tp = b->theap.parentid;
assert(tp >= 0);
tvp = b->tvheap == 0 || b->tvheap->parentid == i ? 0 :
b->tvheap->parentid;
- tmp = b->tmosaic == 0 || b->tmosaic->parentid == i ? 0 :
b->tmosaic->parentid;
refs = ++BBP_refs(i);
- if (refs == 1 && (tp || tvp || tmp)) {
+ if (refs == 1 && (tp || tvp)) {
/* If this is a view, we must load the parent
* BATs, but we must do that outside of the
* lock. Set the BBPLOADING flag so that
@@ -2639,7 +2623,7 @@ decref(bat i, bool logical, bool release
{
int refs = 0;
bool swap = false;
- bat tp = 0, tvp = 0, tmp = 0;
+ bat tp = 0, tvp = 0;
BAT *b;
assert(i > 0);
@@ -2677,13 +2661,11 @@ decref(bat i, bool logical, bool release
} else {
assert(b == NULL || b->theap.parentid == 0 ||
BBP_refs(b->theap.parentid) > 0);
assert(b == NULL || b->tvheap == NULL ||
b->tvheap->parentid == 0 || BBP_refs(b->tvheap->parentid) > 0);
- assert(b == NULL || b->tmosaic == NULL ||
b->tmosaic->parentid == 0 || BBP_refs(b->tmosaic->parentid) > 0);
refs = --BBP_refs(i);
if (b && refs == 0) {
if ((tp = b->theap.parentid) != 0)
b->theap.base = (char *) (b->theap.base
- BBP_cache(tp)->theap.base);
tvp = VIEWvtparent(b);
- tmp = VIEWmosaictparent(b);
}
}
}
@@ -2725,8 +2707,6 @@ decref(bat i, bool logical, bool release
decref(tp, false, false, lock, func);
if (tvp)
decref(tvp, false, false, lock, func);
- if (tmp)
- decref(tmp, false, false, lock, func);
return refs;
}
@@ -2937,7 +2917,6 @@ BBPdestroy(BAT *b)
{
bat tp = b->theap.parentid;
bat vtp = VIEWvtparent(b);
- bat mtp = VIEWmosaictparent(b);
if (isVIEW(b)) { /* a physical view */
VIEWdestroy(b);
@@ -2962,14 +2941,12 @@ BBPdestroy(BAT *b)
GDKunshare(tp);
if (vtp)
GDKunshare(vtp);
- if (mtp)
- GDKunshare(mtp);
}
static gdk_return
BBPfree(BAT *b, const char *calledFrom)
{
- bat bid = b->batCacheid, tp = VIEWtparent(b), vtp = VIEWvtparent(b),
mtp = VIEWmosaictparent(b);
+ bat bid = b->batCacheid, tp = VIEWtparent(b), vtp = VIEWvtparent(b);
gdk_return ret;
assert(bid > 0);
@@ -2998,8 +2975,6 @@ BBPfree(BAT *b, const char *calledFrom)
GDKunshare(tp);
if (ret == GDK_SUCCEED && vtp)
GDKunshare(vtp);
- if (ret == GDK_SUCCEED && mtp)
- GDKunshare(mtp);
return ret;
}
@@ -3848,12 +3823,6 @@ BBPdiskscan(const char *parent, size_t b
} else if (strncmp(p + 1, "theap", 5) == 0) {
BAT *b = getdesc(bid);
delete = (b == NULL || !b->tvheap ||
!b->batCopiedtodisk);
- } else if (strncmp(p + 1, "mosaic", 6) == 0) {
- BAT *b = getdesc(bid);
- delete = (b == NULL || !b->tmosaic ||
!b->batCopiedtodisk);
- } else if (strncmp(p + 1, "vmosaic", 7) == 0) {
- BAT *b = getdesc(bid);
- delete = (b == NULL || !b->tmosaic ||
!b->batCopiedtodisk);
} else if (strncmp(p + 1, "thashl", 6) == 0 ||
strncmp(p + 1, "thashb", 6) == 0) {
#ifdef PERSISTENTHASH
@@ -3864,16 +3833,16 @@ BBPdiskscan(const char *parent, size_t b
#else
delete = true;
#endif
- } else if (strncmp(p + 1, "tmosaic", 9) == 0) {
+ } else if (strncmp(p + 1, "tmosaic", 7) == 0) {
BAT *b = getdesc(bid);
delete = b == NULL;
if (!delete)
b->tmosaic = (Heap *) 1;
- } else if (strncmp(p + 1, "tvmosaic", 10) == 0) {
+ } else if (strncmp(p + 1, "tvmosaic", 8) == 0) {
BAT *b = getdesc(bid);
delete = b == NULL;
if (!delete)
- b->tmosaic = (Heap *) 1;
+ b->tvmosaic = (Heap *) 1;
} else if (strncmp(p + 1, "thash", 5) == 0) {
/* older versions used .thash which we
* can simply ignore */
diff --git a/gdk/gdk_mosaic.c b/gdk/gdk_mosaic.c
--- a/gdk/gdk_mosaic.c
+++ b/gdk/gdk_mosaic.c
@@ -22,16 +22,8 @@ void MOSunsetLock(BAT* b) {
MT_lock_unset(&b->batIdxLock);
}
-static inline void
-MOSsync(int fd) {
-#if defined(NATIVE_WIN32)
- _commit(fd);
-#elif defined(HAVE_FDATASYNC)
- fdatasync(fd);
-#elif defined(HAVE_FSYNC)
- fsync(fd);
-#endif
-}
+
+#define EXT(HEAP) ("t" STRINGIFY(HEAP))
#define HEAP mosaic
#include "gdk_mosaic_templates.h"
@@ -42,28 +34,52 @@ MOSsync(int fd) {
void
MOSdestroy(BAT *bn) {
+ MT_lock_set(&bn->batIdxLock);
MOSdestroy_mosaic(bn);
MOSdestroy_vmosaic(bn);
+ MT_lock_unset(&bn->batIdxLock);
+}
+
+void
+MOSfree(BAT *bn) {
+ MT_lock_set(&bn->batIdxLock);
+ MOSfree_mosaic(bn);
+ MOSfree_vmosaic(bn);
+ MT_lock_unset(&bn->batIdxLock);
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
