Changeset: 2f32cc844ca1 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/2f32cc844ca1
Modified Files:
monetdb5/modules/kernel/batmmath.c
monetdb5/optimizer/opt_pushselect.c
Branch: pushcands
Log Message:
Merged with default
diffs (truncated from 479 to 300 lines):
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -375,6 +375,8 @@ typedef enum { GDK_FAIL, GDK_SUCCEED } g
#define PERFMASK (1<<12)
#define DELTAMASK (1<<13)
#define LOADMASK (1<<14)
+#define PUSHCANDMASK (1<<15) /* used in opt_pushselect.c */
+#define TAILCHKMASK (1<<16) /* check .tail file size during commit */
#define ACCELMASK (1<<20)
#define ALGOMASK (1<<21)
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -1438,6 +1438,9 @@ BBPinit(bool first)
bat nhashbats = 0;
gdk_return res = GDK_SUCCEED;
#endif
+ int dbg = GDKdebug;
+
+ GDKdebug &= ~TAILCHKMASK;
/* the maximum number of BATs allowed in the system and the
* size of the "physical" array are linked in a complicated
@@ -1464,6 +1467,7 @@ BBPinit(bool first)
if (!(bbpdirstr = GDKfilepath(0, BATDIR, "BBP", "dir"))) {
TRC_CRITICAL(GDK, "GDKmalloc failed\n");
MT_lock_unset(&GDKtmLock);
+ GDKdebug = dbg;
return GDK_FAIL;
}
@@ -1471,6 +1475,7 @@ BBPinit(bool first)
GDKfree(bbpdirstr);
TRC_CRITICAL(GDK, "GDKmalloc failed\n");
MT_lock_unset(&GDKtmLock);
+ GDKdebug = dbg;
return GDK_FAIL;
}
@@ -1479,6 +1484,7 @@ BBPinit(bool first)
GDKfree(backupbbpdirstr);
TRC_CRITICAL(GDK, "cannot remove directory %s\n",
TEMPDIR);
MT_lock_unset(&GDKtmLock);
+ GDKdebug = dbg;
return GDK_FAIL;
}
@@ -1487,6 +1493,7 @@ BBPinit(bool first)
GDKfree(backupbbpdirstr);
TRC_CRITICAL(GDK, "cannot remove directory %s\n",
DELDIR);
MT_lock_unset(&GDKtmLock);
+ GDKdebug = dbg;
return GDK_FAIL;
}
@@ -1496,6 +1503,7 @@ BBPinit(bool first)
GDKfree(backupbbpdirstr);
TRC_CRITICAL(GDK, "cannot properly recover_subdir
process %s.", SUBDIR);
MT_lock_unset(&GDKtmLock);
+ GDKdebug = dbg;
return GDK_FAIL;
}
@@ -1513,6 +1521,7 @@ BBPinit(bool first)
GDKfree(backupbbpdirstr);
TRC_CRITICAL(GDK, "cannot open recovered
BBP.dir.");
MT_lock_unset(&GDKtmLock);
+ GDKdebug = dbg;
return GDK_FAIL;
}
} else if ((fp = GDKfilelocate(0, "BBP", "r", "dir")) == NULL) {
@@ -1555,13 +1564,17 @@ BBPinit(bool first)
bbpversion = GDKLIBRARY;
} else {
bbpversion = BBPheader(fp, &lineno, &bbpsize);
- if (bbpversion == 0)
+ if (bbpversion == 0) {
+ GDKdebug = dbg;
return GDK_FAIL;
+ }
}
/* allocate BBP records */
- if (BBPextend(0, false, bbpsize) != GDK_SUCCEED)
+ if (BBPextend(0, false, bbpsize) != GDK_SUCCEED) {
+ GDKdebug = dbg;
return GDK_FAIL;
+ }
ATOMIC_SET(&BBPsize, bbpsize);
if (!GDKinmemory(0)) {
@@ -1569,8 +1582,10 @@ BBPinit(bool first)
#ifdef GDKLIBRARY_HASHASH
, &hashbats, &nhashbats
#endif
- ) != GDK_SUCCEED)
+ ) != GDK_SUCCEED) {
+ GDKdebug = dbg;
return GDK_FAIL;
+ }
fclose(fp);
}
@@ -1581,6 +1596,7 @@ BBPinit(bool first)
#ifdef GDKLIBRARY_HASHASH
GDKfree(hashbats);
#endif
+ GDKdebug = dbg;
return GDK_FAIL;
}
MT_lock_unset(&BBPnameLock);
@@ -1595,6 +1611,7 @@ BBPinit(bool first)
GDKfree(hashbats);
#endif
TRC_CRITICAL(GDK, "cannot properly prepare process
%s.", BAKDIR);
+ GDKdebug = dbg;
return rc;
}
}
@@ -1603,6 +1620,7 @@ BBPinit(bool first)
#ifdef GDKLIBRARY_HASHASH
GDKfree(hashbats);
#endif
+ GDKdebug = dbg;
return GDK_FAIL;
}
@@ -1622,6 +1640,7 @@ BBPinit(bool first)
#ifdef GDKLIBRARY_HASHASH
GDKfree(hashbats);
#endif
+ GDKdebug = dbg;
return GDK_FAIL;
}
close(fd);
@@ -1642,6 +1661,7 @@ BBPinit(bool first)
#ifdef GDKLIBRARY_HASHASH
GDKfree(hashbats);
#endif
+ GDKdebug = dbg;
return GDK_FAIL;
}
}
@@ -1658,6 +1678,7 @@ BBPinit(bool first)
if (bbpversion < GDKLIBRARY && TMcommit() != GDK_SUCCEED) {
TRC_CRITICAL(GDK, "TMcommit failed\n");
+ GDKdebug = dbg;
return GDK_FAIL;
}
@@ -1672,6 +1693,7 @@ BBPinit(bool first)
* above */
if (movestrbats() != GDK_SUCCEED) {
GDKfree(needstrbatmove);
+ GDKdebug = dbg;
return GDK_FAIL;
}
MT_remove(needstrbatmove);
@@ -1679,6 +1701,7 @@ BBPinit(bool first)
needstrbatmove = NULL;
}
#endif
+ GDKdebug = dbg;
/* cleanup any leftovers (must be done after BBPrecover) */
for (i = 0; i < MAXFARMS && BBPfarms[i].dirname != NULL; i++) {
@@ -1799,6 +1822,23 @@ heap_entry(FILE *fp, BAT *b, BUN size)
else
free = 0;
}
+
+ if ((GDKdebug & TAILCHKMASK) && free > 0) {
+ char *fname = GDKfilepath(0, BATDIR,
BBP_physical(b->batCacheid), gettailname(b));
+ if (fname != NULL) {
+ struct stat stb;
+ if (stat(fname, &stb) == -1) {
+ assert(0);
+ TRC_WARNING(GDK, "file %s not found (expected
size %zu)\n", fname, free);
+ } else {
+ assert((size_t) stb.st_size >= free);
+ if ((size_t) stb.st_size < free)
+ TRC_WARNING(GDK, "file %s too small
(expected %zu, actual %zu)\n", fname, free, (size_t) stb.st_size);
+ }
+ GDKfree(fname);
+ }
+ }
+
return fprintf(fp, " %s %d %d %d " BUNFMT " " BUNFMT " " BUNFMT " "
BUNFMT " " OIDFMT " %zu %zu %d %" PRIu64" %" PRIu64,
b->ttype >= 0 ? BATatoms[b->ttype].name :
ATOMunknown_name(b->ttype),
@@ -1823,10 +1863,25 @@ heap_entry(FILE *fp, BAT *b, BUN size)
}
static inline int
-vheap_entry(FILE *fp, Heap *h)
+vheap_entry(FILE *fp, Heap *h, BUN size)
{
+ (void) size;
if (h == NULL)
return 0;
+ if ((GDKdebug & TAILCHKMASK) && size > 0) {
+ char *fname = GDKfilepath(0, BATDIR, BBP_physical(h->parentid),
"theap");
+ if (fname != NULL) {
+ struct stat stb;
+ if (stat(fname, &stb) == -1) {
+ assert(0);
+ TRC_WARNING(GDK, "file %s not found (expected
size %zu)\n", fname, h->free);
+ } else if ((size_t) stb.st_size < h->free) {
+ /* no assert since this can actually happen */
+ TRC_WARNING(GDK, "file %s too small (expected
%zu, actual %zu)\n", fname, h->free, (size_t) stb.st_size);
+ }
+ GDKfree(fname);
+ }
+ }
return fprintf(fp, " %zu %zu %d", h->free, h->size, 0);
}
@@ -1860,7 +1915,7 @@ new_bbpentry(FILE *fp, bat i, BUN size)
BBP_desc(i)->batCapacity,
BBP_desc(i)->hseqbase) < 0 ||
heap_entry(fp, BBP_desc(i), size) < 0 ||
- vheap_entry(fp, BBP_desc(i)->tvheap) < 0 ||
+ vheap_entry(fp, BBP_desc(i)->tvheap, size) < 0 ||
(BBP_options(i) && fprintf(fp, " %s", BBP_options(i)) < 0) ||
fprintf(fp, "\n") < 0) {
GDKsyserror("new_bbpentry: Writing BBP.dir entry failed\n");
@@ -1958,14 +2013,116 @@ BBPdir_first(bool subcommit, lng logno,
static bat
BBPdir_step(bat bid, BUN size, int n, char *buf, size_t bufsize,
- FILE **obbpfp, FILE *nbbpf)
+ FILE **obbpfp, FILE *nbbpf, bool subcommit)
{
if (n < -1) /* safety catch */
return n;
while (n >= 0 && n < bid) {
- if (n > 0 && fputs(buf, nbbpf) == EOF) {
- GDKerror("Writing BBP.dir file failed.\n");
- goto bailout;
+ if (n > 0) {
+ if (GDKdebug & TAILCHKMASK) {
+ uint64_t batid, free, vfree;
+ char filename[sizeof(BBP_physical(0))];
+ char type[33];
+ uint16_t width;
+ char *fname;
+ struct stat stb;
+ switch (sscanf(buf, "%" SCNu64 " %*u %*s %19s
%*u %*u %*u %*u %10s %" SCNu16 " %*u %*u %*u %*u %*u %*u %*u %" SCNu64 " %*u
%*u %*u %*u %" SCNu64 " %*u %*u",
+ &batid, filename, type, &width,
&free, &vfree)) {
+ case 5:
+ vfree = 0;
+ /* fall through */
+ case 6:
+ assert(batid == (uint64_t) n);
+ if (free == 0)
+ break;
+ const char *tailname = "tail";
+ if (strcmp(type, "str") == 0) {
+ switch (width) {
+ case 1:
+ tailname = "tail1";
+ break;
+ case 2:
+ tailname = "tail2";
+ break;
+#if SIZEOF_VAR_T == 8
+ case 4:
+ tailname = "tail4";
+ break;
+#endif
+ }
+ }
+ if (subcommit) {
+ char base[32];
+ snprintf(base, sizeof(base),
"%" PRIo64, batid);
+ fname = GDKfilepath(0, BAKDIR,
base, tailname);
+ } else {
+ fname = GDKfilepath(0, BATDIR,
filename, tailname);
+ }
+ if (fname == NULL)
+ break;
+ bool found = true;
+ if (stat(fname, &stb) == -1) {
+ if (subcommit) {
+ char *fname1 =
GDKfilepath(0, BATDIR, filename, tailname);
+ if (fname1 == NULL) {
+ GDKfree(fname);
+ break;
+ }
+ if (stat(fname1, &stb)
== -1) {
+ assert(0);
+ found = false;
+ GDKfree(fname1);
+ } else {
+ GDKfree(fname);
+ fname = fname1;
+ }
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
