Changeset: eb452a025670 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/eb452a025670
Modified Files:
gdk/gdk.h
gdk/gdk_bbp.c
Branch: Jul2021
Log Message:
Add code to optionally check the sizes of .tail files when committing.
diffs (truncated from 355 to 300 lines):
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -376,6 +376,7 @@ typedef enum { GDK_FAIL, GDK_SUCCEED } g
#define DELTAMASK (1<<13)
#define LOADMASK (1<<14)
#define PUSHCANDMASK (1<<15) /* used in opt_pushselect.c */
+#define TAILCHKMASK (1<<16) /* check .tail file size during commit */
#define ACCELMASK (1<<20)
#define ALGOMASK (1<<21)
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -1153,6 +1153,9 @@ BBPinit(bool first)
unsigned bbpversion = 0;
int i;
int lineno = 0;
+ int dbg = GDKdebug;
+
+ GDKdebug &= ~TAILCHKMASK;
/* the maximum number of BATs allowed in the system and the
* size of the "physical" array are linked in a complicated
@@ -1179,6 +1182,7 @@ BBPinit(bool first)
if (!(bbpdirstr = GDKfilepath(0, BATDIR, "BBP", "dir"))) {
TRC_CRITICAL(GDK, "GDKmalloc failed\n");
MT_lock_unset(&GDKtmLock);
+ GDKdebug = dbg;
return GDK_FAIL;
}
@@ -1186,6 +1190,7 @@ BBPinit(bool first)
GDKfree(bbpdirstr);
TRC_CRITICAL(GDK, "GDKmalloc failed\n");
MT_lock_unset(&GDKtmLock);
+ GDKdebug = dbg;
return GDK_FAIL;
}
@@ -1194,6 +1199,7 @@ BBPinit(bool first)
GDKfree(backupbbpdirstr);
TRC_CRITICAL(GDK, "cannot remove directory %s\n",
TEMPDIR);
MT_lock_unset(&GDKtmLock);
+ GDKdebug = dbg;
return GDK_FAIL;
}
@@ -1202,6 +1208,7 @@ BBPinit(bool first)
GDKfree(backupbbpdirstr);
TRC_CRITICAL(GDK, "cannot remove directory %s\n",
DELDIR);
MT_lock_unset(&GDKtmLock);
+ GDKdebug = dbg;
return GDK_FAIL;
}
@@ -1211,6 +1218,7 @@ BBPinit(bool first)
GDKfree(backupbbpdirstr);
TRC_CRITICAL(GDK, "cannot properly recover_subdir
process %s.", SUBDIR);
MT_lock_unset(&GDKtmLock);
+ GDKdebug = dbg;
return GDK_FAIL;
}
@@ -1228,6 +1236,7 @@ BBPinit(bool first)
GDKfree(backupbbpdirstr);
TRC_CRITICAL(GDK, "cannot open recovered
BBP.dir.");
MT_lock_unset(&GDKtmLock);
+ GDKdebug = dbg;
return GDK_FAIL;
}
} else if ((fp = GDKfilelocate(0, "BBP", "r", "dir")) == NULL) {
@@ -1270,18 +1279,24 @@ BBPinit(bool first)
bbpversion = GDKLIBRARY;
} else {
bbpversion = BBPheader(fp, &lineno, &bbpsize);
- if (bbpversion == 0)
+ if (bbpversion == 0) {
+ GDKdebug = dbg;
return GDK_FAIL;
+ }
}
/* allocate BBP records */
- if (BBPextend(0, false, bbpsize) != GDK_SUCCEED)
+ if (BBPextend(0, false, bbpsize) != GDK_SUCCEED) {
+ GDKdebug = dbg;
return GDK_FAIL;
+ }
ATOMIC_SET(&BBPsize, bbpsize);
if (!GDKinmemory(0)) {
- if (BBPreadEntries(fp, bbpversion, lineno) != GDK_SUCCEED)
+ if (BBPreadEntries(fp, bbpversion, lineno) != GDK_SUCCEED) {
+ GDKdebug = dbg;
return GDK_FAIL;
+ }
fclose(fp);
}
@@ -1289,6 +1304,7 @@ BBPinit(bool first)
if (BBPinithash(0, (bat) ATOMIC_GET(&BBPsize)) != GDK_SUCCEED) {
TRC_CRITICAL(GDK, "BBPinithash failed");
MT_lock_unset(&BBPnameLock);
+ GDKdebug = dbg;
return GDK_FAIL;
}
MT_lock_unset(&BBPnameLock);
@@ -1300,12 +1316,15 @@ BBPinit(bool first)
MT_lock_unset(&GDKtmLock);
if (rc != GDK_SUCCEED) {
TRC_CRITICAL(GDK, "cannot properly prepare process
%s.", BAKDIR);
+ GDKdebug = dbg;
return rc;
}
}
- if (BBPcheckbats(bbpversion) != GDK_SUCCEED)
+ if (BBPcheckbats(bbpversion) != GDK_SUCCEED) {
+ GDKdebug = dbg;
return GDK_FAIL;
+ }
#ifdef GDKLIBRARY_TAILN
char *needstrbatmove;
@@ -1320,6 +1339,7 @@ BBPinit(bool first)
if (fd < 0) {
TRC_CRITICAL(GDK, "cannot create signal file
needstrbatmove.\n");
GDKfree(needstrbatmove);
+ GDKdebug = dbg;
return GDK_FAIL;
}
close(fd);
@@ -1337,6 +1357,7 @@ BBPinit(bool first)
} else {
GDKsyserror("unexpected error opening %s\n",
needstrbatmove);
GDKfree(needstrbatmove);
+ GDKdebug = dbg;
return GDK_FAIL;
}
}
@@ -1345,6 +1366,7 @@ BBPinit(bool first)
if (bbpversion < GDKLIBRARY && TMcommit() != GDK_SUCCEED) {
TRC_CRITICAL(GDK, "TMcommit failed\n");
+ GDKdebug = dbg;
return GDK_FAIL;
}
@@ -1359,6 +1381,7 @@ BBPinit(bool first)
* above */
if (movestrbats() != GDK_SUCCEED) {
GDKfree(needstrbatmove);
+ GDKdebug = dbg;
return GDK_FAIL;
}
MT_remove(needstrbatmove);
@@ -1366,6 +1389,7 @@ BBPinit(bool first)
needstrbatmove = NULL;
}
#endif
+ GDKdebug = dbg;
/* cleanup any leftovers (must be done after BBPrecover) */
for (i = 0; i < MAXFARMS && BBPfarms[i].dirname != NULL; i++) {
@@ -1489,6 +1513,23 @@ heap_entry(FILE *fp, BAT *b, BUN size)
else
free = 0;
}
+
+ if ((GDKdebug & TAILCHKMASK) && free > 0) {
+ char *fname = GDKfilepath(0, BATDIR,
BBP_physical(b->batCacheid), gettailname(b));
+ if (fname != NULL) {
+ struct stat stb;
+ if (stat(fname, &stb) == -1) {
+ assert(0);
+ TRC_WARNING(GDK, "file %s not found (expected
size %zu)\n", fname, free);
+ } else {
+ assert((size_t) stb.st_size >= free);
+ if ((size_t) stb.st_size < free)
+ TRC_WARNING(GDK, "file %s too small
(expected %zu, actual %zu)\n", fname, free, (size_t) stb.st_size);
+ }
+ GDKfree(fname);
+ }
+ }
+
return fprintf(fp, " %s %d %d %d " BUNFMT " " BUNFMT " " BUNFMT " "
BUNFMT " " OIDFMT " %zu %zu %d " OIDFMT " " OIDFMT,
b->ttype >= 0 ? BATatoms[b->ttype].name :
ATOMunknown_name(b->ttype),
@@ -1513,10 +1554,25 @@ heap_entry(FILE *fp, BAT *b, BUN size)
}
static inline int
-vheap_entry(FILE *fp, Heap *h)
+vheap_entry(FILE *fp, Heap *h, BUN size)
{
+ (void) size;
if (h == NULL)
return 0;
+ if ((GDKdebug & TAILCHKMASK) && size > 0) {
+ char *fname = GDKfilepath(0, BATDIR, BBP_physical(h->parentid),
"theap");
+ if (fname != NULL) {
+ struct stat stb;
+ if (stat(fname, &stb) == -1) {
+ assert(0);
+ TRC_WARNING(GDK, "file %s not found (expected
size %zu)\n", fname, h->free);
+ } else if ((size_t) stb.st_size < h->free) {
+ /* no assert since this can actually happen */
+ TRC_WARNING(GDK, "file %s too small (expected
%zu, actual %zu)\n", fname, h->free, (size_t) stb.st_size);
+ }
+ GDKfree(fname);
+ }
+ }
return fprintf(fp, " %zu %zu %d", h->free, h->size, 0);
}
@@ -1550,7 +1606,7 @@ new_bbpentry(FILE *fp, bat i, BUN size)
BBP_desc(i)->batCapacity,
BBP_desc(i)->hseqbase) < 0 ||
heap_entry(fp, BBP_desc(i), size) < 0 ||
- vheap_entry(fp, BBP_desc(i)->tvheap) < 0 ||
+ vheap_entry(fp, BBP_desc(i)->tvheap, size) < 0 ||
(BBP_options(i) && fprintf(fp, " %s", BBP_options(i)) < 0) ||
fprintf(fp, "\n") < 0) {
GDKsyserror("new_bbpentry: Writing BBP.dir entry failed\n");
@@ -1648,14 +1704,116 @@ BBPdir_first(bool subcommit, lng logno,
static bat
BBPdir_step(bat bid, BUN size, int n, char *buf, size_t bufsize,
- FILE **obbpfp, FILE *nbbpf)
+ FILE **obbpfp, FILE *nbbpf, bool subcommit)
{
if (n < -1) /* safety catch */
return n;
while (n >= 0 && n < bid) {
- if (n > 0 && fputs(buf, nbbpf) == EOF) {
- GDKerror("Writing BBP.dir file failed.\n");
- goto bailout;
+ if (n > 0) {
+ if (GDKdebug & TAILCHKMASK) {
+ uint64_t batid, free, vfree;
+ char filename[sizeof(BBP_physical(0))];
+ char type[33];
+ uint16_t width;
+ char *fname;
+ struct stat stb;
+ switch (sscanf(buf, "%" SCNu64 " %*u %*s %19s
%*u %*u %*u %*u %10s %" SCNu16 " %*u %*u %*u %*u %*u %*u %*u %" SCNu64 " %*u
%*u %*u %*u %" SCNu64 " %*u %*u",
+ &batid, filename, type, &width,
&free, &vfree)) {
+ case 5:
+ vfree = 0;
+ /* fall through */
+ case 6:
+ assert(batid == (uint64_t) n);
+ if (free == 0)
+ break;
+ const char *tailname = "tail";
+ if (strcmp(type, "str") == 0) {
+ switch (width) {
+ case 1:
+ tailname = "tail1";
+ break;
+ case 2:
+ tailname = "tail2";
+ break;
+#if SIZEOF_VAR_T == 8
+ case 4:
+ tailname = "tail4";
+ break;
+#endif
+ }
+ }
+ if (subcommit) {
+ char base[32];
+ snprintf(base, sizeof(base),
"%" PRIo64, batid);
+ fname = GDKfilepath(0, BAKDIR,
base, tailname);
+ } else {
+ fname = GDKfilepath(0, BATDIR,
filename, tailname);
+ }
+ if (fname == NULL)
+ break;
+ bool found = true;
+ if (stat(fname, &stb) == -1) {
+ if (subcommit) {
+ char *fname1 =
GDKfilepath(0, BATDIR, filename, tailname);
+ if (fname1 == NULL) {
+ GDKfree(fname);
+ break;
+ }
+ if (stat(fname1, &stb)
== -1) {
+ assert(0);
+ found = false;
+ GDKfree(fname1);
+ } else {
+ GDKfree(fname);
+ fname = fname1;
+ }
+ } else {
+ assert(0);
+ found = false;
+ }
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
