Changeset: 254dcf085ee5 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/254dcf085ee5
Modified Files:
gdk/gdk_bbp.c
Branch: Jul2021
Log Message:
Refactor BBP.dir reading code. Use a function to read a single line.
This function is now also used for the TAILCHKMASK check just before the
actual commit.
Manual backport of changeset 11d1a723b150.
diffs (truncated from 786 to 300 lines):
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -428,7 +428,7 @@ static gdk_return BBPrecover_subdir(void
static bool BBPdiskscan(const char *, size_t);
static int
-heapinit(BAT *b, const char *buf, int *hashash, unsigned bbpversion, bat bid,
const char *filename, int lineno)
+heapinit(BAT *b, const char *buf, int *hashash, unsigned bbpversion, const
char *filename, int lineno)
{
int t;
char type[33];
@@ -488,7 +488,7 @@ heapinit(BAT *b, const char *buf, int *h
return -1;
}
} else if (var != (t == TYPE_void || BATatoms[t].atomPut != NULL)) {
- TRC_CRITICAL(GDK, "inconsistent entry in BBP.dir: tvarsized
mismatch for BAT %d on line %d\n", (int) bid, lineno);
+ TRC_CRITICAL(GDK, "inconsistent entry in BBP.dir: tvarsized
mismatch for BAT %d on line %d\n", (int) b->batCacheid, lineno);
return -1;
} else if (var && t != 0 ?
ATOMsize(t) < width ||
@@ -498,7 +498,7 @@ heapinit(BAT *b, const char *buf, int *h
#endif
) :
ATOMsize(t) != width) {
- TRC_CRITICAL(GDK, "inconsistent entry in BBP.dir: tsize
mismatch for BAT %d on line %d\n", (int) bid, lineno);
+ TRC_CRITICAL(GDK, "inconsistent entry in BBP.dir: tsize
mismatch for BAT %d on line %d\n", (int) b->batCacheid, lineno);
return -1;
}
b->ttype = t;
@@ -541,7 +541,7 @@ heapinit(BAT *b, const char *buf, int *h
}
static int
-vheapinit(BAT *b, const char *buf, int hashash, bat bid, const char *filename,
int lineno)
+vheapinit(BAT *b, const char *buf, int hashash, const char *filename, int
lineno)
{
int n = 0;
uint64_t free, size;
@@ -555,11 +555,6 @@ vheapinit(BAT *b, const char *buf, int h
TRC_CRITICAL(GDK, "invalid format for BBP.dir on line
%d", lineno);
return -1;
}
- b->tvheap = GDKmalloc(sizeof(Heap));
- if (b->tvheap == NULL) {
- TRC_CRITICAL(GDK, "cannot allocate memory for heap.");
- return -1;
- }
if (b->ttype >= 0 &&
ATOMstorage(b->ttype) == TYPE_str &&
free < GDK_STRHASHTABLE * sizeof(stridx_t) + BATTINY *
GDK_VARALIGN)
@@ -577,108 +572,216 @@ vheapinit(BAT *b, const char *buf, int h
.cleanhash = true,
.newstorage = STORE_INVALID,
.dirty = false,
- .parentid = bid,
+ .parentid = b->batCacheid,
.farmid = BBPselectfarm(PERSISTENT, b->ttype, varheap),
};
strconcat_len(b->tvheap->filename, sizeof(b->tvheap->filename),
filename, ".theap", NULL);
- ATOMIC_INIT(&b->tvheap->refs, 1);
+ } else {
+ b->tvheap = NULL;
}
return n;
}
+/* read a single line from the BBP.dir file (file pointer fp) and fill
+ * in the structure pointed to by bn and extra information through the
+ * other pointers; this function does not allocate any memory; return 0
+ * on end of file, 1 on success, and -1 on failure */
+static int
+BBPreadBBPline(FILE *fp, unsigned bbpversion, int *lineno, BAT *bn,
+ int *hashash,
+ char *batname, char *filename, char **options)
+{
+ char buf[4096];
+ uint64_t batid;
+ uint16_t status;
+ unsigned int properties;
+ int nread, n;
+ char *s;
+ uint64_t count, capacity = 0, base = 0;
+
+ if (fgets(buf, sizeof(buf), fp) == NULL) {
+ if (ferror(fp)) {
+ TRC_CRITICAL(GDK, "error reading BBP.dir on line %d\n",
*lineno);
+ return -1;
+ }
+ return 0; /* end of file */
+ }
+ (*lineno)++;
+ if ((s = strchr(buf, '\r')) != NULL) {
+ /* convert \r\n into just \n */
+ if (s[1] != '\n') {
+ TRC_CRITICAL(GDK, "invalid format for BBP.dir on line
%d", *lineno);
+ return -1;
+ }
+ *s++ = '\n';
+ *s = 0;
+ }
+
+ if (sscanf(buf,
+ "%" SCNu64 " %" SCNu16 " %128s %19s %u %" SCNu64
+ " %" SCNu64 " %" SCNu64
+ "%n",
+ &batid, &status, batname, filename,
+ &properties,
+ &count, &capacity, &base,
+ &nread) < 8) {
+ TRC_CRITICAL(GDK, "invalid format for BBP.dir on line %d",
*lineno);
+ return -1;
+ }
+
+ if (batid >= N_BBPINIT * BBPINIT) {
+ TRC_CRITICAL(GDK, "bat ID (%" PRIu64 ") too large to accomodate
(max %d), on line %d.", batid, N_BBPINIT * BBPINIT - 1, *lineno);
+ return -1;
+ }
+
+ /* convert both / and \ path separators to our own DIR_SEP */
+#if DIR_SEP != '/'
+ s = filename;
+ while ((s = strchr(s, '/')) != NULL)
+ *s++ = DIR_SEP;
+#endif
+#if DIR_SEP != '\\'
+ s = filename;
+ while ((s = strchr(s, '\\')) != NULL)
+ *s++ = DIR_SEP;
+#endif
+
+ bn->batCacheid = (bat) batid;
+ BATinit_idents(bn);
+ bn->batTransient = false;
+ bn->batCopiedtodisk = true;
+ switch ((properties & 0x06) >> 1) {
+ case 0:
+ bn->batRestricted = BAT_WRITE;
+ break;
+ case 1:
+ bn->batRestricted = BAT_READ;
+ break;
+ case 2:
+ bn->batRestricted = BAT_APPEND;
+ break;
+ default:
+ TRC_CRITICAL(GDK, "incorrect batRestricted value");
+ return -1;
+ }
+ bn->batCount = (BUN) count;
+ bn->batInserted = bn->batCount;
+ /* set capacity to at least count */
+ bn->batCapacity = (BUN) count <= BATTINY ? BATTINY : (BUN) count;
+
+ if (base > (uint64_t) GDK_oid_max) {
+ TRC_CRITICAL(GDK, "head seqbase out of range (ID = %" PRIu64 ",
seq = %" PRIu64 ") on line %d.", batid, base, *lineno);
+ return -1;
+ }
+ bn->hseqbase = (oid) base;
+ n = heapinit(bn, buf + nread,
+ hashash,
+ bbpversion, filename, *lineno);
+ if (n < 0) {
+ return -1;
+ }
+ nread += n;
+ n = vheapinit(bn, buf + nread, *hashash, filename, *lineno);
+ if (n < 0) {
+ return -1;
+ }
+ nread += n;
+
+ if (buf[nread] != '\n' && buf[nread] != ' ') {
+ TRC_CRITICAL(GDK, "invalid format for BBP.dir on line %d",
*lineno);
+ return -1;
+ }
+ *options = (buf[nread] == ' ') ? buf + nread + 1 : NULL;
+ return 1;
+}
+
static gdk_return
BBPreadEntries(FILE *fp, unsigned bbpversion, int lineno)
{
- bat bid = 0;
- char buf[4096];
-
/* read the BBP.dir and insert the BATs into the BBP */
- while (fgets(buf, sizeof(buf), fp) != NULL) {
- BAT *bn;
- uint64_t batid;
- uint16_t status;
+ for (;;) {
+ BAT b;
+ Heap h;
+ Heap vh;
+ vh = h = (Heap) {
+ .free = 0,
+ };
+ b = (BAT) {
+ .theap = &h,
+ .tvheap = &vh,
+ };
+ char *options;
char headname[129];
char filename[sizeof(BBP_physical(0))];
- unsigned int properties;
- int nread, n;
- char *s, *options = NULL;
char logical[1024];
- uint64_t count, capacity, base = 0;
int Thashash;
- lineno++;
- if ((s = strchr(buf, '\r')) != NULL) {
- /* convert \r\n into just \n */
- if (s[1] != '\n') {
- TRC_CRITICAL(GDK, "invalid format for BBP.dir
on line %d", lineno);
- return GDK_FAIL;
- }
- *s++ = '\n';
- *s = 0;
- }
-
- if (sscanf(buf,
- "%" SCNu64 " %" SCNu16 " %128s %19s %u %" SCNu64
- " %" SCNu64 " %" SCNu64
- "%n",
- &batid, &status, headname, filename,
- &properties,
- &count, &capacity, &base,
- &nread) < 8) {
- TRC_CRITICAL(GDK, "invalid format for BBP.dir on line
%d", lineno);
- return GDK_FAIL;
+ switch (BBPreadBBPline(fp, bbpversion, &lineno, &b,
+ &Thashash,
+ headname, filename, &options)) {
+ case 0:
+ /* end of file */
+ return GDK_SUCCEED;
+ case 1:
+ /* successfully read an entry */
+ break;
+ default:
+ /* error */
+ goto bailout;
}
- if (batid >= N_BBPINIT * BBPINIT) {
- TRC_CRITICAL(GDK, "bat ID (%" PRIu64 ") too large to
accomodate (max %d), on line %d.", batid, N_BBPINIT * BBPINIT - 1, lineno);
- return GDK_FAIL;
+ if (b.batCacheid >= N_BBPINIT * BBPINIT) {
+ TRC_CRITICAL(GDK, "bat ID (%d) too large to accommodate
(max %d), on line %d.", b.batCacheid, N_BBPINIT * BBPINIT - 1, lineno);
+ goto bailout;
+ }
+
+ if (b.batCacheid >= (bat) ATOMIC_GET(&BBPsize)) {
+ if ((bat) ATOMIC_GET(&BBPsize) + 1 >= BBPlimit &&
+ BBPextend(0, false, b.batCacheid + 1) !=
GDK_SUCCEED)
+ goto bailout;
+ ATOMIC_SET(&BBPsize, b.batCacheid + 1);
}
-
- /* convert both / and \ path separators to our own DIR_SEP */
-#if DIR_SEP != '/'
- s = filename;
- while ((s = strchr(s, '/')) != NULL)
- *s++ = DIR_SEP;
-#endif
-#if DIR_SEP != '\\'
- s = filename;
- while ((s = strchr(s, '\\')) != NULL)
- *s++ = DIR_SEP;
-#endif
-
- bid = (bat) batid;
- if (batid >= (uint64_t) ATOMIC_GET(&BBPsize)) {
- if ((bat) ATOMIC_GET(&BBPsize) + 1 >= BBPlimit &&
- BBPextend(0, false, bid + 1) != GDK_SUCCEED)
- return GDK_FAIL;
- ATOMIC_SET(&BBPsize, bid + 1);
+ if (BBP_desc(b.batCacheid) != NULL) {
+ TRC_CRITICAL(GDK, "duplicate entry in BBP.dir (ID = "
+ "%d) on line %d.", b.batCacheid, lineno);
+ goto bailout;
}
- if (BBP_desc(bid) != NULL) {
- TRC_CRITICAL(GDK, "duplicate entry in BBP.dir (ID = "
- "%" PRIu64 ") on line %d.", batid, lineno);
- return GDK_FAIL;
- }
+
+ BAT *bn;
+ Heap *hn;
if ((bn = GDKzalloc(sizeof(BAT))) == NULL ||
- (bn->theap = GDKzalloc(sizeof(Heap))) == NULL) {
+ (hn = GDKzalloc(sizeof(Heap))) == NULL) {
GDKfree(bn);
TRC_CRITICAL(GDK, "cannot allocate memory for BAT.");
- return GDK_FAIL;
_______________________________________________
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org
