Changeset: 42fe95ec77a8 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/42fe95ec77a8
Modified Files:
clients/mapiclient/dotmonetdb.c
gdk/gdk_bat.c
gdk/gdk_bbp.c
gdk/gdk_logger.c
gdk/gdk_utils.c
monetdb5/mal/mal_authorize.c
monetdb5/modules/atoms/str.c
sql/storage/objectset.c
sql/storage/store.c
Branch: default
Log Message:
Merge with Sep2022 branch.
diffs (truncated from 551 to 300 lines):
diff --git a/clients/mapiclient/dotmonetdb.c b/clients/mapiclient/dotmonetdb.c
--- a/clients/mapiclient/dotmonetdb.c
+++ b/clients/mapiclient/dotmonetdb.c
@@ -57,7 +57,7 @@ parse_dotmonetdb(DotMonetdb *dotfile)
}
} else if (*cfile != 0 && (config = _wfopen(cfile, L"r")) == NULL) {
fprintf(stderr, "failed to open file '%ls': %s\n",
- cfile, strerror(errno));
+ cfile, strerror(errno));
}
#else
char *cfile;
@@ -85,7 +85,7 @@ parse_dotmonetdb(DotMonetdb *dotfile)
}
} else if (*cfile != 0 && (config = fopen(cfile, "r")) == NULL) {
fprintf(stderr, "failed to open file '%s': %s\n",
- cfile, strerror(errno));
+ cfile, strerror(errno));
}
#endif
@@ -103,7 +103,7 @@ parse_dotmonetdb(DotMonetdb *dotfile)
continue;
if ((q = strchr(buf, '=')) == NULL) {
fprintf(stderr, CF ":%d: syntax error: %s\n",
- cfile, line, buf);
+ cfile, line, buf);
continue;
}
*q++ = '\0';
@@ -126,9 +126,8 @@ parse_dotmonetdb(DotMonetdb *dotfile)
/* make sure we don't set garbage */
if (strcmp(q, "sql") != 0 &&
strcmp(q, "mal") != 0) {
- fprintf(stderr, CF ":%d: unsupported "
- "language: %s\n",
- cfile, line, q);
+ fprintf(stderr, CF ":%d: unsupported
language: %s\n",
+ cfile, line, q);
}
dotfile->language = strdup(q);
q = NULL;
@@ -138,7 +137,7 @@ parse_dotmonetdb(DotMonetdb *dotfile)
dotfile->save_history = true;
q = NULL;
} else if (strcmp(q, "false") == 0 ||
- strcmp(q, "off") == 0) {
+ strcmp(q, "off") == 0) {
dotfile->save_history = false;
q = NULL;
}
@@ -154,7 +153,7 @@ parse_dotmonetdb(DotMonetdb *dotfile)
}
if (q != NULL)
fprintf(stderr, CF ":%d: unknown property:
%s\n",
- cfile, line, buf);
+ cfile, line, buf);
}
fclose(config);
}
diff --git a/gdk/ChangeLog.Sep2022 b/gdk/ChangeLog.Sep2022
--- a/gdk/ChangeLog.Sep2022
+++ b/gdk/ChangeLog.Sep2022
@@ -1,3 +1,10 @@
# ChangeLog file for GDK
# This file is updated with Maddlog
+* Mon Feb 13 2023 Sjoerd Mullender <sjo...@acm.org>
+- When saving a bat failed for some reason during a low-level commit,
+ this was logged in the log file, but the error was then subsequently
+ ignored, possibly leading to files that are too short or even missing.
+- The write-ahead log (WAL) is now rotated a bit more efficiently by
+ doing multiple log files in one go (i.e. in one low-level transaction).
+
diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c
--- a/gdk/gdk_bat.c
+++ b/gdk/gdk_bat.c
@@ -688,11 +688,27 @@ BATfree(BAT *b)
if (nunique != BUN_NONE) {
b->tunique_est = (double) nunique;
}
+ /* wait until there are no other references to the heap; a
+ * reference is possible in e.g. BBPsync that uses a
+ * bat_iterator directly on the BBP_desc, i.e. without fix */
+ while (b->theap && (ATOMIC_GET(&b->theap->refs) & HEAPREFS) > 1) {
+ MT_lock_unset(&b->theaplock);
+ MT_sleep_ms(1);
+ MT_lock_set(&b->theaplock);
+ }
if (b->theap) {
assert((ATOMIC_GET(&b->theap->refs) & HEAPREFS) == 1);
assert(b->theap->parentid == b->batCacheid);
HEAPfree(b->theap, false);
}
+ /* wait until there are no other references to the heap; a
+ * reference is possible in e.g. BBPsync that uses a
+ * bat_iterator directly on the BBP_desc, i.e. without fix */
+ while (b->tvheap && (ATOMIC_GET(&b->tvheap->refs) & HEAPREFS) > 1) {
+ MT_lock_unset(&b->theaplock);
+ MT_sleep_ms(1);
+ MT_lock_set(&b->theaplock);
+ }
if (b->tvheap) {
assert((ATOMIC_GET(&b->tvheap->refs) & HEAPREFS) == 1);
assert(b->tvheap->parentid == b->batCacheid);
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -3590,9 +3590,9 @@ BBPcheckHeap(bool subcommit, Heap *h)
if (path == NULL)
return;
if (MT_stat(path, &statb) < 0) {
- assert(0);
GDKsyserror("cannot stat file %s (expected size
%zu)\n",
path, h->free);
+ assert(0);
GDKfree(path);
return;
}
@@ -3602,9 +3602,9 @@ BBPcheckHeap(bool subcommit, Heap *h)
if (path == NULL)
return;
if (MT_stat(path, &statb) < 0) {
- assert(0);
GDKsyserror("cannot stat file %s (expected size %zu)\n",
path, h->free);
+ assert(0);
GDKfree(path);
return;
}
@@ -3777,60 +3777,54 @@ BBPsync(int cnt, bat *restrict subcommit
&obbpf, &nbbpf);
}
- if (ret == GDK_SUCCEED) {
- int idx = 0;
-
- while (++idx < cnt) {
- bat i = subcommit ? subcommit[idx] : idx;
- /* BBP_desc(i) may be NULL */
- BUN size = sizes ? sizes[idx] : BUN_NONE;
- BATiter bi;
-
- if (BBP_status(i) & BBPPERSISTENT) {
- BAT *b = dirty_bat(&i, subcommit != NULL);
- if (i <= 0) {
- break;
- }
- bi = bat_iterator(BBP_desc(i));
- assert(sizes == NULL || size <= bi.count);
- assert(sizes == NULL || bi.width == 0 ||
(bi.type == TYPE_msk ? ((size + 31) / 32) * 4 : size << bi.shift) <= bi.hfree);
- if (size > bi.count) /* includes sizes==NULL */
- size = bi.count;
- bi.b->batInserted = size;
- if (b && size != 0) {
- /* wait for BBPSAVING so that we
- * can set it, wait for
- * BBPUNLOADING before
- * attempting to save */
- for (;;) {
- if (lock)
-
MT_lock_set(&GDKswapLock(i));
- if (!(BBP_status(i) &
(BBPSAVING|BBPUNLOADING)))
- break;
- if (lock)
-
MT_lock_unset(&GDKswapLock(i));
- BBPspin(i, __func__,
BBPSAVING|BBPUNLOADING);
- }
- BBP_status_on(i, BBPSAVING);
+ for (int idx = 1; ret == GDK_SUCCEED && idx < cnt; idx++) {
+ bat i = subcommit ? subcommit[idx] : idx;
+ /* BBP_desc(i) may be NULL */
+ BUN size = sizes ? sizes[idx] : BUN_NONE;
+ BATiter bi;
+
+ if (BBP_status(i) & BBPPERSISTENT) {
+ BAT *b = dirty_bat(&i, subcommit != NULL);
+ if (i <= 0) {
+ ret = GDK_FAIL;
+ break;
+ }
+ bi = bat_iterator(BBP_desc(i));
+ assert(sizes == NULL || size <= bi.count);
+ assert(sizes == NULL || bi.width == 0 || (bi.type ==
TYPE_msk ? ((size + 31) / 32) * 4 : size << bi.shift) <= bi.hfree);
+ if (size > bi.count) /* includes sizes==NULL */
+ size = bi.count;
+ bi.b->batInserted = size;
+ if (b && size != 0) {
+ /* wait for BBPSAVING so that we
+ * can set it, wait for
+ * BBPUNLOADING before
+ * attempting to save */
+ for (;;) {
+ if (lock)
+ MT_lock_set(&GDKswapLock(i));
+ if (!(BBP_status(i) &
(BBPSAVING|BBPUNLOADING)))
+ break;
if (lock)
MT_lock_unset(&GDKswapLock(i));
- ret = BATsave_iter(b, &bi, size);
- BBP_status_off(i, BBPSAVING);
+ BBPspin(i, __func__,
BBPSAVING|BBPUNLOADING);
}
- } else {
- bi = bat_iterator(NULL);
- }
- if (ret == GDK_SUCCEED) {
- n = BBPdir_step(i, size, n, buf, sizeof(buf),
&obbpf, nbbpf, &bi);
+ BBP_status_on(i, BBPSAVING);
+ if (lock)
+ MT_lock_unset(&GDKswapLock(i));
+ ret = BATsave_iter(b, &bi, size);
+ BBP_status_off(i, BBPSAVING);
}
- bat_iterator_end(&bi);
- if (n == -2)
- break;
- /* we once again have a saved heap */
+ } else {
+ bi = bat_iterator(NULL);
}
- if (idx < cnt) {
- ret = GDK_FAIL;
+ if (ret == GDK_SUCCEED) {
+ n = BBPdir_step(i, size, n, buf, sizeof(buf), &obbpf,
nbbpf, &bi);
+ if (n < -1)
+ ret = GDK_FAIL;
}
+ bat_iterator_end(&bi);
+ /* we once again have a saved heap */
}
TRC_DEBUG(PERF, "write time "LLFMT" usec\n", (t0 = GDKusec()) - t1);
diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -883,9 +883,9 @@ la_apply(logger *lg, logaction *c, int t
static void
la_destroy(logaction *c)
{
- if (c->b && (c->type == LOG_UPDATE || c->type == LOG_UPDATE_BULK))
+ if ((c->type == LOG_UPDATE || c->type == LOG_UPDATE_BULK) && c->b)
logbat_destroy(c->b);
- if (c->uid && c->type == LOG_UPDATE)
+ if (c->type == LOG_UPDATE && c->uid)
logbat_destroy(c->uid);
}
@@ -1496,7 +1496,7 @@ subcommit_list_add(int next, bat *n, BUN
}
static int
-cleanup_and_swap(logger *lg, int *r, const log_bid *bids, lng *lids, lng
*cnts, BAT *catalog_bid, BAT *catalog_id, BAT *dcatalog, int cleanup)
+cleanup_and_swap(logger *lg, int *r, const log_bid *bids, lng *lids, lng
*cnts, BAT *catalog_bid, BAT *catalog_id, BAT *dcatalog, BUN cleanup)
{
BAT *nbids, *noids, *ncnts, *nlids, *ndels;
BUN p, q;
@@ -1624,7 +1624,7 @@ bm_subcommit(logger *lg)
gdk_return res;
const log_bid *bids;
lng *cnts = NULL, *lids = NULL;
- int cleanup = 0;
+ BUN cleanup = 0;
lng t0 = 0;
if (n == NULL || r == NULL || sizes == NULL) {
@@ -1661,6 +1661,8 @@ bm_subcommit(logger *lg)
sizes[i] = BATcount(dcatalog);
n[i++] = dcatalog->batCacheid;
+ if (cleanup < (lg->cnt/2))
+ cleanup = 0;
if (cleanup && (rcnt=cleanup_and_swap(lg, r, bids, lids, cnts,
catalog_bid, catalog_id, dcatalog, cleanup)) < 0) {
GDKfree(n);
GDKfree(r);
@@ -2282,25 +2284,30 @@ log_create(int debug, const char *fn, co
static ulng
log_next_logfile(logger *lg, ulng ts)
{
+ int m = (GDKdebug & FORCEMITOMASK)?1000:10;
if (!lg->pending || !lg->pending->next)
return 0;
- if (lg->pending->last_ts <= ts)
- return lg->pending->id;
+ if (lg->pending != lg->current && lg->pending->last_ts <= ts) {
+ logged_range *p = lg->pending;
+ for(int i = 1; i<m && p->next && p->next != lg->current &&
p->last_ts <= ts; i++)
+ p = p->next;
+ return p->id;
+ }
return 0;
}
static void
-log_cleanup_range(logger *lg)
+log_cleanup_range(logger *lg, ulng id)
{
- if (lg->pending) {
+ log_lock(lg);
+ while (lg->pending && lg->pending->id <= id) {
_______________________________________________
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org
