Changeset: 9bfdf4d1ea96 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/9bfdf4d1ea96
Modified Files:
sql/storage/bat/bat_storage.c
Branch: smart-merge-jan22
Log Message:
Merge with Jan2022.
diffs (truncated from 693 to 300 lines):
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -769,3 +769,4 @@ 1252291e5c0ddc91ccb16d612d04e34e6a7d3bc3
1252291e5c0ddc91ccb16d612d04e34e6a7d3bc3 Jun2020_SP2_release
59de1ee118d4eccc072c0cf3938f90635a7db311 Jan2022_15
59de1ee118d4eccc072c0cf3938f90635a7db311 Jan2022_SP3_release
+38ba6e40ba9148c762fe4a4b40f4937f86e516b6 Jul2021_25
diff --git a/gdk/ChangeLog.Jan2022 b/gdk/ChangeLog.Jan2022
--- a/gdk/ChangeLog.Jan2022
+++ b/gdk/ChangeLog.Jan2022
@@ -1,6 +1,11 @@
# ChangeLog file for GDK
# This file is updated with Maddlog
+* Tue Aug 2 2022 Sjoerd Mullender <sjo...@acm.org>
+- When destroying a bat, make sure there are no files left over in
+ the BACKUP directory since they can cause problems when the bat id
+ gets reused.
+
* Thu Jul 28 2022 Sjoerd Mullender <sjo...@acm.org>
- Fixed an off-by-one error in the logger which caused older log files
to stick around longer in the write-ahead log than necessary.
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -2510,7 +2510,7 @@ BBPinsert(BAT *bn)
MT_lock_unset(&GDKswapLock(i));
#ifdef HAVE_HGE
- if (bn->ttype == TYPE_hge)
+ if (bn->ttype == TYPE_hge && !havehge)
havehge = true;
#endif
diff --git a/gdk/gdk_join.c b/gdk/gdk_join.c
--- a/gdk/gdk_join.c
+++ b/gdk/gdk_join.c
@@ -3162,7 +3162,6 @@ count_unique(BAT *b, BAT *s, BUN *cnt1,
seen = NULL;
} else {
BUN prb;
- BUN p;
BUN mask;
Hash hs = {0};
@@ -3176,7 +3175,7 @@ count_unique(BAT *b, BAT *s, BUN *cnt1,
(hs.heapbckt.farmid = BBPselectfarm(TRANSIENT, b->ttype,
hashheap)) < 0 ||
snprintf(hs.heaplink.filename,
sizeof(hs.heaplink.filename), "%s.thshjnl%x", nme, (unsigned) THRgettid()) >=
(int) sizeof(hs.heaplink.filename) ||
snprintf(hs.heapbckt.filename,
sizeof(hs.heapbckt.filename), "%s.thshjnb%x", nme, (unsigned) THRgettid()) >=
(int) sizeof(hs.heapbckt.filename) ||
- HASHnew(&hs, b->ttype, BUNlast(b), mask, BUN_NONE, false)
!= GDK_SUCCEED) {
+ HASHnew(&hs, b->ttype, ci.ncand, mask, BUN_NONE, false) !=
GDK_SUCCEED) {
GDKerror("cannot allocate hash table\n");
HEAPfree(&hs.heaplink, true);
HEAPfree(&hs.heapbckt, true);
@@ -3192,15 +3191,15 @@ count_unique(BAT *b, BAT *s, BUN *cnt1,
for (hb = HASHget(&hs, prb);
hb != BUN_NONE;
hb = HASHgetlink(&hs, hb)) {
- if (cmp(v, BUNtail(bi, hb)) == 0)
+ BUN p = canditer_idx(&ci, hb) - b->hseqbase;
+ if (cmp(v, BUNtail(bi, p)) == 0)
break;
}
if (hb == BUN_NONE) {
- p = o - b->hseqbase;
cnt++;
/* enter into hash table */
- HASHputlink(&hs, p, HASHget(&hs, prb));
- HASHput(&hs, prb, p);
+ HASHputlink(&hs, i, HASHget(&hs, prb));
+ HASHput(&hs, prb, i);
}
}
*cnt2 = cnt;
diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -209,8 +209,9 @@ internal_find_bat(logger *lg, log_id id,
}
}
MT_rwlock_rdunlock(&cni.b->thashlock);
+ return 0; /* not found */
}
- return 0;
+ return -1; /* error creating hash */
}
static void
@@ -274,6 +275,9 @@ la_bat_clear(logger *lg, logaction *la,
log_bid bid = internal_find_bat(lg, la->cid, tid);
BAT *b;
+ if (bid < 0)
+ return GDK_FAIL;
+
if (lg->debug & 1)
fprintf(stderr, "#la_bat_clear %d\n", la->cid);
@@ -633,8 +637,9 @@ la_bat_update_count(logger *lg, log_id i
}
}
MT_rwlock_rdunlock(&cni.b->thashlock);
+ return GDK_SUCCEED;
}
- return GDK_SUCCEED;
+ return GDK_FAIL;
}
static gdk_return
@@ -643,6 +648,8 @@ la_bat_updates(logger *lg, logaction *la
log_bid bid = internal_find_bat(lg, la->cid, tid);
BAT *b = NULL;
+ if (bid < 0)
+ return GDK_FAIL;
if (bid == 0)
return GDK_SUCCEED; /* ignore bats no longer in the catalog */
@@ -746,6 +753,8 @@ la_bat_destroy(logger *lg, logaction *la
{
log_bid bid = internal_find_bat(lg, la->cid, tid);
+ if (bid < 0)
+ return GDK_FAIL;
if (bid && logger_del_bat(lg, bid) != GDK_SUCCEED)
return GDK_FAIL;
return GDK_SUCCEED;
@@ -2589,6 +2598,10 @@ log_bat_transient(logger *lg, log_id id)
log_bid bid = internal_find_bat(lg, id, -1);
logformat l;
+ if (bid < 0) {
+ logger_unlock(lg);
+ return GDK_FAIL;
+ }
l.flag = LOG_DESTROY;
l.id = id;
@@ -2887,6 +2900,8 @@ logger_add_bat(logger *lg, BAT *b, log_i
b == lg->seqs_val ||
b == lg->dseqs);
assert(b->batRole == PERSISTENT);
+ if (bid < 0)
+ return GDK_FAIL;
if (bid) {
if (bid != b->batCacheid) {
if (logger_del_bat(lg, bid) != GDK_SUCCEED)
diff --git a/gdk/gdk_logger_old.c b/gdk/gdk_logger_old.c
--- a/gdk/gdk_logger_old.c
+++ b/gdk/gdk_logger_old.c
@@ -313,6 +313,7 @@ old_logger_find_bat(old_logger *lg, cons
}
}
MT_rwlock_rdunlock(&cni.b->thashlock);
+ return 0; /* not found */
}
} else {
BATiter cni = bat_iterator_nolock(lg->catalog_oid);
@@ -331,9 +332,10 @@ old_logger_find_bat(old_logger *lg, cons
}
}
MT_rwlock_rdunlock(&cni.b->thashlock);
+ return 0; /* not found */
}
}
- return 0;
+ return -1; /* BAThash failed */
}
static gdk_return
@@ -342,6 +344,9 @@ la_bat_clear(old_logger *lg, logaction *
log_bid bid = old_logger_find_bat(lg, la->name, la->tpe, la->cid);
BAT *b;
+ if (bid < 0)
+ return GDK_FAIL;
+
if (lg->lg->debug & 1)
fprintf(stderr, "#la_bat_clear %s\n", NAME(la->name, la->tpe,
la->cid));
@@ -409,6 +414,8 @@ static log_return
log_read_updates(old_logger *lg, trans *tr, logformat *l, char *name, int tpe,
oid id, int pax)
{
log_bid bid = old_logger_find_bat(lg, name, tpe, id);
+ if (bid < 0)
+ return LOG_ERR;
BAT *b = BATdescriptor(bid);
log_return res = LOG_OK;
int ht = -1, tt = -1, tseq = 0;
@@ -625,6 +632,8 @@ la_bat_updates(old_logger *lg, logaction
log_bid bid = old_logger_find_bat(lg, la->name, la->tpe, la->cid);
BAT *b;
+ if (bid < 0)
+ return GDK_FAIL;
if (bid == 0)
return GDK_SUCCEED; /* ignore bats no longer in the catalog */
@@ -704,6 +713,8 @@ la_bat_destroy(old_logger *lg, logaction
{
log_bid bid = old_logger_find_bat(lg, la->name, la->tpe, la->cid);
+ if (bid < 0)
+ return GDK_FAIL;
if (bid) {
BUN p;
@@ -1494,6 +1505,8 @@ logger_load(const char *fn, char filenam
goto error;
}
snapshots_bid = old_logger_find_bat(lg, "snapshots_bid", 0, 0);
+ if (snapshots_bid < 0)
+ goto error;
if (snapshots_bid == 0) {
lg->snapshots_bid = logbat_new(TYPE_int, 1, TRANSIENT);
lg->snapshots_tid = logbat_new(TYPE_int, 1, TRANSIENT);
@@ -1508,6 +1521,8 @@ logger_load(const char *fn, char filenam
bat snapshots_tid = old_logger_find_bat(lg, "snapshots_tid", 0,
0);
bat dsnapshots = old_logger_find_bat(lg, "dsnapshots", 0, 0);
+ if (snapshots_tid < 0 || dsnapshots < 0)
+ goto error;
GDKdebug &= ~CHECKMASK;
lg->snapshots_bid = BATdescriptor(snapshots_bid);
if (lg->snapshots_bid == NULL) {
@@ -1924,6 +1939,8 @@ logger_add_bat(old_logger *lg, BAT *b, c
b == lg->seqs_val ||
b == lg->dseqs);
assert(b->batRole == PERSISTENT);
+ if (bid < 0)
+ return GDK_FAIL;
if (bid) {
if (bid != b->batCacheid) {
if (logger_del_bat(lg, bid) != GDK_SUCCEED)
diff --git a/gdk/gdk_storage.c b/gdk/gdk_storage.c
--- a/gdk/gdk_storage.c
+++ b/gdk/gdk_storage.c
@@ -974,7 +974,8 @@ BATdelete(BAT *b)
STRMPdestroy(b);
if (b->theap) {
HEAPfree(b->theap, true);
- if ((f = GDKfilepath(b->theap->farmid, BAKDIR, o, "tail1")) !=
NULL) {
+ if (!GDKinmemory(b->theap->farmid) &&
+ (f = GDKfilepath(b->theap->farmid, BAKDIR, o, "tail1")) !=
NULL) {
MT_remove(f);
size_t i = strlen(f) - 1;
f[i] = '2';
@@ -990,7 +991,8 @@ BATdelete(BAT *b)
}
if (b->tvheap) {
HEAPfree(b->tvheap, true);
- if ((f = GDKfilepath(b->theap->farmid, BAKDIR, o, "theap")) !=
NULL) {
+ if (!GDKinmemory(b->tvheap->farmid) &&
+ (f = GDKfilepath(b->theap->farmid, BAKDIR, o, "theap")) !=
NULL) {
MT_remove(f);
GDKfree(f);
}
diff --git a/sql/storage/bat/bat_logger.c b/sql/storage/bat/bat_logger.c
--- a/sql/storage/bat/bat_logger.c
+++ b/sql/storage/bat/bat_logger.c
@@ -78,6 +78,14 @@ bl_preversion(sqlstore *store, int oldve
#define D(schema, table) "D_" schema "_" table
+static BAT *
+log_temp_descriptor(log_bid b)
+{
+ if (b <= 0)
+ return NULL;
+ return temp_descriptor(b);
+}
+
#if defined CATALOG_JUN2020 || defined CATALOG_OCT2020
static gdk_return
tabins(logger *lg, old_logger *old_lg, bool first, int tt, int nid, ...)
@@ -91,7 +99,7 @@ tabins(logger *lg, old_logger *old_lg, b
va_start(va, nid);
while ((cid = va_arg(va, int)) != 0) {
cval = va_arg(va, void *);
- if ((b = temp_descriptor(logger_find_bat(lg, cid))) == NULL) {
+ if ((b = log_temp_descriptor(logger_find_bat(lg, cid))) ==
NULL) {
va_end(va);
return GDK_FAIL;
}
@@ -1335,18 +1343,18 @@ upgrade(old_logger *lg)
BAT *mapold = COLnew(0, TYPE_int, 256, TRANSIENT);
BAT *mapnew = COLnew(0, TYPE_int, 256, TRANSIENT);
- bats[0].nmbat = temp_descriptor(old_logger_find_bat(lg,
"sys_schemas_name", 0, 0));
- bats[0].idbat = temp_descriptor(old_logger_find_bat(lg,
"sys_schemas_id", 0, 0));
+ bats[0].nmbat = log_temp_descriptor(old_logger_find_bat(lg,
"sys_schemas_name", 0, 0));
+ bats[0].idbat = log_temp_descriptor(old_logger_find_bat(lg,
"sys_schemas_id", 0, 0));
bats[0].parbat = NULL;
- bats[0].cands = temp_descriptor(old_logger_find_bat(lg,
"D_sys_schemas", 0, 0));
- bats[1].nmbat = temp_descriptor(old_logger_find_bat(lg,
"sys__tables_name", 0, 0));
- bats[1].idbat = temp_descriptor(old_logger_find_bat(lg,
"sys__tables_id", 0, 0));
- bats[1].parbat = temp_descriptor(old_logger_find_bat(lg,
"sys__tables_schema_id", 0, 0));
- bats[1].cands = temp_descriptor(old_logger_find_bat(lg,
"D_sys__tables", 0, 0));
- bats[2].nmbat = temp_descriptor(old_logger_find_bat(lg,
"sys__columns_name", 0, 0));
_______________________________________________
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org
