Changeset: efe3338196ef for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/efe3338196ef
Modified Files:
gdk/gdk_bbp.c
monetdb5/mal/mal_resource.c
Branch: default
Log Message:
Merge with Jul2021 branch.
diffs (truncated from 360 to 300 lines):
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -2498,7 +2498,7 @@ static inline int
incref(bat i, bool logical, bool lock)
{
int refs;
- bat tp, tvp;
+ bat tp = i, tvp = i;
BAT *b, *pb = NULL, *pvb = NULL;
bool load = false;
@@ -2512,13 +2512,17 @@ incref(bat i, bool logical, bool lock)
* reference, getting the parent BAT descriptor is
* superfluous, but not too expensive, so we do it anyway. */
if (!logical && (b = BBP_desc(i)) != NULL) {
- if (b->theap && b->theap->parentid != i) {
- pb = BATdescriptor(b->theap->parentid);
+ MT_lock_set(&b->theaplock);
+ tp = b->theap ? b->theap->parentid : i;
+ tvp = b->tvheap ? b->tvheap->parentid : i;
+ MT_lock_unset(&b->theaplock);
+ if (tp != i) {
+ pb = BATdescriptor(tp);
if (pb == NULL)
return 0;
}
- if (b->tvheap && b->tvheap->parentid != i) {
- pvb = BATdescriptor(b->tvheap->parentid);
+ if (tvp != i) {
+ pvb = BATdescriptor(tvp);
if (pvb == NULL) {
if (pb)
BBPunfix(pb->batCacheid);
@@ -2551,16 +2555,13 @@ incref(bat i, bool logical, bool lock)
BBP_status(i) & (BBPDELETED | BBPSWAPPED));
if (logical) {
/* parent BATs are not relevant for logical refs */
- tp = tvp = 0;
refs = ++BBP_lrefs(i);
BBP_pid(i) = 0;
} else {
- tp = b->theap == NULL || b->theap->parentid == i ? 0 :
b->theap->parentid;
assert(tp >= 0);
- tvp = b->tvheap == 0 || b->tvheap->parentid == i ? 0 :
b->tvheap->parentid;
refs = ++BBP_refs(i);
unsigned flag = BBPHOT;
- if (refs == 1 && (tp || tvp)) {
+ if (refs == 1 && (tp != i || tvp != i)) {
/* If this is a view, we must load the parent
* BATs, but we must do that outside of the
* lock. Set the BBPLOADING flag so that
@@ -2577,13 +2578,18 @@ incref(bat i, bool logical, bool lock)
if (load) {
/* load the parent BATs */
assert(!logical);
- if (tp) {
+ if (tp != i) {
assert(pb != NULL);
+ /* load being set implies there is no other
+ * thread that has access to this bat, but the
+ * parent is a different matter */
+ MT_lock_set(&pb->theaplock);
if (b->theap != pb->theap) {
HEAPincref(pb->theap);
HEAPdecref(b->theap, false);
b->theap = pb->theap;
}
+ MT_lock_unset(&pb->theaplock);
}
/* done loading, release descriptor */
BBP_status_off(i, BBPLOADING);
@@ -2637,6 +2643,7 @@ decref(bat i, bool logical, bool release
int refs = 0, lrefs;
bool swap = false;
bat tp = 0, tvp = 0;
+ int farmid = 0;
BAT *b;
if (is_bat_nil(i))
@@ -2693,16 +2700,22 @@ decref(bat i, bool logical, bool release
}
}
}
- if (b && b->batCount > b->batInserted && !isVIEW(b)) {
- /* if batCount is larger than batInserted and the dirty
- * bits are off, it may be that a (sub)commit happened
- * in parallel to an update; we must undo the turning
- * off of the dirty bits */
- b->batDirtydesc = true;
- if (b->theap)
- b->theap->dirty = true;
- if (b->tvheap)
- b->tvheap->dirty = true;
+ if (b) {
+ MT_lock_set(&b->theaplock);
+ if (b->batCount > b->batInserted && !isVIEW(b)) {
+ /* if batCount is larger than batInserted and
+ * the dirty bits are off, it may be that a
+ * (sub)commit happened in parallel to an
+ * update; we must undo the turning off of the
+ * dirty bits */
+ b->batDirtydesc = true;
+ if (b->theap)
+ b->theap->dirty = true;
+ if (b->tvheap)
+ b->tvheap->dirty = true;
+ }
+ farmid = b->theap->farmid;
+ MT_lock_unset(&b->theaplock);
}
/* we destroy transients asap and unload persistent bats only
@@ -2713,7 +2726,7 @@ decref(bat i, bool logical, bool release
BATdirty(b) ||
(BBP_status(i) & (BBPHOT | BBPSYNCING)) ||
!(BBP_status(i) & BBPPERSISTENT) ||
- GDKinmemory(b->theap->farmid)))) {
+ GDKinmemory(farmid)))) {
/* bat cannot be swapped out */
} else if (b ? b->batSharecnt == 0 : (BBP_status(i) & BBPTMP)) {
/* bat will be unloaded now. set the UNLOADING bit
diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -1442,11 +1442,11 @@ subcommit_list_add(int next, bat *n, BUN
}
static int
-cleanup_and_swap(logger *lg, const log_bid *bids, lng *lids, lng *cnts, BAT
*catalog_bid, BAT *catalog_id, BAT *dcatalog, int cleanup)
+cleanup_and_swap(logger *lg, int *r, const log_bid *bids, lng *lids, lng
*cnts, BAT *catalog_bid, BAT *catalog_id, BAT *dcatalog, int cleanup)
{
BAT *nbids, *noids, *ncnts, *nlids, *ndels;
BUN p, q;
- int err = 0;
+ int err = 0, rcnt = 0;
oid *poss = Tloc(dcatalog, 0);
BATloop(dcatalog, p, q) {
@@ -1456,20 +1456,15 @@ cleanup_and_swap(logger *lg, const log_b
continue;
if (lids[pos] >= 0) {
- if (lg->debug & 1) {
- fprintf(stderr, "release %d\n", bids[pos]);
- if (BBP_lrefs(bids[pos]) != 2)
- fprintf(stderr, "release %d %d\n",
bids[pos], BBP_lrefs(bids[pos]));
- }
+ lids[pos] = -1; /* mark as transient */
+ r[rcnt++] = bids[pos];
BAT *lb;
- BBPrelease(bids[pos]);
if ((lb = BATdescriptor(bids[pos])) == NULL ||
BATmode(lb, true/*transient*/) != GDK_SUCCEED) {
TRC_WARNING(GDK, "Failed to set bat(%d)
transient\n", bids[pos]);
}
- lids[pos] = -1; /* mark as transient */
logbat_destroy(lb);
}
}
@@ -1532,9 +1527,9 @@ cleanup_and_swap(logger *lg, const log_b
logbat_destroy(nlids);
return -1;
}
- BBPrelease(lg->catalog_bid->batCacheid);
- BBPrelease(lg->catalog_id->batCacheid);
- BBPrelease(lg->dcatalog->batCacheid);
+ r[rcnt++] = lg->catalog_bid->batCacheid;
+ r[rcnt++] = lg->catalog_id->batCacheid;
+ r[rcnt++] = lg->dcatalog->batCacheid;
logbat_destroy(lg->catalog_bid);
logbat_destroy(lg->catalog_id);
@@ -1552,7 +1547,7 @@ cleanup_and_swap(logger *lg, const log_b
lg->cnt = BATcount(lg->catalog_bid);
lg->deleted -= cleanup;
assert(lg->deleted == BATcount(lg->dcatalog));
- return 0;
+ return rcnt;
}
static gdk_return
@@ -1565,16 +1560,18 @@ bm_subcommit(logger *lg)
BAT *dcatalog = lg->dcatalog;
BUN nn = 13 + BATcount(catalog_bid);
bat *n = GDKmalloc(sizeof(bat) * nn);
+ bat *r = GDKmalloc(sizeof(bat) * nn);
BUN *sizes = GDKmalloc(sizeof(BUN) * nn);
- int i = 0;
+ int i = 0, rcnt = 0;
gdk_return res;
const log_bid *bids;
lng *cnts = NULL, *lids = NULL;
int cleanup = 0;
lng t0 = 0;
- if (n == NULL || sizes == NULL) {
+ if (n == NULL || r == NULL || sizes == NULL) {
GDKfree(n);
+ GDKfree(r);
GDKfree(sizes);
logger_unlock(lg);
return GDK_FAIL;
@@ -1606,8 +1603,9 @@ bm_subcommit(logger *lg)
sizes[i] = BATcount(dcatalog);
n[i++] = dcatalog->batCacheid;
- if (cleanup && cleanup_and_swap(lg, bids, lids, cnts, catalog_bid,
catalog_id, dcatalog, cleanup)) {
+ if (cleanup && (rcnt=cleanup_and_swap(lg, r, bids, lids, cnts,
catalog_bid, catalog_id, dcatalog, cleanup)) < 0) {
GDKfree(n);
+ GDKfree(r);
GDKfree(sizes);
logger_unlock(lg);
return GDK_FAIL;
@@ -1629,6 +1627,7 @@ bm_subcommit(logger *lg)
tids = bm_tids(lg->seqs_id, lg->dseqs);
if (tids == NULL) {
GDKfree(n);
+ GDKfree(r);
GDKfree(sizes);
logger_unlock(lg);
return GDK_FAIL;
@@ -1641,6 +1640,7 @@ bm_subcommit(logger *lg)
logbat_destroy(ids);
logbat_destroy(vals);
GDKfree(n);
+ GDKfree(r);
GDKfree(sizes);
logger_unlock(lg);
return GDK_FAIL;
@@ -1652,6 +1652,7 @@ bm_subcommit(logger *lg)
logbat_destroy(ids);
logbat_destroy(vals);
GDKfree(n);
+ GDKfree(r);
GDKfree(sizes);
logger_unlock(lg);
return GDK_FAIL;
@@ -1664,6 +1665,7 @@ bm_subcommit(logger *lg)
logbat_destroy(ids);
logbat_destroy(vals);
GDKfree(n);
+ GDKfree(r);
GDKfree(sizes);
logger_unlock(lg);
return GDK_FAIL;
@@ -1671,6 +1673,9 @@ bm_subcommit(logger *lg)
i = subcommit_list_add(i, n, sizes, ids->batCacheid,
BATcount(ids));
i = subcommit_list_add(i, n, sizes, vals->batCacheid,
BATcount(ids));
+ r[rcnt++] = lg->seqs_id->batCacheid;
+ r[rcnt++] = lg->seqs_val->batCacheid;
+
logbat_destroy(lg->seqs_id);
logbat_destroy(lg->seqs_val);
@@ -1689,7 +1694,18 @@ bm_subcommit(logger *lg)
res = TMsubcommit_list(n, cnts?sizes:NULL, i, lg->saved_id,
lg->saved_tid);
if (lg->debug & 1)
fprintf(stderr, "#subcommit " LLFMT "usec\n", GDKusec() - t0);
+ if (res == GDK_SUCCEED) { /* now cleanup */
+ for(i=0;i<rcnt; i++) {
+ if (lg->debug & 1) {
+ fprintf(stderr, "release %d\n", r[i]);
+ if (BBP_lrefs(r[i]) != 2)
+ fprintf(stderr, "release %d %d\n",
r[i], BBP_lrefs(r[i]));
+ }
+ BBPrelease(r[i]);
+ }
+ }
GDKfree(n);
+ GDKfree(r);
GDKfree(sizes);
if (res != GDK_SUCCEED)
TRC_CRITICAL(GDK, "commit failed\n");
diff --git a/gdk/gdk_logger_old.c b/gdk/gdk_logger_old.c
--- a/gdk/gdk_logger_old.c
+++ b/gdk/gdk_logger_old.c
@@ -1489,10 +1489,6 @@ logger_load(const char *fn, char filenam
GDKerror("Logger_new: failed to create freed bat");
goto error;
}
- strconcat_len(bak, sizeof(bak), fn, "_freed", NULL);
- if (BBPrename(lg->freed->batCacheid, bak) < 0) {
- goto error;
- }
snapshots_bid = old_logger_find_bat(lg, "snapshots_bid", 0, 0);
if (snapshots_bid == 0) {
lg->snapshots_bid = logbat_new(TYPE_int, 1, TRANSIENT);
diff --git a/monetdb5/mal/mal_resource.c b/monetdb5/mal/mal_resource.c
--- a/monetdb5/mal/mal_resource.c
+++ b/monetdb5/mal/mal_resource.c
@@ -69,7 +69,9 @@ getMemoryClaim(MalBlkPtr mb, MalStkPtr s
b = BATdescriptor( stk->stk[getArg(pci, i)].val.bval);
if (b == NULL)
return 0;
+ MT_lock_set(&b->theaplock);
if (flag && isVIEW(b)) {
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
