Changeset: d31a67f9389e for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d31a67f9389e
Modified Files:
clients/Tests/exports.stable.out
gdk/gdk.h
gdk/gdk_bbp.c
gdk/gdk_bbp.h
gdk/gdk_delta.c
gdk/gdk_logger.c
gdk/gdk_tm.c
monetdb5/mal/mal_authorize.c
monetdb5/modules/mal/querylog.c
monetdb5/modules/mal/transaction.c
sql/storage/bat/bat_storage.c
sql/storage/bat/bat_utils.c
Branch: unlock
Log Message:
Extend TMsubcommit_list with an array of sizes of BATs.
The sizes indicate until where the corresponding BAT should be
committed.
If the array pointer is NULL, the complete BATs are committed, also if
an individual size is BUN_NONE (or really, anything larger than the
actual size) the complete BAT is also committed.
diffs (truncated from 533 to 300 lines):
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -111,7 +111,7 @@ BAT *BATcalcxor(BAT *b1, BAT *b2, BAT *s
BAT *BATcalcxorcst(BAT *b, const ValRecord *v, BAT *s, BAT *r);
bool BATcheckorderidx(BAT *b);
gdk_return BATclear(BAT *b, bool force);
-void BATcommit(BAT *b);
+void BATcommit(BAT *b, BUN size);
BAT *BATconstant(oid hseq, int tt, const void *val, BUN cnt, role_t role);
BAT *BATconvert(BAT *b, BAT *s, BAT *r, int tp, bool abort_on_error);
BUN BATcount_no_nil(BAT *b);
@@ -208,7 +208,7 @@ int BBPrename(bat bid, const char *nme);
int BBPretain(bat b);
gdk_return BBPsave(BAT *b);
void BBPshare(bat b);
-gdk_return BBPsync(int cnt, bat *subcommit);
+gdk_return BBPsync(int cnt, bat *restrict subcommit, BUN *restrict sizes);
int BBPunfix(bat b);
void BBPunlock(void);
gdk_return BUNappend(BAT *b, const void *right, bool force)
__attribute__((__warn_unused_result__));
@@ -375,7 +375,7 @@ void THRsetdata(int, void *);
void TMabort(void);
gdk_return TMcommit(void);
gdk_return TMsubcommit(BAT *bl);
-gdk_return TMsubcommit_list(bat *subcommit, int cnt);
+gdk_return TMsubcommit_list(bat *restrict subcommit, BUN *restrict sizes, int
cnt);
void VALclear(ValPtr v);
int VALcmp(const ValRecord *p, const ValRecord *q);
void *VALconvert(int typ, ValPtr t);
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -1832,7 +1832,7 @@ Tpos(BATiter *bi, BUN p)
gdk_export gdk_return TMcommit(void);
gdk_export void TMabort(void);
gdk_export gdk_return TMsubcommit(BAT *bl);
-gdk_export gdk_return TMsubcommit_list(bat *subcommit, int cnt);
+gdk_export gdk_return TMsubcommit_list(bat *restrict subcommit, BUN *restrict
sizes, int cnt);
/*
* @- Delta Management
@@ -1870,7 +1870,7 @@ gdk_export gdk_return TMsubcommit_list(b
* TMcommit does the BATcommits @emph{before} attempting to sync to
* disk instead of @sc{after} doing this.
*/
-gdk_export void BATcommit(BAT *b);
+gdk_export void BATcommit(BAT *b, BUN size);
gdk_export void BATfakeCommit(BAT *b);
gdk_export void BATundo(BAT *b);
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -114,7 +114,7 @@ static void BBPuncacheit(bat bid, bool u
static gdk_return BBPprepare(bool subcommit);
static BAT *getBBPdescriptor(bat i, bool lock);
static gdk_return BBPbackup(BAT *b, bool subcommit);
-static gdk_return BBPdir(int cnt, bat *subcommit);
+static gdk_return BBPdir(int cnt, bat *restrict subcommit, BUN *restrict
sizes);
#ifdef HAVE_HGE
/* start out by saying we have no hge, but as soon as we've seen one,
@@ -1257,7 +1257,7 @@ BBPinit(void)
/* no BBP.bak (nor BBP.dir or BACKUP/BBP.dir):
* create a new one */
TRC_DEBUG(IO_, "initializing BBP.\n"); /*
BBPdir instead of BBPinit for backward compatibility of error messages */
- if (BBPdir(0, NULL) != GDK_SUCCEED) {
+ if (BBPdir(0, NULL, NULL) != GDK_SUCCEED) {
GDKfree(bbpdirstr);
GDKfree(backupbbpdirstr);
goto bailout;
@@ -1427,8 +1427,11 @@ BBPexit(void)
* reclaimed as well.
*/
static inline int
-heap_entry(FILE *fp, BAT *b)
+heap_entry(FILE *fp, BAT *b, BUN size)
{
+ size_t free = b->theap.free;
+ if (b->twidth > 0 && free / b->twidth > size)
+ free = size * b->twidth;
return fprintf(fp, " %s %d %d %d " BUNFMT " " BUNFMT " " BUNFMT " "
BUNFMT " " OIDFMT " %zu %zu %d",
b->ttype >= 0 ? BATatoms[b->ttype].name :
ATOMunknown_name(b->ttype),
@@ -1445,7 +1448,7 @@ heap_entry(FILE *fp, BAT *b)
b->tnosorted,
b->tnorevsorted,
b->tseqbase,
- b->theap.free,
+ free,
b->theap.size,
(int) b->theap.newstorage);
}
@@ -1460,7 +1463,7 @@ vheap_entry(FILE *fp, Heap *h)
}
static gdk_return
-new_bbpentry(FILE *fp, bat i)
+new_bbpentry(FILE *fp, bat i, BUN size)
{
#ifndef NDEBUG
assert(i > 0);
@@ -1476,6 +1479,8 @@ new_bbpentry(FILE *fp, bat i)
}
#endif
+ if (size > BBP_desc(i)->batCount)
+ size = BBP_desc(i)->batCount;
if (fprintf(fp, "%d %u %s %s %d " BUNFMT " " BUNFMT " " OIDFMT,
/* BAT info */
(int) i,
@@ -1483,10 +1488,10 @@ new_bbpentry(FILE *fp, bat i)
BBP_logical(i),
BBP_physical(i),
BBP_desc(i)->batRestricted << 1,
- BBP_desc(i)->batCount,
+ size,
BBP_desc(i)->batCapacity,
BBP_desc(i)->hseqbase) < 0 ||
- heap_entry(fp, BBP_desc(i)) < 0 ||
+ heap_entry(fp, BBP_desc(i), size) < 0 ||
vheap_entry(fp, BBP_desc(i)->tvheap) < 0 ||
(BBP_options(i) && fprintf(fp, " %s", BBP_options(i)) < 0) ||
fprintf(fp, "\n") < 0) {
@@ -1514,7 +1519,7 @@ BBPdir_header(FILE *f, int n)
}
static gdk_return
-BBPdir_subcommit(int cnt, bat *subcommit)
+BBPdir_subcommit(int cnt, bat *restrict subcommit, BUN *restrict sizes)
{
FILE *obbpf, *nbbpf;
bat j = 1;
@@ -1530,8 +1535,6 @@ BBPdir_subcommit(int cnt, bat *subcommit
if ((nbbpf = GDKfilelocate(0, "BBP", "w", "dir")) == NULL)
return GDK_FAIL;
- n = (bat) ATOMIC_GET(&BBPsize);
-
/* we need to copy the backup BBP.dir to the new, but
* replacing the entries for the subcommitted bats */
if ((obbpf = GDKfileopen(0, SUBDIR, "BBP", "dir", "r")) == NULL &&
@@ -1547,7 +1550,10 @@ BBPdir_subcommit(int cnt, bat *subcommit
goto bailout;
}
/* third line contains BBPsize */
- sscanf(buf, "BBPsize=%d", &n);
+ if (sscanf(buf, "BBPsize=%d", &n) != 1) {
+ GDKerror("BBPdir: cannot read BBPsize in backup BBP.dir.");
+ goto bailout;
+ }
if (n < (bat) ATOMIC_GET(&BBPsize))
n = (bat) ATOMIC_GET(&BBPsize);
@@ -1578,7 +1584,7 @@ BBPdir_subcommit(int cnt, bat *subcommit
bat i = subcommit[j];
/* BBP.dir consists of all persistent bats only */
if (BBP_status(i) & BBPPERSISTENT) {
- if (new_bbpentry(nbbpf, i) != GDK_SUCCEED) {
+ if (new_bbpentry(nbbpf, i, sizes ? sizes[j] :
BUN_NONE) != GDK_SUCCEED) {
goto bailout;
}
}
@@ -1629,13 +1635,13 @@ BBPdir_subcommit(int cnt, bat *subcommit
}
gdk_return
-BBPdir(int cnt, bat *subcommit)
+BBPdir(int cnt, bat *restrict subcommit, BUN *restrict sizes)
{
FILE *fp;
bat i;
if (subcommit)
- return BBPdir_subcommit(cnt, subcommit);
+ return BBPdir_subcommit(cnt, subcommit, sizes);
TRC_DEBUG(IO_, "writing BBP.dir (%d bats).\n", (int) (bat)
ATOMIC_GET(&BBPsize));
if ((fp = GDKfilelocate(0, "BBP", "w", "dir")) == NULL) {
@@ -1650,7 +1656,7 @@ BBPdir(int cnt, bat *subcommit)
/* write the entry
* BBP.dir consists of all persistent bats */
if (BBP_status(i) & BBPPERSISTENT) {
- if (new_bbpentry(fp, i) != GDK_SUCCEED) {
+ if (new_bbpentry(fp, i, BUN_NONE) != GDK_SUCCEED) {
goto bailout;
}
}
@@ -3112,7 +3118,7 @@ fail:
* The BBP.dir is also moved into the BAKDIR.
*/
gdk_return
-BBPsync(int cnt, bat *subcommit)
+BBPsync(int cnt, bat *restrict subcommit, BUN *restrict sizes)
{
gdk_return ret = GDK_SUCCEED;
int t0 = 0, t1 = 0;
@@ -3190,7 +3196,7 @@ BBPsync(int cnt, bat *subcommit)
TRC_DEBUG(PERF, "write time %d\n", (t0 = GDKms()) - t1);
if (ret == GDK_SUCCEED) {
- ret = BBPdir(cnt, subcommit);
+ ret = BBPdir(cnt, subcommit, sizes);
}
TRC_DEBUG(PERF, "dir time %d, %d bats\n", (t1 = GDKms()) - t0, (bat)
ATOMIC_GET(&BBPsize));
diff --git a/gdk/gdk_bbp.h b/gdk/gdk_bbp.h
--- a/gdk/gdk_bbp.h
+++ b/gdk/gdk_bbp.h
@@ -69,7 +69,7 @@ gdk_export bat BBPindex(const char *nme)
gdk_export BAT *BBPdescriptor(bat b);
/* swapping interface */
-gdk_export gdk_return BBPsync(int cnt, bat *subcommit);
+gdk_export gdk_return BBPsync(int cnt, bat *restrict subcommit, BUN *restrict
sizes);
gdk_export int BBPfix(bat b);
gdk_export int BBPunfix(bat b);
gdk_export int BBPretain(bat b);
diff --git a/gdk/gdk_delta.c b/gdk/gdk_delta.c
--- a/gdk/gdk_delta.c
+++ b/gdk/gdk_delta.c
@@ -31,7 +31,7 @@
* need to free their heap space (only if necessary).
*/
void
-BATcommit(BAT *b)
+BATcommit(BAT *b, BUN size)
{
if (b == NULL)
return;
@@ -43,7 +43,7 @@ BATcommit(BAT *b)
if (DELTAdirty(b)) {
b->batDirtydesc = true;
}
- b->batInserted = BUNlast(b);
+ b->batInserted = size < BUNlast(b) ? size : BUNlast(b);
TRC_DEBUG(DELTA, "BATcommit2 %s free %zu ins " BUNFMT " base %p\n",
BATgetId(b), b->theap.free, b->batInserted, b->theap.base);
}
@@ -56,7 +56,7 @@ void
BATfakeCommit(BAT *b)
{
if (b) {
- BATcommit(b);
+ BATcommit(b, BUN_NONE);
b->batDirtydesc = b->theap.dirty = false;
if (b->tvheap)
b->tvheap->dirty = false;
diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -1408,9 +1408,9 @@ logger_commit(logger *lg)
BATclear(lg->snapshots_tid, true) != GDK_SUCCEED ||
BATclear(lg->dsnapshots, true) != GDK_SUCCEED)
return GDK_FAIL;
- BATcommit(lg->snapshots_bid);
- BATcommit(lg->snapshots_tid);
- BATcommit(lg->dsnapshots);
+ BATcommit(lg->snapshots_bid, BUN_NONE);
+ BATcommit(lg->snapshots_tid, BUN_NONE);
+ BATcommit(lg->dsnapshots, BUN_NONE);
}
return bm_commit(lg);
}
@@ -1660,14 +1660,14 @@ bm_subcommit(logger *lg, BAT *list_bid,
}
assert((BUN) i <= nn);
- BATcommit(catalog_bid);
- BATcommit(catalog_nme);
+ BATcommit(catalog_bid, BUN_NONE);
+ BATcommit(catalog_nme, BUN_NONE);
if (catalog_tpe) {
- BATcommit(catalog_tpe);
- BATcommit(catalog_oid);
+ BATcommit(catalog_tpe, BUN_NONE);
+ BATcommit(catalog_oid, BUN_NONE);
}
- BATcommit(dcatalog);
- res = TMsubcommit_list(n, i);
+ BATcommit(dcatalog, BUN_NONE);
+ res = TMsubcommit_list(n, NULL, i);
GDKfree(n);
if (res != GDK_SUCCEED)
fprintf(stderr, "!ERROR: bm_subcommit: commit failed\n");
@@ -3164,7 +3164,7 @@ bm_commit(logger *lg)
BBPreclaim(n);
if (res == GDK_SUCCEED) {
BATclear(lg->freed, false);
- BATcommit(lg->freed);
+ BATcommit(lg->freed, BUN_NONE);
return GDK_SUCCEED;
}
return GDK_FAIL;
diff --git a/gdk/gdk_tm.c b/gdk/gdk_tm.c
--- a/gdk/gdk_tm.c
+++ b/gdk/gdk_tm.c
@@ -47,7 +47,7 @@
/* in the commit prelude, the delta status in the memory image of all
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
