Changeset: 362d90a8d17c for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=362d90a8d17c
Modified Files:
gdk/gdk_batop.c
gdk/gdk_bbp.c
gdk/gdk_logger.c
gdk/gdk_storage.c
monetdb5/mal/mal_authorize.c
monetdb5/modules/mal/tablet.c
sql/test/SQLite_regress/sqllogictest/Tests/select3.test.timeout
sql/test/SQLite_regress/sqllogictest/Tests/select4.test.timeout
sql/test/SQLite_regress/sqllogictest/Tests/select5.test.timeout
testing/Mtest.py.in
Branch: Jul2015
Log Message:
merged changes
diffs (truncated from 474 to 300 lines):
diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c
--- a/gdk/gdk_batop.c
+++ b/gdk/gdk_batop.c
@@ -777,8 +777,6 @@ BATappend(BAT *b, BAT *n, bit force)
r++;
}
}
- if (b->hseqbase != oid_nil)
- b->hrevsorted = 0;
} else {
oid o = MAXoid(b);
BATiter ni = bat_iterator(n);
@@ -789,7 +787,6 @@ BATappend(BAT *b, BAT *n, bit force)
o++;
r++;
}
- b->hrevsorted = 0;
}
} else {
BUN p, q;
@@ -813,7 +810,6 @@ BATappend(BAT *b, BAT *n, bit force)
i++;
}
}
- b->hrevsorted = 0;
} else {
oid on = oid_nil;
@@ -871,6 +867,8 @@ BATappend(BAT *b, BAT *n, bit force)
b->tdense = b->tsorted = b->trevsorted = 0;
}
}
+ if (b->hseqbase != oid_nil)
+ b->hrevsorted = BATcount(b) <= 1;
b->H->nonil &= n->H->nonil;
b->T->nonil &= n->T->nonil;
return GDK_SUCCEED;
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -808,12 +808,13 @@ vheapinit(COLrec *col, const char *buf,
return n;
}
-static void
+static int
BBPreadEntries(FILE *fp, int *min_stamp, int *max_stamp, int oidsize, int
bbpversion)
{
bat bid = 0;
char buf[4096];
BATstore *bs;
+ int needcommit = 0;
/* read the BBP.dir and insert the BATs into the BBP */
while (fgets(buf, sizeof(buf), fp) != NULL) {
@@ -899,6 +900,21 @@ BBPreadEntries(FILE *fp, int *min_stamp,
nread += vheapinit(&bs->H, buf + nread, Hhashash, bid);
nread += vheapinit(&bs->T, buf + nread, Thashash, bid);
+ if (bs->S.count > 1) {
+ /* fix result of bug in BATappend not clearing
+ * revsorted property */
+ if (bs->H.type == TYPE_void && bs->H.seq != oid_nil &&
bs->H.revsorted) {
+ bs->H.revsorted = 0;
+ bs->S.descdirty = 1;
+ needcommit = 1;
+ }
+ if (bs->T.type == TYPE_void && bs->T.seq != oid_nil &&
bs->T.revsorted) {
+ bs->T.revsorted = 0;
+ bs->S.descdirty = 1;
+ needcommit = 1;
+ }
+ }
+
if (buf[nread] != '\n' && buf[nread] != ' ')
GDKfatal("BBPinit: invalid format for BBP.dir\n%s",
buf);
if (buf[nread] == ' ')
@@ -931,6 +947,7 @@ BBPreadEntries(FILE *fp, int *min_stamp,
BBP_refs(bid) = 0;
BBP_lrefs(bid) = 1; /* any BAT we encounter here is
persistent, so has a logical reference */
}
+ return needcommit;
}
#ifdef HAVE_HGE
@@ -1042,6 +1059,7 @@ BBPinit(void)
int bbpversion;
int oidsize;
oid BBPoid;
+ int needcommit;
#ifdef NEED_MT_LOCK_INIT
MT_lock_init(&GDKunloadLock, "GDKunloadLock");
@@ -1093,7 +1111,7 @@ BBPinit(void)
BBPextend(0, FALSE); /* allocate BBP records */
ATOMIC_SET(BBPsize, 1, BBPsizeLock, "BBPinit");
- BBPreadEntries(fp, &min_stamp, &max_stamp, oidsize, bbpversion);
+ needcommit = BBPreadEntries(fp, &min_stamp, &max_stamp, oidsize,
bbpversion);
fclose(fp);
/* normalize saved LRU stamps */
@@ -1122,7 +1140,7 @@ BBPinit(void)
#else
(void) oidsize;
#endif
- if (bbpversion < GDKLIBRARY)
+ if (bbpversion < GDKLIBRARY || needcommit)
TMcommit();
return;
@@ -2489,6 +2507,7 @@ getBBPdescriptor(bat i, int lock)
/* clearing bits can be done without the lock */
BBP_status_off(j, BBPLOADING, "BBPdescriptor");
+ CHECKDEBUG BATassertProps(b);
}
return b;
}
diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -823,6 +823,7 @@ static int
logger_open(logger *lg)
{
char filename[BUFSIZ];
+ bat bid;
snprintf(filename, sizeof(filename), "%s%s." LLFMT, lg->dir, LOGFILE,
lg->id);
@@ -833,6 +834,19 @@ logger_open(logger *lg)
fprintf(stderr, "!ERROR: logger_open: creating %s failed\n",
filename);
return LOG_ERR;
}
+ if ((bid = logger_find_bat(lg, "seqs_id")) != 0) {
+ BAT *b = BATdescriptor(bid);
+ BATmode(b, TRANSIENT);
+ logger_del_bat(lg, bid);
+ logbat_destroy(b);
+ bid = logger_find_bat(lg, "seqs_val");
+ b = BATdescriptor(bid);
+ BATmode(b, TRANSIENT);
+ logger_del_bat(lg, bid);
+ logbat_destroy(b);
+ if (bm_commit(lg) != LOG_OK)
+ return LOG_ERR;
+ }
return LOG_OK;
}
@@ -1452,6 +1466,13 @@ logger_new(int debug, const char *fn, co
BBPincref(bid, TRUE);
}
}
+ lg->freed = logbat_new(TYPE_int, 1, TRANSIENT);
+ if (lg->freed == NULL)
+ logger_fatal("Logger_new: failed to create freed bat", 0, 0, 0);
+ snprintf(bak, sizeof(bak), "%s_freed", fn);
+ if (BBPrename(lg->freed->batCacheid, bak) < 0)
+ logger_fatal("Logger_new: BBPrename to %s failed",
+ bak, 0, 0);
snapshots_bid = logger_find_bat(lg, "snapshots_bid");
if (snapshots_bid == 0) {
lg->seqs_id = logbat_new(TYPE_int, 1, TRANSIENT);
@@ -1503,20 +1524,28 @@ logger_new(int debug, const char *fn, co
bat seqs_val = logger_find_bat(lg, "seqs_val");
bat snapshots_tid = logger_find_bat(lg, "snapshots_tid");
bat dsnapshots = logger_find_bat(lg, "dsnapshots");
+ int needcommit = 0;
+ int dbg = GDKdebug;
if (seqs_id) {
- BAT *o_id = BATdescriptor(seqs_id);
- BAT *o_val = BATdescriptor(seqs_val);
+ BAT *o_id;
+ BAT *o_val;
+
+ /* don't check these bats since they will be fixed */
+ GDKdebug &= ~CHECKMASK;
+ o_id = BATdescriptor(seqs_id);
+ o_val = BATdescriptor(seqs_val);
+ GDKdebug = dbg;
if (o_id == NULL || o_val == NULL)
logger_fatal("Logger_new: inconsistent
database: cannot find seqs bats", 0, 0, 0);
- BATseqbase(o_id, 0);
- BATseqbase(o_val, 0);
lg->seqs_id = BATcopy(o_id, TYPE_void, TYPE_int, 1,
TRANSIENT);
lg->seqs_val = BATcopy(o_val, TYPE_void, TYPE_lng, 1,
TRANSIENT);
BBPunfix(o_id->batCacheid);
BBPunfix(o_val->batCacheid);
+ BATseqbase(lg->seqs_id, 0);
+ BATseqbase(lg->seqs_val, 0);
} else {
lg->seqs_id = logbat_new(TYPE_int, 1, TRANSIENT);
lg->seqs_val = logbat_new(TYPE_lng, 1, TRANSIENT);
@@ -1528,12 +1557,41 @@ logger_new(int debug, const char *fn, co
logger_fatal("Logger_new: cannot create seqs bats",
0, 0, 0);
+ GDKdebug &= ~CHECKMASK;
lg->snapshots_bid = BATdescriptor(snapshots_bid);
if (lg->snapshots_bid == 0)
logger_fatal("Logger_new: inconsistent database,
snapshots_bid does not exist", 0, 0, 0);
lg->snapshots_tid = BATdescriptor(snapshots_tid);
if (lg->snapshots_tid == 0)
logger_fatal("Logger_new: inconsistent database,
snapshots_tid does not exist", 0, 0, 0);
+ GDKdebug = dbg;
+ if (lg->snapshots_bid->htype == TYPE_oid) {
+ BAT *b;
+ assert(lg->snapshots_tid->htype == TYPE_oid);
+ b = BATcopy(lg->snapshots_bid, TYPE_void,
lg->snapshots_bid->ttype, 1, PERSISTENT);
+ BATseqbase(b, 0);
+ BATsetaccess(b, BAT_READ);
+ snprintf(bak, sizeof(bak), "tmp_%o",
lg->snapshots_bid->batCacheid);
+ BBPrename(lg->snapshots_bid->batCacheid, bak);
+ BATmode(lg->snapshots_bid, TRANSIENT);
+ snprintf(bak, sizeof(bak), "%s_snapshots_bid", fn);
+ BBPrename(b->batCacheid, bak);
+ logbat_destroy(lg->snapshots_bid);
+ lg->snapshots_bid = b;
+ logger_add_bat(lg, b, "snapshots_bid");
+ b = BATcopy(lg->snapshots_tid, TYPE_void,
lg->snapshots_tid->ttype, 1, PERSISTENT);
+ BATseqbase(b, 0);
+ BATsetaccess(b, BAT_READ);
+ snprintf(bak, sizeof(bak), "tmp_%o",
lg->snapshots_tid->batCacheid);
+ BBPrename(lg->snapshots_tid->batCacheid, bak);
+ BATmode(lg->snapshots_tid, TRANSIENT);
+ snprintf(bak, sizeof(bak), "%s_snapshots_tid", fn);
+ BBPrename(b->batCacheid, bak);
+ logbat_destroy(lg->snapshots_tid);
+ lg->snapshots_tid = b;
+ logger_add_bat(lg, b, "snapshots_tid");
+ needcommit = 1;
+ }
if (dsnapshots) {
lg->dsnapshots = BATdescriptor(dsnapshots);
@@ -1545,18 +1603,13 @@ logger_new(int debug, const char *fn, co
if (BBPrename(lg->dsnapshots->batCacheid, bak) < 0)
logger_fatal("Logger_new: BBPrename to %s
failed", bak, 0, 0);
logger_add_bat(lg, lg->dsnapshots, "dsnapshots");
-
- if (bm_subcommit(lg->catalog_bid, lg->catalog_nme,
lg->catalog_bid, lg->catalog_nme, lg->dcatalog, NULL, lg->debug) != GDK_SUCCEED)
- logger_fatal("Logger_new: commit failed", 0, 0,
0);
+ needcommit = 1;
}
+ GDKdebug &= ~CHECKMASK;
+ if (needcommit && bm_commit(lg) != LOG_OK)
+ logger_fatal("Logger_new: commit failed", 0, 0, 0);
+ GDKdebug = dbg;
}
- lg->freed = logbat_new(TYPE_int, 1, TRANSIENT);
- if (lg->freed == NULL)
- logger_fatal("Logger_new: failed to create freed bat", 0, 0, 0);
- snprintf(bak, sizeof(bak), "%s_freed", fn);
- if (BBPrename(lg->freed->batCacheid, bak) < 0)
- logger_fatal("Logger_new: BBPrename to %s failed",
- bak, 0, 0);
if (fp != NULL) {
#if SIZEOF_OID == 8
@@ -2280,7 +2333,7 @@ log_abort(logger *lg)
}
static int
-log_sequence_(logger *lg, int seq, lng val)
+log_sequence_(logger *lg, int seq, lng val, int flush)
{
logformat l;
@@ -2293,8 +2346,8 @@ log_sequence_(logger *lg, int seq, lng v
if (log_write_format(lg, &l) == LOG_ERR ||
!mnstr_writeLng(lg->log, val) ||
- mnstr_flush(lg->log) ||
- mnstr_fsync(lg->log) ||
+ (flush && mnstr_flush(lg->log)) ||
+ (flush && mnstr_fsync(lg->log)) ||
pre_allocate(lg) != GDK_SUCCEED) {
fprintf(stderr, "!ERROR: log_sequence_: write failed\n");
return LOG_ERR;
@@ -2316,7 +2369,13 @@ log_sequence_nrs(logger *lg)
oid pos = p;
if (BUNfnd(lg->dseqs, &pos) == BUN_NONE)
- ok &= log_sequence_(lg, *id, *val);
+ ok |= log_sequence_(lg, *id, *val, 0);
+ }
+ if (ok != LOG_OK ||
+ mnstr_flush(lg->log) ||
+ mnstr_fsync(lg->log)) {
+ fprintf(stderr, "!ERROR: log_sequence_nrs: write failed\n");
+ return LOG_ERR;
}
return ok;
}
@@ -2341,7 +2400,7 @@ log_sequence(logger *lg, int seq, lng va
BUNappend(lg->seqs_id, &seq, FALSE);
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
