Changeset: 0be97fc0691d for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/0be97fc0691d
Modified Files:
gdk/gdk.h
gdk/gdk_batop.c
gdk/gdk_bbp.c
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql.c
sql/include/sql_relation.h
sql/server/rel_optimize_sel.c
sql/server/rel_optimizer_private.h
sql/server/rel_rewriter.c
sql/server/rel_rewriter.h
sql/server/rel_statistics.c
sql/server/rel_updates.c
sql/server/sql_mvc.c
sql/storage/bat/bat_storage.c
sql/storage/sql_storage.h
sql/storage/store.c
sql/test/emptydb/Tests/check.stable.out
sql/test/emptydb/Tests/check.stable.out.32bit
sql/test/emptydb/Tests/check.stable.out.int128
Branch: properties
Log Message:
Merged with default
diffs (truncated from 3084 to 300 lines):
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -543,9 +543,9 @@ gdk_return log_bat_transient(logger *lg,
gdk_return log_constant(logger *lg, int type, ptr val, log_id id, lng offset,
lng cnt);
gdk_return log_delta(logger *lg, BAT *uid, BAT *uval, log_id id);
gdk_return log_sequence(logger *lg, int seq, lng id);
-gdk_return log_tdone(logger *lg, ulng commit_ts);
gdk_return log_tend(logger *lg);
-gdk_return log_tstart(logger *lg, bool flush);
+gdk_return log_tflush(logger *lg, ulng log_file_id, ulng commit_ts);
+gdk_return log_tstart(logger *lg, bool flushnow, ulng *log_file_id);
gdk_return logger_activate(logger *lg);
lng logger_changes(logger *lg);
logger *logger_create(int debug, const char *fn, const char *logdir, int
version, preversionfix_fptr prefuncp, postversionfix_fptr postfuncp, void
*funcdata);
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -1415,6 +1415,8 @@ gdk_export void GDKqsort(void *restrict
/* BAT is dense (i.e., BATtvoid() is true and tseqbase is not NIL) */
#define BATtdense(b) (!is_oid_nil((b)->tseqbase) && \
((b)->tvheap == NULL || (b)->tvheap->free == 0))
+#define BATtdensebi(bi) (!is_oid_nil((bi)->tseq) &&
\
+ ((bi)->vh == NULL || (bi)->vhfree == 0))
/* BATtvoid: BAT can be (or actually is) represented by TYPE_void */
#define BATtvoid(b) (BATtdense(b) || (b)->ttype==TYPE_void)
#define BATtkey(b) ((b)->tkey || BATtdense(b))
diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c
--- a/gdk/gdk_batop.c
+++ b/gdk/gdk_batop.c
@@ -736,7 +736,7 @@ BATappend2(BAT *b, BAT *n, BAT *s, bool
/* b does not have storage, keep it that way if we can */
HASHdestroy(b); /* we're not maintaining the hash here */
MT_lock_set(&b->theaplock);
- if (BATtdense(n) && ci.tpe == cand_dense &&
+ if (BATtdensebi(&ni) && ci.tpe == cand_dense &&
(BATcount(b) == 0 ||
(BATtdense(b) &&
b->tseqbase + BATcount(b) == n->tseqbase + ci.seq -
hseq))) {
@@ -776,7 +776,7 @@ BATappend2(BAT *b, BAT *n, BAT *s, bool
if (ci.tpe == cand_dense) {
b->tnosorted = ci.seq - hseq <= n->tnosorted &&
n->tnosorted < ci.seq + ci.ncand - hseq ? n->tnosorted + hseq - ci.seq : 0;
b->tnorevsorted = ci.seq - hseq <= n->tnorevsorted &&
n->tnorevsorted < ci.seq + ci.ncand - hseq ? n->tnorevsorted + hseq - ci.seq :
0;
- if (BATtdense(n)) {
+ if (BATtdensebi(&ni)) {
b->tseqbase = n->tseqbase + ci.seq - hseq;
}
} else {
@@ -812,7 +812,7 @@ BATappend2(BAT *b, BAT *n, BAT *s, bool
BATkey(b, false);
}
if (b->ttype != TYPE_void && b->tsorted && BATtdense(b) &&
- (!BATtdense(n) ||
+ (!BATtdensebi(&ni) ||
ci.tpe != cand_dense ||
1 + *(oid *) BUNtloc(bi, last) != BUNtoid(n, ci.seq -
hseq))) {
b->tseqbase = oid_nil;
@@ -1470,7 +1470,7 @@ BATappend_or_update(BAT *b, BAT *p, cons
* properties */
bi.minpos = ni.minpos;
bi.maxpos = ni.maxpos;
- if (BATtdense(n)) {
+ if (BATtdensebi(&ni)) {
/* replaced all of b with a dense sequence */
MT_lock_set(&b->theaplock);
BATtseqbase(b, ni.tseq);
@@ -1755,7 +1755,7 @@ BATslice(BAT *b, BUN l, BUN h)
BUN p = l;
BUN q = h;
- bn = COLnew((oid) (b->hseqbase + low), BATtdense(b) ? TYPE_void
: b->ttype, h - l, TRANSIENT);
+ bn = COLnew((oid) (b->hseqbase + low), BATtdensebi(&bi) ?
TYPE_void : b->ttype, h - l, TRANSIENT);
if (bn == NULL)
goto doreturn;
@@ -1804,7 +1804,7 @@ BATslice(BAT *b, BUN l, BUN h)
bn->tnosorted = 0;
bn->tnokey[0] = bn->tnokey[1] = 0;
bni = bat_iterator_nolock(bn);
- if (BATtdense(b)) {
+ if (BATtdensebi(&bi)) {
BATtseqbase(bn, (oid) (bi.tseq + low));
} else if (bn->ttype == TYPE_oid) {
if (BATcount(bn) == 0) {
@@ -2919,7 +2919,7 @@ BATcount_no_nil(BAT *b, BAT *s)
t = ATOMbasetype(bi.type);
switch (t) {
case TYPE_void:
- cnt = ci.ncand * BATtdense(b);
+ cnt = ci.ncand * BATtdensebi(&bi);
break;
case TYPE_msk:
cnt = ci.ncand;
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -1863,14 +1863,14 @@ heap_entry(FILE *fp, BATiter *bi, BUN si
(unsigned short) bi->sorted |
((unsigned short) bi->revsorted << 7) |
((unsigned short) bi->key << 8) |
- ((unsigned short) BATtdense(b) << 9) |
+ ((unsigned short) BATtdensebi(bi) << 9) |
((unsigned short) bi->nonil << 10) |
((unsigned short) bi->nil << 11),
b->tnokey[0] >= size || b->tnokey[1] >= size ? 0 :
b->tnokey[0],
b->tnokey[0] >= size || b->tnokey[1] >= size ? 0 :
b->tnokey[1],
b->tnosorted >= size ? 0 : b->tnosorted,
b->tnorevsorted >= size ? 0 : b->tnorevsorted,
- b->tseqbase,
+ bi->tseq,
free,
bi->minpos < size ? (uint64_t) bi->minpos : (uint64_t)
oid_nil,
bi->maxpos < size ? (uint64_t) bi->maxpos : (uint64_t)
oid_nil);
diff --git a/gdk/gdk_calc_convert.c b/gdk/gdk_calc_convert.c
--- a/gdk/gdk_calc_convert.c
+++ b/gdk/gdk_calc_convert.c
@@ -1534,7 +1534,7 @@ BATconvert(BAT *b, BAT *s, int tp, bool
return NULL;
}
if (ATOMstorage(tp) == TYPE_msk) {
- if (BATtdense(b)) {
+ if (BATtdensebi(&bi)) {
/* dense to msk is easy: all values 1, except
* maybe the first */
bn = BATconstant(ci.hseq, tp, &(msk){1}, ci.ncand,
diff --git a/gdk/gdk_join.c b/gdk/gdk_join.c
--- a/gdk/gdk_join.c
+++ b/gdk/gdk_join.c
@@ -2757,7 +2757,7 @@ hashjoin(BAT **r1p, BAT **r2p, BAT *l, B
"existing hash%s\n",
ALGOBATPAR(r),
swapped ? " (swapped)" : "");
- } else if (BATtdense(r)) {
+ } else if (BATtdensebi(&ri)) {
/* no hash, just dense lookup */
MT_thread_setalgorithm(swapped ? "hashjoin on dense (swapped)"
: "hashjoin on dense");
} else {
@@ -2776,7 +2776,7 @@ hashjoin(BAT **r1p, BAT **r2p, BAT *l, B
GDKerror("Hash disappeared for "ALGOBATFMT"\n", ALGOBATPAR(r));
goto bailout;
}
- assert(hsh != NULL || BATtdense(r));
+ assert(hsh != NULL || BATtdensebi(&ri));
if (hsh) {
TRC_DEBUG(ALGO, "hash for " ALGOBATFMT ": nbucket " BUNFMT ",
nunique " BUNFMT ", nheads " BUNFMT "\n", ALGOBATPAR(r), hsh->nbucket,
hsh->nunique, hsh->nheads);
}
@@ -2800,7 +2800,7 @@ hashjoin(BAT **r1p, BAT **r2p, BAT *l, B
false, false, __func__,
t0);
}
}
- } else if (!BATtdense(r)) {
+ } else if (!BATtdensebi(&ri)) {
for (rb = HASHget(hsh, HASHprobe(hsh, nil));
rb != BUN_NONE;
rb = HASHgetlink(hsh, rb)) {
@@ -2850,7 +2850,7 @@ hashjoin(BAT **r1p, BAT **r2p, BAT *l, B
GDK_CHECK_TIMEOUT(timeoffset, counter,
GOTO_LABEL_TIMEOUT_HANDLER(bailout));
lo = canditer_next(lci);
- if (BATtdense(l))
+ if (BATtdensebi(&li))
lval = lo - l->hseqbase + l->tseqbase;
else if (li.type != TYPE_void)
v = VALUE(l, lo - l->hseqbase);
@@ -2877,7 +2877,7 @@ hashjoin(BAT **r1p, BAT **r2p, BAT *l, B
break;
}
} else if (hsh == NULL) {
- assert(BATtdense(r));
+ assert(BATtdensebi(&ri));
ro = *(const oid *) v;
if (ro >= r->tseqbase &&
ro < r->tseqbase + r->batCount) {
@@ -3432,7 +3432,7 @@ thetajoin(BAT **r1p, BAT **r2p, BAT *l,
}
if (BATtvoid(l)) {
- if (!BATtdense(l)) {
+ if (!BATtdensebi(&li)) {
/* trivial: nils don't match anything */
bat_iterator_end(&li);
bat_iterator_end(&ri);
@@ -3442,7 +3442,7 @@ thetajoin(BAT **r1p, BAT **r2p, BAT *l,
loff = (lng) l->tseqbase - (lng) l->hseqbase;
}
if (BATtvoid(r)) {
- if (!BATtdense(r)) {
+ if (!BATtdensebi(&ri)) {
/* trivial: nils don't match anything */
bat_iterator_end(&li);
bat_iterator_end(&ri);
diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -130,29 +130,32 @@ log_find(BAT *b, BAT *d, int val)
assert(b->ttype == TYPE_int);
assert(d->ttype == TYPE_oid);
+ BATiter bi = bat_iterator(b);
if (BAThash(b) == GDK_SUCCEED) {
- BATiter cni = bat_iterator_nolock(b);
- MT_rwlock_rdlock(&cni.b->thashlock);
- HASHloop_int(cni, cni.b->thash, p, &val) {
+ MT_rwlock_rdlock(&b->thashlock);
+ HASHloop_int(bi, b->thash, p, &val) {
oid pos = p;
if (BUNfnd(d, &pos) == BUN_NONE) {
- MT_rwlock_rdunlock(&cni.b->thashlock);
+ MT_rwlock_rdunlock(&b->thashlock);
+ bat_iterator_end(&bi);
return p;
}
}
- MT_rwlock_rdunlock(&cni.b->thashlock);
+ MT_rwlock_rdunlock(&b->thashlock);
} else { /* unlikely: BAThash failed */
- BUN q;
- int *t = (int *) Tloc(b, 0);
+ int *t = (int *) bi.base;
- for (p = 0, q = BATcount(b); p < q; p++) {
+ for (p = 0; p < bi.count; p++) {
if (t[p] == val) {
oid pos = p;
- if (BUNfnd(d, &pos) == BUN_NONE)
+ if (BUNfnd(d, &pos) == BUN_NONE) {
+ bat_iterator_end(&bi);
return p;
+ }
}
}
}
+ bat_iterator_end(&bi);
return BUN_NONE;
}
@@ -2045,7 +2048,12 @@ logger_load(int debug, const char *fn, c
logbat_destroy(lg->seqs_id);
logbat_destroy(lg->seqs_val);
logbat_destroy(lg->dseqs);
+ ATOMIC_DESTROY(&lg->refcount);
MT_lock_destroy(&lg->lock);
+ MT_lock_destroy(&lg->rotation_lock);
+ MT_sema_destroy(&lg->flush_queue_semaphore);
+ MT_lock_destroy(&lg->flush_lock);
+ MT_lock_destroy(&lg->flush_queue_lock);
GDKfree(lg->fn);
GDKfree(lg->dir);
GDKfree(lg->local_dir);
@@ -2105,11 +2113,21 @@ logger_new(int debug, const char *fn, co
GDKfree(lg);
return NULL;
}
- MT_lock_init(&lg->lock, fn);
if (lg->debug & 1) {
fprintf(stderr, "#logger_new dir set to %s\n", lg->dir);
}
+ ATOMIC_INIT(&lg->refcount, 0);
+ MT_lock_init(&lg->lock, fn);
+ MT_lock_init(&lg->rotation_lock, "rotation_lock");
+ MT_sema_init(&lg->flush_queue_semaphore, FLUSH_QUEUE_SIZE,
"flush_queue_semaphore");
+ MT_lock_init(&lg->flush_lock, "flush_lock");
+ MT_lock_init(&lg->flush_queue_lock, "flush_queue_lock");
+
+ // flush variables
+ lg->flush_queue_begin = 0;
+ lg->flush_queue_length = 0;
+
if (logger_load(debug, fn, logdir, lg, filename) == GDK_SUCCEED) {
return lg;
}
@@ -2153,7 +2171,12 @@ logger_destroy(logger *lg)
logbat_destroy(lg->catalog_lid);
logger_unlock(lg);
}
+ ATOMIC_DESTROY(&lg->refcount);
MT_lock_destroy(&lg->lock);
+ MT_lock_destroy(&lg->rotation_lock);
+ MT_sema_destroy(&lg->flush_queue_semaphore);
+ MT_lock_destroy(&lg->flush_lock);
+ MT_lock_destroy(&lg->flush_queue_lock);
GDKfree(lg->fn);
GDKfree(lg->dir);
GDKfree(lg->buf);
@@ -2212,6 +2235,7 @@ logger_cleanup_range(logger *lg)
gdk_return
logger_activate(logger *lg)
{
+ MT_lock_set(&lg->rotation_lock);
logger_lock(lg);
if (lg->end > 0 && lg->saved_id+1 == lg->id) {
lg->id++;
@@ -2219,10 +2243,12 @@ logger_activate(logger *lg)
/* start new file */
if (logger_open_output(lg) != GDK_SUCCEED) {
logger_unlock(lg);
+ MT_lock_unset(&lg->rotation_lock);
return GDK_FAIL;
}
_______________________________________________
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org
