MonetDB: data-vaults - Merge with default

2017-05-09 Thread Panagiotis Koutsourakis 
Changeset: 297acc6530ba for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=297acc6530ba
Added Files:
monetdb5/modules/mal/wlc.c
monetdb5/modules/mal/wlc.h
monetdb5/modules/mal/wlc.mal
monetdb5/optimizer/opt_wlc.c
monetdb5/optimizer/opt_wlc.h
sql/backends/monet5/wlr.c
sql/backends/monet5/wlr.h
sql/backends/monet5/wlr.mal
sql/scripts/60_wlcr.sql
sql/test/wlcr/Tests/All
sql/test/wlcr/Tests/wlc01.py
sql/test/wlcr/Tests/wlc01.stable.err
sql/test/wlcr/Tests/wlc01.stable.out
sql/test/wlcr/Tests/wlc100.py
sql/test/wlcr/Tests/wlc100.stable.err
sql/test/wlcr/Tests/wlc100.stable.out
sql/test/wlcr/Tests/wlc20.py
sql/test/wlcr/Tests/wlc20.stable.err
sql/test/wlcr/Tests/wlc20.stable.out
sql/test/wlcr/Tests/wlc21.py
sql/test/wlcr/Tests/wlc21.stable.err
sql/test/wlcr/Tests/wlc21.stable.out
sql/test/wlcr/Tests/wlc30.py
sql/test/wlcr/Tests/wlc30.stable.err
sql/test/wlcr/Tests/wlc30.stable.out
sql/test/wlcr/Tests/wlc40.py
sql/test/wlcr/Tests/wlc40.stable.err
sql/test/wlcr/Tests/wlc40.stable.out
sql/test/wlcr/Tests/wlc50.py
sql/test/wlcr/Tests/wlc50.stable.err
sql/test/wlcr/Tests/wlc50.stable.out
sql/test/wlcr/Tests/wlc70.py
sql/test/wlcr/Tests/wlc70.stable.err
sql/test/wlcr/Tests/wlc70.stable.out
sql/test/wlcr/Tests/wlcr00.py
sql/test/wlcr/Tests/wlcr00.stable.err
sql/test/wlcr/Tests/wlcr00.stable.out
sql/test/wlcr/Tests/wlr01.py
sql/test/wlcr/Tests/wlr01.reqtests
sql/test/wlcr/Tests/wlr01.stable.err
sql/test/wlcr/Tests/wlr01.stable.out
sql/test/wlcr/Tests/wlr100.py
sql/test/wlcr/Tests/wlr100.stable.err
sql/test/wlcr/Tests/wlr100.stable.out
sql/test/wlcr/Tests/wlr20.py
sql/test/wlcr/Tests/wlr20.reqtests
sql/test/wlcr/Tests/wlr20.stable.err
sql/test/wlcr/Tests/wlr20.stable.out
sql/test/wlcr/Tests/wlr30.py
sql/test/wlcr/Tests/wlr30.reqtests
sql/test/wlcr/Tests/wlr30.stable.err
sql/test/wlcr/Tests/wlr30.stable.out
sql/test/wlcr/Tests/wlr40.py
sql/test/wlcr/Tests/wlr40.reqtests
sql/test/wlcr/Tests/wlr40.stable.err
sql/test/wlcr/Tests/wlr40.stable.out
sql/test/wlcr/Tests/wlr50.py
sql/test/wlcr/Tests/wlr50.stable.err
sql/test/wlcr/Tests/wlr50.stable.out
sql/test/wlcr/Tests/wlr70.py
sql/test/wlcr/Tests/wlr70.stable.err
sql/test/wlcr/Tests/wlr70.stable.out
Modified Files:
.hgtags
MonetDB.spec
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
clients/Tests/exports.stable.out
clients/mapiclient/dotmonetdb.c
clients/mapiclient/mclient.c
common/stream/stream.c
debian/changelog
gdk/gdk_bat.c
gdk/gdk_heap.c
geom/monetdb5/geom.c
geom/monetdb5/geom_upgrade.c
libversions
monetdb5/ChangeLog-Archive
monetdb5/ChangeLog.Dec2016
monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out
monetdb5/extras/rapi/rapi.c
monetdb5/mal/Makefile.ag
monetdb5/mal/mal.c
monetdb5/mal/mal_client.c
monetdb5/mal/mal_client.h
monetdb5/mal/mal_dataflow.c
monetdb5/mal/mal_exception.c
monetdb5/mal/mal_import.c
monetdb5/mal/mal_import.h
monetdb5/mal/mal_instruction.c
monetdb5/mal/mal_interpreter.c
monetdb5/mal/mal_linker.c
monetdb5/mal/mal_listing.c
monetdb5/mal/mal_parser.c
monetdb5/mal/mal_parser.h
monetdb5/mal/mal_private.h
monetdb5/mal/mal_resolve.c
monetdb5/mal/mal_runtime.c
monetdb5/mal/mal_session.c
monetdb5/mal/mal_type.c
monetdb5/modules/atoms/batxml.c
monetdb5/modules/atoms/blob.c
monetdb5/modules/atoms/blob.h
monetdb5/modules/atoms/color.c
monetdb5/modules/atoms/identifier.c
monetdb5/modules/atoms/inet.c
monetdb5/modules/atoms/json.c
monetdb5/modules/atoms/mtime.c
monetdb5/modules/atoms/streams.c
monetdb5/modules/atoms/url.c
monetdb5/modules/atoms/uuid.c
monetdb5/modules/atoms/xml.c
monetdb5/modules/kernel/algebra.c
monetdb5/modules/kernel/bat5.c
monetdb5/modules/mal/Makefile.ag
monetdb5/modules/mal/clients.c
monetdb5/modules/mal/mal_init.mal
monetdb5/modules/mal/mal_io.c
monetdb5/modules/mal/mal_mapi.c
monetdb5/modules/mal/pcre.c
monetdb5/modules/mal/sysmon.c
monetdb5/modules/mal/tablet.c
monetdb5/modules/mal/tablet.h
monetdb5/modules/mal/txtsim.c
monetdb5/optimizer/Makefile.ag
monetdb5/optimizer/opt_emptybind

MonetDB: stratified_sampling - remove CDF-based weighted sampling

2017-05-09 Thread Abe Wits 
Changeset: 9c93f4253ff6 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9c93f4253ff6
Modified Files:
gdk/gdk_sample.c
Branch: stratified_sampling
Log Message:

remove CDF-based weighted sampling


diffs (157 lines):

diff --git a/gdk/gdk_sample.c b/gdk/gdk_sample.c
--- a/gdk/gdk_sample.c
+++ b/gdk/gdk_sample.c
@@ -100,13 +100,10 @@ OIDTreeToBATAntiset(struct oidtreenode *
//TODO create this function
 }*/
 
-/* 
- * _BATsample is the internal (weighted) sampling function without replacement
- * If cdf=NULL, an uniform sample is taken
- * Otherwise it is assumed the cdf increases monotonically
- */
-static BAT *
-_BATsample(BAT *b, BUN n, BAT *cdf)
+
+/* BATsample takes uniform samples of void headed BATs */
+BAT *
+BATsample(BAT *b, BUN n)
 {
BAT *bn;
BUN cnt, slen;
@@ -115,8 +112,6 @@ static BAT *
mtwist *mt_rng;
unsigned int range;
dbl random;
-   dbl cdf_max;
-   dbl* cdf_ptr;
 
BATcheck(b, "BATsample", NULL);
ERRORcheck(n > BUN_MAX, "BATsample: sample size larger than BUN_MAX\n", 
NULL);
@@ -167,48 +162,18 @@ static BAT *

range = maxoid - minoid;

-   /* sample OIDs (method depends on w) */
-   if(cdf == NULL) {
-   /* no weights, hence do uniform sampling */
-
-   /* while we do not have enough sample OIDs yet */
-   for (rescnt = 0; rescnt < n; rescnt++) {
-   oid candoid;
-   do {
-   /* generate a new random OID in 
[minoid, maxoid[
-* that is including minoid, excluding 
maxoid*/
-   candoid = (oid) ( minoid + 
(mtwist_u32rand(mt_rng)%range) );
-   /* if that candidate OID was already
-* generated, try again */
-   } while (!OIDTreeMaybeInsert(tree, candoid, 
rescnt));
-   }
-
-   } else {
-   /* do weighted sampling */
-   
-   cdf_ptr = (dbl*) Tloc(cdf, 0);
-   if (!antiset)
-   cdf_max = cdf_ptr[cnt-1];
-   else
-   cdf_max = cdf_ptr[0];
-  //TODO how to type/cast cdf_max?
-
-   /* generate candoids, using CDF */
-   for (rescnt = 0; rescnt < n; rescnt++) {
-   oid candoid;
-
-   do {
-   random = mtwist_drand(mt_rng)*cdf_max;
-   /* generate a new random OID with 
minoid <= OID < maxoid */
-   /* note that cdf has already been 
adjusted for antiset case */
-   candoid = (oid) ( minoid + (oid) 
SORTfndfirst(cdf, &random) );
-   /* if that candidate OID was already
-* generated, try again */
-   } while (!OIDTreeMaybeInsert(tree, candoid, 
rescnt));
-   }
+   /* while we do not have enough sample OIDs yet */
+   for (rescnt = 0; rescnt < n; rescnt++) {
+   oid candoid;
+   do {
+   /* generate a new random OID in [minoid, maxoid[
+* that is including minoid, excluding maxoid*/
+   candoid = (oid) ( minoid + 
(mtwist_u32rand(mt_rng)%range) );
+   /* if that candidate OID was already
+* generated, try again */
+   } while (!OIDTreeMaybeInsert(tree, candoid, rescnt));
}
 
-
if (!antiset) {
OIDTreeToBAT(tree, bn);
} else {
@@ -227,20 +192,11 @@ static BAT *
return bn;
 }
 
-
-/* BATsample takes uniform samples of void headed BATs */
-BAT *
-BATsample(BAT *b, BUN n)
-{
-   return _BATsample(b, n, NULL);
-}
-
 /* BATweightedsample takes weighted samples of void headed BATs */
 /* Note that the type of w should be castable to doubles */
 BAT *
 BATweightedsample(BAT *b, BUN n, BAT *w)
 {
-   BAT* cdf;
BAT* sample;
dbl* w_ptr;//TODO types of w
dbl* cdf_ptr;
@@ -259,44 +215,14 @@ BATweightedsample(BAT *b, BUN n, BAT *w)
 
antiset = n > cnt / 2;
 
-   cdf = COLnew(0, TYPE_dbl, cnt, TRANSIENT);
-   BATsetcount(cdf, cnt);
-   
-   /* calculate cumilative distribution function */
-   w_ptr = (dbl*) Tloc(w, 0);//TODO support different types w
-   cdf_ptr = (dbl*) Tloc(cdf, 0);
-

MonetDB: stratified_sampling - simplify OIDTreeMaybeInsert

2017-05-09 Thread Abe Wits 
Changeset: 69cd1aa8a456 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=69cd1aa8a456
Modified Files:
gdk/gdk_sample.c
Branch: stratified_sampling
Log Message:

simplify OIDTreeMaybeInsert


diffs (35 lines):

diff --git a/gdk/gdk_sample.c b/gdk/gdk_sample.c
--- a/gdk/gdk_sample.c
+++ b/gdk/gdk_sample.c
@@ -42,23 +42,23 @@ struct oidtreenode {
 static int
 OIDTreeMaybeInsert(struct oidtreenode *tree, oid o, BUN allocated)
 {
-   struct oidtreenode **nodep;
+   struct oidtreenode *nodep;
 
if (allocated == 0) {
tree->left = tree->right = NULL;
tree->o = o;
return 1;
}
-   nodep = &tree;
-   while (*nodep) {
-   if (o == (*nodep)->o)
+   nodep = tree;
+   while (nodep) {
+   if (o == nodep->o)
return 0;
-   if (o < (*nodep)->o)
-   nodep = &(*nodep)->left;
+   if (o < nodep->o)
+   nodep = nodep->left;
else
-   nodep = &(*nodep)->right;
+   nodep = nodep->right;
}
-   *nodep = &tree[allocated];
+   nodep = &tree[allocated];
tree[allocated].left = tree[allocated].right = NULL;
tree[allocated].o = o;
return 1;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: stratified_sampling - write reservoir sampling

2017-05-09 Thread Abe Wits 
Changeset: bb51b8eece7c for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=bb51b8eece7c
Modified Files:
gdk/gdk_sample.c
Branch: stratified_sampling
Log Message:

write reservoir sampling


diffs (140 lines):

diff --git a/gdk/gdk_sample.c b/gdk/gdk_sample.c
--- a/gdk/gdk_sample.c
+++ b/gdk/gdk_sample.c
@@ -32,6 +32,49 @@
 #undef BATsample
 
 
+//TODO share these with gkd_firstn.c
+#define siftup(OPER, START, SWAP)  \
+   do {\
+   pos = (START);  \
+   childpos = (pos << 1) + 1;  \
+   while (childpos < n) {  \
+   /* find most extreme child */   \
+   if (childpos + 1 < n && \
+   !(OPER(childpos + 1, childpos)))\
+   childpos++; \
+   /* compare parent with most extreme child */\
+   if (!OPER(pos, childpos)) { \
+   /* already correctly ordered */ \
+   break;  \
+   }   \
+   /* exchange parent with child and sift child */ \
+   /* further */   \
+   SWAP(pos, childpos);\
+   pos = childpos; \
+   childpos = (pos << 1) + 1;  \
+   }   \
+   } while (0)
+
+#define heapify(OPER, SWAP)\
+   do {\
+   for (i = n / 2; i > 0; i--) \
+   siftup(OPER, i - 1, SWAP);  \
+   } while (0)
+
+#define SWAP3(p1, p2)  \
+   do {\
+   item = oids[p1];\
+   oids[p1] = oids[p2];\
+   oids[p2] = item;\
+   key = keys[p1]; \
+   keys[p1] = keys[p2];\
+   keys[p2] = key; \
+   } while (0)
+
+#define compKeysGT(p1, p2) 
\
+   (keys[p1] > keys[p2] || 
\
+   keys[p1] == keys[p2] && oids[p1] > oids[p2])
+
 /* this is a straightforward implementation of a binary tree */
 struct oidtreenode {
oid o;
@@ -194,14 +237,23 @@ BATsample(BAT *b, BUN n)
 
 /* BATweightedsample takes weighted samples of void headed BATs */
 /* Note that the type of w should be castable to doubles */
+/* based on Alg-A-exp from 'Weighted random sampling with a reservoir' by 
Efraimidis and Spirakis (2006) */
 BAT *
 BATweightedsample(BAT *b, BUN n, BAT *w)
 {
BAT* sample;
+   oid* oids;//points to the oids in sample
dbl* w_ptr;//TODO types of w
-   dbl* cdf_ptr;
-   BUN cnt, i;
+   dbl* keys;//keys as defined in Alg-A-exp
+   BUN cnt, i, j;
bit antiset;
+   mtwist *mt_rng;
+   BUN pos, childpos;
+   oid item;
+   dbl r, xw, r2, key, tw;
+
+   oid minoid = b->hseqbase;
+   oid maxoid = b->hseqbase + cnt;
 
BATcheck(b, "BATsample", NULL);
BATcheck(w, "BATsample", NULL);
@@ -213,10 +265,57 @@ BATweightedsample(BAT *b, BUN n, BAT *w)
 
cnt = BATcount(b);
 
-   antiset = n > cnt / 2;
 
-   /* obtain sample */
-   sample = NULL;//TODO: reservoir sampling with exponential jumps
+   keys = (double*) malloc(sizeof(double)*n);
+   if(keys == NULL)
+   return NULL;
+
+   sample = COLnew(0, TYPE_oid, n, TRANSIENT);
+   if(sample == NULL) {
+   free(keys);
+   return NULL;
+   }
+
+   oids = (oid *)sample->theap.base;
+
+   mt_rng = mtwist_new();
+   mtwist_seed(mt_rng, rand());
+
+   BATsetcount(sample, n);
+   /* obtain sample */
+   //TODO: reservoir sampling with exponential jumps
+   for(j=0; j 0 && j < cnt) {
+   xw -= w[j];
+   j++;
+   }
+   if(j >= cnt) break;
+   tw = pow(keys[oids[0]-minoid], w[j]);
+   r2 = mtwist_drand(mt_rng)*(1-tw)+tw;
+   key = pow(r2, 1/w[j]);
+
+   oids[0] = j+minoid;
+   keys[0] = key;
+   siftup(compKeysGT, 0, SWAP3);//NOTE: writes to 'key'
+   }
+
+   free(keys);
+
+   sample->tr

MonetDB: stratified_sampling - fix compilation errors

2017-05-09 Thread Abe Wits 
Changeset: 23bcb0cddf05 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=23bcb0cddf05
Modified Files:
gdk/gdk_sample.c
Branch: stratified_sampling
Log Message:

fix compilation errors


diffs (144 lines):

diff --git a/gdk/gdk_sample.c b/gdk/gdk_sample.c
--- a/gdk/gdk_sample.c
+++ b/gdk/gdk_sample.c
@@ -23,6 +23,8 @@
  * values that should be omitted from the sample.
  */
 
+#include 
+
 #include "monetdb_config.h"
 #include "gdk.h"
 #include "gdk_private.h"
@@ -61,6 +63,8 @@
siftup(OPER, i - 1, SWAP);  \
} while (0)
 
+
+//CUSTOM SWAP AND COMPARE FUNCTION
 #define SWAP3(p1, p2)  \
do {\
item = oids[p1];\
@@ -72,8 +76,8 @@
} while (0)
 
 #define compKeysGT(p1, p2) 
\
-   (keys[p1] > keys[p2] || 
\
-   keys[p1] == keys[p2] && oids[p1] > oids[p2])
+   ((keys[p1] > keys[p2]) ||   
\
+   (keys[p1] == keys[p2] && oids[p1] > oids[p2]))
 
 /* this is a straightforward implementation of a binary tree */
 struct oidtreenode {
@@ -85,23 +89,23 @@ struct oidtreenode {
 static int
 OIDTreeMaybeInsert(struct oidtreenode *tree, oid o, BUN allocated)
 {
-   struct oidtreenode *nodep;
+   struct oidtreenode **nodep;
 
if (allocated == 0) {
tree->left = tree->right = NULL;
tree->o = o;
return 1;
}
-   nodep = tree;
-   while (nodep) {
-   if (o == nodep->o)
+   nodep = &tree;
+   while (*nodep) {
+   if (o == (*nodep)->o)
return 0;
-   if (o < nodep->o)
-   nodep = nodep->left;
+   if (o < (*nodep)->o)
+   nodep = &(*nodep)->left;
else
-   nodep = nodep->right;
+   nodep = &(*nodep)->right;
}
-   nodep = &tree[allocated];
+   *nodep = &tree[allocated];
tree[allocated].left = tree[allocated].right = NULL;
tree[allocated].o = o;
return 1;
@@ -154,7 +158,6 @@ BATsample(BAT *b, BUN n)
struct oidtreenode *tree = NULL;
mtwist *mt_rng;
unsigned int range;
-   dbl random;
 
BATcheck(b, "BATsample", NULL);
ERRORcheck(n > BUN_MAX, "BATsample: sample size larger than BUN_MAX\n", 
NULL);
@@ -245,15 +248,13 @@ BATweightedsample(BAT *b, BUN n, BAT *w)
oid* oids;//points to the oids in sample
dbl* w_ptr;//TODO types of w
dbl* keys;//keys as defined in Alg-A-exp
-   BUN cnt, i, j;
-   bit antiset;
+   BUN cnt, i;
mtwist *mt_rng;
BUN pos, childpos;
oid item;
dbl r, xw, r2, key, tw;
 
oid minoid = b->hseqbase;
-   oid maxoid = b->hseqbase + cnt;
 
BATcheck(b, "BATsample", NULL);
BATcheck(w, "BATsample", NULL);
@@ -265,7 +266,6 @@ BATweightedsample(BAT *b, BUN n, BAT *w)
 
cnt = BATcount(b);
 
-
keys = (double*) malloc(sizeof(double)*n);
if(keys == NULL)
return NULL;
@@ -277,6 +277,7 @@ BATweightedsample(BAT *b, BUN n, BAT *w)
}
 
oids = (oid *)sample->theap.base;
+   w_ptr = (dbl*) w->theap.base;
 
mt_rng = mtwist_new();
mtwist_seed(mt_rng, rand());
@@ -284,26 +285,27 @@ BATweightedsample(BAT *b, BUN n, BAT *w)
BATsetcount(sample, n);
/* obtain sample */
//TODO: reservoir sampling with exponential jumps
-   for(j=0; j 0 && j < cnt) {
-   xw -= w[j];
-   j++;
+   while(xw > 0 && i < cnt) {
+   xw -= w_ptr[i];
+   i++;
}
-   if(j >= cnt) break;
-   tw = pow(keys[oids[0]-minoid], w[j]);
+   if(i >= cnt) break;
+   //At this point, xw - (w_ptr[c]+w_ptr[c+1]+...+w_ptr[i]) <= 0
+   tw = pow(keys[oids[0]-minoid], w_ptr[i]);
r2 = mtwist_drand(mt_rng)*(1-tw)+tw;
-   key = pow(r2, 1/w[j]);
+   key = pow(r2, 1/w_ptr[i]);
 
-   oids[0] = j+minoid;
+   oids[0] = i+minoid;
keys[0] = key;
siftup(compKeysGT, 0, SWAP3);//NOTE: writes to 'key'
}
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: stratified_sampling - fix reservoir sampling semantic bug

2017-05-09 Thread Abe Wits 
Changeset: f9d71464a676 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f9d71464a676
Modified Files:
gdk/gdk_sample.c
Branch: stratified_sampling
Log Message:

fix reservoir sampling semantic bug


diffs (122 lines):

diff --git a/gdk/gdk_sample.c b/gdk/gdk_sample.c
--- a/gdk/gdk_sample.c
+++ b/gdk/gdk_sample.c
@@ -75,9 +75,9 @@
keys[p2] = key; \
} while (0)
 
-#define compKeysGT(p1, p2) 
\
+#define compKeysGT(p1, p2) 
\
((keys[p1] > keys[p2]) ||   
\
-   (keys[p1] == keys[p2] && oids[p1] > oids[p2]))
+   ((keys[p1] == keys[p2]) && (oids[p1] > oids[p2])))
 
 /* this is a straightforward implementation of a binary tree */
 struct oidtreenode {
@@ -243,12 +243,12 @@ BATsample(BAT *b, BUN n)
 /* based on Alg-A-exp from 'Weighted random sampling with a reservoir' by 
Efraimidis and Spirakis (2006) */
 BAT *
 BATweightedsample(BAT *b, BUN n, BAT *w)
-{
+{//TODO: does not create correct samples yet (so it seems! or is the output 
broken?)
BAT* sample;
oid* oids;//points to the oids in sample
dbl* w_ptr;//TODO types of w
dbl* keys;//keys as defined in Alg-A-exp
-   BUN cnt, i;
+   BUN cnt, i, j;
mtwist *mt_rng;
BUN pos, childpos;
oid item;
@@ -266,7 +266,7 @@ BATweightedsample(BAT *b, BUN n, BAT *w)
 
cnt = BATcount(b);
 
-   keys = (double*) malloc(sizeof(double)*n);
+   keys = (dbl*) malloc(sizeof(dbl)*n);
if(keys == NULL)
return NULL;
 
@@ -276,8 +276,10 @@ BATweightedsample(BAT *b, BUN n, BAT *w)
return NULL;
}
 
-   oids = (oid *)sample->theap.base;
-   w_ptr = (dbl*) w->theap.base;
+
+   //oids = (oid *)sample->theap.base;
+   oids = (oid *) Tloc(sample, 0);
+   w_ptr = (dbl*) Tloc(w, 0);//TODO is this the right way to get w_ptr?
 
mt_rng = mtwist_new();
mtwist_seed(mt_rng, rand());
@@ -285,30 +287,56 @@ BATweightedsample(BAT *b, BUN n, BAT *w)
BATsetcount(sample, n);
/* obtain sample */
//TODO: reservoir sampling with exponential jumps
-   for(i=0; i 0 && i < cnt) {
-   xw -= w_ptr[i];
-   i++;
+   xw = log(r)/log(keys[0]);
+   fprintf(stderr,"THIS SHOULD BE POSITIVE: xw = log(r = 
%f)/log(keys[0] = %f) = %f\n",r,keys[0],xw);
+   for(;j w_ptr[j]; j++) {
+   xw -= w_ptr[j];
+   fprintf(stderr,"j = %d, xw = %f (just subtracted 
%f)\n", (int)j, xw, w_ptr[j]);
}
-   if(i >= cnt) break;
-   //At this point, xw - (w_ptr[c]+w_ptr[c+1]+...+w_ptr[i]) <= 0
-   tw = pow(keys[oids[0]-minoid], w_ptr[i]);
+   if(j >= cnt) break;
+   //At this point, w_ptr[c]+w_ptr[c+1]+...+w_ptr[i-1] < xw < 
w_ptr[c]+w_ptr[c+1]+...+w_ptr[i]
+   tw = pow(keys[0], w_ptr[j]);
r2 = mtwist_drand(mt_rng)*(1-tw)+tw;
-   key = pow(r2, 1/w_ptr[i]);
+   key = pow(r2, 1/w_ptr[j]);
 
-   oids[0] = i+minoid;
+   //Replace element with lowest key in prioqueue
+   oids[0] = (oid)(j+minoid);
keys[0] = key;
siftup(compKeysGT, 0, SWAP3);//NOTE: writes to 'key'
+
+   fprintf(stderr,"\n\n ID | KEY   (<%d,%.2f> INSERTED)\n", 
(int)j, key);
+   fprintf(stderr,"+\n");
+   for(i=0; ihttps://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: stratified_sampling - remove debug output, improve comm...

2017-05-09 Thread Abe Wits 
Changeset: aaaebd745aa0 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=aaaebd745aa0
Modified Files:
gdk/gdk_sample.c
Branch: stratified_sampling
Log Message:

remove debug output, improve comments


diffs (118 lines):

diff --git a/gdk/gdk_sample.c b/gdk/gdk_sample.c
--- a/gdk/gdk_sample.c
+++ b/gdk/gdk_sample.c
@@ -64,7 +64,7 @@
} while (0)
 
 
-//CUSTOM SWAP AND COMPARE FUNCTION
+/* CUSTOM SWAP AND COMPARE FUNCTION */
 #define SWAP3(p1, p2)  \
do {\
item = oids[p1];\
@@ -243,11 +243,11 @@ BATsample(BAT *b, BUN n)
 /* based on Alg-A-exp from 'Weighted random sampling with a reservoir' by 
Efraimidis and Spirakis (2006) */
 BAT *
 BATweightedsample(BAT *b, BUN n, BAT *w)
-{//TODO: does not create correct samples yet (so it seems! or is the output 
broken?)
+{//TODO test correctness extensively
BAT* sample;
-   oid* oids;//points to the oids in sample
+   oid* oids;/* points to the oids in sample */
dbl* w_ptr;//TODO types of w
-   dbl* keys;//keys as defined in Alg-A-exp
+   dbl* keys;/* keys as defined in Alg-A-exp */
BUN cnt, i, j;
mtwist *mt_rng;
BUN pos, childpos;
@@ -263,7 +263,7 @@ BATweightedsample(BAT *b, BUN n, BAT *w)
"BATsample: type of weights not 
castable to doubles\n", NULL);
ERRORcheck(w->ttype != TYPE_dbl,
"BATsample: type of weights must be 
doubles\n", NULL);//TODO types of w (want to remove this)
-
+   //TODO: handle NULL values in w_ptr
cnt = BATcount(b);
 
keys = (dbl*) malloc(sizeof(dbl)*n);
@@ -276,67 +276,49 @@ BATweightedsample(BAT *b, BUN n, BAT *w)
return NULL;
}
 
-
-   //oids = (oid *)sample->theap.base;
oids = (oid *) Tloc(sample, 0);
-   w_ptr = (dbl*) Tloc(w, 0);//TODO is this the right way to get w_ptr?
+   w_ptr = (dbl*) Tloc(w, 0);
 
mt_rng = mtwist_new();
mtwist_seed(mt_rng, rand());
 
BATsetcount(sample, n);
/* obtain sample */
-   //TODO: reservoir sampling with exponential jumps
-   //Initialize prioqueue
-   i=0;//i indices the initial sample (filled with elements with non-zero 
weight)
-   //j indices the oids and weights
+
+   /* Initialize prioqueue */
+   i=0;/* i indices the initial sample (filled with elements with non-zero 
weight) */
+   /* j indices the oids and weights */
for(j=0; i < n && j < cnt; j++) {
if(w_ptr[j] == 0.0)
continue;
oids[i] = (oid)(j+minoid);
-   keys[i] = pow(mtwist_drand(mt_rng),1.0/w_ptr[j]);//TODO cast 
1.0 to dbl? NULL values in w_ptr
+   keys[i] = pow(mtwist_drand(mt_rng),1.0/w_ptr[j]);//TODO cast 
1.0 to dbl?
i++;
}
-   if(i < n)//not enough non-zero weights
+   if(i < n)/* not enough non-zero weights: cannot take sample */
return NULL;
-   heapify(compKeysGT, SWAP3);//NOTE: writes to 'i'
 
-   fprintf(stderr,"\n\n ID | KEY   (INITIAL HEAP)\n");
-   fprintf(stderr,"+\n");
-   for(i=0; i w_ptr[j]; j++) {
+   for(;j w_ptr[j]; j++)
xw -= w_ptr[j];
-   fprintf(stderr,"j = %d, xw = %f (just subtracted 
%f)\n", (int)j, xw, w_ptr[j]);
-   }
if(j >= cnt) break;
-   //At this point, w_ptr[c]+w_ptr[c+1]+...+w_ptr[i-1] < xw < 
w_ptr[c]+w_ptr[c+1]+...+w_ptr[i]
+
+   /* At this point, w_ptr[c]+w_ptr[c+1]+...+w_ptr[i-1] < xw < 
w_ptr[c]+w_ptr[c+1]+...+w_ptr[i] */
tw = pow(keys[0], w_ptr[j]);
r2 = mtwist_drand(mt_rng)*(1-tw)+tw;
key = pow(r2, 1/w_ptr[j]);
 
-   //Replace element with lowest key in prioqueue
+   /* Replace element with lowest key in prioqueue */
oids[0] = (oid)(j+minoid);
keys[0] = key;
-   siftup(compKeysGT, 0, SWAP3);//NOTE: writes to 'key'
-
-   fprintf(stderr,"\n\n ID | KEY   (<%d,%.2f> INSERTED)\n", 
(int)j, key);
-   fprintf(stderr,"+\n");
-   for(i=0; ihttps://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: stratified_sampling - fix memory leak, make mtwist global

2017-05-09 Thread Abe Wits 
Changeset: 10700761ef2a for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=10700761ef2a
Modified Files:
common/utils/mtwist.c
gdk/gdk_sample.c
monetdb5/modules/kernel/microbenchmark.c
monetdb5/modules/mal/sample.c
Branch: stratified_sampling
Log Message:

fix memory leak, make mtwist global


diffs (233 lines):

diff --git a/common/utils/mtwist.c b/common/utils/mtwist.c
--- a/common/utils/mtwist.c
+++ b/common/utils/mtwist.c
@@ -35,8 +35,10 @@
  * @a Dave Beckett, Abe Wits
  * @* Mersenne Twister (MT19937) algorithm
  * 
- * This random number generator has very good statistical properties,
- * and outperforms most stl implementations of rand() in terms of speed
+ * This random number generator outperforms most stl implementations
+ * of rand() in terms of speed. Dieharder tests confirm that the
+ * numbers generated are statistically much closer to true random
+ * numbers than those generated by a typical LCG (including the gcc rand()).
  *
  * http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/emt.html
  * http://en.wikipedia.org/wiki/Mersenne_twister
diff --git a/gdk/gdk_sample.c b/gdk/gdk_sample.c
--- a/gdk/gdk_sample.c
+++ b/gdk/gdk_sample.c
@@ -86,6 +86,8 @@ struct oidtreenode {
struct oidtreenode *right;
 };
 
+mtwist *mt_rng = NULL;
+
 static int
 OIDTreeMaybeInsert(struct oidtreenode *tree, oid o, BUN allocated)
 {
@@ -156,7 +158,7 @@ BATsample(BAT *b, BUN n)
BUN cnt, slen;
BUN rescnt;
struct oidtreenode *tree = NULL;
-   mtwist *mt_rng;
+
unsigned int range;
 
BATcheck(b, "BATsample", NULL);
@@ -201,10 +203,12 @@ BATsample(BAT *b, BUN n)
return NULL;
}

-   /* create and seed Mersenne Twister */
-   mt_rng = mtwist_new();
+   if(!mt_rng) {
+   /* create and seed Mersenne Twister */
+   mt_rng = mtwist_new();
 
-   mtwist_seed(mt_rng, rand());
+   mtwist_seed(mt_rng, rand());
+   }

range = maxoid - minoid;

@@ -249,7 +253,6 @@ BATweightedsample(BAT *b, BUN n, BAT *w)
dbl* w_ptr;//TODO types of w
dbl* keys;/* keys as defined in Alg-A-exp */
BUN cnt, i, j;
-   mtwist *mt_rng;
BUN pos, childpos;
oid item;
dbl r, xw, r2, key, tw;
@@ -266,7 +269,7 @@ BATweightedsample(BAT *b, BUN n, BAT *w)
//TODO: handle NULL values in w_ptr
cnt = BATcount(b);
 
-   keys = (dbl*) malloc(sizeof(dbl)*n);
+   keys = (dbl*) GDKmalloc(sizeof(dbl)*n);
if(keys == NULL)
return NULL;
 
@@ -279,8 +282,10 @@ BATweightedsample(BAT *b, BUN n, BAT *w)
oids = (oid *) Tloc(sample, 0);
w_ptr = (dbl*) Tloc(w, 0);
 
-   mt_rng = mtwist_new();
-   mtwist_seed(mt_rng, rand());
+   if(!mt_rng) {
+   mt_rng = mtwist_new();
+   mtwist_seed(mt_rng, rand());
+   }
 
BATsetcount(sample, n);
/* obtain sample */
@@ -295,19 +300,25 @@ BATweightedsample(BAT *b, BUN n, BAT *w)
keys[i] = pow(mtwist_drand(mt_rng),1.0/w_ptr[j]);//TODO cast 
1.0 to dbl?
i++;
}
-   if(i < n)/* not enough non-zero weights: cannot take sample */
+   if(i < n) {/* not enough non-zero weights: cannot take sample */
+   BBPunfix(sample->batCacheid);//TODO why not unfix?
+   GDKfree(keys);
return NULL;
+   }
 
heapify(compKeysGT, SWAP3);
 
while(true) {
r = mtwist_drand(mt_rng);
xw = log(r)/log(keys[0]);
-   for(;j w_ptr[j]; j++)
+   for(;j= w_ptr[j]; j++)
xw -= w_ptr[j];
if(j >= cnt) break;
 
-   /* At this point, w_ptr[c]+w_ptr[c+1]+...+w_ptr[i-1] < xw < 
w_ptr[c]+w_ptr[c+1]+...+w_ptr[i] */
+   /* At this point:
+*  w_ptr[c]+w_ptr[c+1]+...+w_ptr[i-1]
+*   <  xw (the initial value, log(r)/log(keys[0]))
+*   <= w_ptr[c]+w_ptr[c+1]+...+w_ptr[i] */
tw = pow(keys[0], w_ptr[j]);
r2 = mtwist_drand(mt_rng)*(1-tw)+tw;
key = pow(r2, 1/w_ptr[j]);
@@ -320,7 +331,7 @@ BATweightedsample(BAT *b, BUN n, BAT *w)
j++;/* Increment j so j=c (c is defined in Alg-A-exp) */
}
 
-   free(keys);
+   GDKfree(keys);
 
sample->trevsorted = sample->batCount <= 1;
sample->tsorted = sample->batCount <= 1;
diff --git a/monetdb5/modules/kernel/microbenchmark.c 
b/monetdb5/modules/kernel/microbenchmark.c
--- a/monetdb5/modules/kernel/microbenchmark.c
+++ b/monetdb5/modules/kernel/microbenchmark.c
@@ -22,6 +22,8 @@
 #include "microbenchmark.h"
 #include "mtwist.h"
 
+mtwist *mt_rng = NULL;
+
 static gdk_return
 BATrandom(BAT **bn, oi

MonetDB: Jul2017 - Approved 64 bit output.

2017-05-09 Thread Sjoerd Mullender 
Changeset: a502779e71f8 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a502779e71f8
Modified Files:
sql/benchmarks/tpch/LOCKED/Tests/01-22.stable.out
Branch: Jul2017
Log Message:

Approved 64 bit output.


diffs (73 lines):

diff --git a/sql/benchmarks/tpch/LOCKED/Tests/01-22.stable.out 
b/sql/benchmarks/tpch/LOCKED/Tests/01-22.stable.out
--- a/sql/benchmarks/tpch/LOCKED/Tests/01-22.stable.out
+++ b/sql/benchmarks/tpch/LOCKED/Tests/01-22.stable.out
@@ -701,31 +701,17 @@ Ready.
 % decimal # type
 % 20 # length
 [ 15.48654 ]
-#select * from optimizer_stats() stats;
-% .stats,  .stats # table_name
-% rewrite, count # name
-% clob,int # type
-% 7,   1 # length
-[ "joinidx",   0   ]
-
-# 11:48:06 >  
-# 11:48:06 >  Mtimeout -timeout 60 MapiClient -lsql -umonetdb -Pmonetdb 
--host=localhost --port=35781
-# 11:48:06 >  
 
 % sys.supplier,sys.supplier,   sys.supplier,   sys.supplier,   
sys.revenue0 # table_name
 % s_suppkey,   s_name, s_address,  s_phone,total_revenue # name
 % int, char,   varchar,char,   decimal # type
 % 2,   25, 36, 15, 20 # length
 [ 21,  "Supplier#00021",   "81CavellcrJ0PQ3CPBID0Z0JwyJm0ka5igEs", 
"12-253-590-5816",  1161099.4636]
-% .stats,  .stats # table_name
-% rewrite, count # name
-% clob,int # type
-% 7,   1 # length
-[ "joinidx",   0   ]
+#drop view revenue0;
 
-# 00:16:03 >  
-# 00:16:03 >  Mtimeout -timeout 60 mclient -lsql -ftest -i -e --host=alf 
--port=39190  < 
/net/alf.ins.cwi.nl/export/scratch1/niels/rc/MonetDB/sql/src/benchmarks/tpch/LOCKED/16.sql
-# 00:16:03 >  
+# 11:44:39 >  
+# 11:44:39 >  Mtimeout -timeout 60 mclient -lsql -ftest -Eutf-8 -i -e 
--host=/var/tmp/mtest-29401 --port=32058 < 
/net/madrid/export/scratch1/sjoerd/src/MonetDB/candidate/sql/benchmarks/tpch/LOCKED/../16.sql
+# 11:44:39 >  
 
 #select
 #  p_brand,
@@ -1159,30 +1145,16 @@ Ready.
 % decimal # type
 % 20 # length
 [ 22923.0280   ]
-% .stats,  .stats # table_name
-% rewrite, count # name
-% clob,int # type
-% 7,   1 # length
-[ "joinidx",   0   ]
-
-# 11:48:06 >  
-# 11:48:06 >  Mtimeout -timeout 60 MapiClient -lsql -umonetdb -Pmonetdb 
--host=localhost --port=35781
-# 11:48:06 >  
 
 % sys.supplier,sys.supplier # table_name
 % s_name,  s_address # name
 % char,varchar # type
 % 25,  30 # length
 [ "Supplier#00013","HK71HQyWoqRWOX8GI FpgAifW,2PoH"]
-% .stats,  .stats # table_name
-% rewrite, count # name
-% clob,int # type
-% 7,   1 # length
-[ "joinidx",   0   ]
 
-# 11:48:06 >  
-# 11:48:06 >  Mtimeout -timeout 60 MapiClient -lsql -umonetdb -Pmonetdb 
--host=localhost --port=35781
-# 11:48:06 >  
+# 11:44:39 >  
+# 11:44:39 >  Mtimeout -timeout 60 mclient -lsql -ftest -Eutf-8 -i -e 
--host=/var/tmp/mtest-29401 --port=32058 < 
/net/madrid/export/scratch1/sjoerd/src/MonetDB/candidate/sql/benchmarks/tpch/LOCKED/../21.sql
+# 11:44:39 >  
 
 #select
 #  s_name,
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Jul2017 - Fix leak: if loading fails, free the heap sin...

2017-05-09 Thread Sjoerd Mullender 
Changeset: 35bf81017569 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=35bf81017569
Modified Files:
gdk/gdk_storage.c
Branch: Jul2017
Log Message:

Fix leak: if loading fails, free the heap since no reference is returned.


diffs (19 lines):

diff --git a/gdk/gdk_storage.c b/gdk/gdk_storage.c
--- a/gdk/gdk_storage.c
+++ b/gdk/gdk_storage.c
@@ -798,6 +798,7 @@ BATload_intern(bat bid, int lock)
/* LOAD bun heap */
if (b->ttype != TYPE_void) {
if (HEAPload(&b->theap, nme, "tail", b->batRestricted == 
BAT_READ) != GDK_SUCCEED) {
+   HEAPfree(&b->theap, 0);
return NULL;
}
assert(b->theap.size >> b->tshift <= BUN_MAX);
@@ -810,6 +811,7 @@ BATload_intern(bat bid, int lock)
if (ATOMvarsized(b->ttype)) {
if (HEAPload(b->tvheap, nme, "theap", b->batRestricted == 
BAT_READ) != GDK_SUCCEED) {
HEAPfree(&b->theap, 0);
+   HEAPfree(b->tvheap, 0);
return NULL;
}
if (ATOMstorage(b->ttype) == TYPE_str) {
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Jul2017 - Add conditions.

2017-05-09 Thread Sjoerd Mullender 
Changeset: 6212add62441 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6212add62441
Modified Files:
sql/test/BugTracker-2017/Tests/All
Branch: Jul2017
Log Message:

Add conditions.


diffs (14 lines):

diff --git a/sql/test/BugTracker-2017/Tests/All 
b/sql/test/BugTracker-2017/Tests/All
--- a/sql/test/BugTracker-2017/Tests/All
+++ b/sql/test/BugTracker-2017/Tests/All
@@ -24,8 +24,8 @@ dce_bug-6177
 crash_on_NULL_ptr.Bug-6130
 HAVE_SAMTOOLS?sqlsmith01
 HAVE_NETCDF?sqlsmith02
-sqlsmith03
-sqlsmith04
+HAVE_GEOM?sqlsmith03
+HAVE_GEOM?sqlsmith04
 nested_with.Bug-6187
 prepare.Bug-6133
 null-quantile.Bug-6218
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - Merge with Jul2017 branch.

2017-05-09 Thread Sjoerd Mullender 
Changeset: 91e672a02d3c for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=91e672a02d3c
Modified Files:
gdk/gdk_atoms.c
gdk/gdk_bbp.c
gdk/gdk_storage.c
monetdb5/mal/mal_atom.c
monetdb5/modules/atoms/str.c
monetdb5/modules/mal/manifold.c
sql/benchmarks/tpch/LOCKED/Tests/01-22.stable.out
sql/test/BugTracker-2017/Tests/All
Branch: default
Log Message:

Merge with Jul2017 branch.


diffs (truncated from 309 to 300 lines):

diff --git a/gdk/gdk_atoms.c b/gdk/gdk_atoms.c
--- a/gdk/gdk_atoms.c
+++ b/gdk/gdk_atoms.c
@@ -165,20 +165,25 @@ ATOMallocate(const char *id)
 {
int t;
 
+   if (strlen(id) >= IDLENGTH) {
+   GDKerror("ATOMallocate: name too long");
+   return int_nil;
+   }
+
MT_lock_set(&GDKthreadLock);
t = ATOMindex(id);
-
if (t < 0) {
t = -t;
if (t == GDKatomcnt) {
+   if (GDKatomcnt == MAXATOMS) {
+   MT_lock_unset(&GDKthreadLock);
+   GDKerror("ATOMallocate: too many types");
+   return int_nil;
+   }
GDKatomcnt++;
}
-   if (GDKatomcnt == MAXATOMS)
-   GDKfatal("ATOMallocate: too many types");
-   if (strlen(id) >= IDLENGTH)
-   GDKfatal("ATOMallocate: name too long");
memset(BATatoms + t, 0, sizeof(atomDesc));
-   snprintf(BATatoms[t].name, sizeof(BATatoms[t].name), "%s", id);
+   strcpy(BATatoms[t].name, id);
BATatoms[t].size = sizeof(int); /* default */
BATatoms[t].align = sizeof(int);/* default */
BATatoms[t].linear = 1; /* default */
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -284,7 +284,7 @@ BBPunlock(void)
 }
 
 
-static void
+static gdk_return
 BBPinithash(int j)
 {
bat i = (bat) ATOMIC_GET(BBPsize, BBPsizeLock);
@@ -293,8 +293,10 @@ BBPinithash(int j)
for (BBP_mask = 1; (BBP_mask << 1) <= BBPlimit; BBP_mask <<= 1)
;
BBP_hash = (bat *) GDKzalloc(BBP_mask * sizeof(bat));
-   if (BBP_hash == NULL)
-   GDKfatal("BBPinithash: cannot allocate memory\n");
+   if (BBP_hash == NULL) {
+   GDKerror("BBPinithash: cannot allocate memory\n");
+   return GDK_FAIL;
+   }
BBP_mask--;
 
while (--i > 0) {
@@ -311,6 +313,7 @@ BBPinithash(int j)
j = 0;
}
}
+   return GDK_SUCCEED;
 }
 
 int
@@ -360,7 +363,8 @@ BBPextend(int idx, int buildhash)
BBP_hash = NULL;
for (i = 0; i <= BBP_THREADMASK; i++)
BBP_free(i) = 0;
-   BBPinithash(idx);
+   if (BBPinithash(idx) != GDK_SUCCEED)
+   return GDK_FAIL;
}
return GDK_SUCCEED;
 }
@@ -1402,7 +1406,8 @@ BBPinit(void)
BBPreadEntries(fp, bbpversion);
fclose(fp);
 
-   BBPinithash(0);
+   if (BBPinithash(0) != GDK_SUCCEED)
+   GDKfatal("BBPinit: BBPinithash failed");
 
/* will call BBPrecover if needed */
if (BBPprepare(FALSE) != GDK_SUCCEED)
diff --git a/gdk/gdk_storage.c b/gdk/gdk_storage.c
--- a/gdk/gdk_storage.c
+++ b/gdk/gdk_storage.c
@@ -798,6 +798,7 @@ BATload_intern(bat bid, int lock)
/* LOAD bun heap */
if (b->ttype != TYPE_void) {
if (HEAPload(&b->theap, nme, "tail", b->batRestricted == 
BAT_READ) != GDK_SUCCEED) {
+   HEAPfree(&b->theap, 0);
return NULL;
}
assert(b->theap.size >> b->tshift <= BUN_MAX);
@@ -810,6 +811,7 @@ BATload_intern(bat bid, int lock)
if (ATOMvarsized(b->ttype)) {
if (HEAPload(b->tvheap, nme, "theap", b->batRestricted == 
BAT_READ) != GDK_SUCCEED) {
HEAPfree(&b->theap, 0);
+   HEAPfree(b->tvheap, 0);
return NULL;
}
if (ATOMstorage(b->ttype) == TYPE_str) {
diff --git a/monetdb5/mal/mal_atom.c b/monetdb5/mal/mal_atom.c
--- a/monetdb5/mal/mal_atom.c
+++ b/monetdb5/mal/mal_atom.c
@@ -184,6 +184,10 @@ malAtomDefinition(stream *out, str name,
if (strlen(name) >= sizeof(BATatoms[0].name))
return -1;
i = ATOMallocate(name);
+   if (i == int_nil) {
+   showException(out, TYPE, "atomDefinition", "Could not allocate 
atom '%s'", name);
+   return -1;
+   }
/* overload atom ? */
if (tpe) {
BATatoms[i] = BATatoms[tpe];
diff --git a/monetdb5/modules/atoms/str.c b/monetdb5/modules/atoms/str.c
--- a/monetdb5/modules/atoms/str.c
+++ b/monetdb5/modules/atoms

MonetDB: stratified_sampling - Add weighted sampling to the SQL ...

2017-05-09 Thread Mark Raasveldt 
Changeset: 8eb4729bbdb8 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8eb4729bbdb8
Modified Files:
monetdb5/optimizer/opt_candidates.c
monetdb5/optimizer/opt_prelude.c
monetdb5/optimizer/opt_prelude.h
monetdb5/optimizer/opt_support.c
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql_statement.c
sql/backends/monet5/sql_statement.h
sql/server/rel_select.c
sql/server/sql_parser.h
sql/server/sql_parser.y
sql/server/sql_scan.c
Branch: stratified_sampling
Log Message:

Add weighted sampling to the SQL layer.

The syntax for weighted sampling is:

SELECT * FROM  SAMPLE  WITH WEIGHTS ;


diffs (232 lines):

diff --git a/monetdb5/optimizer/opt_candidates.c 
b/monetdb5/optimizer/opt_candidates.c
--- a/monetdb5/optimizer/opt_candidates.c
+++ b/monetdb5/optimizer/opt_candidates.c
@@ -64,7 +64,7 @@ OPTcandidatesImplementation(Client cntxt
setVarCList(mb,getArg(p,0));
}
else if (getModuleId(p) == sampleRef) {
-   if (getFunctionId(p) == subuniformRef)
+   if (getFunctionId(p) == subuniformRef || 
getFunctionId(p) == subweightedRef)
setVarCList(mb, getArg(p, 0));
}
else if (getModuleId(p) == groupRef && p->retc > 1) {
diff --git a/monetdb5/optimizer/opt_prelude.c b/monetdb5/optimizer/opt_prelude.c
--- a/monetdb5/optimizer/opt_prelude.c
+++ b/monetdb5/optimizer/opt_prelude.c
@@ -286,6 +286,7 @@ str not_uniqueRef;
 str sampleRef;
 str uniqueRef;
 str subuniformRef;
+str subweightedRef;
 str unlockRef;
 str unpackRef;
 str unpinRef;
@@ -567,6 +568,7 @@ void optimizerInit(void)
not_uniqueRef= putName("not_unique");
sampleRef= putName("sample");
uniqueRef= putName("unique");
+   subweightedRef = putName("subweighted");
subuniformRef= putName("subuniform");
unlockRef= putName("unlock");
unpackRef = putName("unpack");
diff --git a/monetdb5/optimizer/opt_prelude.h b/monetdb5/optimizer/opt_prelude.h
--- a/monetdb5/optimizer/opt_prelude.h
+++ b/monetdb5/optimizer/opt_prelude.h
@@ -280,6 +280,7 @@ mal_export  str not_uniqueRef;
 mal_export  str sampleRef;
 mal_export  str uniqueRef;
 mal_export  str subuniformRef;
+mal_export  str subweightedRef;
 mal_export  str unpackRef;
 mal_export  str unpinRef;
 mal_export  str unlockRef;
diff --git a/monetdb5/optimizer/opt_support.c b/monetdb5/optimizer/opt_support.c
--- a/monetdb5/optimizer/opt_support.c
+++ b/monetdb5/optimizer/opt_support.c
@@ -549,7 +549,9 @@ isSlice(InstrPtr p)
 int 
 isSample(InstrPtr p)
 {
-   return (getModuleId(p) == sampleRef && getFunctionId(p) == 
subuniformRef);
+   return (getModuleId(p) == sampleRef && 
+   (getFunctionId(p) == subuniformRef || 
+getFunctionId(p) == subweightedRef));
 }
 
 int isOrderby(InstrPtr p){
diff --git a/sql/backends/monet5/rel_bin.c b/sql/backends/monet5/rel_bin.c
--- a/sql/backends/monet5/rel_bin.c
+++ b/sql/backends/monet5/rel_bin.c
@@ -530,9 +530,8 @@ exp_bin(backend *be, sql_exp *e, stmt *l
list *ops;
node *n;
int first = 1;
-
-   ops = sa_list(sql->sa);
-   args = e->l;
+   ops = sa_list(sql->sa);
+   args = e->l;
for( n = args->h; n; n = n->next ) {
s = NULL;
if (!swapped)
@@ -2794,12 +2793,23 @@ rel2bin_sample(backend *be, sql_rel *rel
const char *tname = table_name(sql->sa, sc);
 
s = exp_bin(be, rel->exps->h->data, NULL, NULL, NULL, NULL, 
NULL, NULL);
-
if (!s)
s = stmt_atom_lng_nil(be);
 
sc = column(be, sc);
-   sample = stmt_sample(be, stmt_alias(be, sc, tname, cname),s);
+
+   if (rel->exps->h->next) {
+   stmt* left = rel_bin(be, rel->l);
+   stmt* right = rel_bin(be, rel->r);
+   // weighted sampling
+   stmt* weights = exp_bin(be, rel->exps->h->next->data, 
left, right, NULL, NULL, NULL, NULL);
+   if (!weights)
+   return NULL;
+
+   sample = stmt_weighted_sample(be, stmt_alias(be, sc, 
tname, cname), s, weights);
+   } else {
+   sample = stmt_sample(be, stmt_alias(be, sc, tname, 
cname), s);
+   }
 
for ( ; n; n = n->next) {
stmt *sc = n->data;
diff --git a/sql/backends/monet5/sql_statement.c 
b/sql/backends/monet5/sql_statement.c
--- a/sql/backends/monet5/sql_statement.c
+++ b/sql/backends/monet5/sql_statement.c
@@ -1013,6 +1013,34 @@ stmt_sample(backend *be, s

MonetDB: timetrails - Another simple example

2017-05-09 Thread Martin Kersten 
Changeset: dd95771746c3 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=dd95771746c3
Added Files:
sql/test/timetrails/Tests/extend.sql
sql/test/timetrails/Tests/extend.stable.err
sql/test/timetrails/Tests/extend.stable.out
Modified Files:
sql/test/timetrails/Tests/All
Branch: timetrails
Log Message:

Another simple example


diffs (116 lines):

diff --git a/sql/test/timetrails/Tests/All b/sql/test/timetrails/Tests/All
--- a/sql/test/timetrails/Tests/All
+++ b/sql/test/timetrails/Tests/All
@@ -7,6 +7,7 @@ time_rollup
 derivative
 windowfunction
 rieman
+extend
 microbatch
 summary
 retention
diff --git a/sql/test/timetrails/Tests/extend.sql 
b/sql/test/timetrails/Tests/extend.sql
new file mode 100644
--- /dev/null
+++ b/sql/test/timetrails/Tests/extend.sql
@@ -0,0 +1,9 @@
+--Extend s table with the period
+CREATE FUNCTION room_period(stride integer)
+RETURNS TABLE(period int,tick timestamp,room string,level int,temp float)
+BEGIN
+   RETURN SELECT epoch(tick)/60 AS period, min(tick), 
+   room, level, avg( temp) FROM rooms GROUP BY period, room, level;
+END;
+
+SELECT * FROM room_period(60);
diff --git a/sql/test/timetrails/Tests/extend.stable.err 
b/sql/test/timetrails/Tests/extend.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/test/timetrails/Tests/extend.stable.err
@@ -0,0 +1,34 @@
+stderr of test 'extend` in directory 'sql/test/timetrails` itself:
+
+
+# 16:26:31 >  
+# 16:26:31 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=34445" "--set" 
"mapi_usock=/var/tmp/mtest-3509/.s.monetdb.34445" "--set" "monet_prompt=" 
"--forcemito" 
"--dbpath=/export/scratch1/mk/timetrails//Linux/var/MonetDB/mTests_sql_test_timetrails"
+# 16:26:31 >  
+
+# builtin opt  gdk_dbpath = 
/export/scratch1/mk/timetrails//Linux/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 34445
+# cmdline opt  mapi_usock = /var/tmp/mtest-3509/.s.monetdb.34445
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = 
/export/scratch1/mk/timetrails//Linux/var/MonetDB/mTests_sql_test_timetrails
+# cmdline opt  gdk_debug = 536870922
+
+# 16:26:32 >  
+# 16:26:32 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-3509" "--port=34445"
+# 16:26:32 >  
+
+
+# 16:26:32 >  
+# 16:26:32 >  "Done."
+# 16:26:32 >  
+
diff --git a/sql/test/timetrails/Tests/extend.stable.out 
b/sql/test/timetrails/Tests/extend.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/test/timetrails/Tests/extend.stable.out
@@ -0,0 +1,47 @@
+stdout of test 'extend` in directory 'sql/test/timetrails` itself:
+
+
+# 16:26:31 >  
+# 16:26:31 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=34445" "--set" 
"mapi_usock=/var/tmp/mtest-3509/.s.monetdb.34445" "--set" "monet_prompt=" 
"--forcemito" 
"--dbpath=/export/scratch1/mk/timetrails//Linux/var/MonetDB/mTests_sql_test_timetrails"
+# 16:26:31 >  
+
+# MonetDB 5 server v11.28.0
+# This is an unreleased version
+# Serving database 'mTests_sql_test_timetrails', using 8 threads
+# Compiled for x86_64-unknown-linux-gnu/64bit with 128bit integers
+# Found 15.589 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2017 MonetDB B.V., all rights reserved
+# Visit http://www.monetdb.org/ for further information
+# Listening for connection requests on mapi:monetdb://vienna.da.cwi.nl:34445/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-3509/.s.monetdb.34445
+# MonetDB/GIS module loaded
+# MonetDB/SQL module loaded
+
+Ready.
+
+# 16:26:32 >  
+# 16:26:32 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-3509" "--port=34445"
+# 16:26:32 >  
+
+#CREATE FUNCTION room_period(stride integer)
+#RETURNS TABLE(period int,tick timestamp,room string,level int,temp float)
+#BEGIN
+#  RETURN SELECT epoch(tick)/60 AS period, min(tick), 
+#  room, level, avg( temp) FROM rooms GROUP BY period, room, level;
+#END;
+#SELECT * FROM room_period(60);
+% .L2, .L2,.L2,.L2,.L2 # table_name
+% period,  tick,   room,   level,  temp # name
+% int, timestamp,  clob,   int,double # type
+% 8,   26, 4,  1,  24 # length
+[ 24721020,2017-01-01 09:00:00.00, "L302", 3,  13.875  ]
+[ 24721080,2017-01-01 10:00:00.00, "L302", 3,  18.775  ]
+[ 24721140,2017-01-01 11:00:00.00, "L302", 3,  21.425  ]
+[ 24721200,2017-01-01 12:00:00.00, "L302", 3,  21.5]
+[ 24721260,2017-01-01 13:00:00.00, "L

MonetDB: Dec2016 - Ported to GCC 7.1.1 which uses -Wimplicit-fal...

2017-05-09 Thread Sjoerd Mullender 
Changeset: 34e7480c7085 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=34e7480c7085
Modified Files:
clients/odbc/samples/arraytest.c
clients/odbc/samples/odbcsample1.c
clients/odbc/samples/testgetinfo.c
gdk/gdk_search.c
monetdb5/mal/mal_function.c
monetdb5/mal/mal_parser.c
monetdb5/modules/atoms/inet.c
sql/backends/monet5/UDF/pyapi/unicode.c
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql_statistics.c
sql/server/rel_exp.c
sql/server/rel_optimizer.c
sql/server/rel_planner.c
sql/server/rel_rel.c
sql/server/rel_select.c
tools/merovingian/client/monetdb.c
tools/merovingian/utils/glob.c
Branch: Dec2016
Log Message:

Ported to GCC 7.1.1 which uses -Wimplicit-fallthough=3 when using -Wextra.
There must be a quite particular comment or __attribute__ as the last
thing before the next case/default label when the code falls through.


diffs (truncated from 325 to 300 lines):

diff --git a/clients/odbc/samples/arraytest.c b/clients/odbc/samples/arraytest.c
--- a/clients/odbc/samples/arraytest.c
+++ b/clients/odbc/samples/arraytest.c
@@ -36,6 +36,7 @@ prerr(SQLSMALLINT tpe, SQLHANDLE hnd, co
case SQL_SUCCESS_WITH_INFO:
if (msglen >= (signed int) sizeof msg)
fprintf(stderr, "(message truncated)\n");
+   /* fall through */
case SQL_SUCCESS:
fprintf(stderr,
"%s: %s: SQLstate %s, Errnr %d, Message %s\n",
diff --git a/clients/odbc/samples/odbcsample1.c 
b/clients/odbc/samples/odbcsample1.c
--- a/clients/odbc/samples/odbcsample1.c
+++ b/clients/odbc/samples/odbcsample1.c
@@ -34,6 +34,7 @@ prerr(SQLSMALLINT tpe, SQLHANDLE hnd, co
case SQL_SUCCESS_WITH_INFO:
if (msglen >= (signed int) sizeof(msg))
fprintf(stderr, "(message truncated)\n");
+   /* fall through */
case SQL_SUCCESS:
fprintf(stderr, "%s: %s: SQLstate %s, Errnr %d, Message %s\n", 
func, pref, (char*)state, (int)errnr, (char*)msg);
break;
diff --git a/clients/odbc/samples/testgetinfo.c 
b/clients/odbc/samples/testgetinfo.c
--- a/clients/odbc/samples/testgetinfo.c
+++ b/clients/odbc/samples/testgetinfo.c
@@ -31,6 +31,7 @@ prerr(SQLSMALLINT tpe, SQLHANDLE hnd, co
case SQL_SUCCESS_WITH_INFO:
if (msglen >= (signed int) sizeof(msg))
fprintf(stderr, "(message truncated)\n");
+   /* fall through */
case SQL_SUCCESS:
fprintf(stderr, "%s: %s: SQLstate %s, Errnr %d, Message %s\n", 
func, pref, (char*)state, (int)errnr, (char*)msg);
break;
diff --git a/gdk/gdk_search.c b/gdk/gdk_search.c
--- a/gdk/gdk_search.c
+++ b/gdk/gdk_search.c
@@ -113,6 +113,7 @@ SORTfndwhich(BAT *b, const void *v, enum
case FIND_FIRST:
if (*(const oid *) v == oid_nil)
return lo;
+   /* fall through */
case FIND_LAST:
return hi;
default:
diff --git a/monetdb5/mal/mal_function.c b/monetdb5/mal/mal_function.c
--- a/monetdb5/mal/mal_function.c
+++ b/monetdb5/mal/mal_function.c
@@ -179,6 +179,7 @@ void chkFlow(stream *out, MalBlkPtr mb)
}
yieldseen= TRUE;
 }
+   /* fall through */
case RETURNsymbol:
{
InstrPtr ps = getInstrPtr(mb, 0);
diff --git a/monetdb5/mal/mal_parser.c b/monetdb5/mal/mal_parser.c
--- a/monetdb5/mal/mal_parser.c
+++ b/monetdb5/mal/mal_parser.c
@@ -376,6 +376,7 @@ cstToken(Client cntxt, ValPtr cst)
case '-':
i++;
s++;
+   /* fall through */
case '0':
if ((s[1] == 'x' || s[1] == 'X')) {
/* deal with hex */
@@ -383,6 +384,7 @@ cstToken(Client cntxt, ValPtr cst)
i += 2;
s += 2;
}
+   /* fall through */
case '1': case '2': case '3': case '4': case '5':
case '6': case '7': case '8': case '9':
if (hex)
@@ -401,6 +403,7 @@ cstToken(Client cntxt, ValPtr cst)
 
if (hex)
goto handleInts;
+   /* fall through */
case '.':
if (*s == '.' && isdigit(*(s + 1))) {
i++;
@@ -1919,6 +1922,7 @@ parseMAL(Client cntxt, Symbol curPrg, in
cntrl = YIELDsymbol;
goto allLeft;
}
+   /* fall through */
default: allLeft :
parseAssign(cntxt, cntrl);
cntrl = 0;
diff --git a/monetdb5

MonetDB: Jul2017 - Ported to GCC 7.1.1 which uses -Wimplicit-fal...

2017-05-09 Thread Sjoerd Mullender 
Changeset: 3afd6b9f2b9c for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=3afd6b9f2b9c
Modified Files:
clients/odbc/samples/arraytest.c
clients/odbc/samples/odbcsample1.c
clients/odbc/samples/testgetinfo.c
gdk/gdk_search.c
monetdb5/mal/mal_function.c
monetdb5/mal/mal_parser.c
monetdb5/modules/atoms/inet.c
sql/backends/monet5/UDF/pyapi/unicode.c
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql_statement.c
sql/backends/monet5/sql_statistics.c
sql/server/rel_exp.c
sql/server/rel_optimizer.c
sql/server/rel_planner.c
sql/server/rel_rel.c
sql/server/rel_select.c
tools/merovingian/client/monetdb.c
tools/merovingian/utils/glob.c
Branch: Jul2017
Log Message:

Ported to GCC 7.1.1 which uses -Wimplicit-fallthough=3 when using -Wextra.
There must be a quite particular comment or __attribute__ as the last
thing before the next case/default label when the code falls through.
(Merge with Dec2016 branch.)


diffs (truncated from 334 to 300 lines):

diff --git a/clients/odbc/samples/arraytest.c b/clients/odbc/samples/arraytest.c
--- a/clients/odbc/samples/arraytest.c
+++ b/clients/odbc/samples/arraytest.c
@@ -36,6 +36,7 @@ prerr(SQLSMALLINT tpe, SQLHANDLE hnd, co
case SQL_SUCCESS_WITH_INFO:
if (msglen >= (signed int) sizeof msg)
fprintf(stderr, "(message truncated)\n");
+   /* fall through */
case SQL_SUCCESS:
fprintf(stderr,
"%s: %s: SQLstate %s, Errnr %d, Message %s\n",
diff --git a/clients/odbc/samples/odbcsample1.c 
b/clients/odbc/samples/odbcsample1.c
--- a/clients/odbc/samples/odbcsample1.c
+++ b/clients/odbc/samples/odbcsample1.c
@@ -34,6 +34,7 @@ prerr(SQLSMALLINT tpe, SQLHANDLE hnd, co
case SQL_SUCCESS_WITH_INFO:
if (msglen >= (signed int) sizeof(msg))
fprintf(stderr, "(message truncated)\n");
+   /* fall through */
case SQL_SUCCESS:
fprintf(stderr, "%s: %s: SQLstate %s, Errnr %d, Message %s\n", 
func, pref, (char*)state, (int)errnr, (char*)msg);
break;
diff --git a/clients/odbc/samples/testgetinfo.c 
b/clients/odbc/samples/testgetinfo.c
--- a/clients/odbc/samples/testgetinfo.c
+++ b/clients/odbc/samples/testgetinfo.c
@@ -31,6 +31,7 @@ prerr(SQLSMALLINT tpe, SQLHANDLE hnd, co
case SQL_SUCCESS_WITH_INFO:
if (msglen >= (signed int) sizeof(msg))
fprintf(stderr, "(message truncated)\n");
+   /* fall through */
case SQL_SUCCESS:
fprintf(stderr, "%s: %s: SQLstate %s, Errnr %d, Message %s\n", 
func, pref, (char*)state, (int)errnr, (char*)msg);
break;
diff --git a/monetdb5/mal/mal_function.c b/monetdb5/mal/mal_function.c
--- a/monetdb5/mal/mal_function.c
+++ b/monetdb5/mal/mal_function.c
@@ -179,6 +179,7 @@ void chkFlow(stream *out, MalBlkPtr mb)
}
yieldseen= TRUE;
 }
+   /* fall through */
case RETURNsymbol:
{
InstrPtr ps = getInstrPtr(mb, 0);
diff --git a/monetdb5/mal/mal_parser.c b/monetdb5/mal/mal_parser.c
--- a/monetdb5/mal/mal_parser.c
+++ b/monetdb5/mal/mal_parser.c
@@ -380,6 +380,7 @@ cstToken(Client cntxt, ValPtr cst)
case '-':
i++;
s++;
+   /* fall through */
case '0':
if ((s[1] == 'x' || s[1] == 'X')) {
/* deal with hex */
@@ -387,6 +388,7 @@ cstToken(Client cntxt, ValPtr cst)
i += 2;
s += 2;
}
+   /* fall through */
case '1': case '2': case '3': case '4': case '5':
case '6': case '7': case '8': case '9':
if (hex)
@@ -405,6 +407,7 @@ cstToken(Client cntxt, ValPtr cst)
 
if (hex)
goto handleInts;
+   /* fall through */
case '.':
if (*s == '.' && isdigit(*(s + 1))) {
i++;
@@ -1934,6 +1937,7 @@ parseMAL(Client cntxt, Symbol curPrg, in
cntrl = YIELDsymbol;
goto allLeft;
}
+   /* fall through */
default: allLeft :
parseAssign(cntxt, cntrl);
cntrl = 0;
diff --git a/monetdb5/modules/atoms/inet.c b/monetdb5/modules/atoms/inet.c
--- a/monetdb5/modules/atoms/inet.c
+++ b/monetdb5/modules/atoms/inet.c
@@ -170,8 +170,10 @@ INETfromString(const char *src, int *len
switch (type) {
case 1:
(*retval)->q2 = (un

MonetDB: default - Ported to GCC 7.1.1 which uses -Wimplicit-fal...

2017-05-09 Thread Sjoerd Mullender 
Changeset: 6c5f42ea5a45 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6c5f42ea5a45
Modified Files:
clients/odbc/samples/arraytest.c
clients/odbc/samples/odbcsample1.c
clients/odbc/samples/testgetinfo.c
gdk/gdk_search.c
monetdb5/mal/mal_function.c
monetdb5/mal/mal_parser.c
monetdb5/modules/atoms/inet.c
monetdb5/modules/mal/wlc.c
sql/backends/monet5/UDF/pyapi/unicode.c
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql_statement.c
sql/backends/monet5/sql_statistics.c
sql/server/rel_exp.c
sql/server/rel_optimizer.c
sql/server/rel_planner.c
sql/server/rel_rel.c
sql/server/rel_select.c
tools/merovingian/client/monetdb.c
tools/merovingian/utils/glob.c
Branch: default
Log Message:

Ported to GCC 7.1.1 which uses -Wimplicit-fallthough=3 when using -Wextra.
There must be a quite particular comment or __attribute__ as the last
thing before the next case/default label when the code falls through.
(Merge with Jul2017 branch.)


diffs (truncated from 345 to 300 lines):

diff --git a/clients/odbc/samples/arraytest.c b/clients/odbc/samples/arraytest.c
--- a/clients/odbc/samples/arraytest.c
+++ b/clients/odbc/samples/arraytest.c
@@ -36,6 +36,7 @@ prerr(SQLSMALLINT tpe, SQLHANDLE hnd, co
case SQL_SUCCESS_WITH_INFO:
if (msglen >= (signed int) sizeof msg)
fprintf(stderr, "(message truncated)\n");
+   /* fall through */
case SQL_SUCCESS:
fprintf(stderr,
"%s: %s: SQLstate %s, Errnr %d, Message %s\n",
diff --git a/clients/odbc/samples/odbcsample1.c 
b/clients/odbc/samples/odbcsample1.c
--- a/clients/odbc/samples/odbcsample1.c
+++ b/clients/odbc/samples/odbcsample1.c
@@ -34,6 +34,7 @@ prerr(SQLSMALLINT tpe, SQLHANDLE hnd, co
case SQL_SUCCESS_WITH_INFO:
if (msglen >= (signed int) sizeof(msg))
fprintf(stderr, "(message truncated)\n");
+   /* fall through */
case SQL_SUCCESS:
fprintf(stderr, "%s: %s: SQLstate %s, Errnr %d, Message %s\n", 
func, pref, (char*)state, (int)errnr, (char*)msg);
break;
diff --git a/clients/odbc/samples/testgetinfo.c 
b/clients/odbc/samples/testgetinfo.c
--- a/clients/odbc/samples/testgetinfo.c
+++ b/clients/odbc/samples/testgetinfo.c
@@ -31,6 +31,7 @@ prerr(SQLSMALLINT tpe, SQLHANDLE hnd, co
case SQL_SUCCESS_WITH_INFO:
if (msglen >= (signed int) sizeof(msg))
fprintf(stderr, "(message truncated)\n");
+   /* fall through */
case SQL_SUCCESS:
fprintf(stderr, "%s: %s: SQLstate %s, Errnr %d, Message %s\n", 
func, pref, (char*)state, (int)errnr, (char*)msg);
break;
diff --git a/monetdb5/mal/mal_function.c b/monetdb5/mal/mal_function.c
--- a/monetdb5/mal/mal_function.c
+++ b/monetdb5/mal/mal_function.c
@@ -179,6 +179,7 @@ void chkFlow(stream *out, MalBlkPtr mb)
}
yieldseen= TRUE;
 }
+   /* fall through */
case RETURNsymbol:
{
InstrPtr ps = getInstrPtr(mb, 0);
diff --git a/monetdb5/mal/mal_parser.c b/monetdb5/mal/mal_parser.c
--- a/monetdb5/mal/mal_parser.c
+++ b/monetdb5/mal/mal_parser.c
@@ -380,6 +380,7 @@ cstToken(Client cntxt, ValPtr cst)
case '-':
i++;
s++;
+   /* fall through */
case '0':
if ((s[1] == 'x' || s[1] == 'X')) {
/* deal with hex */
@@ -387,6 +388,7 @@ cstToken(Client cntxt, ValPtr cst)
i += 2;
s += 2;
}
+   /* fall through */
case '1': case '2': case '3': case '4': case '5':
case '6': case '7': case '8': case '9':
if (hex)
@@ -405,6 +407,7 @@ cstToken(Client cntxt, ValPtr cst)
 
if (hex)
goto handleInts;
+   /* fall through */
case '.':
if (*s == '.' && isdigit(*(s + 1))) {
i++;
@@ -1935,6 +1938,7 @@ parseMAL(Client cntxt, Symbol curPrg, in
cntrl = YIELDsymbol;
goto allLeft;
}
+   /* fall through */
default: allLeft :
parseAssign(cntxt, cntrl);
cntrl = 0;
diff --git a/monetdb5/modules/atoms/inet.c b/monetdb5/modules/atoms/inet.c
--- a/monetdb5/modules/atoms/inet.c
+++ b/monetdb5/modules/atoms/inet.c
@@ -170,8 +170,10 @@ INETfromString(const char *src, int *len
switch (type) {
case 1:

MonetDB: exanest - Merge branch Dec2016 into exanest

2017-05-09 Thread Panagiotis Koutsourakis 
Changeset: 601ab8baa2d1 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=601ab8baa2d1
Added Files:
sql/test/BugTracker-2017/Tests/crash_in_in_handling.Bug-6260.sql
sql/test/BugTracker-2017/Tests/crash_in_in_handling.Bug-6260.stable.err
sql/test/BugTracker-2017/Tests/crash_in_in_handling.Bug-6260.stable.out
sql/test/BugTracker-2017/Tests/function_and_mergetable.Bug-6288.sql

sql/test/BugTracker-2017/Tests/function_and_mergetable.Bug-6288.stable.err

sql/test/BugTracker-2017/Tests/function_and_mergetable.Bug-6288.stable.out
sql/test/BugTracker-2017/Tests/nestedoperators.Bug-6292.sql
Removed Files:
monetdb5/scheduler/srvpool.c
monetdb5/scheduler/srvpool.h
monetdb5/scheduler/srvpool.mal
Modified Files:
.hgtags
MonetDB.spec
NT/installer32/MonetDB-ODBC-Installer.vdproj
NT/installer32/MonetDB5-Geom-Module.vdproj
NT/installer32/MonetDB5-SQL-Installer.vdproj
NT/installer64/MonetDB-ODBC-Installer.vdproj
NT/installer64/MonetDB5-Geom-Module.vdproj
NT/installer64/MonetDB5-SQL-Installer.vdproj
NT/monetdb_config.h.in
NT/rules.msc
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
clients/Tests/exports.stable.out
clients/mapiclient/dotmonetdb.c
clients/mapiclient/dump.c
clients/mapiclient/mclient.c
clients/mapilib/mapi.rc
clients/odbc/driver/driver.rc
clients/odbc/winsetup/setup.rc
common/stream/stream.c
configure.ag
debian/changelog
gdk/gdk_bat.c
gdk/gdk_heap.c
gdk/gdk_logger.c
gdk/libbat.rc
geom/monetdb5/Makefile.ag
geom/monetdb5/geom.c
geom/monetdb5/geom_upgrade.c
libversions
monetdb5/ChangeLog-Archive
monetdb5/extras/mal_optimizer_template/Makefile.ag
monetdb5/extras/rapi/rapi.c
monetdb5/extras/sphinx/Makefile.ag
monetdb5/mal/mal_builder.c
monetdb5/mal/mal_client.c
monetdb5/mal/mal_dataflow.c
monetdb5/mal/mal_exception.c
monetdb5/mal/mal_import.c
monetdb5/mal/mal_instruction.c
monetdb5/mal/mal_linker.c
monetdb5/mal/mal_listing.c
monetdb5/mal/mal_parser.c
monetdb5/mal/mal_resolve.c
monetdb5/mal/mal_runtime.c
monetdb5/mal/mal_session.c
monetdb5/mal/mal_stack.c
monetdb5/mal/mal_type.c
monetdb5/modules/atoms/batxml.c
monetdb5/modules/atoms/blob.c
monetdb5/modules/atoms/color.c
monetdb5/modules/atoms/identifier.c
monetdb5/modules/atoms/inet.c
monetdb5/modules/atoms/json.c
monetdb5/modules/atoms/mtime.c
monetdb5/modules/atoms/str.c
monetdb5/modules/atoms/streams.c
monetdb5/modules/atoms/url.c
monetdb5/modules/atoms/uuid.c
monetdb5/modules/atoms/xml.c
monetdb5/modules/kernel/Makefile.ag
monetdb5/modules/kernel/algebra.c
monetdb5/modules/mal/Makefile.ag
monetdb5/modules/mal/clients.c
monetdb5/modules/mal/mal_init.mal
monetdb5/modules/mal/mal_io.c
monetdb5/modules/mal/mal_mapi.c
monetdb5/modules/mal/pcre.c
monetdb5/modules/mal/remote.c
monetdb5/modules/mal/sysmon.c
monetdb5/modules/mal/tablet.c
monetdb5/modules/mal/txtsim.c
monetdb5/optimizer/opt_emptybind.c
monetdb5/optimizer/opt_evaluate.c
monetdb5/optimizer/opt_macro.c
monetdb5/optimizer/opt_mergetable.c
monetdb5/optimizer/opt_multiplex.c
monetdb5/optimizer/opt_support.c
monetdb5/scheduler/Makefile.ag
monetdb5/scheduler/run_adder.c
monetdb5/tools/libmonetdb5.rc
sql/backends/monet5/LSST/Makefile.ag
sql/backends/monet5/Makefile.ag
sql/backends/monet5/UDF/pyapi/convert_loops.h
sql/backends/monet5/UDF/pyapi/pyapi.c
sql/backends/monet5/generator/generator.c
sql/backends/monet5/gsl/Makefile.ag
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql_execute.c
sql/backends/monet5/sql_gencode.c
sql/backends/monet5/sql_optimizer.c
sql/backends/monet5/sql_scenario.c
sql/backends/monet5/sql_upgrades.c
sql/backends/monet5/sql_user.c
sql/backends/monet5/vaults/Makefile.ag
sql/backends/monet5/vaults/bam/bam_export.c
sql/backends/monet5/vaults/bam/bam_globals.h
sql/backends/monet5/vaults/bam/bam_loader.c
sql/backends/monet5/vaults/bam/bam_wrapper.c
sql/backends/monet5/vaults/fits/Makefile.ag
sql/backends/monet5/vaults/netcdf/Makefile.ag
sql/backends/monet5/vaults/shp/shp.c
sql/common/sql_mem.c
sql/server/rel_optimizer.c
sql/server/rel_psm.c
sql/server/rel_select.c
sql/server/sql_atom.c
sql/test/BugTracker-2016

MonetDB: timetrails - Fixing some tests

2017-05-09 Thread Martin Kersten 
Changeset: 171c9203904b for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=171c9203904b
Added Files:
sql/test/timetrails/Tests/onoff.sql
sql/test/timetrails/Tests/onoff.stable.err
sql/test/timetrails/Tests/onoff.stable.out
sql/test/timetrails/Tests/retention.stable.err
sql/test/timetrails/Tests/retention.stable.out
Modified Files:
sql/test/timetrails/Tests/All
sql/test/timetrails/Tests/retention.sql
sql/test/timetrails/Tests/simple_queries.sql
sql/test/timetrails/Tests/simple_queries.stable.out
sql/test/timetrails/Tests/summary.sql
Branch: timetrails
Log Message:

Fixing some tests


diffs (truncated from 307 to 300 lines):

diff --git a/sql/test/timetrails/Tests/All b/sql/test/timetrails/Tests/All
--- a/sql/test/timetrails/Tests/All
+++ b/sql/test/timetrails/Tests/All
@@ -11,3 +11,4 @@ extend
 microbatch
 summary
 retention
+onoff
diff --git a/sql/test/timetrails/Tests/onoff.sql 
b/sql/test/timetrails/Tests/onoff.sql
new file mode 100644
--- /dev/null
+++ b/sql/test/timetrails/Tests/onoff.sql
@@ -0,0 +1,9 @@
+-- on-off query problem. Find all the values that form a consecutive list
+
+CREATE TABLE coil( tick integer, level integer);
+
+INSERT INTO coil
+VALUES
+(1,0),(2,0),(3,0),(4,0),(5,1),(6,1),(7,1),(8,0),(9,2),(10,2),(11,2),(12,2),(13,0),(14,0),(15,3),(16,0),(17,0),(18,0),(19,4),(20,4),(21,4),(22,5),(23,5),(24,5),(25,0),(26,0),(27,6),(28,0),(29,6),(30,0),(31,7),(32,7),(33,7),(34,7),(35,8),(36,8),(37,8),(38,8),(39,8),(40,7),(41,7),(42,7),(43,7),(44,9),(45,9),(46,9),(47,0),(48,0),(49,0);
+
+select A.level, count(*)from coil A, coil B where A.tick+1 = B.tick and 
A.level <> B.level group by A.level;
diff --git a/sql/test/timetrails/Tests/onoff.stable.err 
b/sql/test/timetrails/Tests/onoff.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/test/timetrails/Tests/onoff.stable.err
@@ -0,0 +1,34 @@
+stderr of test 'onoff` in directory 'sql/test/timetrails` itself:
+
+
+# 20:56:51 >  
+# 20:56:51 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=31197" "--set" 
"mapi_usock=/var/tmp/mtest-14521/.s.monetdb.31197" "--set" "monet_prompt=" 
"--forcemito" 
"--dbpath=/export/scratch1/mk/timetrails//Linux/var/MonetDB/mTests_sql_test_timetrails"
+# 20:56:51 >  
+
+# builtin opt  gdk_dbpath = 
/export/scratch1/mk/timetrails//Linux/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 31197
+# cmdline opt  mapi_usock = /var/tmp/mtest-14521/.s.monetdb.31197
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = 
/export/scratch1/mk/timetrails//Linux/var/MonetDB/mTests_sql_test_timetrails
+# cmdline opt  gdk_debug = 536870922
+
+# 20:56:51 >  
+# 20:56:51 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-14521" "--port=31197"
+# 20:56:51 >  
+
+
+# 20:56:51 >  
+# 20:56:51 >  "Done."
+# 20:56:51 >  
+
diff --git a/sql/test/timetrails/Tests/onoff.stable.out 
b/sql/test/timetrails/Tests/onoff.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/test/timetrails/Tests/onoff.stable.out
@@ -0,0 +1,51 @@
+stdout of test 'onoff` in directory 'sql/test/timetrails` itself:
+
+
+# 20:56:51 >  
+# 20:56:51 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=31197" "--set" 
"mapi_usock=/var/tmp/mtest-14521/.s.monetdb.31197" "--set" "monet_prompt=" 
"--forcemito" 
"--dbpath=/export/scratch1/mk/timetrails//Linux/var/MonetDB/mTests_sql_test_timetrails"
+# 20:56:51 >  
+
+# MonetDB 5 server v11.28.0
+# This is an unreleased version
+# Serving database 'mTests_sql_test_timetrails', using 8 threads
+# Compiled for x86_64-unknown-linux-gnu/64bit with 128bit integers
+# Found 15.589 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2017 MonetDB B.V., all rights reserved
+# Visit http://www.monetdb.org/ for further information
+# Listening for connection requests on mapi:monetdb://vienna.da.cwi.nl:31197/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-14521/.s.monetdb.31197
+# MonetDB/GIS module loaded
+# MonetDB/SQL module loaded
+
+Ready.
+
+# 20:56:51 >  
+# 20:56:51 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-14521" "--port=31197"
+# 20:56:51 >  
+
+#CREATE TABLE coil( tick integer, level integer);
+#INSERT INTO coil
+#VALUES
+#(1,0),(2,0),(3,0),(4,0),(5,1),(6,1),(7,1),(8,0),(9,2),(10,2),(11,2),(12,2),(13,0),(14,0),(15,3),(16,0),(17,0),(18,0),(19,4),(20,4),(21,4),(22,5),(23,5),(24,5),(25,0),(26,0),(27,6),(28,0),(29,6),(30,0),(31,7),(32

MonetDB: mosaic - Compilation issues

2017-05-09 Thread Martin Kersten 
Changeset: c2572f48beeb for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c2572f48beeb
Modified Files:
monetdb5/modules/mosaic/mosaic.h
monetdb5/modules/mosaic/mosaic_prefix.c
Branch: mosaic
Log Message:

Compilation issues


diffs (24 lines):

diff --git a/monetdb5/modules/mosaic/mosaic.h b/monetdb5/modules/mosaic/mosaic.h
--- a/monetdb5/modules/mosaic/mosaic.h
+++ b/monetdb5/modules/mosaic/mosaic.h
@@ -180,7 +180,7 @@ typedef struct MOSTASK{
 
 #ifndef HAVE_NEXTAFTERF
 #define nextafter  _nextafter
-float nextafterf(float x, float y);
+mal_export float nextafterf(float x, float y);
 #endif
 
 #define PREVVALUEbit(x) ((x) - 1)
diff --git a/monetdb5/modules/mosaic/mosaic_prefix.c 
b/monetdb5/modules/mosaic/mosaic_prefix.c
--- a/monetdb5/modules/mosaic/mosaic_prefix.c
+++ b/monetdb5/modules/mosaic/mosaic_prefix.c
@@ -350,7 +350,7 @@ findPrefixLng(Client cntxt, ulng *v, int
 } 
 
 
-#define LOOKAHEAD  (limit <10? limit:10)
+#define LOOKAHEAD  (int)(limit <10? limit:10)
 // calculate the expected reduction 
 flt
 MOSestimate_prefix(Client cntxt, MOStask task)
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: mosaic - Merge with default

2017-05-09 Thread Martin Kersten 
Changeset: 237391267472 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=237391267472
Modified Files:
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
clients/Tests/exports.stable.out
clients/mapiclient/dotmonetdb.c
clients/mapiclient/mclient.c
clients/odbc/samples/arraytest.c
clients/odbc/samples/odbcsample1.c
clients/odbc/samples/testgetinfo.c
common/stream/stream.c
gdk/gdk_atoms.c
gdk/gdk_bat.c
gdk/gdk_bbp.c
gdk/gdk_heap.c
gdk/gdk_logger.c
gdk/gdk_search.c
gdk/gdk_storage.c
geom/monetdb5/geom.c
geom/monetdb5/geom_upgrade.c

monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out.Windows
monetdb5/extras/rapi/rapi.c
monetdb5/mal/mal_atom.c
monetdb5/mal/mal_builder.c
monetdb5/mal/mal_client.c
monetdb5/mal/mal_dataflow.c
monetdb5/mal/mal_exception.c
monetdb5/mal/mal_function.c
monetdb5/mal/mal_import.c
monetdb5/mal/mal_instruction.c
monetdb5/mal/mal_linker.c
monetdb5/mal/mal_listing.c
monetdb5/mal/mal_parser.c
monetdb5/mal/mal_profiler.c
monetdb5/mal/mal_resolve.c
monetdb5/mal/mal_runtime.c
monetdb5/mal/mal_session.c
monetdb5/mal/mal_type.c
monetdb5/modules/atoms/batxml.c
monetdb5/modules/atoms/blob.c
monetdb5/modules/atoms/blob.h
monetdb5/modules/atoms/color.c
monetdb5/modules/atoms/identifier.c
monetdb5/modules/atoms/inet.c
monetdb5/modules/atoms/json.c
monetdb5/modules/atoms/mtime.c
monetdb5/modules/atoms/str.c
monetdb5/modules/atoms/streams.c
monetdb5/modules/atoms/url.c
monetdb5/modules/atoms/uuid.c
monetdb5/modules/atoms/xml.c
monetdb5/modules/kernel/algebra.c
monetdb5/modules/kernel/bat5.c
monetdb5/modules/mal/clients.c
monetdb5/modules/mal/mal_io.c
monetdb5/modules/mal/mal_mapi.c
monetdb5/modules/mal/manifold.c
monetdb5/modules/mal/pcre.c
monetdb5/modules/mal/remote.c
monetdb5/modules/mal/sysmon.c
monetdb5/modules/mal/tablet.c
monetdb5/modules/mal/tablet.h
monetdb5/modules/mal/txtsim.c
monetdb5/modules/mal/wlc.c
monetdb5/optimizer/opt_emptybind.c
monetdb5/optimizer/opt_evaluate.c
monetdb5/optimizer/opt_macro.c
monetdb5/optimizer/opt_mergetable.c
monetdb5/optimizer/opt_multiplex.c
monetdb5/optimizer/opt_reorder.c
monetdb5/optimizer/opt_support.c
monetdb5/scheduler/run_adder.c
sql/backends/monet5/UDF/pyapi/convert_loops.h
sql/backends/monet5/UDF/pyapi/unicode.c
sql/backends/monet5/generator/generator.c
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql.mal
sql/backends/monet5/sql_cast.c
sql/backends/monet5/sql_execute.c
sql/backends/monet5/sql_gencode.c
sql/backends/monet5/sql_optimizer.c
sql/backends/monet5/sql_scenario.c
sql/backends/monet5/sql_statement.c
sql/backends/monet5/sql_statistics.c
sql/backends/monet5/sql_upgrades.c
sql/backends/monet5/sql_user.c
sql/backends/monet5/vaults/bam/bam_export.c
sql/backends/monet5/vaults/bam/bam_globals.h
sql/backends/monet5/vaults/bam/bam_wrapper.c
sql/backends/monet5/vaults/shp/shp.c
sql/benchmarks/tpch/LOCKED/Tests/01-22.stable.out
sql/benchmarks/tpch/Tests/01-22.stable.out
sql/common/sql_mem.c
sql/common/sql_types.c
sql/server/rel_exp.c
sql/server/rel_optimizer.c
sql/server/rel_planner.c
sql/server/rel_psm.c
sql/server/rel_rel.c
sql/server/rel_select.c
sql/server/rel_updates.c
sql/storage/store.c
sql/test/BugTracker-2015/Tests/sys_tracelog.Bug-3887.stable.err
sql/test/BugTracker-2015/Tests/sys_tracelog.Bug-3887.stable.out
sql/test/BugTracker-2017/Tests/All

sql/test/BugTracker-2017/Tests/crash_correlated_subqueries_in_select.Bug-6254.stable.out

sql/test/BugTracker-2017/Tests/wrong_aggregation_count.Bug-6257.stable.out
sql/test/Dump/Tests/dump.stable.out
sql/test/Tests/setoptimizer.stable.out
sql/test/Tests/systemfunctions.stable.out
sql/test/Tests/systemfunctions.stable.out.int128
sql/test/emptydb-upgrade-chain-hge/Tests/check.stable.out.int128
sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade-chain/Tests/check.stable.out
sql/test/emptydb-upgrade-chain/Tests/check.stable.out.32bit
sql/test/emptydb-upgrade-chain/Tests/check.stable.out.int128
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out

MonetDB: timetrails - Merge with default

2017-05-09 Thread Martin Kersten 
Changeset: 281c6d02902f for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=281c6d02902f
Modified Files:
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
clients/Tests/exports.stable.out
clients/mapiclient/dotmonetdb.c
clients/mapiclient/mclient.c
common/stream/stream.c
gdk/gdk_bat.c
gdk/gdk_heap.c
gdk/gdk_logger.c
geom/monetdb5/geom.c
geom/monetdb5/geom_upgrade.c
monetdb5/extras/rapi/rapi.c
monetdb5/mal/mal_builder.c
monetdb5/mal/mal_client.c
monetdb5/mal/mal_dataflow.c
monetdb5/mal/mal_exception.c
monetdb5/mal/mal_import.c
monetdb5/mal/mal_instruction.c
monetdb5/mal/mal_linker.c
monetdb5/mal/mal_listing.c
monetdb5/mal/mal_resolve.c
monetdb5/mal/mal_runtime.c
monetdb5/mal/mal_session.c
monetdb5/mal/mal_type.c
monetdb5/modules/atoms/batxml.c
monetdb5/modules/atoms/blob.c
monetdb5/modules/atoms/blob.h
monetdb5/modules/atoms/color.c
monetdb5/modules/atoms/identifier.c
monetdb5/modules/atoms/inet.c
monetdb5/modules/atoms/json.c
monetdb5/modules/atoms/mtime.c
monetdb5/modules/atoms/str.c
monetdb5/modules/atoms/streams.c
monetdb5/modules/atoms/url.c
monetdb5/modules/atoms/uuid.c
monetdb5/modules/atoms/xml.c
monetdb5/modules/kernel/algebra.c
monetdb5/modules/kernel/bat5.c
monetdb5/modules/mal/clients.c
monetdb5/modules/mal/mal_io.c
monetdb5/modules/mal/mal_mapi.c
monetdb5/modules/mal/pcre.c
monetdb5/modules/mal/remote.c
monetdb5/modules/mal/sysmon.c
monetdb5/modules/mal/tablet.c
monetdb5/modules/mal/tablet.h
monetdb5/modules/mal/txtsim.c
monetdb5/optimizer/opt_emptybind.c
monetdb5/optimizer/opt_evaluate.c
monetdb5/optimizer/opt_macro.c
monetdb5/optimizer/opt_mergetable.c
monetdb5/optimizer/opt_multiplex.c
monetdb5/optimizer/opt_reorder.c
monetdb5/optimizer/opt_support.c
monetdb5/scheduler/run_adder.c
sql/backends/monet5/UDF/pyapi/convert_loops.h
sql/backends/monet5/generator/generator.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql.mal
sql/backends/monet5/sql_cast.c
sql/backends/monet5/sql_execute.c
sql/backends/monet5/sql_gencode.c
sql/backends/monet5/sql_optimizer.c
sql/backends/monet5/sql_scenario.c
sql/backends/monet5/sql_upgrades.c
sql/backends/monet5/sql_user.c
sql/backends/monet5/vaults/bam/bam_export.c
sql/backends/monet5/vaults/bam/bam_globals.h
sql/backends/monet5/vaults/bam/bam_wrapper.c
sql/backends/monet5/vaults/shp/shp.c
sql/common/sql_mem.c
sql/common/sql_types.c
sql/server/rel_optimizer.c
sql/server/rel_psm.c
sql/server/rel_updates.c
sql/storage/store.c

sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-0.Bug-3063.stable.out

sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-1.Bug-3063.stable.out

sql/test/BugTracker-2017/Tests/wrong_aggregation_count.Bug-6257.stable.out
sql/test/Dependencies/Tests/Dependencies.stable.out
sql/test/Dependencies/Tests/Dependencies.stable.out.int128
sql/test/DropCascade/Tests/drop_columns_cascade.stable.out
sql/test/DropCascade/Tests/drop_pkey_cascade.stable.out
sql/test/Dump/Tests/dump.stable.out
sql/test/Tests/keys.stable.out
sql/test/Tests/systemfunctions.stable.out
sql/test/Tests/systemfunctions.stable.out.int128
sql/test/emptydb-upgrade-chain-hge/Tests/check.stable.out.int128
sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade-chain/Tests/check.stable.out
sql/test/emptydb-upgrade-chain/Tests/check.stable.out.32bit
sql/test/emptydb-upgrade-chain/Tests/check.stable.out.int128
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade-hge/Tests/check.stable.out.int128
sql/test/emptydb-upgrade/Tests/check.stable.out
sql/test/emptydb-upgrade/Tests/check.stable.out.32bit
sql/test/emptydb-upgrade/Tests/check.stable.out.int128
sql/test/emptydb/Tests/check.stable.out
sql/test/emptydb/Tests/check.stable.out.32bit
sql/test/emptydb/Tests/check.stable.out.int128
sql/test/leaks/Tests/check1.stable.out
sql/test/leaks/Tests/check1.stable.out.int128
sql/test/leaks/Tests/check2.stable.out
sql/test/leaks/Tests/check2.stable.out.int128
sql/test/leaks/Tests/check3.stable.out
sql/test/leaks/Tests/check3.stable.out.int128

MonetDB: malparsing - Merge with default

2017-05-09 Thread Martin Kersten 
Changeset: 50ae2d43314b for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=50ae2d43314b
Modified Files:
clients/Tests/exports.stable.out
clients/odbc/samples/arraytest.c
clients/odbc/samples/odbcsample1.c
clients/odbc/samples/testgetinfo.c
gdk/gdk_atoms.c
gdk/gdk_bbp.c
gdk/gdk_search.c
gdk/gdk_storage.c

monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out.Windows
monetdb5/mal/mal_atom.c
monetdb5/mal/mal_function.c
monetdb5/mal/mal_parser.c
monetdb5/mal/mal_profiler.c
monetdb5/modules/atoms/inet.c
monetdb5/modules/atoms/str.c
monetdb5/modules/mal/manifold.c
monetdb5/modules/mal/tablet.c
monetdb5/modules/mal/tablet.h
monetdb5/modules/mal/wlc.c
sql/backends/monet5/UDF/pyapi/unicode.c
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql_statement.c
sql/backends/monet5/sql_statistics.c
sql/benchmarks/tpch/LOCKED/Tests/01-22.stable.out
sql/benchmarks/tpch/Tests/01-22.stable.out
sql/server/rel_exp.c
sql/server/rel_optimizer.c
sql/server/rel_planner.c
sql/server/rel_rel.c
sql/server/rel_select.c
sql/test/BugTracker-2015/Tests/sys_tracelog.Bug-3887.stable.err
sql/test/BugTracker-2015/Tests/sys_tracelog.Bug-3887.stable.out
sql/test/BugTracker-2017/Tests/All
sql/test/mapi/Tests/sql_int128.stable.out.int128
tools/merovingian/client/monetdb.c
tools/merovingian/utils/glob.c
Branch: malparsing
Log Message:

Merge with default


diffs (truncated from 906 to 300 lines):

diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -1704,7 +1704,7 @@ str SERVERsuspend(void *ret);
 str SERVERtrace(void *ret, int *mid, int *flag);
 int SQLBLOBfromstr(char *instr, int *l, blob **val);
 int SQLBLOBtostr(str *tostr, int *l, const blob *pin);
-BUN SQLload_file(Client cntxt, Tablet *as, bstream *b, stream *out, char 
*csep, char *rsep, char quote, lng skip, lng maxrow, int best);
+BUN SQLload_file(Client cntxt, Tablet *as, bstream *b, stream *out, const char 
*csep, const char *rsep, char quote, lng skip, lng maxrow, int best);
 str STRBytes(int *res, const str *arg1);
 str STRConcat(str *res, const str *val1, const str *val2);
 str STRFromWChr(str *res, const int *at);
@@ -1801,7 +1801,6 @@ str SYSmem_usage(bat *ret, bat *ret2, co
 str SYSsetmem_maxsize(void *ret, const lng *num);
 str SYSsetvm_maxsize(void *ret, const lng *num);
 str SYSvm_usage(bat *ret, bat *ret2, const lng *minsize);
-int TABLETadt_toStr(void *extra, char **buf, int *len, int type, ptr a);
 str TABLETcollect(BAT **bats, Tablet *as);
 str TABLETcollect_parts(BAT **bats, Tablet *as, BUN offset);
 str TABLETcreate_bats(Tablet *as, BUN est);
diff --git a/clients/odbc/samples/arraytest.c b/clients/odbc/samples/arraytest.c
--- a/clients/odbc/samples/arraytest.c
+++ b/clients/odbc/samples/arraytest.c
@@ -36,6 +36,7 @@ prerr(SQLSMALLINT tpe, SQLHANDLE hnd, co
case SQL_SUCCESS_WITH_INFO:
if (msglen >= (signed int) sizeof msg)
fprintf(stderr, "(message truncated)\n");
+   /* fall through */
case SQL_SUCCESS:
fprintf(stderr,
"%s: %s: SQLstate %s, Errnr %d, Message %s\n",
diff --git a/clients/odbc/samples/odbcsample1.c 
b/clients/odbc/samples/odbcsample1.c
--- a/clients/odbc/samples/odbcsample1.c
+++ b/clients/odbc/samples/odbcsample1.c
@@ -34,6 +34,7 @@ prerr(SQLSMALLINT tpe, SQLHANDLE hnd, co
case SQL_SUCCESS_WITH_INFO:
if (msglen >= (signed int) sizeof(msg))
fprintf(stderr, "(message truncated)\n");
+   /* fall through */
case SQL_SUCCESS:
fprintf(stderr, "%s: %s: SQLstate %s, Errnr %d, Message %s\n", 
func, pref, (char*)state, (int)errnr, (char*)msg);
break;
diff --git a/clients/odbc/samples/testgetinfo.c 
b/clients/odbc/samples/testgetinfo.c
--- a/clients/odbc/samples/testgetinfo.c
+++ b/clients/odbc/samples/testgetinfo.c
@@ -31,6 +31,7 @@ prerr(SQLSMALLINT tpe, SQLHANDLE hnd, co
case SQL_SUCCESS_WITH_INFO:
if (msglen >= (signed int) sizeof(msg))
fprintf(stderr, "(message truncated)\n");
+   /* fall through */
case SQL_SUCCESS:
fprintf(stderr, "%s: %s: SQLstate %s, Errnr %d, Message %s\n", 
func, pref, (char*)state, (int)errnr, (char*)msg);
break;
diff --git a/gdk/gdk_atoms.c b/gdk/gdk_atoms.c
--- a/gdk/gdk_atoms.c
+++ b/gdk/gdk_atoms.c
@@ -165,20 +165,25 @@ ATOMallocate(const char *id)
 {
int t;
 
+   if (strlen(id) >= IDLENGTH) {
+   GDKerror("ATOMallocate: name too long");
+   return int_nil;
+   }
+