MonetDB: cleanup_types - try to use smallest output possible for...

2024-01-10 Thread Niels Nes via checkin-list 
Changeset: 1c4f0555e35c for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/1c4f0555e35c
Modified Files:
sql/server/rel_basetable.c
sql/server/rel_exp.c
sql/server/rel_exp.h
sql/server/rel_select.c
sql/server/rel_statistics.c
Branch: cleanup_types
Log Message:

try to use smallest output possible for sum


diffs (141 lines):

diff --git a/sql/server/rel_basetable.c b/sql/server/rel_basetable.c
--- a/sql/server/rel_basetable.c
+++ b/sql/server/rel_basetable.c
@@ -17,6 +17,7 @@
 #include "rel_basetable.h"
 #include "rel_remote.h"
 #include "rel_statistics.h"
+#include "rel_rewriter.h"
 #include "sql_privileges.h"
 
 #define USED_LEN(nr) ((nr+31)/32)
@@ -106,10 +107,13 @@ rel_basetable(mvc *sql, sql_table *t, co
sql_rel *rel = rel_create(sa);
int nrcols = ol_length(t->columns), end = nrcols + 1 + 
ol_length(t->idxs);
rel_base_t *ba = (rel_base_t*)sa_zalloc(sa, sizeof(rel_base_t) + 
sizeof(int)*USED_LEN(end));
+   sqlstore *store = sql->session->tr->store;
 
if(!rel || !ba)
return NULL;
 
+   if (isTable(t) && t->s && !isDeclaredTable(t)) /* count active rows 
only */
+   set_count_prop(sql->sa, rel, 
(BUN)store->storage_api.count_col(sql->session->tr, 
ol_first_node(t->columns)->data, 10));
assert(atname);
if (strcmp(atname, t->base.name) != 0)
ba->name = sa_strdup(sa, atname);
diff --git a/sql/server/rel_exp.c b/sql/server/rel_exp.c
--- a/sql/server/rel_exp.c
+++ b/sql/server/rel_exp.c
@@ -13,6 +13,7 @@
 #include "monetdb_config.h"
 #include "sql_relation.h"
 #include "sql_semantic.h"
+#include "sql_decimal.h"
 #include "rel_exp.h"
 #include "rel_rel.h"
 #include "rel_basetable.h"
@@ -3113,6 +3114,43 @@ exps_inout(sql_subfunc *f, list *exps)
res->digits = digits;
 }
 
+/* for aggregates we can reduce the result types size based on real 
digits/bits used number of known input rows */
+void
+exps_largest_int(sql_subfunc *f, list *exps, lng cnt)
+{
+   if (!f->func->res || cnt == 0)
+   return;
+   sql_subtype *res = f->res->h->data;
+   if (res->type->eclass != EC_DEC && res->type->eclass != EC_NUM)
+   return;
+   bool is_decimal = (res->type->eclass == EC_DEC);
+   unsigned int digits = 0, scale = 0, mdigits = is_decimal ? 
decimal_digits(cnt) : number_bits(cnt);
+   sql_type *largesttype = NULL;
+   for(node *n = exps->h; n; n = n->next) {
+   sql_subtype *t = exp_subtype(n->data);
+
+   if (!t)
+   continue;
+
+   largesttype = t->type;
+   if (is_decimal && t->type->eclass == EC_NUM) {
+   unsigned int d = bits2digits(t->digits);
+   digits = d>digits?d:digits;
+   } else if (digits < t->digits)
+   digits = t->digits;
+   if (scale < t->scale)
+   scale = t->scale;
+   break;
+   }
+   digits += mdigits;
+   if (largesttype && digits <= largesttype->digits)
+   sql_init_subtype(res, largesttype, digits, scale);
+   else if (is_decimal)
+   sql_find_subtype(res, res->type->base.name, digits, scale);
+   else
+   sql_find_numeric(res, 1, digits);
+}
+
 #define is_addition(fname) (strcmp(fname, "sql_add") == 0)
 #define is_subtraction(fname) (strcmp(fname, "sql_sub") == 0)
 void
diff --git a/sql/server/rel_exp.h b/sql/server/rel_exp.h
--- a/sql/server/rel_exp.h
+++ b/sql/server/rel_exp.h
@@ -210,6 +210,7 @@ extern void exps_sum_scales(sql_subfunc 
 extern sql_exp *exps_scale_algebra(mvc *sql, sql_subfunc *f, sql_rel *rel, 
list *exps);
 extern void exps_digits_add(sql_subfunc *f, list *exps);
 extern void exps_inout(sql_subfunc *f, list *exps);
+extern void exps_largest_int(sql_subfunc *f, list *exps, lng cnt);
 
 extern int exp_aggr_is_count(sql_exp *e);
 extern list *check_distinct_exp_names(mvc *sql, list *exps);
diff --git a/sql/server/rel_select.c b/sql/server/rel_select.c
--- a/sql/server/rel_select.c
+++ b/sql/server/rel_select.c
@@ -416,6 +416,23 @@ exp_fix_scale(mvc *sql, sql_subtype *ct,
return e;
 }
 
+static lng
+rel_get_count(sql_rel *rel)
+{
+   if (!rel)
+   return 0;
+   prop *p = NULL;
+   if (rel->p && (p = find_prop(rel->p, PROP_COUNT)) != NULL)
+   return p->value.lval;
+   else if(rel->l) {
+   if (is_select(rel->op) || is_project(rel->op))
+   return rel_get_count(rel->l);
+   }
+   return 0;
+}
+
+#define is_sum_aggr(f) (f->type == F_AGGR && strcmp(f->base.name, "sum") == 0)
+
 list *
 check_arguments_and_find_largest_any_type(mvc *sql, sql_rel *rel, list *exps, 
sql_subfunc *sf, int maybe_zero_or_one, bool internal)
 {
@@ -507,7 +524,9 @@ check_arguments_and_find_largest_any_typ
exps_digits_add(sf, nexps);
} else if (sf->func->fix

MonetDB: default - Merge with Dec2023 branch.

2024-01-10 Thread Sjoerd Mullender via checkin-list 
Changeset: d965f8f9b5b3 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/d965f8f9b5b3
Branch: default
Log Message:

Merge with Dec2023 branch.


diffs (98 lines):

diff --git a/sql/backends/monet5/rel_bin.c b/sql/backends/monet5/rel_bin.c
--- a/sql/backends/monet5/rel_bin.c
+++ b/sql/backends/monet5/rel_bin.c
@@ -4333,7 +4333,7 @@ rel2bin_select(backend *be, sql_rel *rel
sql_exp *e = en->data;
prop *p;
 
-   if ((p=find_prop(e->p, PROP_HASHCOL)) != NULL) {
+   if ((p=find_prop(e->p, PROP_HASHCOL)) != NULL && !is_anti(e)) {
sql_idx *i = p->value.pval;
int oldvtop = be->mb->vtop, oldstop = be->mb->stop, 
oldvid = be->mb->vid;
 
@@ -5143,6 +5143,7 @@ update_check_ukey(backend *be, stmt **up
*/
if (!isNew(k)) {
stmt *nu_tids = stmt_tdiff(be, dels, u_tids, NULL); /* 
not updated ids */
+   nu_tids = stmt_project(be, nu_tids, dels);
list *lje = sa_list(sql->sa);
list *rje = sa_list(sql->sa);
 
@@ -5256,6 +5257,7 @@ update_check_ukey(backend *be, stmt **up
/* s should be empty */
if (!isNew(k)) {
stmt *nu_tids = stmt_tdiff(be, dels, u_tids, NULL); /* 
not updated ids */
+   nu_tids = stmt_project(be, nu_tids, dels);
assert (updates);
 
h = updates[c->c->colnr];
diff --git a/sql/test/BugTracker-2024/Tests/All 
b/sql/test/BugTracker-2024/Tests/All
new file mode 100644
--- /dev/null
+++ b/sql/test/BugTracker-2024/Tests/All
@@ -0,0 +1,2 @@
+inequality-hash-issue-7426
+pkey-check-failed-7425
diff --git a/sql/test/BugTracker-2024/Tests/SingleServer 
b/sql/test/BugTracker-2024/Tests/SingleServer
new file mode 100644
diff --git a/sql/test/BugTracker-2024/Tests/inequality-hash-issue-7426.test 
b/sql/test/BugTracker-2024/Tests/inequality-hash-issue-7426.test
new file mode 100644
--- /dev/null
+++ b/sql/test/BugTracker-2024/Tests/inequality-hash-issue-7426.test
@@ -0,0 +1,34 @@
+
+statement ok
+CREATE TABLE t1(c1 INTEGER)
+
+statement ok
+CREATE TABLE t0(c0 BOOL, c1 INTEGER)
+
+statement ok
+INSERT INTO t1 (c1) VALUES (0)
+
+statement ok
+CREATE UNIQUE INDEX i0 ON t0(c1 , c0 )
+
+statement ok
+INSERT INTO t0 (c0, c1) VALUES (true, 0)
+
+query III
+SELECT t0.c0, t0.c1, t1.c1 FROM t1 INNER JOIN t0 ON t0.c0
+
+1
+0
+0
+
+query I
+SELECT (t0.c1) IS NOT NULL FROM t1 INNER  JOIN t0 ON t0.c0
+
+1
+
+query III
+SELECT t0.c0, t0.c1, t1.c1 FROM t1 INNER  JOIN t0 ON t0.c0 WHERE (t0.c1) IS 
NOT NULL
+
+1
+0
+0
diff --git a/sql/test/BugTracker-2024/Tests/pkey-check-failed-7425.test 
b/sql/test/BugTracker-2024/Tests/pkey-check-failed-7425.test
new file mode 100644
--- /dev/null
+++ b/sql/test/BugTracker-2024/Tests/pkey-check-failed-7425.test
@@ -0,0 +1,17 @@
+statement ok
+create table ttt(id int primary key,k int NOT NULL DEFAULT '0')
+
+statement ok
+insert into ttt values(1,3),(2,3),(3,3),(4,3),(5,3)
+
+statement ok
+update ttt set k=k+1 where id=3
+
+statement ok
+delete from ttt where id=2
+
+statement ok
+update ttt set id = 3 where id = 3
+
+statement ok
+drop table ttt
___
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org

MonetDB: client_interrupts - Implemented abort in COPY INTO ON C...

2024-01-10 Thread Sjoerd Mullender via checkin-list 
Changeset: 94542d05b6dc for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/94542d05b6dc
Modified Files:
clients/mapiclient/mclient.c
clients/mapilib/mapi.c
common/stream/mapi_stream.c
monetdb5/modules/mal/tablet.c
Branch: client_interrupts
Log Message:

Implemented abort in COPY INTO ON CLIENT.


diffs (91 lines):

diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c
--- a/clients/mapiclient/mclient.c
+++ b/clients/mapiclient/mclient.c
@@ -3135,6 +3135,10 @@ getfile(void *data, const char *filename
return (char*) mnstr_peek_error(NULL);
}
while (offset > 1) {
+   if (state == INTERRUPT) {
+   close_stream(f);
+   return "interrupted";
+   }
s = mnstr_readline(f, buf, READSIZE);
if (s < 0) {
close_stream(f);
@@ -3158,6 +3162,11 @@ getfile(void *data, const char *filename
return NULL;
}
}
+   if (state == INTERRUPT) {
+   close_stream(f);
+   priv->f = NULL;
+   return "interrupted";
+   }
s = mnstr_read(f, buf, 1, READSIZE);
if (s <= 0) {
close_stream(f);
diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -3340,6 +3340,12 @@ read_file(MapiHdl hdl, uint64_t off, cha
data = mid->getfilecontent(mid->filecontentprivate, 
NULL, false, 0, &size);
}
}
+   if (data != NULL && size == 0) {
+   /* some error occurred */
+   mnstr_clearerr(mid->from);
+   if (mid->oobintr)
+   mnstr_putoob(mid->to, 1);
+   }
mnstr_flush(mid->to, MNSTR_FLUSH_DATA);
line = read_line(mid);
if (line == NULL)
diff --git a/common/stream/mapi_stream.c b/common/stream/mapi_stream.c
--- a/common/stream/mapi_stream.c
+++ b/common/stream/mapi_stream.c
@@ -150,6 +150,7 @@ setup_transfer(const char *req, const ch
ssize_t nwritten;
ssize_t nread;
bool ok;
+   int oob = 0;
 
while (!bs->eof)
bstream_next(bs);
@@ -169,9 +170,19 @@ setup_transfer(const char *req, const ch
 
char buf[256];
nread = mnstr_readline(rs, buf, sizeof(buf));
-   ok = (nread == 0 || (nread == 1 && buf[0] == '\n'));
+   ok = (nread == 0 || (nread == 1 && buf[0] == '\n') || !(oob = 
mnstr_getoob(rs)));
if (!ok) {
-   msg = buf;
+   switch (oob) {
+   case 1: /* client side 
interrupt */
+   msg = "Query aborted";
+   break;
+   case 2:
+   msg = "Read error on client";
+   break;
+   default:
+   msg = nread > 0 ? buf : "Unknown error";
+   break;
+   }
discard(rs);
goto end;
}
diff --git a/monetdb5/modules/mal/tablet.c b/monetdb5/modules/mal/tablet.c
--- a/monetdb5/modules/mal/tablet.c
+++ b/monetdb5/modules/mal/tablet.c
@@ -1249,6 +1249,12 @@ SQLproducer(void *p)
 
// we may be reading from standard input and may be out of input
// warn the consumers
+   if (bstream_getoob(task->cntxt->fdin)) {
+   tablet_error(task, rowno, lineno, int_nil,
+"problem reported by client", 
s);
+   goto reportlackofinput;
+   }
+
if (ateof[cur] && partial) {
if (unlikely(partial)) {
tablet_error(task, rowno, lineno, int_nil,
___
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org

MonetDB: client_interrupts - Restore signal mask when longjmping.

2024-01-10 Thread Sjoerd Mullender via checkin-list 
Changeset: b6a6a46a2480 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/b6a6a46a2480
Modified Files:
clients/mapiclient/ReadlineTools.c
Branch: client_interrupts
Log Message:

Restore signal mask when longjmping.


diffs (12 lines):

diff --git a/clients/mapiclient/ReadlineTools.c 
b/clients/mapiclient/ReadlineTools.c
--- a/clients/mapiclient/ReadlineTools.c
+++ b/clients/mapiclient/ReadlineTools.c
@@ -440,7 +440,7 @@ char *
 call_readline(const char *prompt)
 {
char *res;
-   if (sigsetjmp(readline_jumpbuf, 0) != 0)
+   if (sigsetjmp(readline_jumpbuf, 1) != 0)
return (char *) -1; /* interrupted */
mayjump = true;
res = readline(prompt); /* normal code path */
___
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org

MonetDB: client_interrupts - Implemented abort on interrupt of O...

2024-01-10 Thread Sjoerd Mullender via checkin-list 
Changeset: 930a4f72e4b4 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/930a4f72e4b4
Modified Files:
clients/mapiclient/mclient.c
clients/mapilib/mapi.c
clients/mapilib/mapi.h
clients/mapilib/mapi_intern.h
monetdb5/modules/mal/tablet.c
monetdb5/modules/mal/tablet.h
sql/backends/monet5/sql.c
sql/backends/monet5/sql_result.c
sql/backends/monet5/sql_scenario.c
sql/server/sql_parser.y
sql/server/sql_scan.c
sql/server/sql_scan.h
Branch: client_interrupts
Log Message:

Implemented abort on interrupt of ON CLIENT processing.
Also implemented abort when ON CLIENT reading fails for some other
reason half way through.


diffs (truncated from 573 to 300 lines):

diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c
--- a/clients/mapiclient/mclient.c
+++ b/clients/mapiclient/mclient.c
@@ -2420,9 +2420,12 @@ doFile(Mapi mid, stream *fp, bool useins
mnstr_write(toConsole, "\n", 1, 1);
if (hdl) {
/* on interrupt when continuing a 
query, force an error */
-   buf[0] = '\200';
-   buf[1] = '\n';
-   l = 2;
+   l = 0;
+   if (mapi_query_abort(hdl, 1) != MOK) {
+   /* if abort failed, insert 
something not allowed */
+   buf[l++] = '\200';
+   }
+   buf[l++] = '\n';
length = 0;
} else {
/* not continuing; just repeat */
@@ -3060,6 +3063,7 @@ doFile(Mapi mid, stream *fp, bool useins
 struct privdata {
stream *f;
char *buf;
+   Mapi mid;
 };
 
 #define READSIZE   (1 << 16)
@@ -3165,13 +3169,18 @@ getfile(void *data, const char *filename
if (state == INTERRUPT) {
close_stream(f);
priv->f = NULL;
+   (void) mapi_query_abort(mapi_get_active(priv->mid), 1);
return "interrupted";
}
s = mnstr_read(f, buf, 1, READSIZE);
if (s <= 0) {
close_stream(f);
priv->f = NULL;
-   return s < 0 ? "error reading file" : NULL;
+   if (s < 0) {
+   (void) mapi_query_abort(mapi_get_active(priv->mid), 
state == INTERRUPT ? 1 : 2);
+   return "error reading file";
+   }
+   return NULL;
}
if (size)
*size = (size_t) s;
@@ -3198,6 +3207,8 @@ putfile(void *data, const char *filename
}
}
 #endif
+   if (state == INTERRUPT)
+   goto interrupted;
if (buf == NULL || bufsize == 0)
return NULL; /* successfully opened file */
} else if (buf == NULL) {
@@ -3207,6 +3218,18 @@ putfile(void *data, const char *filename
priv->f = NULL;
return flush < 0 ? "error writing output" : NULL;
}
+   if (state == INTERRUPT) {
+   char *filename;
+ interrupted:
+   filename = strdup(mnstr_name(priv->f));
+   close_stream(priv->f);
+   priv->f = NULL;
+   if (filename) {
+   MT_remove(filename);
+   free(filename);
+   }
+   return "query aborted";
+   }
if (mnstr_write(priv->f, buf, 1, bufsize) < (ssize_t) bufsize) {
close_stream(priv->f);
priv->f = NULL;
@@ -3716,7 +3739,7 @@ main(int argc, char **argv)
}
 
struct privdata priv;
-   priv = (struct privdata) {0};
+   priv = (struct privdata) {.mid = mid};
mapi_setfilecallback2(mid, getfile, putfile, &priv);
 
mapi_trace(mid, trace);
diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -1652,12 +1652,11 @@ mapi_new_handle(Mapi mid)
mapi_setError(mid, "Memory allocation failure", __func__, 
MERROR);
return NULL;
}
+   /* initialize and add to doubly-linked list */
*hdl = (struct MapiStatement) {
.mid = mid,
-   .needmore = false,
+   .next = mid->first,
};
-   /* add to doubly-linked list */
-   hdl->next = mid->first;
mid->first = hdl;
if (hdl->next)
hdl->next->prev = hdl;
@@ -2181,8 +2180,7 @@ mapi_disconnect(Mapi mid)
  * The arguments are:
  * private - the value of the filecontentprivate argument to
  *   mapi

MonetDB: client_interrupts - Merge with default branch.

2024-01-10 Thread Sjoerd Mullender via checkin-list 
Changeset: 04182e80d280 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/04182e80d280
Modified Files:
clients/Tests/exports.stable.out
gdk/gdk_system.h
Branch: client_interrupts
Log Message:

Merge with default branch.


diffs (truncated from 4212 to 300 lines):

diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -385,6 +385,7 @@ const char *MT_thread_getalgorithm(void)
 void *MT_thread_getdata(void);
 const char *MT_thread_getname(void);
 bool MT_thread_init(void);
+gdk_return MT_thread_init_add_callback(void (*init)(void *), void 
(*destroy)(void *), void *data);
 bool MT_thread_register(void);
 void MT_thread_set_qry_ctx(QryCtx *ctx);
 void MT_thread_setalgorithm(const char *algo);
diff --git a/gdk/gdk_system.c b/gdk/gdk_system.c
--- a/gdk/gdk_system.c
+++ b/gdk/gdk_system.c
@@ -184,10 +184,18 @@ GDKlockstatistics(int what)
 
 #endif /* LOCK_STATS */
 
+struct thread_funcs {
+   void (*init)(void *);
+   void (*destroy)(void *);
+   void *data;
+};
+
 static struct mtthread {
struct mtthread *next;
void (*func) (void *);  /* function to be called */
void *data; /* and its data */
+   struct thread_funcs *thread_funcs; /* callback funcs */
+   int nthread_funcs;
MT_Lock *lockwait;  /* lock we're waiting for */
MT_Sema *semawait;  /* semaphore we're waiting for */
MT_Cond *condwait;  /* condition variable we're waiting for */
@@ -686,6 +694,36 @@ MT_thread_override_limits(void)
return self && self->limit_override;
 }
 
+static struct thread_init_cb {
+   struct thread_init_cb *next;
+   void (*init)(void *);
+   void (*destroy)(void *);
+   void *data;
+} *init_cb;
+static MT_Lock thread_init_lock = MT_LOCK_INITIALIZER(thread_init_lock);
+
+gdk_return
+MT_thread_init_add_callback(void (*init)(void *), void (*destroy)(void *), 
void *data)
+{
+   struct thread_init_cb *p = GDKmalloc(sizeof(struct thread_init_cb));
+
+   if (p == NULL)
+   return GDK_FAIL;
+   *p = (struct thread_init_cb) {
+   .init = init,
+   .destroy = destroy,
+   .next = NULL,
+   .data = data,
+   };
+   MT_lock_set(&thread_init_lock);
+   struct thread_init_cb **pp = &init_cb;
+   while (*pp)
+   pp = &(*pp)->next;
+   *pp = p;
+   MT_lock_unset(&thread_init_lock);
+   return GDK_SUCCEED;
+}
+
 #ifdef HAVE_PTHREAD_H
 static void *
 #else
@@ -719,7 +757,16 @@ thread_starter(void *arg)
self->data = NULL;
self->sp = THRsp();
thread_setself(self);
+   for (int i = 0; i < self->nthread_funcs; i++) {
+   if (self->thread_funcs[i].init)
+   
(*self->thread_funcs[i].init)(self->thread_funcs[i].data);
+   }
(*self->func)(data);
+   for (int i = 0; i < self->nthread_funcs; i++) {
+   if (self->thread_funcs[i].destroy)
+   
(*self->thread_funcs[i].destroy)(self->thread_funcs[i].data);
+   }
+   free(self->thread_funcs);
ATOMIC_SET(&self->exited, 1);
TRC_DEBUG(THRD, "Exit thread \"%s\"\n", self->threadname);
return 0;   /* NULL for pthreads, 0 for Windows */
@@ -843,6 +890,33 @@ MT_create_thread(MT_Id *t, void (*f) (vo
.refs = 1,
.tid = mtid,
};
+   MT_lock_set(&thread_init_lock);
+   /* remember the list of callback functions we need to call for
+* this thread (i.e. anything registered so far) */
+   for (struct thread_init_cb *p = init_cb; p; p = p->next)
+   self->nthread_funcs++;
+   if (self->nthread_funcs > 0) {
+   self->thread_funcs = malloc(self->nthread_funcs * 
sizeof(*self->thread_funcs));
+   if (self->thread_funcs == NULL) {
+   GDKsyserror("Cannot allocate memory\n");
+   MT_lock_unset(&thread_init_lock);
+   free(self);
+#ifdef HAVE_PTHREAD_H
+   pthread_attr_destroy(&attr);
+#endif
+   return -1;
+   }
+   int n = 0;
+   for (struct thread_init_cb *p = init_cb; p; p = p->next) {
+   self->thread_funcs[n++] = (struct thread_funcs) {
+   .init = p->init,
+   .destroy = p->destroy,
+   .data = p->data,
+   };
+   }
+   }
+   MT_lock_unset(&thread_init_lock);
+
ATOMIC_INIT(&self->exited, 0);
strcpy_len(self->threadname, threadname, sizeof(self->threadname));
char *p;
@@ -869,6 +943,7 @@ MT_create_thread(MT_Id *t, void (*f) (vo
 #endif
if (ret != 0) {
GDKsyserr(ret, "Cannot start thread");
+   fre

MonetDB: client_interrupts - Approve API change.

2024-01-10 Thread Sjoerd Mullender via checkin-list 
Changeset: 828aa13dbfe9 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/828aa13dbfe9
Modified Files:
clients/Tests/exports.stable.out
Branch: client_interrupts
Log Message:

Approve API change.


diffs (20 lines):

diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -681,6 +681,7 @@ MapiMsg mapi_ping(Mapi mid) __attribute_
 MapiHdl mapi_prepare(Mapi mid, const char *cmd) 
__attribute__((__nonnull__(1)));
 MapiMsg mapi_prepare_handle(MapiHdl hdl, const char *cmd) 
__attribute__((__nonnull__(1)));
 MapiHdl mapi_query(Mapi mid, const char *cmd) __attribute__((__nonnull__(1)));
+MapiMsg mapi_query_abort(MapiHdl hdl, int reason) 
__attribute__((__nonnull__(1)));
 MapiMsg mapi_query_done(MapiHdl hdl) __attribute__((__nonnull__(1)));
 MapiMsg mapi_query_handle(MapiHdl hdl, const char *cmd) 
__attribute__((__nonnull__(1)));
 MapiMsg mapi_query_part(MapiHdl hdl, const char *cmd, size_t size) 
__attribute__((__nonnull__(1)));
@@ -797,7 +798,7 @@ BUN SQLload_file(Client cntxt, Tablet *a
 str TABLETcollect(BAT **bats, Tablet *as);
 str TABLETcreate_bats(Tablet *as, BUN est);
 void TABLETdestroy_format(Tablet *as);
-int TABLEToutput_file(Tablet *as, BAT *order, stream *s);
+int TABLEToutput_file(Tablet *as, BAT *order, stream *s, bstream *in);
 int TRACEtable(Client cntxt, BAT **r);
 int TYPE_xml;
 int UTF8_strlen(const char *restrict s);
___
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org

MonetDB: cleanup_types - removed useless SCALE_NOFIX, use SCALE_...

2024-01-10 Thread Niels Nes via checkin-list 
Changeset: 6e970b1b1309 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/6e970b1b1309
Modified Files:
sql/common/sql_types.c
sql/include/sql_catalog.h
sql/test/emptydb/Tests/check.stable.out
sql/test/emptydb/Tests/check.stable.out.32bit
sql/test/emptydb/Tests/check.stable.out.int128
Branch: cleanup_types
Log Message:

removed useless SCALE_NOFIX, use SCALE_NONE instead


diffs (88 lines):

diff --git a/sql/common/sql_types.c b/sql/common/sql_types.c
--- a/sql/common/sql_types.c
+++ b/sql/common/sql_types.c
@@ -995,8 +995,8 @@ sqltypeinit( sql_allocator *sa)
/* this requires a type definition */
 
floats = t;
-   FLT = *t++ = sql_create_type(sa, "REAL", 24, SCALE_NOFIX, 2, EC_FLT, 
"flt");
-   DBL = *t++ = sql_create_type(sa, "DOUBLE", 53, SCALE_NOFIX, 2, EC_FLT, 
"dbl");
+   FLT = *t++ = sql_create_type(sa, "REAL", 24, SCALE_NONE, 2, EC_FLT, 
"flt");
+   DBL = *t++ = sql_create_type(sa, "DOUBLE", 53, SCALE_NONE, 2, EC_FLT, 
"dbl");
 
dates = t;
MONINT = *t++ = sql_create_type(sa, "MONTH_INTERVAL", 3, 0, 10, 
EC_MONTH, "int"); /* 1 .. 13 enumerates the 13 different interval types */
diff --git a/sql/include/sql_catalog.h b/sql/include/sql_catalog.h
--- a/sql/include/sql_catalog.h
+++ b/sql/include/sql_catalog.h
@@ -82,13 +82,12 @@ typedef enum sql_dependency {
 #define SCALE_NONE 0
 #define SCALE_FIX  1   /* many numerical functions require equal
   scales/precision for all 
their inputs */
-#define SCALE_NOFIX2
-#define MAX_BITS   3
-#define SCALE_MUL  4   /* multiplication gives the sum of scales */
-#define SCALE_DIV  5   /* div on the other hand reduces the scales */
-#define DIGITS_ADD 6   /* some types grow under functions (concat) */
-#define INOUT  7   /* output type equals input type (of first 
input) */
-#define SCALE_EQ   8   /* user defined functions need equal scales */
+#define MAX_BITS   2
+#define SCALE_MUL  3   /* multiplication gives the sum of scales */
+#define SCALE_DIV  4   /* div on the other hand reduces the scales */
+#define DIGITS_ADD 5   /* some types grow under functions (concat) */
+#define INOUT  6   /* output type equals input type (of first 
input) */
+#define SCALE_EQ   7   /* user defined functions need equal scales */
 
 #define RDONLY 0
 #define RD_INS 1
@@ -500,7 +499,7 @@ typedef struct sql_func {
private:1;  /* certain functions cannot be bound from user queries 
*/
int fix_scale;
/*
-  SCALE_NOFIX/SCALE_NONE => nothing
+  SCALE_NONE => nothing
   SCALE_FIX => input scale fixing,
   SCALE_ADD => leave inputs as is and do add scales
   example numerical multiplication
diff --git a/sql/test/emptydb/Tests/check.stable.out 
b/sql/test/emptydb/Tests/check.stable.out
--- a/sql/test/emptydb/Tests/check.stable.out
+++ b/sql/test/emptydb/Tests/check.stable.out
@@ -5174,8 +5174,8 @@ select 'null in fkeys.delete_action', de
 [ "sys.types", "sys",  "date", "date", 0,  0,  0,  "DATE"  ]
 [ "sys.types", "sys",  "daytime",  "time", 7,  0,  0,  "TIME"  
]
 [ "sys.types", "sys",  "daytime",  "timetz",   7,  1,  0,  
"TIME_TZ"   ]
-[ "sys.types", "sys",  "dbl",  "double",   53, 2,  2,  "FLT"   
]
-[ "sys.types", "sys",  "flt",  "real", 24, 2,  2,  "FLT"   ]
+[ "sys.types", "sys",  "dbl",  "double",   53, 0,  2,  "FLT"   
]
+[ "sys.types", "sys",  "flt",  "real", 24, 0,  2,  "FLT"   ]
 [ "sys.types", "sys",  "inet", "inet", 0,  0,  0,  "EXTERNAL"  
]
 [ "sys.types", "sys",  "int",  "decimal",  9,  1,  10, "DEC"   
]
 [ "sys.types", "sys",  "int",  "int",  32, 1,  2,  "NUM"   ]
diff --git a/sql/test/emptydb/Tests/check.stable.out.32bit 
b/sql/test/emptydb/Tests/check.stable.out.32bit
--- a/sql/test/emptydb/Tests/check.stable.out.32bit
+++ b/sql/test/emptydb/Tests/check.stable.out.32bit
@@ -4832,8 +4832,8 @@ select 'null in fkeys.delete_action', de
 [ "sys.types", "sys",  "date", "date", 0,  0,  0,  "DATE"  ]
 [ "sys.types", "sys",  "daytime",  "time", 7,  0,  0,  "TIME"  
]
 [ "sys.types", "sys",  "daytime",  "timetz",   7,  1,  0,  
"TIME_TZ"   ]
-[ "sys.types", "sys",  "dbl",  "double",   53, 2,  2,  "FLT"   
]
-[ "sys.types", "sys",  "flt",  "real", 24, 2,  2,  "FLT"   ]
+[ "sys.types", "sys",  "dbl",  "double",   53, 0,  2,  "FLT"   
]
+[ "sys.types", "sys",  "flt",  "real", 24, 0,  2,  "FLT"   ]
 [ "sys.types", "sys",  "inet", "inet", 0,  0,  0,  "EXTERNAL"  
]
 [ "sys.types", "sys",  "int",  "decimal

MonetDB: cleanup_types - merged with default

2024-01-10 Thread Niels Nes via checkin-list 
Changeset: fd71f4732897 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/fd71f4732897
Modified Files:
sql/backends/monet5/rel_bin.c
Branch: cleanup_types
Log Message:

merged with default


diffs (truncated from 4223 to 300 lines):

diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -385,6 +385,7 @@ const char *MT_thread_getalgorithm(void)
 void *MT_thread_getdata(void);
 const char *MT_thread_getname(void);
 bool MT_thread_init(void);
+gdk_return MT_thread_init_add_callback(void (*init)(void *), void 
(*destroy)(void *), void *data);
 bool MT_thread_register(void);
 void MT_thread_set_qry_ctx(QryCtx *ctx);
 void MT_thread_setalgorithm(const char *algo);
diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -2024,6 +2024,7 @@ mapi_destroy(Mapi mid)
free(mid->server);
free(mid->uri);
free(mid->tracebuffer);
+   free(mid->noexplain);
if (mid->errorstr && mid->errorstr != mapi_nomem)
free(mid->errorstr);
 
diff --git a/gdk/gdk_system.c b/gdk/gdk_system.c
--- a/gdk/gdk_system.c
+++ b/gdk/gdk_system.c
@@ -184,10 +184,18 @@ GDKlockstatistics(int what)
 
 #endif /* LOCK_STATS */
 
+struct thread_funcs {
+   void (*init)(void *);
+   void (*destroy)(void *);
+   void *data;
+};
+
 static struct mtthread {
struct mtthread *next;
void (*func) (void *);  /* function to be called */
void *data; /* and its data */
+   struct thread_funcs *thread_funcs; /* callback funcs */
+   int nthread_funcs;
MT_Lock *lockwait;  /* lock we're waiting for */
MT_Sema *semawait;  /* semaphore we're waiting for */
MT_Cond *condwait;  /* condition variable we're waiting for */
@@ -686,6 +694,36 @@ MT_thread_override_limits(void)
return self && self->limit_override;
 }
 
+static struct thread_init_cb {
+   struct thread_init_cb *next;
+   void (*init)(void *);
+   void (*destroy)(void *);
+   void *data;
+} *init_cb;
+static MT_Lock thread_init_lock = MT_LOCK_INITIALIZER(thread_init_lock);
+
+gdk_return
+MT_thread_init_add_callback(void (*init)(void *), void (*destroy)(void *), 
void *data)
+{
+   struct thread_init_cb *p = GDKmalloc(sizeof(struct thread_init_cb));
+
+   if (p == NULL)
+   return GDK_FAIL;
+   *p = (struct thread_init_cb) {
+   .init = init,
+   .destroy = destroy,
+   .next = NULL,
+   .data = data,
+   };
+   MT_lock_set(&thread_init_lock);
+   struct thread_init_cb **pp = &init_cb;
+   while (*pp)
+   pp = &(*pp)->next;
+   *pp = p;
+   MT_lock_unset(&thread_init_lock);
+   return GDK_SUCCEED;
+}
+
 #ifdef HAVE_PTHREAD_H
 static void *
 #else
@@ -719,7 +757,16 @@ thread_starter(void *arg)
self->data = NULL;
self->sp = THRsp();
thread_setself(self);
+   for (int i = 0; i < self->nthread_funcs; i++) {
+   if (self->thread_funcs[i].init)
+   
(*self->thread_funcs[i].init)(self->thread_funcs[i].data);
+   }
(*self->func)(data);
+   for (int i = 0; i < self->nthread_funcs; i++) {
+   if (self->thread_funcs[i].destroy)
+   
(*self->thread_funcs[i].destroy)(self->thread_funcs[i].data);
+   }
+   free(self->thread_funcs);
ATOMIC_SET(&self->exited, 1);
TRC_DEBUG(THRD, "Exit thread \"%s\"\n", self->threadname);
return 0;   /* NULL for pthreads, 0 for Windows */
@@ -843,6 +890,33 @@ MT_create_thread(MT_Id *t, void (*f) (vo
.refs = 1,
.tid = mtid,
};
+   MT_lock_set(&thread_init_lock);
+   /* remember the list of callback functions we need to call for
+* this thread (i.e. anything registered so far) */
+   for (struct thread_init_cb *p = init_cb; p; p = p->next)
+   self->nthread_funcs++;
+   if (self->nthread_funcs > 0) {
+   self->thread_funcs = malloc(self->nthread_funcs * 
sizeof(*self->thread_funcs));
+   if (self->thread_funcs == NULL) {
+   GDKsyserror("Cannot allocate memory\n");
+   MT_lock_unset(&thread_init_lock);
+   free(self);
+#ifdef HAVE_PTHREAD_H
+   pthread_attr_destroy(&attr);
+#endif
+   return -1;
+   }
+   int n = 0;
+   for (struct thread_init_cb *p = init_cb; p; p = p->next) {
+   self->thread_funcs[n++] = (struct thread_funcs) {
+   .init = p->init,
+   .destroy = p->destroy,
+   .data = p->data,
+   };
+   }
+   }
+

MonetDB: client_interrupts - Don't shadow variable names; only n...

2024-01-10 Thread Sjoerd Mullender via checkin-list 
Changeset: 54858f81f090 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/54858f81f090
Modified Files:
clients/mapiclient/mclient.c
Branch: client_interrupts
Log Message:

Don't shadow variable names; only need to send abort on followup call.


diffs (26 lines):

diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c
--- a/clients/mapiclient/mclient.c
+++ b/clients/mapiclient/mclient.c
@@ -3219,15 +3219,17 @@ putfile(void *data, const char *filename
return flush < 0 ? "error writing output" : NULL;
}
if (state == INTERRUPT) {
-   char *filename;
+   char *fname;
  interrupted:
-   filename = strdup(mnstr_name(priv->f));
+   fname = strdup(mnstr_name(priv->f));
close_stream(priv->f);
priv->f = NULL;
-   if (filename) {
-   MT_remove(filename);
-   free(filename);
+   if (fname) {
+   MT_remove(fname);
+   free(fname);
}
+   if (filename == NULL)
+   (void) mapi_query_abort(mapi_get_active(priv->mid), 1);
return "query aborted";
}
if (mnstr_write(priv->f, buf, 1, bufsize) < (ssize_t) bufsize) {
___
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org

MonetDB: client_interrupts - Stop sending columnar (binary) resu...

2024-01-10 Thread Sjoerd Mullender via checkin-list 
Changeset: cdd7c7912b94 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/cdd7c7912b94
Modified Files:
sql/backends/monet5/sql_result.c
Branch: client_interrupts
Log Message:

Stop sending columnar (binary) results on interrupt.
Untested.


diffs (48 lines):

diff --git a/sql/backends/monet5/sql_result.c b/sql/backends/monet5/sql_result.c
--- a/sql/backends/monet5/sql_result.c
+++ b/sql/backends/monet5/sql_result.c
@@ -660,7 +660,7 @@ mvc_export_warning(stream *s, str w)
 }
 
 static int
-mvc_export_binary_bat(stream *s, BAT* bn)
+mvc_export_binary_bat(stream *s, BAT* bn, bstream *in)
 {
BATiter bni = bat_iterator(bn);
bool sendtheap = bni.type != TYPE_void, sendtvheap = sendtheap && 
bni.vh;
@@ -703,6 +703,8 @@ mvc_export_binary_bat(stream *s, BAT* bn
}
}
bat_iterator_end(&bni);
+   if (bstream_getoob(in))
+   return -5;
return 0;
 }
 
@@ -1134,7 +1136,7 @@ mvc_export_row(backend *b, stream *s, re
 }
 
 static int
-mvc_export_table_columnar(stream *s, res_table *t)
+mvc_export_table_columnar(stream *s, res_table *t, bstream *in)
 {
int i, res = 0;
 
@@ -1151,7 +1153,7 @@ mvc_export_table_columnar(stream *s, res
if (b == NULL)
return -2;
 
-   res = mvc_export_binary_bat(s, b);
+   res = mvc_export_binary_bat(s, b, in);
BBPunfix(b->batCacheid);
if (res < 0)
return res;
@@ -1738,7 +1740,7 @@ mvc_export_result(backend *b, stream *s,
if (b->client->protocol == PROTOCOL_COLUMNAR) {
if (mnstr_flush(s, MNSTR_FLUSH_DATA) < 0)
return -4;
-   return mvc_export_table_columnar(s, t);
+   return mvc_export_table_columnar(s, t, m->scanner.rs);
}
 
count = m->reply_size;
___
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org

MonetDB: client_interrupts - Thinko.

2024-01-10 Thread Sjoerd Mullender via checkin-list 
Changeset: 291429165bff for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/291429165bff
Modified Files:
common/stream/mapi_stream.c
Branch: client_interrupts
Log Message:

Thinko.


diffs (12 lines):

diff --git a/common/stream/mapi_stream.c b/common/stream/mapi_stream.c
--- a/common/stream/mapi_stream.c
+++ b/common/stream/mapi_stream.c
@@ -170,7 +170,7 @@ setup_transfer(const char *req, const ch
 
char buf[256];
nread = mnstr_readline(rs, buf, sizeof(buf));
-   ok = (nread == 0 || (nread == 1 && buf[0] == '\n') || !(oob = 
mnstr_getoob(rs)));
+   ok = ((nread == 0 || (nread == 1 && buf[0] == '\n')) && !(oob = 
mnstr_getoob(rs)));
if (!ok) {
switch (oob) {
case 1: /* client side 
interrupt */
___
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org

MonetDB: client_interrupts - Changelog and manual update.

2024-01-10 Thread Sjoerd Mullender via checkin-list 
Changeset: ca392ca1e4e5 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/ca392ca1e4e5
Modified Files:
clients/ChangeLog
clients/mapiclient/mclient.1
Branch: client_interrupts
Log Message:

Changelog and manual update.


diffs (29 lines):

diff --git a/clients/ChangeLog b/clients/ChangeLog
--- a/clients/ChangeLog
+++ b/clients/ChangeLog
@@ -1,3 +1,12 @@
 # ChangeLog file for clients
 # This file is updated with Maddlog
 
+* Wed Jan 10 2024 Sjoerd Mullender 
+- Implemented interrupt handling in mclient.  When using mclient
+  interactively, an interrupt (usually control-C) stops whatever the
+  client is doing.  When editing a line, the line is discarded; when
+  editing a second or later line of a query, the whole query is discarded;
+  when a query is being executed, the server is asked to stop the query
+  at its earliest convenience.  Stopping a running query can only be
+  done with an up-to-date server.  All of this does not work on Windows.
+
diff --git a/clients/mapiclient/mclient.1 b/clients/mapiclient/mclient.1
--- a/clients/mapiclient/mclient.1
+++ b/clients/mapiclient/mclient.1
@@ -107,6 +107,9 @@ To disable reading the
 file, set the variable
 .B DOTMONETDBFILE
 to the empty string in the environment.
+.PP
+When working interactively, an interrupt (usually control-C) will clear
+any query being edited and will stop any running query.
 .SH OPTIONS
 .SS
 General Options
___
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org