Changeset: 33be2b324bb6 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=33be2b324bb6 Added Files: sql/test/BugTracker-2020/Tests/global_table_propagation.Bug-6846.py sql/test/BugTracker-2020/Tests/global_table_propagation.Bug-6846.stable.err sql/test/BugTracker-2020/Tests/global_table_propagation.Bug-6846.stable.out Modified Files: clients/Tests/All clients/Tests/MAL-signatures.stable.out clients/Tests/MAL-signatures.stable.out.int128 clients/Tests/SingleServer monetdb5/modules/mal/tablet.c monetdb5/optimizer/opt_pushselect.c sql/ChangeLog.Jun2020 sql/backends/monet5/sql.c sql/backends/monet5/sql_cat.c sql/backends/monet5/sql_result.c sql/server/rel_schema.c sql/server/sql_mvc.c sql/server/sql_partition.c sql/storage/bat/bat_storage.c sql/storage/sql_storage.h sql/storage/store.c sql/test/BugTracker-2018/Tests/nested-merge-tables.Bug-6585.stable.err sql/test/BugTracker-2019/Tests/grant-select-column.Bug-6765.py sql/test/BugTracker-2019/Tests/grant-select-column.Bug-6765.stable.err sql/test/BugTracker-2019/Tests/grant-select-column.Bug-6765.stable.out sql/test/BugTracker-2019/Tests/python-loader-string.Bug-6759.py sql/test/BugTracker-2019/Tests/python-loader-string.Bug-6759.stable.err sql/test/BugTracker-2019/Tests/python-loader-string.Bug-6759.stable.out sql/test/BugTracker-2020/Tests/All sql/test/BugTracker-2020/Tests/alter-table-add-column.Bug-6816.py sql/test/BugTracker-2020/Tests/alter-table-add-column.Bug-6816.stable.err sql/test/BugTracker-2020/Tests/alter-table-add-column.Bug-6816.stable.out sql/test/BugTracker-2020/Tests/functions-not-persist.Bug-6819.SQL.py sql/test/BugTracker-2020/Tests/functions-not-persist.Bug-6819.stable.err sql/test/BugTracker-2020/Tests/functions-not-persist.Bug-6819.stable.out sql/test/merge-partitions/Tests/mergepart05.stable.err sql/test/merge-partitions/Tests/mergepart07.stable.err sql/test/mergetables/Tests/mergedropcrash.stable.err sql/test/pg_regress/Tests/alter_table.stable.err sql/test/pg_regress/Tests/drop.stable.err Branch: default Log Message:
Merged with Jun2020 diffs (truncated from 1895 to 300 lines): diff --git a/clients/Tests/All b/clients/Tests/All --- a/clients/Tests/All +++ b/clients/Tests/All @@ -1,4 +1,4 @@ exports -HAVE_FITS&HAVE_GEOM&HAVE_LIBR&HAVE_LIDAR&HAVE_NETCDF&HAVE_SHP?MAL-signatures +HAVE_FITS&HAVE_GEOM&HAVE_LIBR&HAVE_LIBPY3&HAVE_LIDAR&HAVE_NETCDF&HAVE_SHP&NOT_WIN32?MAL-signatures MERCURIAL?malcheck mclient-uri diff --git a/clients/Tests/MAL-signatures.stable.out b/clients/Tests/MAL-signatures.stable.out --- a/clients/Tests/MAL-signatures.stable.out +++ b/clients/Tests/MAL-signatures.stable.out @@ -64,7 +64,7 @@ stdout of test 'MAL-signatures` in direc % .%1, .%1, .%1, .%1, .%1 # table_name % module, function, signature, address, comment # name % clob, clob, clob, clob, clob # type -% 10, 28, 364, 38, 874 # length +% 12, 28, 364, 38, 874 # length [ "aggr", "allnotequal", "command aggr.allnotequal(l:bat[:any_1], r:bat[:any_1]):bit ", "SQLallnotequal;", "if all values in r are not equal to l return true, else if r has nil nil else false" ] [ "aggr", "anyequal", "pattern aggr.anyequal(l:any_1, r:any_1):bit ", "CMDvarEQ;", "" ] [ "aggr", "anyequal", "command aggr.anyequal(l:bat[:any_1], r:bat[:any_1]):bit ", "SQLanyequal;", "if any value in r is equal to l return true, else if r has nil nil else false" ] @@ -11242,6 +11242,14 @@ stdout of test 'MAL-signatures` in direc [ "batmtime", "year", "command batmtime.year(d:bat[:date]):bat[:int] ", "MTIMEdate_extract_year_bulk;", "" ] [ "batpcre", "replace", "command batpcre.replace(orig:bat[:str], pat:str, repl:str, flag:str):bat[:str] ", "PCREreplace_bat_wrap;", "" ] [ "batpcre", "replace_first", "command batpcre.replace_first(orig:bat[:str], pat:str, repl:str, flag:str):bat[:str] ", "PCREreplacefirst_bat_wrap;", "" ] +[ "batpyapi3", "eval", "unsafe pattern batpyapi3.eval(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalStd;", "Execute a simple Python script value" ] +[ "batpyapi3", "eval_aggr", "unsafe pattern batpyapi3.eval_aggr(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalAggr;", "grouped aggregates through Python" ] +[ "batpyapi3", "eval_loader", "unsafe pattern batpyapi3.eval_loader(fptr:ptr, expr:str):any... ", "PYAPI3PyAPIevalLoader;", "loader functions through Python" ] +[ "batpyapi3", "eval_loader", "unsafe pattern batpyapi3.eval_loader(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalLoader;", "loader functions through Python" ] +[ "batpyapi3", "subeval_aggr", "unsafe pattern batpyapi3.subeval_aggr(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalAggr;", "grouped aggregates through Python" ] +[ "batpyapi3map", "eval", "pattern batpyapi3map.eval(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalStdMap;", "Execute a simple Python script value" ] +[ "batpyapi3map", "eval_aggr", "pattern batpyapi3map.eval_aggr(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalAggrMap;", "grouped aggregates through Python" ] +[ "batpyapi3map", "subeval_aggr", "pattern batpyapi3map.subeval_aggr(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalAggrMap;", "grouped aggregates through Python" ] [ "batrapi", "eval", "pattern batrapi.eval(fptr:ptr, expr:str, arg:any...):any... ", "RAPIevalStd;", "Execute a simple R script value" ] [ "batrapi", "eval_aggr", "pattern batrapi.eval_aggr(fptr:ptr, expr:str, arg:any...):any... ", "RAPIevalAggr;", "grouped aggregates through R" ] [ "batrapi", "subeval_aggr", "pattern batrapi.subeval_aggr(fptr:ptr, expr:str, arg:any...):any... ", "RAPIevalAggr;", "grouped aggregates through R" ] @@ -13949,6 +13957,17 @@ stdout of test 'MAL-signatures` in direc [ "profiler", "starttrace", "pattern profiler.starttrace():void ", "CMDstartTrace;", "Start collecting trace information" ] [ "profiler", "stop", "pattern profiler.stop():void ", "CMDstopProfiler;", "Stop offline performance profiling" ] [ "profiler", "stoptrace", "pattern profiler.stoptrace():void ", "CMDstopTrace;", "Stop collecting trace information" ] +[ "pyapi3", "eval", "unsafe pattern pyapi3.eval(fptr:ptr, expr:str):any ", "PYAPI3PyAPIevalStd;", "Execute a simple Python script returning a single value" ] +[ "pyapi3", "eval", "unsafe pattern pyapi3.eval(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalStd;", "Execute a simple Python script value" ] +[ "pyapi3", "eval_aggr", "unsafe pattern pyapi3.eval_aggr(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalAggr;", "grouped aggregates through Python" ] +[ "pyapi3", "eval_loader", "unsafe pattern pyapi3.eval_loader(fptr:ptr, expr:str):any... ", "PYAPI3PyAPIevalLoader;", "loader functions through Python" ] +[ "pyapi3", "eval_loader", "unsafe pattern pyapi3.eval_loader(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalLoader;", "loader functions through Python" ] +[ "pyapi3", "prelude", "command pyapi3.prelude():void ", "PYAPI3PyAPIprelude;", "" ] +[ "pyapi3", "subeval_aggr", "unsafe pattern pyapi3.subeval_aggr(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalAggr;", "grouped aggregates through Python" ] +[ "pyapi3map", "eval", "pattern pyapi3map.eval(fptr:ptr, expr:str):any ", "PYAPI3PyAPIevalStdMap;", "Execute a simple Python script returning a single value" ] +[ "pyapi3map", "eval", "pattern pyapi3map.eval(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalStdMap;", "Execute a simple Python script value" ] +[ "pyapi3map", "eval_aggr", "pattern pyapi3map.eval_aggr(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalAggrMap;", "grouped aggregates through Python" ] +[ "pyapi3map", "subeval_aggr", "pattern pyapi3map.subeval_aggr(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalAggrMap;", "grouped aggregates through Python" ] [ "querylog", "append", "pattern querylog.append(q:str, pipe:str, usr:str, tick:timestamp):void ", "QLOGappend;", "Add a new query call to the query log" ] [ "querylog", "call", "pattern querylog.call(tick1:timestamp, tick2:timestamp, arg:str, tuples:lng, xtime:lng, rtime:lng, cpu:int, iowait:int):void ", "QLOGcall;", "Add a new query call to the query log" ] [ "querylog", "context", "command querylog.context(release:str, version:str, revision:str, uri:str):void ", "QLOGcontextNaive;", "Noop operation, just marking the query" ] diff --git a/clients/Tests/MAL-signatures.stable.out.int128 b/clients/Tests/MAL-signatures.stable.out.int128 --- a/clients/Tests/MAL-signatures.stable.out.int128 +++ b/clients/Tests/MAL-signatures.stable.out.int128 @@ -68,7 +68,7 @@ stdout of test 'MAL-signatures` in direc % .%1, .%1, .%1, .%1, .%1 # table_name % module, function, signature, address, comment # name % clob, clob, clob, clob, clob # type -% 10, 28, 364, 38, 874 # length +% 12, 28, 364, 38, 874 # length [ "aggr", "allnotequal", "command aggr.allnotequal(l:bat[:any_1], r:bat[:any_1]):bit ", "SQLallnotequal;", "if all values in r are not equal to l return true, else if r has nil nil else false" ] [ "aggr", "anyequal", "pattern aggr.anyequal(l:any_1, r:any_1):bit ", "CMDvarEQ;", "" ] [ "aggr", "anyequal", "command aggr.anyequal(l:bat[:any_1], r:bat[:any_1]):bit ", "SQLanyequal;", "if any value in r is equal to l return true, else if r has nil nil else false" ] @@ -15997,6 +15997,14 @@ stdout of test 'MAL-signatures` in direc [ "batmtime", "year", "command batmtime.year(d:bat[:date]):bat[:int] ", "MTIMEdate_extract_year_bulk;", "" ] [ "batpcre", "replace", "command batpcre.replace(orig:bat[:str], pat:str, repl:str, flag:str):bat[:str] ", "PCREreplace_bat_wrap;", "" ] [ "batpcre", "replace_first", "command batpcre.replace_first(orig:bat[:str], pat:str, repl:str, flag:str):bat[:str] ", "PCREreplacefirst_bat_wrap;", "" ] +[ "batpyapi3", "eval", "unsafe pattern batpyapi3.eval(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalStd;", "Execute a simple Python script value" ] +[ "batpyapi3", "eval_aggr", "unsafe pattern batpyapi3.eval_aggr(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalAggr;", "grouped aggregates through Python" ] +[ "batpyapi3", "eval_loader", "unsafe pattern batpyapi3.eval_loader(fptr:ptr, expr:str):any... ", "PYAPI3PyAPIevalLoader;", "loader functions through Python" ] +[ "batpyapi3", "eval_loader", "unsafe pattern batpyapi3.eval_loader(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalLoader;", "loader functions through Python" ] +[ "batpyapi3", "subeval_aggr", "unsafe pattern batpyapi3.subeval_aggr(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalAggr;", "grouped aggregates through Python" ] +[ "batpyapi3map", "eval", "pattern batpyapi3map.eval(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalStdMap;", "Execute a simple Python script value" ] +[ "batpyapi3map", "eval_aggr", "pattern batpyapi3map.eval_aggr(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalAggrMap;", "grouped aggregates through Python" ] +[ "batpyapi3map", "subeval_aggr", "pattern batpyapi3map.subeval_aggr(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalAggrMap;", "grouped aggregates through Python" ] [ "batrapi", "eval", "pattern batrapi.eval(fptr:ptr, expr:str, arg:any...):any... ", "RAPIevalStd;", "Execute a simple R script value" ] [ "batrapi", "eval_aggr", "pattern batrapi.eval_aggr(fptr:ptr, expr:str, arg:any...):any... ", "RAPIevalAggr;", "grouped aggregates through R" ] [ "batrapi", "subeval_aggr", "pattern batrapi.subeval_aggr(fptr:ptr, expr:str, arg:any...):any... ", "RAPIevalAggr;", "grouped aggregates through R" ] @@ -19401,6 +19409,17 @@ stdout of test 'MAL-signatures` in direc [ "profiler", "starttrace", "pattern profiler.starttrace():void ", "CMDstartTrace;", "Start collecting trace information" ] [ "profiler", "stop", "pattern profiler.stop():void ", "CMDstopProfiler;", "Stop offline performance profiling" ] [ "profiler", "stoptrace", "pattern profiler.stoptrace():void ", "CMDstopTrace;", "Stop collecting trace information" ] +[ "pyapi3", "eval", "unsafe pattern pyapi3.eval(fptr:ptr, expr:str):any ", "PYAPI3PyAPIevalStd;", "Execute a simple Python script returning a single value" ] +[ "pyapi3", "eval", "unsafe pattern pyapi3.eval(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalStd;", "Execute a simple Python script value" ] +[ "pyapi3", "eval_aggr", "unsafe pattern pyapi3.eval_aggr(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalAggr;", "grouped aggregates through Python" ] +[ "pyapi3", "eval_loader", "unsafe pattern pyapi3.eval_loader(fptr:ptr, expr:str):any... ", "PYAPI3PyAPIevalLoader;", "loader functions through Python" ] +[ "pyapi3", "eval_loader", "unsafe pattern pyapi3.eval_loader(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalLoader;", "loader functions through Python" ] +[ "pyapi3", "prelude", "command pyapi3.prelude():void ", "PYAPI3PyAPIprelude;", "" ] +[ "pyapi3", "subeval_aggr", "unsafe pattern pyapi3.subeval_aggr(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalAggr;", "grouped aggregates through Python" ] +[ "pyapi3map", "eval", "pattern pyapi3map.eval(fptr:ptr, expr:str):any ", "PYAPI3PyAPIevalStdMap;", "Execute a simple Python script returning a single value" ] +[ "pyapi3map", "eval", "pattern pyapi3map.eval(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalStdMap;", "Execute a simple Python script value" ] +[ "pyapi3map", "eval_aggr", "pattern pyapi3map.eval_aggr(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalAggrMap;", "grouped aggregates through Python" ] +[ "pyapi3map", "subeval_aggr", "pattern pyapi3map.subeval_aggr(fptr:ptr, expr:str, arg:any...):any... ", "PYAPI3PyAPIevalAggrMap;", "grouped aggregates through Python" ] [ "querylog", "append", "pattern querylog.append(q:str, pipe:str, usr:str, tick:timestamp):void ", "QLOGappend;", "Add a new query call to the query log" ] [ "querylog", "call", "pattern querylog.call(tick1:timestamp, tick2:timestamp, arg:str, tuples:lng, xtime:lng, rtime:lng, cpu:int, iowait:int):void ", "QLOGcall;", "Add a new query call to the query log" ] [ "querylog", "context", "command querylog.context(release:str, version:str, revision:str, uri:str):void ", "QLOGcontextNaive;", "Noop operation, just marking the query" ] diff --git a/clients/Tests/SingleServer b/clients/Tests/SingleServer --- a/clients/Tests/SingleServer +++ b/clients/Tests/SingleServer @@ -1,2 +1,3 @@ ---set embedded_py=2 +--set embedded_py=3 --set embedded_r=true +--set embedded_c=true diff --git a/monetdb5/modules/mal/tablet.c b/monetdb5/modules/mal/tablet.c --- a/monetdb5/modules/mal/tablet.c +++ b/monetdb5/modules/mal/tablet.c @@ -1304,80 +1304,71 @@ SQLproducer(void *p) goto reportlackofinput; } for (e = s; *e && e < end && cnt < task->maxrow;) { - /* tokenize the record completely the format of the input - * should comply to the following grammar rule [ - * [[quote][[esc]char]*[quote]csep]*rsep]* where quote is - * a single user defined character within the quoted - * fields a character may be escaped with a backslash The - * user should supply the correct number of fields. - * In the first phase we simply break the lines at the - * record boundary. */ + /* tokenize the record completely + * + * The format of the input should comply to the following + * grammar rule [ [[quote][[esc]char]*[quote]csep]*rsep]* + * where quote is a single user-defined character. + * Within the quoted fields a character may be escaped + * with a backslash. The correct number of fields should + * be supplied. In the first phase we simply break the + * lines at the record boundary. */ int nutf = 0; int m = 0; bool bs = false; char q = 0; size_t i = 0; while (*e) { - /* check for correctly encoded UTF-8 */ - if (nutf > 0) { - if ((*e & 0xC0) != 0x80) - goto badutf8; - if (m != 0 && (*e & m) == 0) - goto badutf8; - m = 0; - nutf--; - } else if ((*e & 0xE0) == 0xC0) { - nutf = 1; - if ((e[0] & 0x1E) == 0) + if (task->skip > 0) { + /* no interpretation of data we're skipping, just + * look for newline */ + if (*e == '\n') + break; + } else { + /* check for correctly encoded UTF-8 */ + if (nutf > 0) { + if ((*e & 0xC0) != 0x80) + goto badutf8; + if (m != 0 && (*e & m) == 0) + goto badutf8; + m = 0; + nutf--; + } else if ((*e & 0xE0) == 0xC0) { + nutf = 1; + if ((e[0] & 0x1E) == 0) + goto badutf8; + } else if ((*e & 0xF0) == 0xE0) { + nutf = 2; + if ((e[0] & 0x0F) == 0) + m = 0x20; + } else if ((*e & 0xF8) == 0xF0) { + nutf = 3; + if ((e[0] & 0x07) == 0) + m = 0x30; + } else if ((*e & 0x80) != 0) { goto badutf8; - } else if ((*e & 0xF0) == 0xE0) { - nutf = 2; - if ((e[0] & 0x0F) == 0) - m = 0x20; - } else if ((*e & 0xF8) == 0xF0) { - nutf = 3; - if ((e[0] & 0x07) == 0) - m = 0x30; - } else if ((*e & 0x80) != 0) { - goto badutf8; - } - /* check for quoting and the row separator */ - if (bs) { - bs = false; - } else if (*e == '\\') { - bs = true; - i = 0; - } else if (*e == q) { - q = 0; - } else if (*e == quote) { - q = quote; - i = 0; - } else if (q == 0) { - i = rdfa[i][(unsigned char) *e]; - if (i == rseplen) - break; + } + /* check for quoting and the row separator */ + if (bs) { + bs = false; + } else if (*e == '\\') { + bs = true; + i = 0; + } else if (*e == q) { + q = 0; + } else if (*e == quote) { + q = quote; + i = 0; + } else if (q == 0) { + i = rdfa[i][(unsigned char) *e]; + if (i == rseplen) + break; + } } e++; } if (*e == 0) { partial = e - s; - e = NULL; /* nonterminated record, we need more */ - } - /* check for incomplete line and end of buffer condition */ - if (e) { - rowno++; - /* found a complete record, do we need to skip it? */ - if (--task->skip < 0 && cnt < task->maxrow) { - task->lines[cur][task->top[cur]++] = s; - cnt++; - } - *(e + 1 - rseplen) = 0; - s = ++e; - task->b->pos += (size_t) (e - base); - base = e; - if (task->top[cur] == task->limit) - break; - } else { /* found an incomplete record, saved for next round */ if (s+partial < end) { /* found a EOS in the input */ @@ -1386,6 +1377,22 @@ SQLproducer(void *p) goto reportlackofinput; } break; + } else { + rowno++; + if (task->skip > 0) { + task->skip--; + } else { + if (cnt < task->maxrow) { + task->lines[cur][task->top[cur]++] = s; + cnt++; + } + *(e + 1 - rseplen) = 0; + } + s = ++e; + task->b->pos += (size_t) (e - base); + base = e; + if (task->top[cur] == task->limit) + break; } } diff --git a/monetdb5/optimizer/opt_pushselect.c b/monetdb5/optimizer/opt_pushselect.c --- a/monetdb5/optimizer/opt_pushselect.c +++ b/monetdb5/optimizer/opt_pushselect.c @@ -297,7 +297,7 @@ OPTpushselectImplementation(Client cntxt } if (subselects.nr) { - if ((!nr_topn && !nr_likes) || newMalBlkStmt(mb, mb->ssize) <0 ) { + if (newMalBlkStmt(mb, mb->ssize) <0 ) { GDKfree(vars); goto wrapup; } @@ -429,8 +429,6 @@ OPTpushselectImplementation(Client cntxt for (; i<limit; i++) if (old[i]) pushInstruction(mb,old[i]); - GDKfree(slices); - GDKfree(rslices); GDKfree(old); GDKfree(vars); throw(MAL,"optimizer.pushselect", SQLSTATE(HY013) MAL_MALLOC_FAIL); @@ -469,7 +467,7 @@ OPTpushselectImplementation(Client cntxt slices = (int*) GDKzalloc(sizeof(int)* mb->vtop); rslices = (char*) GDKzalloc(sizeof(char)* mb->vtop); oclean = (char*) GDKzalloc(sizeof(char)* mb->vtop); - if (!nvars || !slices || !rslices || !oclean || newMalBlkStmt(mb, mb->stop+(5*push_down_delta)) <0 ) { + if (!nvars || !slices || !rslices || !oclean || newMalBlkStmt(mb, mb->stop+(5*push_down_delta)+(2*nr_topn)) <0 ) { mb->stmt = old; GDKfree(vars); GDKfree(nvars); @@ -697,6 +695,7 @@ OPTpushselectImplementation(Client cntxt actions++; } _______________________________________________ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list