Changeset: cbbe10a4b78d for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=cbbe10a4b78d
Modified Files:
clients/mapiclient/dump.c
clients/mapiclient/msqldump.c
sql/backends/monet5/UDF/pyapi/conversion.c
sql/backends/monet5/UDF/pyapi/convert_loops.h
sql/backends/monet5/UDF/pyapi/pyapi.c
sql/backends/monet5/UDF/pyapi/pyloader.c
sql/backends/monet5/UDF/pyapi/pytypes.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_result.c
sql/backends/monet5/sql_scenario.c
sql/backends/monet5/sql_upgrades.c
Branch: default
Log Message:
Merge with Nov2019
diffs (truncated from 586 to 300 lines):
diff --git a/clients/mapiclient/dump.c b/clients/mapiclient/dump.c
--- a/clients/mapiclient/dump.c
+++ b/clients/mapiclient/dump.c
@@ -128,7 +128,17 @@ squoted_print(stream *f, const char *s,
if (mnstr_printf(f, "%c", quote) < 0)
return -1;
while (*s) {
+ size_t n = strcspn(s, "\\'\"\177"
+ "\001\002\003\004\005\006\007"
+ "\010\011\012\013\014\015\016\017"
+ "\020\021\022\023\024\025\026\027"
+ "\030\031\032\033\034\035\036\037");
+ if (n > 0 && mnstr_write(f, s, 1, n) < 0)
+ return -1;
+ s += n;
switch (*s) {
+ case '\0':
+ continue;
case '\\':
if (mnstr_write(f, "\\\\", 1, 2) < 0)
return -1;
@@ -148,13 +158,8 @@ squoted_print(stream *f, const char *s,
return -1;
break;
default:
- if ((0 < *s && *s < 32) || *s == '\177') {
- if (mnstr_printf(f, "\\%03o", (uint8_t) *s) < 0)
- return -1;
- } else {
- if (mnstr_write(f, s, 1, 1) < 0)
- return -1;
- }
+ if (mnstr_printf(f, "\\%03o", (uint8_t) *s) < 0)
+ return -1;
break;
}
s++;
@@ -1783,14 +1788,13 @@ dump_table_data(Mapi mid, const char *sc
strcmp(tp, "url") == 0 ||
strcmp(tp, "uuid") == 0 ||
string[i])
- squoted_print(toConsole, s, useInserts
? '\'' : '"');
+ squoted_print(toConsole, s, '\'');
else
mnstr_printf(toConsole, "%s", s);
} else if (string[i]) {
- /* write double or single-quoted
- string with certain characters
- escaped */
- squoted_print(toConsole, s, useInserts ? '\'' :
'"');
+ /* write double-quoted string with
+ certain characters escaped */
+ squoted_print(toConsole, s, '"');
} else
mnstr_printf(toConsole, "%s", s);
diff --git a/clients/mapiclient/msqldump.c b/clients/mapiclient/msqldump.c
--- a/clients/mapiclient/msqldump.c
+++ b/clients/mapiclient/msqldump.c
@@ -190,7 +190,7 @@ main(int argc, char **argv)
fprintf(stderr, "%s", motd);
}
mapi_trace(mid, trace);
- mapi_cache_limit(mid, 10000);
+ mapi_cache_limit(mid, -1);
out = file_wastream(stdout, "stdout");
if (out == NULL) {
diff --git a/sql/backends/monet5/UDF/pyapi/conversion.c
b/sql/backends/monet5/UDF/pyapi/conversion.c
--- a/sql/backends/monet5/UDF/pyapi/conversion.c
+++ b/sql/backends/monet5/UDF/pyapi/conversion.c
@@ -37,6 +37,11 @@ static bool IsBlobType(int type)
return type == TYPE_blob;
}
+static bool IsVoidType(int type)
+{
+ return type == TYPE_void;
+}
+
PyObject *PyArrayObject_FromScalar(PyInput *inp, char **return_message)
{
PyObject *vararray = NULL;
@@ -44,6 +49,18 @@ PyObject *PyArrayObject_FromScalar(PyInp
assert(inp->scalar); // input has to be a scalar
switch (inp->bat_type) {
+ case TYPE_void:
+ vararray = PyArray_Arange(0, 1, 1,
+#if SIZEOF_OID == SIZEOF_INT
+
NPY_UINT
+#else
+
NPY_ULONGLONG
+#endif
+ );
+ break;
+ case TYPE_oid:
+ vararray = PyInt_FromLong((long)(*(oid *)inp->dataptr));
+ break;
case TYPE_bit:
vararray = PyInt_FromLong((long)(*(bit *)inp->dataptr));
break;
@@ -165,7 +182,7 @@ PyObject *PyArrayObject_FromBAT(PyInput
goto wrapup;
}
- if (!IsBlobType(inp->bat_type) &&
+ if (!IsVoidType(inp->bat_type) && !IsBlobType(inp->bat_type) &&
(!IsStandardBATType(inp->bat_type) ||
ConvertableSQLType(inp->sql_subtype))) { // if the sql type is
set, we
// have to do some conversion
@@ -205,6 +222,24 @@ PyObject *PyArrayObject_FromBAT(PyInput
}
} else {
switch (inp->bat_type) {
+ case TYPE_void:
+ BAT_TO_NP_CREATE_ALWAYS(b,
+#if SIZEOF_OID == SIZEOF_INT
+ NPY_UINT
+#else
+ NPY_ULONGLONG
+#endif
+ );
+ break;
+ case TYPE_oid:
+ BAT_TO_NP(b, oid,
+#if SIZEOF_OID == SIZEOF_INT
+ NPY_UINT32
+#else
+ NPY_UINT64
+#endif
+ );
+ break;
case TYPE_bit:
BAT_TO_NP(b, bit, NPY_INT8);
break;
@@ -927,6 +962,9 @@ BAT *PyObject_ConvertToBAT(PyReturn *ret
BATsettrivprop(b);
} else {
switch (bat_type) {
+ case TYPE_void:
+ NP_CREATE_EMPTY_BAT(b, oid);
+ break;
case TYPE_bit:
NP_CREATE_BAT(b, bit);
break;
diff --git a/sql/backends/monet5/UDF/pyapi/convert_loops.h
b/sql/backends/monet5/UDF/pyapi/convert_loops.h
--- a/sql/backends/monet5/UDF/pyapi/convert_loops.h
+++ b/sql/backends/monet5/UDF/pyapi/convert_loops.h
@@ -12,6 +12,9 @@
* these are in a separate header because they are used in multiple places
*/
+#define BAT_TO_NP_CREATE_ALWAYS(bat, nptpe)
\
+ vararray = PyArray_Arange(0, (double)bat->batCount, 1, nptpe);
+
#define BAT_TO_NP(bat, mtpe, nptpe)
\
if (copy) {
\
vararray = PyArray_EMPTY(1, elements, nptpe, 0);
\
@@ -561,6 +564,22 @@ convert_and_append(BAT* b, const char* t
#define NOT_HGE(mtpe) true
#endif
+#define NP_CREATE_EMPTY_BAT(bat, mtpe) \
+ { \
+ bat = COLnew(seqbase, TYPE_##mtpe, (BUN)ret->count, TRANSIENT);
\
+ if (bat == NULL) {
\
+ msg = createException(MAL, "pyapi.eval",
SQLSTATE(PY000) "Cannot create column"); \
+ goto wrapup;
\
+ }
\
+ bat->tkey = false;
\
+ bat->tsorted = false;
\
+ bat->trevsorted = false;
\
+ bat->tnil = false;
\
+ bat->tnonil = true;
\
+ BATsetcount(bat, (BUN)ret->count);
\
+ BATsettrivprop(bat);
\
+ }
+
// This very big #define combines all the previous #defines for one big #define
// that is responsible for converting a Numpy array (described in the PyReturn
// object 'ret')
diff --git a/sql/backends/monet5/UDF/pyapi/pyapi.c
b/sql/backends/monet5/UDF/pyapi/pyapi.c
--- a/sql/backends/monet5/UDF/pyapi/pyapi.c
+++ b/sql/backends/monet5/UDF/pyapi/pyapi.c
@@ -281,7 +281,7 @@ static str PyAPIeval(Client cntxt, MalBl
}
seqbase = b->hseqbase;
inp->count = BATcount(b);
- inp->bat_type = ATOMstorage(getBatType(getArgType(mb,
pci, i)));
+ inp->bat_type = b->ttype;
inp->bat = b;
if (inp->count == 0) {
// one of the input BATs is empty, don't
execute the function at
@@ -635,9 +635,10 @@ static str PyAPIeval(Client cntxt, MalBl
result_array = PyArrayObject_FromScalar(
&pyinput_values[i - (pci->retc + 2)], &msg);
} else {
+ int type = pyinput_values[i - (pci->retc + 2)].bat_type;
result_array = PyMaskedArray_FromBAT(
&pyinput_values[i - (pci->retc + 2)], t_start,
t_end, &msg,
- !enable_zerocopy_input);
+ !enable_zerocopy_input && type != TYPE_void);
}
if (result_array == NULL) {
if (msg == MAL_SUCCEED) {
@@ -756,6 +757,9 @@ static str PyAPIeval(Client cntxt, MalBl
if (!input.scalar) {
switch (input.bat_type) {
+ case TYPE_void:
+ NP_SPLIT_BAT(oid);
+ break;
case TYPE_bit:
NP_SPLIT_BAT(bit);
break;
@@ -1072,7 +1076,8 @@ static str PyAPIeval(Client cntxt, MalBl
// if someone has some problem with memory
size exploding when
// using PYTHON_MAP but it being fine in
regular PYTHON this
// is probably the issue
- int bat_type = getBatType(getArgType(mb, pci,
i));
+ PyInput *inp = &pyinput_values[i - (pci->retc +
2)];
+ int bat_type = inp->bat_type;
PyObject *new_array = PyArray_FromAny(
ret->numpy_array,
PyArray_DescrFromType(BatType_ToPyType(bat_type)), 1, 1,
@@ -1502,6 +1507,38 @@ static void ComputeParallelAggregation(A
} else {
npy_intp elements[1] = {group_elements};
switch (input.bat_type) {
+ case TYPE_void:
+ vararray = PyArray_New(
+ &PyArray_Type, 1,
elements,
+#if SIZEOF_OID == SIZEOF_INT
+ NPY_UINT
+#else
+ NPY_ULONGLONG
+#endif
+ ,
+ NULL,
+ ((oid
***)(*p->split_bats))[group_it][i], 0,
+ NPY_ARRAY_CARRAY ||
!NPY_ARRAY_WRITEABLE, NULL);
+ break;
+ case TYPE_oid:
+ vararray = PyArray_New(
+ &PyArray_Type, 1,
elements,
+#if SIZEOF_OID == SIZEOF_INT
+ NPY_UINT32
+#else
+ NPY_UINT64
+#endif
+ ,
+ NULL,
+ ((oid
***)(*p->split_bats))[group_it][i], 0,
+ NPY_ARRAY_CARRAY ||
!NPY_ARRAY_WRITEABLE, NULL);
+ break;
+ case TYPE_bit:
+ vararray = PyArray_New(
+ &PyArray_Type, 1,
elements, NPY_BOOL, NULL,
+ ((bit
***)(*p->split_bats))[group_it][i], 0,
+ NPY_ARRAY_CARRAY ||
!NPY_ARRAY_WRITEABLE, NULL);
+ break;
case TYPE_bte:
vararray = PyArray_New(
&PyArray_Type, 1,
elements, NPY_INT8, NULL,
diff --git a/sql/backends/monet5/UDF/pyapi/pyloader.c
b/sql/backends/monet5/UDF/pyapi/pyloader.c
--- a/sql/backends/monet5/UDF/pyapi/pyloader.c
+++ b/sql/backends/monet5/UDF/pyapi/pyloader.c
@@ -138,7 +138,7 @@ PYFUNCNAME(PyAPIevalLoader)(Client cntxt
}
inp.scalar = false;
inp.count = BATcount(b);
- inp.bat_type = ATOMstorage(getBatType(getArgType(mb,
pci, i)));
+ inp.bat_type = getBatType(getArgType(mb, pci, i));
inp.bat = b;
val = PyMaskedArray_FromBAT(
diff --git a/sql/backends/monet5/UDF/pyapi/pytypes.c
b/sql/backends/monet5/UDF/pyapi/pytypes.c
--- a/sql/backends/monet5/UDF/pyapi/pytypes.c
+++ b/sql/backends/monet5/UDF/pyapi/pytypes.c
@@ -113,6 +113,8 @@ char *BatType_Format(int type)
return "BLOB";
}
switch (type) {
+ case TYPE_void:
+ return "VOID";
case TYPE_bit:
return "BOOL";
case TYPE_bte:
@@ -157,12 +159,9 @@ int PyType_ToBat(int type)
#endif
case NPY_LONGLONG:
return TYPE_lng;
- case NPY_UBYTE:
- case NPY_USHORT:
- case NPY_UINT:
- case NPY_ULONG:
- case NPY_ULONGLONG:
- return TYPE_void;
+ case NPY_UINT32:
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
