Changeset: 7b7fc905a056 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7b7fc905a056 Modified Files: monetdb5/modules/atoms/blob.c sql/backends/monet5/Tests/pyloader04.stable.err sql/backends/monet5/UDF/pyapi/convert_loops.h sql/backends/monet5/UDF/pyapi/emit.c sql/backends/monet5/UDF/pyapi/pyapi.c sql/backends/monet5/UDF/pyapi/pyloader.c Branch: default Log Message:
Merge with Dec2016 branch.
diffs (truncated from 476 to 300 lines):
diff --git a/monetdb5/modules/atoms/blob.c b/monetdb5/modules/atoms/blob.c
--- a/monetdb5/modules/atoms/blob.c
+++ b/monetdb5/modules/atoms/blob.c
@@ -99,7 +99,7 @@ blob_cmp(blob *l, blob *r)
size_t len = l->nitems;
if (len != r->nitems)
- return len - r->nitems;
+ return len < r->nitems ? -1 : len > r->nitems ? 1 : 0;
if (len == ~(size_t) 0)
return (0);
diff --git a/sql/backends/monet5/Tests/pyloader04.stable.err
b/sql/backends/monet5/Tests/pyloader04.stable.err
--- a/sql/backends/monet5/Tests/pyloader04.stable.err
+++ b/sql/backends/monet5/Tests/pyloader04.stable.err
@@ -32,16 +32,18 @@ stderr of test 'pyloader04` in directory
MAPI = (monetdb) /var/tmp/mtest-6147/.s.monetdb.38930
QUERY = COPY LOADER INTO pyloader04table FROM pyloader04();
ERROR = !Python exception
+ !
! 1. def pyfun(_emit,_conn):
- ! 2. _emit.emit({'a1': 3, 'a2': 4, 'a3': 5})
- !> 3.
+ !> 2. _emit.emit({'a1': 3, 'a2': 4, 'a3': 5})
+ ! 3.
!Unmatched element "a3" in dict
MAPI = (monetdb) /var/tmp/mtest-6147/.s.monetdb.38930
QUERY = COPY LOADER INTO pyloader04table FROM pyloader04();
ERROR = !Python exception
+ !
! 1. def pyfun(_emit,_conn):
- ! 2. _emit.emit({'a1': 3, 'a2': 4, 3: 5})
- !> 3.
+ !> 2. _emit.emit({'a1': 3, 'a2': 4, 3: 5})
+ ! 3.
!Unmatched element "3" in dict
MAPI = (monetdb) /var/tmp/mtest-79373/.s.monetdb.33370
QUERY = COPY LOADER INTO pyloader04table FROM pyloader04();
@@ -74,7 +76,7 @@ ERROR = !Python exception
! 1. def pyfun(_emit,_conn):
!> 2. _emit.emit({'a1': 'hello'})
! 3.
- !Conversion Failed: Error converting string.
+ !Failed conversion: Error converting string.
# 12:30:44 >
# 12:30:44 > "Done."
diff --git a/sql/backends/monet5/UDF/pyapi/convert_loops.h
b/sql/backends/monet5/UDF/pyapi/convert_loops.h
--- a/sql/backends/monet5/UDF/pyapi/convert_loops.h
+++ b/sql/backends/monet5/UDF/pyapi/convert_loops.h
@@ -253,8 +253,10 @@
for (iu = 0; iu < ret->count; iu++)
\
{
\
snprintf(utf8_string, utf8string_minlength, fmt,
*((mtpe*)&data[(index_offset * ret->count + iu) * ret->memory_size]));
\
- if (BUNappend(bat, utf8_string, FALSE) != GDK_SUCCEED) \
- goto bunins_failed; \
+ if (BUNappend(bat, utf8_string, FALSE) != GDK_SUCCEED) { \
+ msg = createException(MAL, "pyapi.eval", "BUNappend
failed.\n"); \
+ goto wrapup;
\
+ }
\
}
\
}
\
else
\
@@ -264,14 +266,18 @@
if (mask[index_offset * ret->count + iu] == TRUE)
\
{
\
bat->tnil = 1;
\
- if (BUNappend(bat, str_nil, FALSE) != GDK_SUCCEED) \
- goto bunins_failed; \
+ if (BUNappend(bat, str_nil, FALSE) != GDK_SUCCEED) { \
+ msg = createException(MAL, "pyapi.eval", "BUNappend
failed.\n"); \
+ goto wrapup;
\
+ }
\
}
\
else
\
{
\
snprintf(utf8_string, utf8string_minlength, fmt,
*((mtpe*)&data[(index_offset * ret->count + iu) * ret->memory_size]));
\
- if (BUNappend(bat, utf8_string, FALSE) != GDK_SUCCEED) \
- goto bunins_failed; \
+ if (BUNappend(bat, utf8_string, FALSE) != GDK_SUCCEED) { \
+ msg = createException(MAL, "pyapi.eval", "BUNappend
failed.\n"); \
+ goto wrapup;
\
+ }
\
}
\
}
\
}
@@ -326,15 +332,19 @@
for (iu = 0; iu < ret->count; iu++) {
\
if (mask != NULL && (mask[index_offset * ret->count + iu])
== TRUE) {
\
b->tnil = 1;
\
- if (BUNappend(b, str_nil, FALSE) != GDK_SUCCEED) \
- goto bunins_failed; \
+ if (BUNappend(b, str_nil, FALSE) != GDK_SUCCEED) { \
+ msg = createException(MAL, "pyapi.eval", "BUNappend
failed.\n"); \
+ goto wrapup;
\
+ }
\
} else {
\
if (!string_copy(&data[(index_offset * ret->count + iu)
* ret->memory_size], utf8_string, ret->memory_size, false)) {
\
msg = createException(MAL, "pyapi.eval", "Invalid
string encoding used. Please return a regular ASCII string, or a Numpy_Unicode
object.\n"); \
goto wrapup;
\
}
\
- if (BUNappend(b, utf8_string, FALSE) != GDK_SUCCEED) \
- goto bunins_failed; \
+ if (BUNappend(b, utf8_string, FALSE) != GDK_SUCCEED) { \
+ msg = createException(MAL, "pyapi.eval", "BUNappend
failed.\n"); \
+ goto wrapup;
\
+ }
\
}
\
}
\
break;
\
@@ -342,12 +352,16 @@
for (iu = 0; iu < ret->count; iu++) {
\
if (mask != NULL && (mask[index_offset * ret->count + iu])
== TRUE) {
\
b->tnil = 1;
\
- if (BUNappend(b, str_nil, FALSE) != GDK_SUCCEED) \
- goto bunins_failed; \
+ if (BUNappend(b, str_nil, FALSE) != GDK_SUCCEED) { \
+ msg = createException(MAL, "pyapi.eval", "BUNappend
failed.\n"); \
+ goto wrapup;
\
+ }
\
} else {
\
utf32_to_utf8(0, ret->memory_size / 4, utf8_string,
(const Py_UNICODE*)(&data[(index_offset * ret->count + iu) *
ret->memory_size])); \
- if (BUNappend(b, utf8_string, FALSE) != GDK_SUCCEED) \
- goto bunins_failed; \
+ if (BUNappend(b, utf8_string, FALSE) != GDK_SUCCEED) { \
+ msg = createException(MAL, "pyapi.eval", "BUNappend
failed.\n"); \
+ goto wrapup;
\
+ }
\
}
\
}
\
break;
\
@@ -369,13 +383,17 @@
for (iu = 0; iu < ret->count; iu++) {
\
if (mask != NULL && (mask[index_offset * ret->count + iu])
== TRUE) {
\
b->tnil = 1;
\
- if (BUNappend(b, str_nil, FALSE) != GDK_SUCCEED) \
- goto bunins_failed; \
+ if (BUNappend(b, str_nil, FALSE) != GDK_SUCCEED) { \
+ msg = createException(MAL, "pyapi.eval", "BUNappend
failed.\n"); \
+ goto wrapup;
\
+ }
\
} else {
\
/* we try to handle as many types as possible */
\
pyobject_to_str(((PyObject**) &data[(index_offset *
ret->count + iu) * ret->memory_size]), utf8_size, &utf8_string);
\
- if (BUNappend(b, utf8_string, FALSE) != GDK_SUCCEED) \
- goto bunins_failed; \
+ if (BUNappend(b, utf8_string, FALSE) != GDK_SUCCEED) { \
+ msg = createException(MAL, "pyapi.eval", "BUNappend
failed.\n"); \
+ goto wrapup;
\
+ }
\
}
\
}
\
break;
\
diff --git a/sql/backends/monet5/UDF/pyapi/emit.c
b/sql/backends/monet5/UDF/pyapi/emit.c
--- a/sql/backends/monet5/UDF/pyapi/emit.c
+++ b/sql/backends/monet5/UDF/pyapi/emit.c
@@ -15,12 +15,12 @@
#include "unicode.h"
-#define scalar_convert(tpe) {\
- tpe val = (tpe) tpe##_nil; msg = pyobject_to_##tpe(&dictEntry, 42, &val); \
- BUNappend(self->cols[i].b, &val, 0); \
- if (msg != MAL_SUCCEED) { \
- PyErr_Format(PyExc_TypeError, "Conversion Failed: %s", msg); \
- return NULL; \
+#define scalar_convert(tpe) { \
+ tpe val = (tpe) tpe##_nil; \
+ msg = pyobject_to_##tpe(&dictEntry, 42, &val); \
+ if (msg != MAL_SUCCEED || BUNappend(self->cols[i].b, &val, 0) !=
GDK_SUCCEED) { \
+ if (msg == MAL_SUCCEED) msg = GDKstrdup("BUNappend failed."); \
+ goto wrapup; \
}}
@@ -30,6 +30,7 @@ PyEmit_Emit(PyEmitObject *self, PyObject
ssize_t el_count = -1; // the amount of elements this emit call will write
to the table
size_t dict_elements, matched_elements;
str msg = MAL_SUCCEED; // return message
+ bool error = false;
if (!PyDict_Check(args)) {
PyErr_SetString(PyExc_TypeError, "need dict");
@@ -80,6 +81,10 @@ PyEmit_Emit(PyEmitObject *self, PyObject
if (matched_elements != dict_elements) {
// not all elements in the dictionary were matched, look for the
element that was not matched
PyObject *keys = PyDict_Keys(args);
+ if (!keys) {
+ msg = GDKstrdup(MAL_MALLOC_FAIL);
+ goto wrapup;
+ }
for(i = 0; i < (size_t) PyList_Size(keys); i++) {
PyObject *key = PyList_GetItem(keys, i);
char *val = NULL;
@@ -100,6 +105,7 @@ PyEmit_Emit(PyEmitObject *self, PyObject
if (!found) {
// the current element was present in the dictionary, but
it has no matching column
PyErr_Format(PyExc_TypeError, "Unmatched element \"%s\" in
dict", val);
+ error = true;
goto loop_end;
}
}
@@ -114,7 +120,7 @@ PyEmit_Emit(PyEmitObject *self, PyObject
// allocate space for new columns (if any new columns show up)
sql_emit_col *old = self->cols;
// FIXME unchecked_malloc GDKmalloc can return NULL
- self->cols = GDKmalloc(sizeof(sql_emit_col) * potential_size);
+ self->cols = GDKzalloc(sizeof(sql_emit_col) * potential_size);
if (old) {
memcpy(self->cols, old, sizeof(sql_emit_col) * self->maxcols);
GDKfree(old);
@@ -132,6 +138,7 @@ PyEmit_Emit(PyEmitObject *self, PyObject
if (msg != MAL_SUCCEED) {
// one of the keys in the dictionary was not a string
PyErr_Format(PyExc_TypeError, "Could not convert object type
%s to a string: %s", PyString_AsString(PyObject_Str(PyObject_Type(key))), msg);
+ error = true;
Py_DECREF(keys);
goto wrapup;
}
@@ -152,7 +159,8 @@ PyEmit_Emit(PyEmitObject *self, PyObject
int bat_type = TYPE_int;
if (!array) {
PyErr_Format(PyExc_TypeError, "Failed to create NumPy
array.");
- return NULL;
+ error = true;
+ goto wrapup;
}
array_type = (PyArray_Descr*)
PyArray_DESCR((PyArrayObject*)array);
bat_type = PyType_ToBat(array_type->type_num);
@@ -163,8 +171,10 @@ PyEmit_Emit(PyEmitObject *self, PyObject
if (self->nvals > 0) {
// insert NULL values up until the current entry
for (ai = 0; ai < self->nvals; ai++) {
- // FIXME unchecked_malloc ATOMnil can return
NULL
- BUNappend(self->cols[self->ncols].b,
ATOMnil(self->cols[self->ncols].b->ttype), 0);
+ if (BUNappend(self->cols[self->ncols].b,
ATOMnil(self->cols[self->ncols].b->ttype), 0) != GDK_SUCCEED) {
+ msg = GDKstrdup("BUNappend failed.");
+ goto wrapup;
+ }
}
self->cols[i].b->tnil = 1;
self->cols[i].b->tnonil = 0;
@@ -213,20 +223,24 @@ PyEmit_Emit(PyEmitObject *self, PyObject
case TYPE_str:
{
str val = NULL;
+ gdk_return retval;
msg = pyobject_to_str(&dictEntry, 42, &val);
- BUNappend(self->cols[i].b, val, 0);
- if (val) {
- free(val);
+ if (msg != MAL_SUCCEED) {
+ goto wrapup;
}
- if (msg != MAL_SUCCEED) {
- PyErr_Format(PyExc_TypeError, "Conversion Failed:
%s", msg);
- return NULL;
+ assert(val);
+ retval = BUNappend(self->cols[i].b, val, 0);
+ free(val);
+ if (retval != GDK_SUCCEED) {
+ msg = GDKstrdup("BUNappend failed.");
+ goto wrapup;
}
}
break;
default:
PyErr_Format(PyExc_TypeError, "Unsupported BAT Type
%s", BatType_Format(self->cols[i].b->ttype));
- return NULL;
+ error = true;
+ goto wrapup;
}
} else {
bool *mask = NULL;
@@ -236,17 +250,18 @@ PyEmit_Emit(PyEmitObject *self, PyObject
size_t index_offset = 0;
size_t iu = 0;
if (BATextend(self->cols[i].b, self->nvals + el_count) !=
GDK_SUCCEED) {
- PyErr_Format(PyExc_TypeError, "Failed to allocate memory
to extend BAT.");
- return NULL;
+ msg = GDKstrdup("Failed to allocate memory to extend
BAT.");
+ goto wrapup;
}
msg = PyObject_GetReturnValues(dictEntry, ret);
- if (ret->mask_data != NULL) {
- mask = (bool*) ret->mask_data;
+ if (msg != MAL_SUCCEED) {
+ goto wrapup;
}
if (ret->array_data == NULL) {
- msg = createException(MAL, "pyapi.eval", "No return value
stored in the structure.\n");
+ msg = GDKstrdup("No return value stored in the
structure.");
goto wrapup;
}
+ mask = (bool*) ret->mask_data;
data = (char*) ret->array_data;
assert((size_t) el_count == (size_t) ret->count);
switch (self->cols[i].b->ttype) {
@@ -292,13 +307,16 @@ PyEmit_Emit(PyEmitObject *self, PyObject
break;
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
