Changeset: 2405d7e48215 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=2405d7e48215
Added Files:
sql/backends/monet5/Tests/pyloader04.sql
sql/backends/monet5/Tests/pyloader04.stable.err
sql/backends/monet5/Tests/pyloader04.stable.out
Modified Files:
monetdb5/extras/pyapi/emit.c
sql/backends/monet5/Tests/All
Branch: pythonloader
Log Message:
Added testcase for various edge cases and added proper error messages for them.
diffs (truncated from 400 to 300 lines):
diff --git a/monetdb5/extras/pyapi/emit.c b/monetdb5/extras/pyapi/emit.c
--- a/monetdb5/extras/pyapi/emit.c
+++ b/monetdb5/extras/pyapi/emit.c
@@ -29,41 +29,73 @@
static PyObject *
_emit_emit(Py_EmitObject *self, PyObject *args) {
- size_t i, ai;
- ssize_t el_count = -1;
- str msg = MAL_SUCCEED;
- (void) self;
+ size_t i, ai; // iterators
+ ssize_t el_count = -1; // the amount of elements this emit call will write
to the table
+ size_t dict_elements, matched_elements;
+ str msg = MAL_SUCCEED; // return message
+
if (!PyDict_Check(args)) {
PyErr_SetString(PyExc_TypeError, "need dict");
return NULL;
}
+ matched_elements = 0;
+ dict_elements = PyDict_Size(args);
+ if (dict_elements == 0) {
+ PyErr_SetString(PyExc_TypeError, "dict must contain at least one
element");
+ return NULL;
+ }
for (i = 0; i < self->ncols; i++) {
PyObject *dictEntry = PyDict_GetItemString(args, self->cols[i].name);
- ssize_t this_size = 1;
if (dictEntry) {
+ ssize_t this_size = 1;
+ matched_elements++;
this_size = PyType_Size(dictEntry);
if (this_size < 0) {
PyErr_Format(PyExc_TypeError, "Unsupported Python Object %s",
PyString_AsString(PyObject_Str(PyObject_Type(dictEntry))));
return NULL;
}
- if (el_count < 0) el_count = this_size;
- else {
- if (el_count != this_size) {
- PyErr_Format(PyExc_TypeError, "Element %s has size %zu,
but expected an element with size %zu", self->cols[i].name, this_size,
el_count);
- return NULL;
- }
+ if (el_count < 0) {
+ el_count = this_size;
+ } else if (el_count != this_size) {
+ PyErr_Format(PyExc_TypeError, "Element %s has size %zu, but
expected an element with size %zu", self->cols[i].name, this_size, el_count);
+ return NULL;
}
}
}
- if (el_count < 1) {
- PyErr_SetString(PyExc_TypeError, "need at least some values");
- // todo: better error message!
+ if (el_count == 0) {
+ PyErr_SetString(PyExc_TypeError, "Empty input values supplied");
return NULL;
}
-
- // TODO: check for dict entries not matched by any column and complain if
present
-
+ if (matched_elements != dict_elements) {
+ // not all elements in the dictionary were matched, look for the
element that was not matched
+ PyObject *keys = PyDict_Keys(args);
+ for(i = 0; i < (size_t) PyList_Size(keys); i++) {
+ PyObject *key = PyList_GetItem(keys, i);
+ char *val;
+ if (!PyString_CheckExact(key)) {
+ // one of the keys in the dictionary was not a string
+ PyErr_Format(PyExc_TypeError, "Expected a string as dict key,
but found %s", PyString_AsString(PyObject_Str(PyObject_Type(key))));
+ goto loop_end;
+ }
+ val = PyString_AsString(key);
+ bool found = false;
+ for(ai = 0; ai < self->ncols; ai++) {
+ if (strcmp(val, self->cols[ai].name) == 0) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ // the current element was present in the dictionary, but it
has no matching column
+ PyErr_Format(PyExc_TypeError, "Unmatched element \"%s\" in
dict", val);
+ goto loop_end;
+ }
+ }
+loop_end:
+ Py_DECREF(keys);
+ goto wrapup;
+ }
for (i = 0; i < self->ncols; i++) {
PyObject *dictEntry = PyDict_GetItemString(args, self->cols[i].name);
@@ -187,7 +219,6 @@ static PyObject *
}
self->nvals += el_count;
Py_RETURN_NONE;
-
wrapup:
if (msg != MAL_SUCCEED) {
PyErr_Format(PyExc_TypeError, "Failed conversion: %s", msg);
diff --git a/sql/backends/monet5/Tests/All b/sql/backends/monet5/Tests/All
--- a/sql/backends/monet5/Tests/All
+++ b/sql/backends/monet5/Tests/All
@@ -54,6 +54,7 @@ HAVE_LIBPY?pyapi29
HAVE_LIBPY?pyloader01
HAVE_LIBPY?pyloader02
HAVE_LIBPY?pyloader03
+HAVE_LIBPY?pyloader04
# should this work?
diff --git a/sql/backends/monet5/Tests/pyloader04.sql
b/sql/backends/monet5/Tests/pyloader04.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/pyloader04.sql
@@ -0,0 +1,69 @@
+
+# unmatched element (a3) present in dict, this should throw an error
+START TRANSACTION;
+CREATE TABLE pyloader04table(a1 int, a2 int);
+CREATE LOADER pyloader04() LANGUAGE PYTHON {
+ _emit.emit({'a1': 3, 'a2': 4, 'a3': 5})
+};
+COPY INTO pyloader04table FROM LOADER pyloader04();
+ROLLBACK;
+
+# use non-string type as key
+START TRANSACTION;
+CREATE TABLE pyloader04table(a1 int, a2 int);
+CREATE LOADER pyloader04() LANGUAGE PYTHON {
+ _emit.emit({'a1': 3, 'a2': 4, 3: 5})
+};
+COPY INTO pyloader04table FROM LOADER pyloader04();
+ROLLBACK;
+
+# return empty list
+START TRANSACTION;
+CREATE TABLE pyloader04table(a1 int, a2 int);
+CREATE LOADER pyloader04() LANGUAGE PYTHON {
+ _emit.emit({'a1': [], 'a2': numpy.array([])})
+};
+COPY INTO pyloader04table FROM LOADER pyloader04();
+ROLLBACK;
+
+# empty dictionary
+START TRANSACTION;
+CREATE TABLE pyloader04table(a1 int, a2 int);
+CREATE LOADER pyloader04() LANGUAGE PYTHON {
+ _emit.emit({})
+};
+COPY INTO pyloader04table FROM LOADER pyloader04();
+ROLLBACK;
+
+# unsupported python object
+START TRANSACTION;
+CREATE TABLE pyloader04table(a1 int, a2 int);
+CREATE LOADER pyloader04() LANGUAGE PYTHON {
+ class MyClass:
+ i = 1234
+
+ _emit.emit({'a1': MyClass()})
+};
+COPY INTO pyloader04table FROM LOADER pyloader04();
+ROLLBACK;
+
+# fail str -> int conversion
+START TRANSACTION;
+CREATE TABLE pyloader04table(a1 int, a2 int);
+CREATE LOADER pyloader04() LANGUAGE PYTHON {
+ _emit.emit({'a1': 'hello'})
+};
+COPY INTO pyloader04table FROM LOADER pyloader04();
+ROLLBACK;
+
+# test quoted names
+START TRANSACTION;
+CREATE TABLE pyloader04table("select" int, "from" int);
+CREATE LOADER pyloader04() LANGUAGE PYTHON {
+ _emit.emit({'select': 3, 'from': 4})
+};
+COPY INTO pyloader04table FROM LOADER pyloader04();
+SELECT * FROM pyloader04table;
+DROP TABLE pyloader04table;
+DROP LOADER pyloader04;
+ROLLBACK;
diff --git a/sql/backends/monet5/Tests/pyloader04.stable.err
b/sql/backends/monet5/Tests/pyloader04.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/pyloader04.stable.err
@@ -0,0 +1,82 @@
+stderr of test 'pyloader04` in directory 'sql/backends/monet5` itself:
+
+
+# 12:30:43 >
+# 12:30:43 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set"
"mapi_open=true" "--set" "mapi_port=34261" "--set"
"mapi_usock=/var/tmp/mtest-7433/.s.monetdb.34261" "--set" "monet_prompt="
"--forcemito"
"--dbpath=/home/mytherin/opt/var/MonetDB/mTests_sql_backends_monet5" "--set"
"embedded_r=yes" "--set" "embedded_py=true"
+# 12:30:43 >
+
+# builtin opt gdk_dbpath = /home/mytherin/opt/var/monetdb5/dbfarm/demo
+# builtin opt gdk_debug = 0
+# builtin opt gdk_vmtrim = no
+# builtin opt monet_prompt = >
+# builtin opt monet_daemon = no
+# builtin opt mapi_port = 50000
+# builtin opt mapi_open = false
+# builtin opt mapi_autosense = false
+# builtin opt sql_optimizer = default_pipe
+# builtin opt sql_debug = 0
+# cmdline opt gdk_nr_threads = 0
+# cmdline opt mapi_open = true
+# cmdline opt mapi_port = 34261
+# cmdline opt mapi_usock = /var/tmp/mtest-7433/.s.monetdb.34261
+# cmdline opt monet_prompt =
+# cmdline opt gdk_dbpath =
/home/mytherin/opt/var/MonetDB/mTests_sql_backends_monet5
+# cmdline opt embedded_r = yes
+# cmdline opt embedded_py = true
+# cmdline opt gdk_debug = 536870922
+
+# 12:30:43 >
+# 12:30:43 > "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e"
"--host=/var/tmp/mtest-7433" "--port=34261"
+# 12:30:43 >
+
+MAPI = (monetdb) /var/tmp/mtest-7433/.s.monetdb.34261
+QUERY = COPY INTO pyloader04table FROM LOADER pyloader04();
+ERROR = !Python exception
+ ! 1. def pyfun(_emit,_conn):
+ ! 2. _emit.emit({'a1': 3, 'a2': 4, 'a3': 5})
+ !> 3.
+ !Unmatched element "a3" in dict
+MAPI = (monetdb) /var/tmp/mtest-7433/.s.monetdb.34261
+QUERY = COPY INTO pyloader04table FROM LOADER pyloader04();
+ERROR = !Python exception
+ !def pyfun(_emit,_conn):
+ ! _emit.emit({'a1': 3, 'a2': 4, 3: 5})
+ !
+ !Expected a string as dict key, but found <type 'int'>
+MAPI = (monetdb) /var/tmp/mtest-7433/.s.monetdb.34261
+QUERY = COPY INTO pyloader04table FROM LOADER pyloader04();
+ERROR = !Python exception
+ !
+ ! 1. def pyfun(_emit,_conn):
+ !> 2. _emit.emit({'a1': [], 'a2': numpy.array([])})
+ ! 3.
+ !Empty input values supplied
+MAPI = (monetdb) /var/tmp/mtest-7433/.s.monetdb.34261
+QUERY = COPY INTO pyloader04table FROM LOADER pyloader04();
+ERROR = !Python exception
+ !
+ ! 1. def pyfun(_emit,_conn):
+ !> 2. _emit.emit({})
+ ! 3.
+ !dict must contain at least one element
+MAPI = (monetdb) /var/tmp/mtest-7433/.s.monetdb.34261
+QUERY = COPY INTO pyloader04table FROM LOADER pyloader04();
+ERROR = !Python exception
+ ! 2. class MyClass:
+ ! 3. i = 1234
+ !> 4. _emit.emit({'a1': MyClass()})
+ ! 5.
+ !Unsupported Python Object <type 'instance'>
+MAPI = (monetdb) /var/tmp/mtest-7433/.s.monetdb.34261
+QUERY = COPY INTO pyloader04table FROM LOADER pyloader04();
+ERROR = !Python exception
+ !
+ ! 1. def pyfun(_emit,_conn):
+ !> 2. _emit.emit({'a1': 'hello'})
+ ! 3.
+ !Conversion Failed: Error converting string.
+
+# 12:30:44 >
+# 12:30:44 > "Done."
+# 12:30:44 >
+
diff --git a/sql/backends/monet5/Tests/pyloader04.stable.out
b/sql/backends/monet5/Tests/pyloader04.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/pyloader04.stable.out
@@ -0,0 +1,121 @@
+stdout of test 'pyloader04` in directory 'sql/backends/monet5` itself:
+
+
+# 12:30:43 >
+# 12:30:43 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set"
"mapi_open=true" "--set" "mapi_port=34261" "--set"
"mapi_usock=/var/tmp/mtest-7433/.s.monetdb.34261" "--set" "monet_prompt="
"--forcemito"
"--dbpath=/home/mytherin/opt/var/MonetDB/mTests_sql_backends_monet5" "--set"
"embedded_r=yes" "--set" "embedded_py=true"
+# 12:30:43 >
+
+# MonetDB 5 server v11.24.0
+# This is an unreleased version
+# Serving database 'mTests_sql_backends_monet5', using 8 threads
+# Compiled for x86_64-unknown-linux-gnu/64bit with 64bit OIDs and 128bit
integers dynamically linked
+# Found 7.682 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2016 MonetDB B.V., all rights reserved
+# Visit http://www.monetdb.org/ for further information
+# Listening for connection requests on mapi:monetdb://mytherin:34261/
+# Listening for UNIX domain connection requests on
mapi:monetdb:///var/tmp/mtest-7433/.s.monetdb.34261
+# MonetDB/SQL module loaded
+# MonetDB/Python module loaded
+# MonetDB/R module loaded
+
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
