Changeset: d5f5b22d73e9 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d5f5b22d73e9
Added Files:
sql/backends/monet5/Tests/pyapi30.sql
Modified Files:
monetdb5/extras/pyapi/Makefile.ag
monetdb5/extras/pyapi/convert_loops.h
sql/backends/monet5/Tests/All
sql/backends/monet5/Tests/pyapi30.stable.out
Branch: pythonloader
Log Message:
Don't assume BAT contains no nils when a NumPy array without mask is returned
(in case there are values such as -2^31).
diffs (165 lines):
diff --git a/monetdb5/extras/pyapi/Makefile.ag
b/monetdb5/extras/pyapi/Makefile.ag
--- a/monetdb5/extras/pyapi/Makefile.ag
+++ b/monetdb5/extras/pyapi/Makefile.ag
@@ -24,7 +24,7 @@ MTSAFE
lib__pyapi = {
MODULE
DIR = libdir/monetdb5
- SOURCES = pyapi.c pyapi.h unicode.c unicode.h pytypes.c pytypes.h
type_conversion.c type_conversion.h formatinput.c formatinput.h connection.c
connection.h unspecified_evil.h pyloader.c emit.h emit.c
+ SOURCES = pyapi.c pyapi.h unicode.c unicode.h pytypes.c pytypes.h
type_conversion.c type_conversion.h formatinput.c formatinput.h connection.c
connection.h unspecified_evil.h pyloader.c emit.h emit.c convert_loops.h
XDEPS = $(libpy_LIBDEP)
LIBS = ../../tools/libmonetdb5 \
../../../gdk/libbat \
diff --git a/monetdb5/extras/pyapi/convert_loops.h
b/monetdb5/extras/pyapi/convert_loops.h
--- a/monetdb5/extras/pyapi/convert_loops.h
+++ b/monetdb5/extras/pyapi/convert_loops.h
@@ -27,8 +27,11 @@
bat->T->nil = 1;
\
}
\
}
\
+ bat->T->nonil = 1 - bat->T->nil;
\
+ } else {
\
+ bat->T->nil = 0; bat->T->nonil = 0;
\
}
\
- bat->T->nonil = 1 - bat->T->nil;
\
+
\
/*When we create a BAT a small part of memory is allocated, free it*/
\
GDKfree(bat->T->heap.base);
\
bat->T->heap.base = &data[(index_offset * ret->count) *
ret->memory_size]; \
@@ -312,8 +315,8 @@
goto wrapup;
\
}
\
data = (char*) ret->array_data;
\
- if (!copy && ret->count > 0 && TYPE_##mtpe ==
PyType_ToBat(ret->result_type) && (ret->count * ret->memory_size < BUN_MAX) &&
\
- (ret->numpy_array == NULL ||
PyArray_FLAGS((PyArrayObject*)ret->numpy_array) & NPY_ARRAY_OWNDATA)) {
\
+ if (!copy && ret->count > 0 && TYPE_##mtpe ==
PyType_ToBat(ret->result_type) && (ret->count * ret->memory_size < BUN_MAX) &&
\
+ (ret->numpy_array == NULL ||
PyArray_FLAGS((PyArrayObject*)ret->numpy_array) & NPY_ARRAY_OWNDATA)) {
\
/*We can only create a direct map if the numpy array type and
target BAT type*/
\
/*are identical, otherwise we have to do a conversion.*/
\
if (ret->numpy_array == NULL) {
\
@@ -325,13 +328,14 @@
CREATE_BAT_ZEROCOPY(bat, mtpe, STORE_CMEM);
\
}
\
} else {
\
- bat = BATnew(TYPE_void, TYPE_##mtpe, (BUN) ret->count, TRANSIENT);
\
- BATseqbase(bat, seqbase); bat->T->nil = 0; bat->T->nonil = 1;
\
+ bat = BATnew(TYPE_void, TYPE_##mtpe, (BUN) ret->count, TRANSIENT);
\
+ BATseqbase(bat, seqbase);
\
if (NOT_HGE(mtpe) && TYPE_##mtpe !=
PyType_ToBat(ret->result_type)) WARNING_MESSAGE("!PERFORMANCE WARNING: You are
returning a Numpy Array of type %s, which has to be converted to a BAT of type
%s. If you return a Numpy\
Array of type %s no copying will be needed.\n",
PyType_Format(ret->result_type), BatType_Format(TYPE_##mtpe),
PyType_Format(BatType_ToPyType(TYPE_##mtpe))); \
bat->tkey = 0; bat->tsorted = 0; bat->trevsorted = 0;
\
- NP_INSERT_BAT(bat, mtpe, 0);
\
- BATsetcount(bat, (BUN) ret->count);
\
+ NP_INSERT_BAT(bat, mtpe, 0);
\
+ if (!mask) { bat->T->nil = 0; bat->T->nonil = 0; }
\
+ BATsetcount(bat, (BUN) ret->count);
\
BATsettrivprop(bat);
\
}
\
}
diff --git a/sql/backends/monet5/Tests/All b/sql/backends/monet5/Tests/All
--- a/sql/backends/monet5/Tests/All
+++ b/sql/backends/monet5/Tests/All
@@ -50,6 +50,8 @@ HAVE_LIBPY?pyapi26
HAVE_LIBPY?pyapi27
HAVE_LIBPY?pyapi28
HAVE_LIBPY?pyapi29
+HAVE_LIBPY?pyapi29
+HAVE_LIBPY?pyapi30
HAVE_LIBPY?pyloader01
HAVE_LIBPY?pyloader02
diff --git a/sql/backends/monet5/Tests/pyapi30.sql
b/sql/backends/monet5/Tests/pyapi30.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/pyapi30.sql
@@ -0,0 +1,16 @@
+
+# test returning of numpy.nan values
+
+START TRANSACTION;
+
+CREATE FUNCTION pyapi32() RETURNS TABLE(i INTEGER, j DOUBLE) LANGUAGE PYTHON {
+ return {'i': numpy.nan, 'j': numpy.nan }
+};
+
+SELECT * FROM pyapi32();
+SELECT * FROM pyapi32() WHERE i IS NULL;
+SELECT * FROM pyapi32() WHERE j IS NULL;
+
+DROP FUNCTION pyapi32;
+
+ROLLBACK;
diff --git a/sql/backends/monet5/Tests/pyapi30.stable.out
b/sql/backends/monet5/Tests/pyapi30.stable.out
--- a/sql/backends/monet5/Tests/pyapi30.stable.out
+++ b/sql/backends/monet5/Tests/pyapi30.stable.out
@@ -66,51 +66,25 @@ Ready.
#CREATE LOADER myfunc() LANGUAGE PYTHON {
# _emit.emit({'a':42,'d':1})
#};
-#CREATE LOADER myfunc1(i integer) LANGUAGE PYTHON {
-# _emit.emit({'a':i,'d':2})
-#};
-#CREATE LOADER myfunc2(i integer, f string) LANGUAGE PYTHON {
-# _emit.emit({'a':i,'d':3})
-#};
-#CREATE LOADER myfunc3(i integer, f string, d double) LANGUAGE PYTHON {
-# _emit.emit({'a':i,'d':4})
-#};
-#SELECT name,func,mod,language,type,side_effect,varres,vararg FROM functions
WHERE name='myfunc';
-% sys.functions, sys.functions, sys.functions, sys.functions,
sys.functions, sys.functions, sys.functions, sys.functions # table_name
-% name, func, mod, language, type, side_effect, varres,
vararg # name
-% varchar, varchar, varchar, int, int, boolean,
boolean, boolean # type
-% 6, 32, 5, 1, 1, 5, 5, 5 # length
-[ "myfunc", "{\n\t_emit.emit({'a':42,'d':1})\n};", "pyapi", 6,
7, true, true, false ]
-#COPY INTO mytable FROM LOADER myfunc3(46, 'asdf', 3.2);
-[ 1 ]
-#COPY INTO mytable FROM LOADER myfunc2(45, 'asdf');
-[ 1 ]
-#COPY INTO mytable FROM LOADER myfunc1(44);
-[ 1 ]
-#COPY INTO mytable FROM LOADER myfunc();
-[ 1 ]
-#SELECT * FROM mytable;
-% sys.mytable, sys.mytable, sys.mytable # table_name
-% a, d, s # name
-% double, int, clob # type
-% 24, 1, 5 # length
-[ 46, 4, "hello" ]
-[ 45, 3, NULL ]
-[ 44, 2, NULL ]
-[ 42, 1, NULL ]
-#DROP TABLE mytable;
-#DROP ALL LOADER myfunc;
-#CREATE LOADER myfunc() LANGUAGE PYTHON {
-#};
-#DROP LOADER myfunc;
-#DROP LOADER myfunc1;
-#DROP LOADER myfunc2;
-#DROP LOADER myfunc3;
-#SELECT * FROM functions WHERE name='myfunc';
-% sys.functions, sys.functions, sys.functions, sys.functions,
sys.functions, sys.functions, sys.functions, sys.functions, sys.functions,
sys.functions # table_name
-% id, name, func, mod, language, type, side_effect, varres,
vararg, schema_id # name
-% int, varchar, varchar, varchar, int, int,
boolean, boolean, boolean, int # type
-% 1, 0, 0, 0, 1, 1, 5, 5, 5, 1 #
length
+#SELECT * FROM pyapi32();
+% ., . # table_name
+% i, j # name
+% int, double # type
+% 1, 24 # length
+[ NULL, NULL ]
+#SELECT * FROM pyapi32() WHERE i IS NULL;
+% ., . # table_name
+% i, j # name
+% int, double # type
+% 1, 24 # length
+[ NULL, NULL ]
+#SELECT * FROM pyapi32() WHERE j IS NULL;
+% ., . # table_name
+% i, j # name
+% int, double # type
+% 1, 24 # length
+[ NULL, NULL ]
+#DROP FUNCTION pyapi32;
#ROLLBACK;
# 17:27:12 >
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
