Changeset: 51ad9f806cc6 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=51ad9f806cc6
Modified Files:
gdk/shared_memory.c
gdk/shared_memory.h
monetdb5/extras/pyapi/Tests/pyapi_returntypes.malC
monetdb5/extras/pyapi/Tests/pyapi_returntypes.stable.err
monetdb5/extras/pyapi/pyapi.c
sql/backends/monet5/Tests/pyapi10.stable.err
Branch: pyapi
Log Message:
Cleaned up code and added a lot of comments.
diffs (truncated from 1228 to 300 lines):
diff --git a/gdk/shared_memory.c b/gdk/shared_memory.c
--- a/gdk/shared_memory.c
+++ b/gdk/shared_memory.c
@@ -1,7 +1,15 @@
#include "shared_memory.h"
-#ifndef WIN32
+#ifndef false
+#define false 0
+#endif
+
+#ifndef true
+#define true 1
+#endif
+
+#ifndef _WIN32
#include <stdlib.h>
#include <assert.h>
@@ -27,12 +35,12 @@ static void **shm_ptrs;
static int shm_unique_id = 1;
static int shm_current_id = 0;
static int shm_max_id = 32;
-static bool shm_is_initialized = false;
+static int shm_is_initialized = false;
static char shm_keystring[] = ".";
void *init_shared_memory(int id, size_t size, int flags);
void store_shared_memory(int memory_id, void *ptr);
-bool release_shared_memory_id(int memory_id, void *ptr);
+int release_shared_memory_id(int memory_id, void *ptr);
int init_process_semaphore(int id, int count, int flags);
@@ -146,7 +154,7 @@ void *init_shared_memory(int id, size_t
return ptr;
}
-bool release_shared_memory(void *ptr)
+int release_shared_memory(void *ptr)
{
int i = 0;
int memory_id = -1;
@@ -170,7 +178,7 @@ bool release_shared_memory(void *ptr)
return release_shared_memory_id(memory_id, ptr);
}
-bool release_shared_memory_id(int memory_id, void *ptr)
+int release_shared_memory_id(int memory_id, void *ptr)
{
if (shmctl(memory_id, IPC_RMID, NULL) == -1)
{
@@ -222,7 +230,7 @@ int get_semaphore_value(int sem_id, int
return semval;
}
-bool change_semaphore_value(int sem_id, int number, int change)
+int change_semaphore_value(int sem_id, int number, int change)
{
struct sembuf buffer;
buffer.sem_num = number;
@@ -237,7 +245,7 @@ bool change_semaphore_value(int sem_id,
return true;
}
-bool release_process_semaphore(int sem_id)
+int release_process_semaphore(int sem_id)
{
if (semctl(sem_id, 0, IPC_RMID) < 0)
{
@@ -266,7 +274,7 @@ void* create_shared_memory(int id, size_
return NULL;
}
-bool release_shared_memory(void *ptr)
+int release_shared_memory(void *ptr)
{
(void) ptr;
NOTIMPLEMENTED();
@@ -309,14 +317,14 @@ int get_semaphore_value(int sem_id, int
return -1;
}
-bool change_semaphore_value(int sem_id, int number, int change)
+int change_semaphore_value(int sem_id, int number, int change)
{
(void) sem_id; (void) number; (void) change;
NOTIMPLEMENTED();
return false;
}
-bool release_process_semaphore(int sem_id)
+int release_process_semaphore(int sem_id)
{
(void) sem_id;
NOTIMPLEMENTED();
diff --git a/gdk/shared_memory.h b/gdk/shared_memory.h
--- a/gdk/shared_memory.h
+++ b/gdk/shared_memory.h
@@ -14,23 +14,14 @@
#ifndef _SHAREDMEMORY_LIB_
#define _SHAREDMEMORY_LIB_
-#include <stdbool.h>
-#include <stdio.h>
-
-/*
-//! Returns the ID of the shared memory, returns -1 on failure
-int init_shared_memory(char *keystring, int id, size_t size, int flags);
-//! Returns the pointer to the shared memory, argument is the return value of
init_shared_memory
-void *get_shared_memory_address(int memory_id);
-//! Release the chunk of shared memory
-bool release_shared_memory(int memory_id, void *ptr);*/
+#include <stddef.h>
//! Initialize the shared memory module
void initialize_shared_memory(void);
//! Not thread safe
void* create_shared_memory(int id, size_t size);
//! Not thread safe
-bool release_shared_memory(void *ptr);
+int release_shared_memory(void *ptr);
//! Not thread safe
int get_unique_shared_memory_id(int offset);
//! This is thread safe
@@ -43,9 +34,9 @@ int get_process_semaphore(int id, int co
//! Returns value of semaphore <number> at semaphore id <sem_id>
int get_semaphore_value(int sem_id, int number);
//! Change the semaphore <number> at semaphore id <sem_id> value by <change>
(change = 1 means +1, not set the value to 1)
-bool change_semaphore_value(int sem_id, int number, int change);
+int change_semaphore_value(int sem_id, int number, int change);
//! Release semaphore at sem_id
-bool release_process_semaphore(int sem_id);
+int release_process_semaphore(int sem_id);
diff --git a/monetdb5/extras/pyapi/Tests/pyapi_returntypes.malC
b/monetdb5/extras/pyapi/Tests/pyapi_returntypes.malC
--- a/monetdb5/extras/pyapi/Tests/pyapi_returntypes.malC
+++ b/monetdb5/extras/pyapi/Tests/pyapi_returntypes.malC
@@ -183,6 +183,12 @@ r:bat[:oid,:int] := pyapi.eval(nil:ptr,"
# return a single UTF-32 encoded string
a:bat[:oid,:str] := pyapi.eval(nil:ptr,"x =
unicode(\"hello\")\nreturn(x.encode(\"utf32\"))");
+# Incorrect syntax
+bb:bat[:oid,:int] := pyapi.eval(nil:ptr,"return (1");
+
+# Incorrect indentation
+cc:bat[:oid,:int] := pyapi.eval(nil:ptr,"x = 4\n x++\n\treturn (x)");
+
# return a UTF-32 encoded string in a numpy array
(str1:bat[:oid,:str], str2:bat[:oid,:str]) := pyapi.eval(nil:ptr,"x =
unicode(\"hello\")\nreturn(numpy.array([[x.encode(\"utf32\")],
[x.encode(\"utf32\")]]))");
diff --git a/monetdb5/extras/pyapi/Tests/pyapi_returntypes.stable.err
b/monetdb5/extras/pyapi/Tests/pyapi_returntypes.stable.err
--- a/monetdb5/extras/pyapi/Tests/pyapi_returntypes.stable.err
+++ b/monetdb5/extras/pyapi/Tests/pyapi_returntypes.stable.err
@@ -31,34 +31,48 @@ stderr of test 'pyapi_returntypes` in di
# 23:33:07 > "mclient" "-lmal" "-ftest" "-Eutf-8" "-i" "-e"
"--host=/var/tmp/mtest-20340" "--port=36739"
# 23:33:07 >
-MAPI = (monetdb) /var/tmp/mtest-9199/.s.monetdb.37865
+MAPI = (monetdb) /var/tmp/mtest-25907/.s.monetdb.33825
QUERY = r:bat[:oid,:int] := pyapi.eval(nil:ptr,"return(\"Test\")");
ERROR = !MALException:pyapi.eval:Could not convert from type STRING to type int
-MAPI = (monetdb) /var/tmp/mtest-9199/.s.monetdb.37865
+MAPI = (monetdb) /var/tmp/mtest-25907/.s.monetdb.33825
QUERY = (r:bat[:oid,:int], s:bat[:oid,:int]) := pyapi.eval(nil:ptr,"class
NewClass:\n\tx = 5\n\n\treturn(NewClass())");
ERROR = !MALException:pyapi.eval:Unsupported result object. Expected either an
array, a numpy array, a numpy masked array or a pandas data frame, but received
an object of type "<type 'instance'>"
-MAPI = (monetdb) /var/tmp/mtest-9199/.s.monetdb.37865
+MAPI = (monetdb) /var/tmp/mtest-25907/.s.monetdb.33825
QUERY = (r:bat[:oid,:int], s:bat[:oid,:int]) :=
pyapi.eval(nil:ptr,"return(12)");
ERROR = !MALException:pyapi.eval:A single scalar was returned, yet we expect a
list of 2 columns. We can only convert a single scalar into a single column,
thus the result is invalid.
-MAPI = (monetdb) /var/tmp/mtest-9199/.s.monetdb.37865
+MAPI = (monetdb) /var/tmp/mtest-25907/.s.monetdb.33825
QUERY = (r:bat[:oid,:int], s:bat[:oid,:int]) := pyapi.eval(nil:ptr,"import
pandas as pd\ndf = pd.DataFrame({\'Group\': arg1, \'Values\': arg2, 'Values2':
arg2})\nreturn(df)", g, b);
ERROR = !MALException:pyapi.eval:An array of size 3 was returned, yet we
expect a list of 2 columns. The result is invalid.
-MAPI = (monetdb) /var/tmp/mtest-9199/.s.monetdb.37865
+MAPI = (monetdb) /var/tmp/mtest-25907/.s.monetdb.33825
QUERY = (r:bat[:oid,:int], s:bat[:oid,:int]) :=
pyapi.eval(nil:ptr,"return(numpy.array([12]))");
ERROR = !MALException:pyapi.eval:A single array was returned, yet we expect a
list of 2 columns. The result is invalid.
-MAPI = (monetdb) /var/tmp/mtest-9199/.s.monetdb.37865
+MAPI = (monetdb) /var/tmp/mtest-25907/.s.monetdb.33825
QUERY = (r:bat[:oid,:int], s:bat[:oid,:int]) :=
pyapi.eval(nil:ptr,"return([12])");
ERROR = !MALException:pyapi.eval:A single array was returned, yet we expect a
list of 2 columns. The result is invalid.
-MAPI = (monetdb) /var/tmp/mtest-9199/.s.monetdb.37865
+MAPI = (monetdb) /var/tmp/mtest-25907/.s.monetdb.33825
QUERY = r:bat[:oid,:int] := pyapi.eval(nil:ptr,"return([[33,24,55],
[44,66,345]])");
ERROR = !MALException:pyapi.eval:An array of size 2 was returned, yet we
expect a list of 1 columns. The result is invalid.
-MAPI = (monetdb) /var/tmp/mtest-9199/.s.monetdb.37865
+MAPI = (monetdb) /var/tmp/mtest-25907/.s.monetdb.33825
QUERY = a:bat[:oid,:str] := pyapi.eval(nil:ptr,"x =
unicode(\"hello\")\nreturn(x.encode(\"utf32\"))");
ERROR = !MALException:pyapi.eval:Invalid string encoding used. Please return a
regular ASCII string, or a Numpy_Unicode object.
-MAPI = (monetdb) /var/tmp/mtest-9199/.s.monetdb.37865
+MAPI = (monetdb) /var/tmp/mtest-25907/.s.monetdb.33825
+QUERY = bb:bat[:oid,:int] := pyapi.eval(nil:ptr,"return (1");
+ERROR = !MALException:pyapi.eval:Could not parse Python code
+ !def pyfun():
+ ! return (1
+ !invalid syntax (<string>, line 2)
+MAPI = (monetdb) /var/tmp/mtest-25907/.s.monetdb.33825
+QUERY = cc:bat[:oid,:int] := pyapi.eval(nil:ptr,"x = 4\n x++\n\treturn (x)");
+ERROR = !MALException:pyapi.eval:Could not parse Python code
+ !def pyfun():
+ ! x = 4
+ ! x++
+ ! return (x)
+ !unexpected indent (<string>, line 3)
+MAPI = (monetdb) /var/tmp/mtest-25907/.s.monetdb.33825
QUERY = (str1:bat[:oid,:str], str2:bat[:oid,:str]) := pyapi.eval(nil:ptr,"x =
unicode(\"hello\")\nreturn(numpy.array([[x.encode(\"utf32\")],
[x.encode(\"utf32\")]]))");
ERROR = !MALException:pyapi.eval:Invalid string encoding used. Please return a
regular ASCII string, or a Numpy_Unicode object.
-MAPI = (monetdb) /var/tmp/mtest-9199/.s.monetdb.37865
+MAPI = (monetdb) /var/tmp/mtest-25907/.s.monetdb.33825
QUERY = (str1:bat[:oid,:str], str2:bat[:oid,:str]) :=
pyapi.eval(nil:ptr,"return(numpy.array([[\"Hëllo\", \"Hello Again\"], [\"Hello
Again Again\",\"That's quite enough.\"]]))");
ERROR = !MALException:pyapi.eval:Invalid string encoding used. Please return a
regular ASCII string, or a Numpy_Unicode object.
diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c
--- a/monetdb5/extras/pyapi/pyapi.c
+++ b/monetdb5/extras/pyapi/pyapi.c
@@ -31,8 +31,6 @@
//#define _PYAPI_VERBOSE_
#define _PYAPI_DEBUG_
-bool memory_mapping = TRUE;
-
#include <stdint.h>
#include <stdio.h>
@@ -46,6 +44,9 @@ bool memory_mapping = TRUE;
#endif
const char* pyapi_enableflag = "embedded_py";
+const char* zerocopy_disableflag = "disable_pyzerocopy";
+const char* verbose_enableflag = "enable_pyverbose";
+const char* debug_enableflag = "enable_pydebug";
#ifdef _PYAPI_VERBOSE_
#define VERBOSE_MESSAGE(...) { \
@@ -59,8 +60,7 @@ const char* pyapi_enableflag = "embedded
#define GDK_Alloc(var, size) { \
var = GDKzalloc(size); \
- if (var == NULL) \
- { \
+ if (var == NULL) { \
msg = createException(MAL, "pyapi.eval", MAL_MALLOC_FAIL); \
goto wrapup; \
} \
@@ -98,7 +98,6 @@ struct _PyReturn{
bool *mask_data;
size_t count;
size_t memory_size;
- BAT *bat_return;
int result_type;
bool multidimensional;
};
@@ -256,7 +255,7 @@ static int pyapiInitialized = FALSE;
goto wrapup;
\
}
\
data = (char*) ret->array_data;
\
- if (memory_mapping && ret->count > 0 && TYPE_##mtpe ==
PyType_ToBat(ret->result_type) && (ret->count * ret->memory_size < BUN_MAX) &&
\
+ if (zero_copy && ret->count > 0 && TYPE_##mtpe ==
PyType_ToBat(ret->result_type) && (ret->count * ret->memory_size < BUN_MAX) &&
\
(ret->numpy_array == NULL || PyArray_FLAGS(ret->numpy_array) &
NPY_ARRAY_OWNDATA))
\
{
\
/*We can only create a direct map if the numpy array type and
target BAT type*/
\
@@ -345,6 +344,7 @@ typedef enum {
} pyapi_scan_state;
bool PyType_IsPyScalar(PyObject *object);
+char *PyError_CreateException(char *error_text, char *error_text_2);
str PyAPIeval(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, bit grouped, bit
mapped) {
sql_func * sqlfun = *(sql_func**) getArgReference(stk, pci, pci->retc);
@@ -368,13 +368,13 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
PyReturn *pyreturn_values = NULL;
PyInput *pyinput_values = NULL;
int seqbase = 0;
-
+ bool zero_copy = !(GDKgetenv_isyes(zerocopy_disableflag) ||
GDKgetenv_istrue(zerocopy_disableflag));
#ifndef WIN32
bool single_fork = mapped == 1;
int shm_id = -1;
int sem_id = -1;
int process_id = 0;
- int memory_size;
+ int memory_size = 0;
int process_count;
#endif
@@ -769,25 +769,26 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
#endif
}
- //VERBOSE_MESSAGE("Attempt to acquire GIL.\n");
- //gstate = AcquireLock(&holds_gil);
-
-
VERBOSE_MESSAGE("Loading data from the database into Python.\n");
- // create function argument tuple, we pass a tuple of numpy arrays
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
