Changeset: ae661d755579 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ae661d755579 Added Files: monetdb5/extras/pyapi/pytypes.c monetdb5/extras/pyapi/type_conversion.c Modified Files: gdk/gdk.h gdk/gdk_heap.c monetdb5/extras/pyapi/Makefile.ag monetdb5/extras/pyapi/Tests/pyapi_types_boolean.malC monetdb5/extras/pyapi/Tests/random_file.py monetdb5/extras/pyapi/pyapi.c monetdb5/extras/pyapi/pytypes.h monetdb5/extras/pyapi/type_conversion.h monetdb5/extras/pyapi/unicode.c Branch: pyapi Log Message:
Return types are now directly mapped into BATs instead of copied if possible.
Also added some fixes for other compilers.
diffs (truncated from 1617 to 300 lines):
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -644,6 +644,8 @@ typedef enum {
STORE_MEM = 0, /* load into GDKmalloced memory */
STORE_MMAP = 1, /* mmap() into virtual memory */
STORE_PRIV = 2, /* BAT copy of copy-on-write mmap */
+ STORE_CMEM = 3, /* Indicates the value is stored in regular C
memory rather than GDK memory.*/
+ STORE_NOWN = 4, /* Indicates that the bat does not own the chunk of
memory and is not in charge of freeing it.*/
STORE_INVALID /* invalid value, used to indicate error */
} storage_t;
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -551,12 +551,15 @@ HEAPcopy(Heap *dst, Heap *src)
int
HEAPfree(Heap *h, int remove)
{
- if (h->base) {
+ if (h->base && h->storage != STORE_NOWN) {
if (h->storage == STORE_MEM) { /* plain memory */
HEAPDEBUG fprintf(stderr, "#HEAPfree " SZFMT
" " PTRFMT "\n",
h->size, PTRFMTCAST h->base);
GDKfree(h->base);
+ } else if (h->storage == STORE_CMEM) {
+ //heap is stored in regular C memory rather than GDK
memory
+ free(h->base);
} else { /* mapped file, or STORE_PRIV */
gdk_return ret = GDKmunmap(h->base, h->size);
diff --git a/monetdb5/extras/pyapi/Makefile.ag
b/monetdb5/extras/pyapi/Makefile.ag
--- a/monetdb5/extras/pyapi/Makefile.ag
+++ b/monetdb5/extras/pyapi/Makefile.ag
@@ -17,7 +17,7 @@ MTSAFE
lib__pyapi = {
MODULE
DIR = libdir/monetdb5
- SOURCES = pyapi.c pyapi.h unicode.c unicode.h
+ SOURCES = pyapi.c pyapi.h unicode.c unicode.h pytypes.c pytypes.h
type_conversion.c type_conversion.h
XDEPS = $(libpy_LIBDEP)
LIBS = ../../tools/libmonetdb5 \
../../../gdk/libbat \
diff --git a/monetdb5/extras/pyapi/Tests/pyapi_types_boolean.malC
b/monetdb5/extras/pyapi/Tests/pyapi_types_boolean.malC
--- a/monetdb5/extras/pyapi/Tests/pyapi_types_boolean.malC
+++ b/monetdb5/extras/pyapi/Tests/pyapi_types_boolean.malC
@@ -1,4 +1,15 @@
+#########################
+#########################
+## --CORRECT RETURNS-- ##
+#########################
+#########################
-# return a string when an int is expected
-r:bat[:oid,:int] := pyapi.eval(nil:ptr,"import
os\nprint(os.getcwd())\nreturn(33)");
-io.print(r);
+# int with null value
+bint:= bat.new(:oid,:int);
+bat.append(bint,1804289383:int);
+bat.append(bint,846930886:int);
+bat.append(bint,1681692777:int);
+bat.append(bint,1714636915:int);
+bat.append(bint,nil:int);
+rint:bat[:oid,:int] := pyapi.eval(nil:ptr,"return(arg1)",bint);
+io.print(rint);
diff --git a/monetdb5/extras/pyapi/Tests/random_file.py
b/monetdb5/extras/pyapi/Tests/random_file.py
--- a/monetdb5/extras/pyapi/Tests/random_file.py
+++ b/monetdb5/extras/pyapi/Tests/random_file.py
@@ -1,15 +1,18 @@
-for j in range(1,2):
- f = open('workfile' + str(j) + '.sql', 'w')
- import random
- random.seed()
- f.write("START TRANSACTION;\n")
- for i in range(1, 50000):
- f.write("INSERT INTO rval VALUES ")
- x = random.randint(0,100)
- y = random.randint(0,100)
- f.write("(" + str(x) + "," + str(y) + ");\n")
- f.write("COMMIT;\n")
- f.close()
+
+f = open('workfile.sql', 'w')
+import random
+random.seed()
+records = 15000000
+f.write("START TRANSACTION;\n")
+f.write("CREATE TABLE rval2 (a integer, b integer);\n");
+f.write("copy " + str(records) + " records into rval2 from stdin;\n");
+for i in range(0, records):
+ x = random.randint(0,100)
+ y = random.randint(0,100)
+ f.write(str(x) + "|" + str(y) + "\n")
+
+f.write("COMMIT;\n")
+f.close()
\<workfile1.sql
diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c
--- a/monetdb5/extras/pyapi/pyapi.c
+++ b/monetdb5/extras/pyapi/pyapi.c
@@ -38,7 +38,6 @@
const char* pyapi_enableflag = "embedded_py";
-
#ifdef _PYAPI_VERBOSE_
#define VERBOSE_MESSAGE(...) { \
printf(__VA_ARGS__); \
@@ -71,25 +70,61 @@ static MT_Lock pyapiLock;
static MT_Lock pyapiSluice;
static int pyapiInitialized = FALSE;
-#define BAT_TO_NP(bat, mtpe, nptpe) \
- PyArray_New(&PyArray_Type, 1, (npy_intp[1]) {BATcount(bat)}, \
- nptpe, NULL, (mtpe*) Tloc(bat, BUNfirst(bat)), 0, \
+#define BAT_TO_NP(bat, mtpe, nptpe)
\
+ PyArray_New(&PyArray_Type, 1, (npy_intp[1]) {BATcount(bat)},
\
+ nptpe, NULL, (mtpe*) Tloc(bat, BUNfirst(bat)), 0,
\
NPY_ARRAY_CARRAY || !NPY_ARRAY_WRITEABLE, NULL);
+#define BAT_MMAP(bat, mtpe) {
\
+ bat = BATnew(TYPE_void, TYPE_##mtpe, 0, TRANSIENT);
\
+ BATseqbase(bat, 0); bat->T->nil = 0; bat->T->nonil = 1;
\
+ bat->tkey = 0; bat->tsorted = 0; bat->trevsorted = 0;
\
+ /*Change nil values to the proper values, if they exist*/
\
+ if (mask != NULL)
\
+ {
\
+ for (j = 0; j < ret->count; j++)
\
+ {
\
+ if (mask[index_offset * ret->count + j] == TRUE)
\
+ {
\
+ (*(mtpe*)(&data[(index_offset * ret->count + j) *
ret->memory_size])) = mtpe##_nil; \
+ bat->T->nil = 1;
\
+ }
\
+ }
\
+ }
\
+ bat->T->nonil = 1 - bat->T->nil;
\
+ /*When we create a BAT a small part of memory is allocated, free it*/
\
+ GDKfree(bat->T->heap.base);
\
+
\
+ bat->T->heap.base = &data[(index_offset * ret->count) *
ret->memory_size]; \
+ bat->T->heap.size = ret->count * ret->memory_size;
\
+ bat->T->heap.free = bat->T->heap.size; /*There are no free places in
the array*/ \
+ /*If index_offset > 0, we are mapping part of a multidimensional
array.*/ \
+ /*The entire array will be cleared when the part with index_offset=0
is freed*/ \
+ /*So we set this part of the mapping to 'NOWN'*/
\
+ if (index_offset > 0) bat->T->heap.storage = STORE_NOWN;
\
+ else bat->T->heap.storage = STORE_CMEM; /*Numpy allocated the array
with malloc*/ \
+ bat->T->heap.newstorage = STORE_MEM;
\
+ bat->S->count = ret->count;
\
+ bat->S->capacity = ret->count;
\
+ bat->S->copiedtodisk = false;
\
+
\
+ /*Take over the data from the numpy array*/
\
+ PyArray_CLEARFLAGS(ret->numpy_array, NPY_ARRAY_OWNDATA);
\
+ }
-#define NP_COL_BAT_LOOP(bat, mtpe_to, mtpe_from) {
\
- if (mask == NULL)
\
- {
\
- for (j = 0; j < ret->count; j++)
\
- {
\
- ((mtpe_to*) Tloc(bat, BUNfirst(bat)))[j] =
(mtpe_to)(*(mtpe_from*)(&data[(index_offset * ret->count + j) *
ret->memory_size])); \
- }
\
- }
\
- else
\
- {
\
- for (j = 0; j < ret->count; j++)
\
- {
\
- if (mask[index_offset * ret->count + j] == TRUE)
\
+#define NP_COL_BAT_LOOP(bat, mtpe_to, mtpe_from) {
\
+ if (mask == NULL)
\
+ {
\
+ for (j = 0; j < ret->count; j++)
\
+ {
\
+ ((mtpe_to*) Tloc(bat, BUNfirst(bat)))[j] =
(mtpe_to)(*(mtpe_from*)(&data[(index_offset * ret->count + j) *
ret->memory_size])); \
+ }
\
+ }
\
+ else
\
+ {
\
+ for (j = 0; j < ret->count; j++)
\
+ {
\
+ if (mask[index_offset * ret->count + j] == TRUE)
\
{
\
bat->T->nil = 1;
\
((mtpe_to*) Tloc(bat, BUNfirst(bat)))[j] = mtpe_to##_nil;
\
@@ -101,110 +136,120 @@ static int pyapiInitialized = FALSE;
}
\
} }
-#define NP_COL_BAT_LOOP_FUNC(bat, mtpe_to, func) {
\
- mtpe_to value;
\
- if (mask == NULL)
\
- {
\
- for (j = 0; j < ret->count; j++)
\
- {
\
- if (!func(&data[(index_offset * ret->count + j) *
ret->memory_size], ret->memory_size, &value)) \
- {
\
- msg = createException(MAL, "pyapi.eval", "Could not convert
from type %s to type %s", PyType_Format(ret->result_type), #mtpe_to); \
- goto wrapup;
\
- }
\
- ((mtpe_to*) Tloc(bat, BUNfirst(bat)))[j] = value;
\
- }
\
- }
\
- else
\
- {
\
- for (j = 0; j < ret->count; j++)
\
- {
\
- if (mask[index_offset * ret->count + j] == TRUE)
\
- {
\
- bat->T->nil = 1;
\
- ((mtpe_to*) Tloc(bat, BUNfirst(bat)))[j] = mtpe_to##_nil;
\
- }
\
- else
\
- {
\
- if (!func(&data[(index_offset * ret->count + j) *
ret->memory_size], ret->memory_size, &value)) \
- {
\
+#define NP_COL_BAT_LOOP_FUNC(bat, mtpe_to, func) {
\
+ mtpe_to value;
\
+ if (mask == NULL)
\
+ {
\
+ for (j = 0; j < ret->count; j++)
\
+ {
\
+ if (!func(&data[(index_offset * ret->count + j) *
ret->memory_size], ret->memory_size, &value))
\
+ {
\
+ msg = createException(MAL, "pyapi.eval", "Could not convert
from type %s to type %s", PyType_Format(ret->result_type), #mtpe_to); \
+ goto wrapup;
\
+ }
\
+ ((mtpe_to*) Tloc(bat, BUNfirst(bat)))[j] = value;
\
+ }
\
+ }
\
+ else
\
+ {
\
+ for (j = 0; j < ret->count; j++)
\
+ {
\
+ if (mask[index_offset * ret->count + j] == TRUE)
\
+ {
\
+ bat->T->nil = 1;
\
+ ((mtpe_to*) Tloc(bat, BUNfirst(bat)))[j] = mtpe_to##_nil;
\
+ }
\
+ else
\
+ {
\
+ if (!func(&data[(index_offset * ret->count + j) *
ret->memory_size], ret->memory_size, &value))
\
+ {
\
msg = createException(MAL, "pyapi.eval", "Could not
convert from type %s to type %s", PyType_Format(ret->result_type), #mtpe_to); \
- goto wrapup;
\
- }
\
- ((mtpe_to*) Tloc(bat, BUNfirst(bat)))[j] = value;
\
- }
\
- }
\
+ goto wrapup;
\
+ }
\
+ ((mtpe_to*) Tloc(bat, BUNfirst(bat)))[j] = value;
\
+ }
\
+ }
\
} }
-#define NP_COL_BAT_STR_LOOP(bat, mtpe, conv)
\
- if (mask == NULL)
\
- {
\
- for (j = 0; j < ret->count; j++)
\
- {
\
- conv(utf8_string, *((mtpe*)&data[(index_offset * ret->count + j) *
ret->memory_size])); \
- BUNappend(bat, utf8_string, FALSE);
\
- }
\
- }
\
- else
\
- {
\
- for (j = 0; j < ret->count; j++)
\
- {
\
- if (mask[index_offset * ret->count + j] == TRUE)
\
- {
\
- bat->T->nil = 1;
\
- BUNappend(b, str_nil, FALSE);
\
- }
\
- else
\
- {
\
- conv(utf8_string, *((mtpe*)&data[(index_offset * ret->count +
j) * ret->memory_size])); \
- BUNappend(bat, utf8_string, FALSE);
\
- }
\
- }
\
+#define NP_COL_BAT_STR_LOOP(bat, mtpe, conv)
\
+ if (mask == NULL)
\
+ {
\
+ for (j = 0; j < ret->count; j++)
\
+ {
\
+ conv(utf8_string, *((mtpe*)&data[(index_offset * ret->count + j) *
ret->memory_size])); \
+ BUNappend(bat, utf8_string, FALSE);
\
+ }
\
+ }
\
+ else
\
+ {
\
+ for (j = 0; j < ret->count; j++)
\
+ {
\
+ if (mask[index_offset * ret->count + j] == TRUE)
\
+ {
\
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
