Changeset: 665a95d54a0c for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=665a95d54a0c
Modified Files:
monetdb5/extras/pyapi/Benchmarks/pyapi_test.sh
monetdb5/extras/pyapi/pyapi.c
Branch: pyapi
Log Message:
Make numpy.string the default instead of numpy.object
diffs (truncated from 305 to 300 lines):
diff --git a/monetdb5/extras/pyapi/Benchmarks/pyapi_test.sh
b/monetdb5/extras/pyapi/Benchmarks/pyapi_test.sh
--- a/monetdb5/extras/pyapi/Benchmarks/pyapi_test.sh
+++ b/monetdb5/extras/pyapi/Benchmarks/pyapi_test.sh
@@ -1,7 +1,7 @@
# The base directory of testing, a new folder is created in this base
directory [$PYAPI_TEST_DIR], and everything is done in that new folder
-export PYAPI_BASE_DIR=/export/scratch1/raasveld
+export PYAPI_BASE_DIR=/tmp
# The terminal to start mserver with, examples are gnome-terminal, xterm,
konsole
export TERMINAL=x-terminal-emulator
# Port used by the MSERVER
@@ -11,39 +11,39 @@ export MSERVERTEST='netstat -ant | grep
# Testing parameters
# Input test (zero copy vs copy)
# The input sizes to test (in MB)
-export INPUT_TESTING_SIZES="10"
+export INPUT_TESTING_SIZES="1000 10000 100000"
# Amount of tests to run for each size
-export INPUT_TESTING_NTESTS=10
+export INPUT_TESTING_NTESTS=3
# Output test (zero copy vs copy)
# The output sizes to test (in MB)
-export OUTPUT_TESTING_SIZES="0.1 1 10 100 1000"
+export OUTPUT_TESTING_SIZES="1000 10000 100000"
# Amount of tests to run for each size
-export OUTPUT_TESTING_NTESTS=10
+export OUTPUT_TESTING_NTESTS=3
# String tests
# Strings of the same length (mb, length)
-export STRINGSAMELENGTH_TESTING_SIZES="(100,1) (100,10) (100,100) (100,250)
(100,500) (100,750) (100,1000) (100,1000) (100,10000) (100,100000)"
+export STRINGSAMELENGTH_TESTING_SIZES="(1000,1) (1000,10) (1000,100)
(1000,1000) (1000,10000) (1000,100000)"
export STRINGSAMELENGTH_TESTING_NTESTS=3
# Extreme length string testing (all strings have length 1 except for one
string, which has EXTREME length)
# Arguments are (Extreme Length, String Count)
export STRINGEXTREMELENGTH_TESTING_SIZES="(10,1000000) (100,1000000)
(1000,1000000) (10000,1000000)"
-export STRINGEXTREMELENGTH_TESTING_NTESTS=1
+export STRINGEXTREMELENGTH_TESTING_NTESTS=3
# Check Unicode vs Always Unicode (ASCII) (mb, length)
-export STRINGUNICODE_TESTING_SIZES="(1000,10) (10000,10) (1000,100)
(10000,100)"
-export STRINGUNICODE_TESTING_NTESTS=1
+export STRINGUNICODE_TESTING_SIZES="(1000,10) (1000,100) (1000,1000)
(1000,10000)"
+export STRINGUNICODE_TESTING_NTESTS=3
# Multithreading tests
-export MULTITHREADING_NR_THREADS="1 2 3 4 5 6 7 8"
+export MULTITHREADING_NR_THREADS="1 2 4 8 16 32 64 96"
export MULTITHREADING_TESTING_SIZES="1"
#amount of tests for each thread
-export MULTITHREADING_TESTING_NTESTS=10
+export MULTITHREADING_TESTING_NTESTS=3
# Quantile speedtest
# The input sizes to test (in MB)
-export QUANTILE_TESTING_SIZES="0.1 1 10 100 1000"
+export QUANTILE_TESTING_SIZES="1000 8000 40000"
# Amount of tests to run for each size
-export QUANTILE_TESTING_NTESTS=10
+export QUANTILE_TESTING_NTESTS=3
# PyAPI TAR url
export PYAPI_BRANCH_NAME=pyapi
@@ -268,58 +268,58 @@ function pyapi_test_string_samelength()
return 1
fi
- pyapi_run_single_test "String Testing (NPY_OBJECT, Same Length)" ""
"STRING_SAMELENGTH" string_samelength_npyobject
"$STRINGSAMELENGTH_TESTING_NTESTS" "$STRINGSAMELENGTH_TESTING_SIZES"
+ pyapi_run_single_test "String Testing (NPY_OBJECT, Same Length)" "--set
disable_numpystringarray=true" "STRING_SAMELENGTH" string_samelength_npyobject
"$STRINGSAMELENGTH_TESTING_NTESTS" "$STRINGSAMELENGTH_TESTING_SIZES"
if [ $? -ne 0 ]; then
return 1
fi
- pyapi_run_single_test "String Testing (NPY_STRING, Same Length)" "--set
enable_numpystringarray=true" "STRING_SAMELENGTH" string_samelength_npystring
"$STRINGSAMELENGTH_TESTING_NTESTS" "$STRINGSAMELENGTH_TESTING_SIZES"
+ pyapi_run_single_test "String Testing (NPY_STRING, Same Length)" ""
"STRING_SAMELENGTH" string_samelength_npystring
"$STRINGSAMELENGTH_TESTING_NTESTS" "$STRINGSAMELENGTH_TESTING_SIZES"
if [ $? -ne 0 ]; then
return 1
fi
}
function pyapi_test_string_extreme() {
- pyapi_run_single_test "String Testing (NPY_OBJECT, Extreme Length)" ""
"STRING_EXTREMELENGTH" string_extremelength_npyobject
"$STRINGEXTREMELENGTH_TESTING_NTESTS" "$STRINGEXTREMELENGTH_TESTING_SIZES"
+ pyapi_run_single_test "String Testing (NPY_OBJECT, Extreme Length)" "--set
disable_numpystringarray=true" "STRING_EXTREMELENGTH"
string_extremelength_npyobject "$STRINGEXTREMELENGTH_TESTING_NTESTS"
"$STRINGEXTREMELENGTH_TESTING_SIZES"
if [ $? -ne 0 ]; then
return 1
fi
- pyapi_run_single_test "String Testing (NPY_STRING, Extreme Length)" "--set
enable_numpystringarray=true" "STRING_EXTREMELENGTH"
string_extremelength_npystring "$STRINGEXTREMELENGTH_TESTING_NTESTS"
"$STRINGEXTREMELENGTH_TESTING_SIZES"
+ pyapi_run_single_test "String Testing (NPY_STRING, Extreme Length)" ""
"STRING_EXTREMELENGTH" string_extremelength_npystring
"$STRINGEXTREMELENGTH_TESTING_NTESTS" "$STRINGEXTREMELENGTH_TESTING_SIZES"
if [ $? -ne 0 ]; then
return 1
fi
}
function pyapi_test_string_unicode_ascii() {
- pyapi_run_single_test "String Testing (Check Unicode, ASCII)" ""
"STRING_SAMELENGTH" string_unicode_ascii_check "$STRINGUNICODE_TESTING_NTESTS"
"$STRINGUNICODE_TESTING_SIZES"
+ pyapi_run_single_test "String Testing (Check Unicode, ASCII)" "--set
disable_numpystringarray=true" "STRING_SAMELENGTH" string_unicode_ascii_check
"$STRINGUNICODE_TESTING_NTESTS" "$STRINGUNICODE_TESTING_SIZES"
if [ $? -ne 0 ]; then
return 1
fi
- pyapi_run_single_test "String Testing (Always Unicode, ASCII)" "--set
enable_alwaysunicode=true" "STRING_SAMELENGTH" string_unicode_ascii_always
"$STRINGUNICODE_TESTING_NTESTS" "$STRINGUNICODE_TESTING_SIZES"
+ pyapi_run_single_test "String Testing (Always Unicode, ASCII)" "--set
disable_numpystringarray=true --set enable_alwaysunicode=true"
"STRING_SAMELENGTH" string_unicode_ascii_always "$STRINGUNICODE_TESTING_NTESTS"
"$STRINGUNICODE_TESTING_SIZES"
if [ $? -ne 0 ]; then
return 1
fi
- pyapi_run_single_test "String Testing (Check Unicode, Extreme)" ""
"STRING_EXTREMEUNICODE" string_unicode_extreme_check
"$STRINGUNICODE_TESTING_NTESTS" "$STRINGUNICODE_TESTING_SIZES"
+ pyapi_run_single_test "String Testing (Check Unicode, Extreme)" "--set
disable_numpystringarray=true" "STRING_EXTREMEUNICODE"
string_unicode_extreme_check "$STRINGUNICODE_TESTING_NTESTS"
"$STRINGUNICODE_TESTING_SIZES"
if [ $? -ne 0 ]; then
return 1
fi
- pyapi_run_single_test "String Testing (Always Unicode, Extreme)" ""
"STRING_EXTREMEUNICODE" string_unicode_extreme_always
"$STRINGUNICODE_TESTING_NTESTS" "$STRINGUNICODE_TESTING_SIZES"
+ pyapi_run_single_test "String Testing (Always Unicode, Extreme)" "--set
disable_numpystringarray=true" "STRING_EXTREMEUNICODE"
string_unicode_extreme_always "$STRINGUNICODE_TESTING_NTESTS"
"$STRINGUNICODE_TESTING_SIZES"
if [ $? -ne 0 ]; then
return 1
fi
}
function pyapi_test_bytearray_vs_string() {
- pyapi_run_single_test "String Testing (ByteArray Object)" "--set
enable_bytearray=true" "STRING_SAMELENGTH" string_bytearrayobject
"$STRINGSAMELENGTH_TESTING_NTESTS" "$STRINGSAMELENGTH_TESTING_SIZES"
+ pyapi_run_single_test "String Testing (ByteArray Object)" "--set
disable_numpystringarray=true --set enable_bytearray=true" "STRING_SAMELENGTH"
string_bytearrayobject "$STRINGSAMELENGTH_TESTING_NTESTS"
"$STRINGSAMELENGTH_TESTING_SIZES"
if [ $? -ne 0 ]; then
return 1
fi
- pyapi_run_single_test "String Testing (String Object)" ""
"STRING_SAMELENGTH" string_stringobject "$STRINGSAMELENGTH_TESTING_NTESTS"
"$STRINGSAMELENGTH_TESTING_SIZES"
+ pyapi_run_single_test "String Testing (String Object)" "--set
disable_numpystringarray=true" "STRING_SAMELENGTH" string_stringobject
"$STRINGSAMELENGTH_TESTING_NTESTS" "$STRINGSAMELENGTH_TESTING_SIZES"
if [ $? -ne 0 ]; then
return 1
fi
@@ -367,18 +367,16 @@ function pyapi_run_tests() {
echo "Failed to create output directory."
return 1
fi
- cp /local/raasveld/monetdb_testing.py
/tmp/monetdb_pyapi_test/MonetDB-$PYAPI_BRANCH_NAME/monetdb5/extras/pyapi/Benchmarks/
- cp /local/raasveld/randomstrings.c
/tmp/monetdb_pyapi_test/MonetDB-$PYAPI_BRANCH_NAME/monetdb5/extras/pyapi/Benchmarks/
-
- #pyapi_test_input
- #pyapi_test_input_null
- #pyapi_test_output
+
+ pyapi_test_input
+ pyapi_test_input_null
+ pyapi_test_output
pyapi_test_string_samelength
- #pyapi_test_string_extreme
- #pyapi_test_string_unicode_ascii
- #pyapi_test_bytearray_vs_string
- #pyapi_test_quantile
- #pyapi_test_threads
+ pyapi_test_string_extreme
+ pyapi_test_string_unicode_ascii
+ pyapi_test_bytearray_vs_string
+ pyapi_test_quantile
+ pyapi_test_threads
}
function pyapi_graph() {
diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c
--- a/monetdb5/extras/pyapi/pyapi.c
+++ b/monetdb5/extras/pyapi/pyapi.c
@@ -51,7 +51,7 @@ const char* debug_enableflag = "enable_p
#ifdef _PYAPI_TESTING_
const char* zerocopyinput_disableflag = "disable_pyzerocopyinput";
const char* zerocopyoutput_disableflag = "disable_pyzerocopyoutput";
-const char* numpy_string_array_enableflag = "enable_numpystringarray";
+const char* numpy_string_array_disableflag = "disable_numpystringarray";
const char* alwaysunicode_enableflag = "enable_alwaysunicode";
const char* lazyarray_enableflag = "enable_lazyarray";
const char* oldnullmask_enableflag = "enable_oldnullmask";
@@ -60,7 +60,7 @@ const char* benchmark_output_flag = "pya
const char* disable_malloc_tracking = "disable_malloc_tracking";
static bool option_zerocopyinput = true;
static bool option_zerocopyoutput = true;
-static bool option_numpy_string_array = false;
+static bool option_numpy_string_array = true;
static bool option_bytearray = false;
static bool option_lazyarray = false;
static bool option_oldnullmask = false;
@@ -1254,29 +1254,6 @@ str
}
MT_lock_unset(&pyapiLock, "pyapi.evaluate");
fprintf(stdout, "# MonetDB/Python module loaded\n");
-#ifdef _PYAPI_VERBOSE_
- option_verbose = GDKgetenv_isyes(verbose_enableflag) ||
GDKgetenv_istrue(verbose_enableflag);
-#endif
-#ifdef _PYAPI_DEBUG_
- option_debug = GDKgetenv_isyes(debug_enableflag) ||
GDKgetenv_istrue(debug_enableflag);
- (void) option_debug;
-#endif
-#ifdef _PYAPI_WARNINGS_
- option_warning = GDKgetenv_isyes(warning_enableflag) ||
GDKgetenv_istrue(warning_enableflag);
-#endif
-#ifdef _PYAPI_TESTING_
- //These flags are for testing purposes, they shouldn't be used for
normal purposes
- option_zerocopyinput = !(GDKgetenv_isyes(zerocopyinput_disableflag) ||
GDKgetenv_istrue(zerocopyinput_disableflag));
- option_zerocopyoutput = !(GDKgetenv_isyes(zerocopyoutput_disableflag)
|| GDKgetenv_istrue(zerocopyoutput_disableflag));
- option_numpy_string_array =
GDKgetenv_isyes(numpy_string_array_enableflag) ||
GDKgetenv_istrue(numpy_string_array_enableflag);
- option_bytearray = GDKgetenv_isyes(bytearray_enableflag) ||
GDKgetenv_istrue(bytearray_enableflag);
- option_oldnullmask = GDKgetenv_isyes(oldnullmask_enableflag) ||
GDKgetenv_istrue(oldnullmask_enableflag);
- option_lazyarray = GDKgetenv_isyes(lazyarray_enableflag) ||
GDKgetenv_istrue(lazyarray_enableflag);
- option_alwaysunicode = (GDKgetenv_isyes(alwaysunicode_enableflag) ||
GDKgetenv_istrue(alwaysunicode_enableflag));
- benchmark_output = GDKgetenv(benchmark_output_flag);
- option_disablemalloctracking =
(GDKgetenv_isyes(disable_malloc_tracking) ||
GDKgetenv_istrue(disable_malloc_tracking));
- fprintf(stdout, "# MonetDB/Python testing enabled.\n");
-#endif
}
#else
if (!pyapiInitialized) {
@@ -1284,6 +1261,7 @@ str
import_array1(iar);
pyapiInitialized++;
}
+#endif
#ifdef _PYAPI_VERBOSE_
option_verbose = GDKgetenv_isyes(verbose_enableflag) ||
GDKgetenv_istrue(verbose_enableflag);
#endif
@@ -1294,6 +1272,18 @@ str
#ifdef _PYAPI_WARNINGS_
option_warning = GDKgetenv_isyes(warning_enableflag) ||
GDKgetenv_istrue(warning_enableflag);
#endif
+#ifdef _PYAPI_TESTING_
+ //These flags are for testing purposes, they shouldn't be used for normal
purposes
+ option_zerocopyinput = !(GDKgetenv_isyes(zerocopyinput_disableflag) ||
GDKgetenv_istrue(zerocopyinput_disableflag));
+ option_zerocopyoutput = !(GDKgetenv_isyes(zerocopyoutput_disableflag) ||
GDKgetenv_istrue(zerocopyoutput_disableflag));
+ option_numpy_string_array =
!(GDKgetenv_isyes(numpy_string_array_disableflag) ||
GDKgetenv_istrue(numpy_string_array_disableflag));
+ option_bytearray = GDKgetenv_isyes(bytearray_enableflag) ||
GDKgetenv_istrue(bytearray_enableflag);
+ option_oldnullmask = GDKgetenv_isyes(oldnullmask_enableflag) ||
GDKgetenv_istrue(oldnullmask_enableflag);
+ option_lazyarray = GDKgetenv_isyes(lazyarray_enableflag) ||
GDKgetenv_istrue(lazyarray_enableflag);
+ option_alwaysunicode = (GDKgetenv_isyes(alwaysunicode_enableflag) ||
GDKgetenv_istrue(alwaysunicode_enableflag));
+ benchmark_output = GDKgetenv(benchmark_output_flag);
+ option_disablemalloctracking = (GDKgetenv_isyes(disable_malloc_tracking)
|| GDKgetenv_istrue(disable_malloc_tracking));
+ fprintf(stdout, "# MonetDB/Python testing enabled.\n");
#endif
return MAL_SUCCEED;
}
@@ -1552,8 +1542,10 @@ PyObject *PyArrayObject_FromBAT(PyInput
BAT_TO_NP(b, dbl, NPY_FLOAT64);
break;
case TYPE_str:
- #ifdef _PYAPI_TESTING_
- if (option_numpy_string_array) {
+#ifdef _PYAPI_TESTING_
+ if (option_numpy_string_array)
+#endif
+ {
bool unicode = false;
size_t maxsize = 0;
li = bat_iterator(b);
@@ -1620,7 +1612,6 @@ PyObject *PyArrayObject_FromBAT(PyInput
j++;
}
} else {
-
VERBOSE_MESSAGE("- ASCII string!\n");
//create a NPY_STRING array object
vararray = PyArray_New(
@@ -1653,14 +1644,10 @@ PyObject *PyArrayObject_FromBAT(PyInput
}
}
}
+#ifdef _PYAPI_TESTING_
else
-#endif
{
-#ifdef _PYAPI_TESTING_
bool unicode = option_alwaysunicode;
-#else
- bool unicode = false;
-#endif
li = bat_iterator(b);
//create a NPY_OBJECT array object
vararray = PyArray_New(
@@ -1674,9 +1661,7 @@ PyObject *PyArrayObject_FromBAT(PyInput
0,
NULL);
-#ifdef _PYAPI_TESTING_
if (!option_alwaysunicode)
-#endif
{
j = 0;
BATloop(b, p, q) {
@@ -1710,10 +1695,8 @@ PyObject *PyArrayObject_FromBAT(PyInput
obj = PyUnicode_FromString(t);
}
} else {
-#ifdef _PYAPI_TESTING_
if (option_bytearray) obj = PyByteArray_FromString(t);
else
-#endif
{
obj = PyString_FromString(t);
}
@@ -1730,6 +1713,7 @@ PyObject *PyArrayObject_FromBAT(PyInput
j++;
}
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
