Changeset: c941812354a0 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c941812354a0
Added Files:
sql/test/configurable_rss/Tests/All
sql/test/configurable_rss/Tests/cap_rss.py
sql/test/configurable_rss/Tests/cap_rss.stable.err
sql/test/configurable_rss/Tests/cap_rss.stable.out
sql/test/configurable_rss/Tests/no_cap_rss.sql
sql/test/configurable_rss/Tests/no_cap_rss.stable.err
sql/test/configurable_rss/Tests/no_cap_rss.stable.out
sql/test/configurable_rss/Tests/push_and_check_rss.sql
Modified Files:
gdk/gdk_utils.c
monetdb5/modules/kernel/status.c
monetdb5/modules/kernel/status.h
monetdb5/modules/kernel/status.mal
Branch: config_rss
Log Message:
Migrate changes configurable_working_Set/default to config_rss/Mar2018.
diffs (truncated from 403 to 300 lines):
diff --git a/gdk/gdk_utils.c b/gdk/gdk_utils.c
--- a/gdk/gdk_utils.c
+++ b/gdk/gdk_utils.c
@@ -47,6 +47,10 @@ static char THRprintbuf[BUFSIZ];
#define chdir _chdir
#endif
+#ifdef WIN32
+# include <Windows.h>
+#endif
+
static volatile ATOMIC_FLAG GDKstopped = ATOMIC_FLAG_INIT;
static void GDKunlockHome(int farmid);
@@ -549,6 +553,16 @@ GDKinit(opt *set, int setlen)
if (strcmp("gdk_mem_maxsize", n[i].name) == 0) {
GDK_mem_maxsize = (size_t) strtoll(n[i].value, NULL,
10);
GDK_mem_maxsize = MAX(1 << 26, GDK_mem_maxsize);
+#ifdef WIN32 // On windows we can use a system call to limit the maximum
working set size.
+ const HANDLE process = GetCurrentProcess();
+
+ if (!SetProcessWorkingSetSizeEx(process, 1,
GDK_mem_maxsize, QUOTA_LIMITS_HARDWS_MAX_ENABLE))
+ {
+ printf("Something went wrong while setting the
limits on the working set: %d\n", GetLastError());
+
+ return 1;
+ }
+#endif
} else if (strcmp("gdk_vm_maxsize", n[i].name) == 0) {
GDK_vm_maxsize = (size_t) strtoll(n[i].value, NULL, 10);
GDK_vm_maxsize = MAX(1 << 30, GDK_vm_maxsize);
diff --git a/monetdb5/modules/kernel/status.c b/monetdb5/modules/kernel/status.c
--- a/monetdb5/modules/kernel/status.c
+++ b/monetdb5/modules/kernel/status.c
@@ -46,6 +46,13 @@ pseudo(bat *ret, bat *ret2, BAT *bn, BAT
}
str
+SYSgetrss_cursize(lng *num)
+{
+ *num = (lng) (MT_getrss() >> 10);
+ return MAL_SUCCEED;
+}
+
+str
SYSgetmem_cursize(lng *num)
{
*num = GDKmem_cursize();
diff --git a/monetdb5/modules/kernel/status.h b/monetdb5/modules/kernel/status.h
--- a/monetdb5/modules/kernel/status.h
+++ b/monetdb5/modules/kernel/status.h
@@ -9,6 +9,7 @@
#ifndef _SYS_H_
#define _SYS_H_
+mal_export str SYSgetrss_cursize(lng *num);
mal_export str SYSgetmem_cursize(lng *num);
mal_export str SYSgetmem_maxsize(lng *num);
mal_export str SYSsetmem_maxsize(void *ret, const lng *num);
diff --git a/monetdb5/modules/kernel/status.mal
b/monetdb5/modules/kernel/status.mal
--- a/monetdb5/modules/kernel/status.mal
+++ b/monetdb5/modules/kernel/status.mal
@@ -29,6 +29,10 @@ command getThreads()(:bat[:int],:bat[:st
address SYSgdkThread
comment "Produce overview of active threads";
+command rss_cursize():lng
+address SYSgetrss_cursize
+comment "The resident set size in KB";
+
command mem_cursize():lng
address SYSgetmem_cursize
comment "The amount of physical swapspace in KB that is currently in use";
diff --git a/sql/test/configurable_rss/Tests/All
b/sql/test/configurable_rss/Tests/All
new file mode 100644
--- /dev/null
+++ b/sql/test/configurable_rss/Tests/All
@@ -0,0 +1,2 @@
+NATIVE_WIN32?cap_rss
+NATIVE_WIN32?no_cap_rss
diff --git a/sql/test/configurable_rss/Tests/cap_rss.py
b/sql/test/configurable_rss/Tests/cap_rss.py
new file mode 100644
--- /dev/null
+++ b/sql/test/configurable_rss/Tests/cap_rss.py
@@ -0,0 +1,33 @@
+import os, sys
+try:
+ from MonetDBtesting import process
+except ImportError:
+ import process
+
+def server_start(args):
+ sys.stderr.write('#mserver: "%s"\n' % ' '.join(args))
+ sys.stderr.flush()
+ srv = process.server(args = args, stdin = process.PIPE,
+ stdout = process.PIPE, stderr = process.PIPE)
+ return srv
+
+def client(lang, file):
+ sys.stderr.write('#client: "%s"\n' % file)
+ sys.stderr.flush()
+ clt = process.client(lang.lower(), stdin = open(file),
+ stdout = process.PIPE, stderr = process.PIPE)
+ return clt.communicate()
+
+def main():
+ srv = server_start(["--set", "gdk_mem_maxsize=20000000"])
+
+ out, err = client('SQL',
+ os.path.join(os.getenv('RELSRCDIR'),
+ 'push_and_check_rss.sql'))
+ sys.stdout.write(out)
+ sys.stderr.write(err)
+ out, err = srv.communicate()
+ sys.stdout.write(out)
+ sys.stderr.write(err)
+
+main()
diff --git a/sql/test/configurable_rss/Tests/cap_rss.stable.err
b/sql/test/configurable_rss/Tests/cap_rss.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/test/configurable_rss/Tests/cap_rss.stable.err
@@ -0,0 +1,31 @@
+stderr of test 'cap_rss` in directory 'sql/test/configurable_rss` itself:
+
+
+# 20:34:03 >
+# 20:34:03 > "C:\Python27\python.exe" "cap_rss.py" "cap_rss"
+# 20:34:03 >
+
+#mserver: "--set gdk_mem_maxsize=70000000"
+#client:
"..\..\..\..\..\Sources\monetdb\sql\test\configurable_rss\Tests\push_and_check_rss.sql"
+# builtin opt gdk_dbpath =
C:\Users\aris\monetdb-installation\var\monetdb5\dbfarm\demo
+# builtin opt gdk_debug = 0
+# builtin opt gdk_vmtrim = no
+# builtin opt monet_prompt = >
+# builtin opt monet_daemon = no
+# builtin opt mapi_port = 50000
+# builtin opt mapi_open = false
+# builtin opt mapi_autosense = false
+# builtin opt sql_optimizer = default_pipe
+# builtin opt sql_debug = 0
+# cmdline opt gdk_nr_threads = 0
+# cmdline opt mapi_open = true
+# cmdline opt mapi_port = 32637
+# cmdline opt monet_prompt =
+# cmdline opt gdk_dbpath =
C:\Users\aris\monetdb-installation\var\MonetDB\mTests_sql_test_configurable_rss
+# cmdline opt gdk_mem_maxsize = 70000000
+# cmdline opt gdk_debug = 553648138
+
+# 20:34:21 >
+# 20:34:21 > "Done."
+# 20:34:21 >
+
diff --git a/sql/test/configurable_rss/Tests/cap_rss.stable.out
b/sql/test/configurable_rss/Tests/cap_rss.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/test/configurable_rss/Tests/cap_rss.stable.out
@@ -0,0 +1,82 @@
+stdout of test 'cap_rss` in directory 'sql/test/configurable_rss` itself:
+
+
+# 20:34:03 >
+# 20:34:03 > "C:\Python27\python.exe" "cap_rss.py" "cap_rss"
+# 20:34:03 >
+
+#create function getrss()
+#returns bigint external name status.rss_cursize;
+#create function getvm()
+#returns bigint external name status.mem_cursize;
+#create table test(a int, b int, c double);
+#insert into test values (1, 0, 1);
+[ 1 ]
+#create procedure loop_insert(maximum_size bigint)
+#begin
+# declare size bigint;
+# set size = (select count(*) from test);
+#
+# while size < maximum_size do
+# insert into test (select a+1, b+2, rand()*c from test);
+#
+# set size = (select count(*) from test);
+# end while;
+#end;
+#select getrss() < 20000 as resident_set_size_is_less_then_20kbytes,
quantile(c/a, 0.8) from test;
+% .L2, sys.L6 # table_name
+% resident_set_size_is_less_then_20kbytes, L6 # name
+% boolean, double # type
+% 5, 24 # length
+[ true, 0 ]
+#drop table test cascade;
+#drop function getrss;
+# MonetDB 5 server v11.30.0
+# This is an unreleased version
+# Serving database 'mTests_sql_test_configurable_rss', using 2 threads
+# Compiled for x86_64-pc-winnt/64bit
+# Found 8.000 GiB available main-memory.
+# Copyright (c) 1993 - July 2008 CWI.
+# Copyright (c) August 2008 - 2018 MonetDB B.V., all rights reserved
+# Visit https://www.monetdb.org/ for further information
+# Listening for connection requests on mapi:monetdb://DESKTOP-J5OE0E3:32637/
+# MonetDB/GIS module loaded
+# MonetDB/SQL module loaded
+# SQL catalog created, loading sql scripts once
+# loading sql script: 09_like.sql
+# loading sql script: 10_math.sql
+# loading sql script: 11_times.sql
+# loading sql script: 12_url.sql
+# loading sql script: 13_date.sql
+# loading sql script: 14_inet.sql
+# loading sql script: 15_querylog.sql
+# loading sql script: 16_tracelog.sql
+# loading sql script: 17_temporal.sql
+# loading sql script: 18_index.sql
+# loading sql script: 20_vacuum.sql
+# loading sql script: 21_dependency_functions.sql
+# loading sql script: 21_dependency_views.sql
+# loading sql script: 22_clients.sql
+# loading sql script: 23_skyserver.sql
+# loading sql script: 25_debug.sql
+# loading sql script: 26_sysmon.sql
+# loading sql script: 27_rejects.sql
+# loading sql script: 39_analytics.sql
+# loading sql script: 40_geom.sql
+# loading sql script: 40_json.sql
+# loading sql script: 41_md5sum.sql
+# loading sql script: 45_uuid.sql
+# loading sql script: 46_profiler.sql
+# loading sql script: 51_sys_schema_extension.sql
+# loading sql script: 60_wlcr.sql
+# loading sql script: 75_storagemodel.sql
+# loading sql script: 80_statistics.sql
+# loading sql script: 80_udf.sql
+# loading sql script: 90_generator.sql
+# loading sql script: 97_comments.sql
+# loading sql script: 99_system.sql
+
+# 20:34:21 >
+# 20:34:21 > "Done."
+# 20:34:21 >
+
diff --git a/sql/test/configurable_rss/Tests/no_cap_rss.sql
b/sql/test/configurable_rss/Tests/no_cap_rss.sql
new file mode 100644
--- /dev/null
+++ b/sql/test/configurable_rss/Tests/no_cap_rss.sql
@@ -0,0 +1,26 @@
+create function getrss()
+returns bigint external name status.rss_cursize;
+
+create table test(a int, b int, c double);
+
+insert into test values (1, 0, 1);
+
+create procedure loop_insert(maximum_size bigint)
+begin
+ declare size bigint;
+ set size = (select count(*) from test);
+
+ while size < maximum_size do
+ insert into test (select a+1, b+2, rand()*c from test);
+
+ set size = (select count(*) from test);
+ end while;
+end;
+
+call loop_insert(1000000);
+
+-- it seems that it requires an analytical query to keep memory in ram.
+select getrss() > 20000 as resident_set_size_is_bigger_then_20kbytes,
quantile(c/a, 0.8) * 0 from test;
+
+drop table test cascade;
+drop function getrss;
diff --git a/sql/test/configurable_rss/Tests/no_cap_rss.stable.err
b/sql/test/configurable_rss/Tests/no_cap_rss.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/test/configurable_rss/Tests/no_cap_rss.stable.err
@@ -0,0 +1,33 @@
+stderr of test 'no_cap_rss` in directory 'sql/test/configurable_rss` itself:
+
+
+# 20:52:50 >
+# 20:52:50 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set"
"mapi_open=true" "--set" "mapi_port=31687" "--set" "monet_prompt="
"--forcemito"
"--dbpath=C:\Users\aris\monetdb-installation\var\MonetDB\mTests_sql_test_configurable_rss"
+# 20:52:50 >
+
+# builtin opt gdk_dbpath =
C:\Users\aris\monetdb-installation\var\monetdb5\dbfarm\demo
+# builtin opt gdk_debug = 0
+# builtin opt gdk_vmtrim = no
+# builtin opt monet_prompt = >
+# builtin opt monet_daemon = no
+# builtin opt mapi_port = 50000
+# builtin opt mapi_open = false
+# builtin opt mapi_autosense = false
+# builtin opt sql_optimizer = default_pipe
+# builtin opt sql_debug = 0
+# cmdline opt gdk_nr_threads = 0
+# cmdline opt mapi_open = true
+# cmdline opt mapi_port = 31687
+# cmdline opt monet_prompt =
+# cmdline opt gdk_dbpath =
C:\Users\aris\monetdb-installation\var\MonetDB\mTests_sql_test_configurable_rss
+# cmdline opt gdk_debug = 553648138
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
