Changeset: a8556e9e6991 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a8556e9e6991
Modified Files:
clients/Tests/exports.stable.out
monetdb5/mal/mal_client.c
monetdb5/mal/mal_client.h
monetdb5/mal/mal_dataflow.c
monetdb5/mal/mal_profiler.c
monetdb5/mal/mal_profiler.h
monetdb5/mal/mal_runtime.c
Branch: mal-profiler-resources
Log Message:
Set process resource track on a per client basis instead of global variables.
This avoids potential race conditions while profiling multiple clients.
diffs (truncated from 479 to 300 lines):
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -2515,9 +2515,9 @@ str printRef;
void printSignature(stream *fd, Symbol s, int flg);
void printStack(stream *f, MalBlkPtr mb, MalStkPtr s);
str prodRef;
-void profilerEvent(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, int start, str
usrname);
+void profilerEvent(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, int start, str
usrname, void *res_usage);
void profilerGetCPUStat(lng *user, lng *nice, lng *sys, lng *idle, lng
*iowait);
-void profilerHeartbeatEvent(char *alter);
+void profilerHeartbeatEvent(char *alter, void *res_usage);
str profilerRef;
str projectRef;
str projectdeltaRef;
diff --git a/monetdb5/mal/mal_client.c b/monetdb5/mal/mal_client.c
--- a/monetdb5/mal/mal_client.c
+++ b/monetdb5/mal/mal_client.c
@@ -266,6 +266,9 @@ MCinitClientRecord(Client c, oid user, b
c->filetrans = false;
MT_sema_init(&c->s, 0, "Client->s");
+#ifdef HAVE_SYS_RESOURCE_H
+ memset(&(c->res_usage), 0, sizeof(struct rusage));
+#endif
return c;
}
@@ -430,6 +433,9 @@ freeClient(Client c)
c->wlc_kind = 0;
c->wlc = NULL;
MT_sema_destroy(&c->s);
+#ifdef HAVE_SYS_RESOURCE_H
+ memset(&(c->res_usage), 0, sizeof(struct rusage));
+#endif
c->mode = MCshutdowninprogress()? BLOCKCLIENT: FREECLIENT;
}
diff --git a/monetdb5/mal/mal_client.h b/monetdb5/mal/mal_client.h
--- a/monetdb5/mal/mal_client.h
+++ b/monetdb5/mal/mal_client.h
@@ -15,6 +15,10 @@
#include "mal_resolve.h"
#include "mal_profiler.h"
+#ifdef HAVE_SYS_RESOURCE_H
+#include <sys/resource.h>
+#endif
+
#define SCENARIO_PROPERTIES 8
#define CONSOLE 0
@@ -191,6 +195,10 @@ typedef struct CLIENT {
size_t blocksize;
protocol_version protocol;
bool filetrans; /* whether the client can read
files for us */
+
+#ifdef HAVE_SYS_RESOURCE_H
+ struct rusage res_usage; /* used by the mal_profiler */
+#endif
} *Client, ClientRec;
mal_export void MCinit(void);
diff --git a/monetdb5/mal/mal_dataflow.c b/monetdb5/mal/mal_dataflow.c
--- a/monetdb5/mal/mal_dataflow.c
+++ b/monetdb5/mal/mal_dataflow.c
@@ -26,6 +26,7 @@
* access it mostly without expensive locking.
*/
#include "monetdb_config.h"
+#include "mal_client.h"
#include "mal_dataflow.h"
#include "mal_private.h"
#include "mal_runtime.h"
@@ -471,7 +472,13 @@ DFLOWworker(void *T)
last = todo->last;
MT_lock_unset(&todo->l);
if (last == 0)
- profilerHeartbeatEvent("wait");
+ profilerHeartbeatEvent("wait",
+#ifdef HAVE_SYS_RESOURCE_H
+
&(cntxt->res_usage)
+#else
+ NULL
+#endif
+ );
}
}
GDKfree(GDKerrbuf);
diff --git a/monetdb5/mal/mal_profiler.c b/monetdb5/mal/mal_profiler.c
--- a/monetdb5/mal/mal_profiler.c
+++ b/monetdb5/mal/mal_profiler.c
@@ -16,6 +16,7 @@
*/
#include "monetdb_config.h"
#include "mutils.h" /* mercurial_revision */
+#include "mal_client.h"
#include "mal_function.h"
#include "mal_listing.h"
#include "mal_profiler.h"
@@ -29,7 +30,7 @@
#include <string.h>
-static void cachedProfilerEvent(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
+static void cachedProfilerEvent(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci,
void *res_usage);
stream *eventstream = 0;
@@ -47,14 +48,9 @@ static struct timeval startup_time;
static volatile ATOMIC_TYPE hbdelay = 0;
-#ifdef HAVE_SYS_RESOURCE_H
-struct rusage infoUsage;
-static struct rusage prevUsage;
-#endif
-
static struct{
lng user, nice, system, idle, iowait;
- double load;
+ dbl load;
} corestat[256];
@@ -138,7 +134,7 @@ EXAMPLE:
}
*/
static void
-renderProfilerEvent(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, int start, str
usrname)
+renderProfilerEvent(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, int start, str
usrname, void *res_usage)
{
(void)usrname;
char logbuffer[LOGLEN], *logbase;
@@ -147,6 +143,9 @@ renderProfilerEvent(MalBlkPtr mb, MalStk
str stmtq;
lng usec= GDKusec();
uint64_t microseconds = (uint64_t)startup_time.tv_sec*1000000 +
(uint64_t)startup_time.tv_usec + (uint64_t)usec;
+#ifdef HAVE_SYS_RESOURCE_H
+ struct rusage infoUsage, *prevUsage = (struct rusage*) res_usage;
+#endif
// ignore generation of events for instructions that are called too
often
if(highwatermark && highwatermark + (start == 0) < pci->calls)
@@ -199,17 +198,19 @@ renderProfilerEvent(MalBlkPtr mb, MalStk
#ifdef HAVE_SYS_RESOURCE_H
getrusage(RUSAGE_SELF, &infoUsage);
- if(infoUsage.ru_inblock - prevUsage.ru_inblock)
- logadd("\"inblock\":%ld,%s", infoUsage.ru_inblock -
prevUsage.ru_inblock, prettify);
- if(infoUsage.ru_oublock - prevUsage.ru_oublock)
- logadd("\"oublock\":%ld,%s", infoUsage.ru_oublock -
prevUsage.ru_oublock, prettify);
- if(infoUsage.ru_majflt - prevUsage.ru_majflt)
- logadd("\"majflt\":%ld,%s", infoUsage.ru_majflt -
prevUsage.ru_majflt, prettify);
- if(infoUsage.ru_nswap - prevUsage.ru_nswap)
- logadd("\"nswap\":%ld,s%sn", infoUsage.ru_nswap -
prevUsage.ru_nswap, prettify);
- if(infoUsage.ru_nvcsw - prevUsage.ru_nvcsw)
- logadd("\"nvcsw\":%ld,%s", infoUsage.ru_nvcsw -
prevUsage.ru_nvcsw +infoUsage.ru_nivcsw - prevUsage.ru_nivcsw, prettify);
- prevUsage = infoUsage;
+ if(infoUsage.ru_inblock - prevUsage->ru_inblock)
+ logadd("\"inblock\":%ld,%s", infoUsage.ru_inblock -
prevUsage->ru_inblock, prettify);
+ if(infoUsage.ru_oublock - prevUsage->ru_oublock)
+ logadd("\"oublock\":%ld,%s", infoUsage.ru_oublock -
prevUsage->ru_oublock, prettify);
+ if(infoUsage.ru_majflt - prevUsage->ru_majflt)
+ logadd("\"majflt\":%ld,%s", infoUsage.ru_majflt -
prevUsage->ru_majflt, prettify);
+ if(infoUsage.ru_nswap - prevUsage->ru_nswap)
+ logadd("\"nswap\":%ld,s%sn", infoUsage.ru_nswap -
prevUsage->ru_nswap, prettify);
+ if(infoUsage.ru_nvcsw - prevUsage->ru_nvcsw)
+ logadd("\"nvcsw\":%ld,%s", infoUsage.ru_nvcsw -
prevUsage->ru_nvcsw +infoUsage.ru_nivcsw - prevUsage->ru_nivcsw, prettify);
+ memcpy(prevUsage, &infoUsage, sizeof(struct rusage));
+#else
+ (void) res_usage;
#endif
if( mb){
@@ -417,7 +418,7 @@ getCPULoad(char cpuload[BUFSIZ]){
goto skip;
newload = (user - corestat[cpu].user + nice -
corestat[cpu].nice + system - corestat[cpu].system);
if ( newload)
- corestat[cpu].load = (double) newload /
(newload + idle - corestat[cpu].idle + iowait - corestat[cpu].iowait);
+ corestat[cpu].load = (dbl) newload / (newload +
idle - corestat[cpu].idle + iowait - corestat[cpu].iowait);
corestat[cpu].user = user;
corestat[cpu].nice = nice;
corestat[cpu].system = system;
@@ -441,13 +442,16 @@ getCPULoad(char cpuload[BUFSIZ]){
}
void
-profilerHeartbeatEvent(char *alter)
+profilerHeartbeatEvent(char *alter, void *res_usage)
{
char cpuload[BUFSIZ];
char logbuffer[LOGLEN], *logbase;
int loglen;
lng usec = GDKusec();
uint64_t microseconds = (uint64_t)startup_time.tv_sec*1000000 +
(uint64_t)startup_time.tv_usec + (uint64_t)usec;
+#ifdef HAVE_SYS_RESOURCE_H
+ struct rusage infoUsage, *prevUsage = (struct rusage*) res_usage;
+#endif
if (ATOMIC_GET(hbdelay, mal_beatLock) == 0 || eventstream == NULL)
return;
@@ -465,18 +469,21 @@ profilerHeartbeatEvent(char *alter)
logadd("\"ctime\":%"PRIu64",%s", microseconds, prettify);
logadd("\"rss\":%zu,%s", MT_getrss()/1024/1024, prettify);
#ifdef HAVE_SYS_RESOURCE_H
+ assert(res_usage);
getrusage(RUSAGE_SELF, &infoUsage);
- if(infoUsage.ru_inblock - prevUsage.ru_inblock)
- logadd("\"inblock\":%ld,%s", infoUsage.ru_inblock -
prevUsage.ru_inblock, prettify);
- if(infoUsage.ru_oublock - prevUsage.ru_oublock)
- logadd("\"oublock\":%ld,%s", infoUsage.ru_oublock -
prevUsage.ru_oublock, prettify);
- if(infoUsage.ru_majflt - prevUsage.ru_majflt)
- logadd("\"majflt\":%ld,%s", infoUsage.ru_majflt -
prevUsage.ru_majflt, prettify);
- if(infoUsage.ru_nswap - prevUsage.ru_nswap)
- logadd("\"nswap\":%ld,%s", infoUsage.ru_nswap -
prevUsage.ru_nswap, prettify);
- if(infoUsage.ru_nvcsw - prevUsage.ru_nvcsw)
- logadd("\"nvcsw\":%ld,%s", infoUsage.ru_nvcsw -
prevUsage.ru_nvcsw +infoUsage.ru_nivcsw - prevUsage.ru_nivcsw, prettify);
- prevUsage = infoUsage;
+ if(infoUsage.ru_inblock - prevUsage->ru_inblock)
+ logadd("\"inblock\":%ld,%s", infoUsage.ru_inblock -
prevUsage->ru_inblock, prettify);
+ if(infoUsage.ru_oublock - prevUsage->ru_oublock)
+ logadd("\"oublock\":%ld,%s", infoUsage.ru_oublock -
prevUsage->ru_oublock, prettify);
+ if(infoUsage.ru_majflt - prevUsage->ru_majflt)
+ logadd("\"majflt\":%ld,%s", infoUsage.ru_majflt -
prevUsage->ru_majflt, prettify);
+ if(infoUsage.ru_nswap - prevUsage->ru_nswap)
+ logadd("\"nswap\":%ld,%s", infoUsage.ru_nswap -
prevUsage->ru_nswap, prettify);
+ if(infoUsage.ru_nvcsw - prevUsage->ru_nvcsw)
+ logadd("\"nvcsw\":%ld,%s", infoUsage.ru_nvcsw -
prevUsage->ru_nvcsw +infoUsage.ru_nivcsw - prevUsage->ru_nivcsw, prettify);
+ memcpy(prevUsage, &infoUsage, sizeof(struct rusage));
+#else
+ (void) res_usage;
#endif
logadd("\"state\":\"%s\",%s",alter,prettify);
logadd("\"cpuload\":%s%s",cpuload,prettify);
@@ -485,7 +492,7 @@ profilerHeartbeatEvent(char *alter)
}
void
-profilerEvent(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, int start, str
usrname)
+profilerEvent(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, int start, str
usrname, void *res_usage)
{
if (stk == NULL) return;
if (pci == NULL) return;
@@ -493,14 +500,14 @@ profilerEvent(MalBlkPtr mb, MalStkPtr st
return;
if( sqlProfiling && !start )
- cachedProfilerEvent(mb, stk, pci);
+ cachedProfilerEvent(mb, stk, pci, res_usage);
if( eventstream) {
- renderProfilerEvent(mb, stk, pci, start, usrname);
+ renderProfilerEvent(mb, stk, pci, start, usrname, res_usage);
if ( start && pci->pc ==0)
- profilerHeartbeatEvent("ping");
+ profilerHeartbeatEvent("ping", res_usage);
if ( !start && pci->token == ENDsymbol)
- profilerHeartbeatEvent("ping");
+ profilerHeartbeatEvent("ping", res_usage);
}
}
@@ -518,10 +525,6 @@ openProfilerStream(stream *fd, int mode)
int i,j;
Client c;
-#ifdef HAVE_SYS_RESOURCE_H
- getrusage(RUSAGE_SELF, &infoUsage);
- prevUsage = infoUsage;
-#endif
if (myname == 0){
myname = putName("profiler");
eventcounter = 0;
@@ -537,11 +540,21 @@ openProfilerStream(stream *fd, int mode)
if( (mode & PROFSHOWRUNNING) > 0){
for (i = 0; i < MAL_MAXCLIENTS; i++) {
c = mal_clients+i;
- if ( c->active )
+ if ( c->active ) {
+#ifdef HAVE_SYS_RESOURCE_H
+ getrusage(RUSAGE_SELF, &(c->res_usage));
+#endif
for(j = 0; j <THREADS; j++)
- if( c->inprogress[j].mb)
- /* show the event */
- profilerEvent(c->inprogress[j].mb,
c->inprogress[j].stk, c->inprogress[j].pci, 1, c->username);
+ if( c->inprogress[j].mb)
+ /* show the event */
+
profilerEvent(c->inprogress[j].mb, c->inprogress[j].stk, c->inprogress[j].pci,
1, c->username,
+#ifdef HAVE_SYS_RESOURCE_H
+
&(c->res_usage)
+#else
+ NULL
+#endif
+ );
+ }
}
}
return MAL_SUCCEED;
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
