Changeset: 1e0de7ef57df for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1e0de7ef57df
Modified Files:
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
Branch: default
Log Message:
Merged branch 'userstats' into 'default'
diffs (truncated from 679 to 300 lines):
diff --git a/ChangeLog.userstats b/ChangeLog.userstats
new file mode 100644
--- /dev/null
+++ b/ChangeLog.userstats
@@ -0,0 +1,19 @@
+# ChangeLog file for userstats
+# This file is updated with Maddlog
+
+* Mon Aug 10 2020 Ying Zhang <y.zh...@cwi.nl>
+- Finished a first version of the new monitoring function
+ user_statistics(), which is only intended for the DBAs.
+ For each database user who has logged in during the current mserver5
+ session, it returns
+ "username": login name of the database user,
+ "querycount": the number of queries this user has executed since his/her
+ first login,
+ "totalticks": the total execution time (in microsecond) of the queries ran
+ by this user,
+ "maxquery": the query with the longest execution time (if two queries have
+ the same execution time, the newer overwrites the older),
+ "maxticks": the execution time of the 'maxquery' (in microsecond),
+ "started": the start timestamp of the 'maxquery',
+ "finished": the finish timestamp of the 'maxquery'.
+
diff --git a/clients/Tests/MAL-signatures.stable.out
b/clients/Tests/MAL-signatures.stable.out
--- a/clients/Tests/MAL-signatures.stable.out
+++ b/clients/Tests/MAL-signatures.stable.out
@@ -8766,6 +8766,7 @@ stdout of test 'MAL-signatures` in direc
[ "sysmon", "stop", "pattern sysmon.stop(X_1:int):void ", "SYSMONstop;",
"" ]
[ "sysmon", "stop", "pattern sysmon.stop(X_1:lng):void ", "SYSMONstop;",
"" ]
[ "sysmon", "stop", "pattern sysmon.stop(X_1:sht):void ", "SYSMONstop;",
"" ]
+[ "sysmon", "user_statistics", "pattern sysmon.user_statistics()
(X_0:bat[:str], X_1:bat[:lng], X_2:bat[:lng], X_3:bat[:timestamp],
X_4:bat[:timestamp], X_5:bat[:lng], X_6:bat[:str]) ", "SYSMONstatistics;",
"" ]
[ "tokenizer", "append", "command tokenizer.append(X_1:str):oid ",
"TKNZRappend;", "" ]
[ "tokenizer", "close", "command tokenizer.close():void ",
"TKNZRclose;", "" ]
[ "tokenizer", "depositFile", "command tokenizer.depositFile(X_1:str):void ",
"TKNZRdepositFile;", "" ]
diff --git a/clients/Tests/MAL-signatures.stable.out.int128
b/clients/Tests/MAL-signatures.stable.out.int128
--- a/clients/Tests/MAL-signatures.stable.out.int128
+++ b/clients/Tests/MAL-signatures.stable.out.int128
@@ -11730,6 +11730,7 @@ stdout of test 'MAL-signatures` in direc
[ "sysmon", "stop", "pattern sysmon.stop(X_1:int):void ", "SYSMONstop;",
"" ]
[ "sysmon", "stop", "pattern sysmon.stop(X_1:lng):void ", "SYSMONstop;",
"" ]
[ "sysmon", "stop", "pattern sysmon.stop(X_1:sht):void ", "SYSMONstop;",
"" ]
+[ "sysmon", "user_statistics", "pattern sysmon.user_statistics()
(X_0:bat[:str], X_1:bat[:lng], X_2:bat[:lng], X_3:bat[:timestamp],
X_4:bat[:timestamp], X_5:bat[:lng], X_6:bat[:str]) ", "SYSMONstatistics;",
"" ]
[ "tokenizer", "append", "command tokenizer.append(X_1:str):oid ",
"TKNZRappend;", "" ]
[ "tokenizer", "close", "command tokenizer.close():void ",
"TKNZRclose;", "" ]
[ "tokenizer", "depositFile", "command tokenizer.depositFile(X_1:str):void ",
"TKNZRdepositFile;", "" ]
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -1126,6 +1126,7 @@ str STRtostr(str *res, const str *src);
str SYSMONpause(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
str SYSMONqueue(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
str SYSMONresume(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
+str SYSMONstatistics(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
str SYSMONstop(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
str SYScpuStatistics(bat *ret, bat *ret2);
str SYSgdkEnv(bat *ret, bat *ret2);
@@ -1176,6 +1177,7 @@ str TRNtrans_commit(Client cntxt, MalBlk
int TYPE_blob;
int TYPE_uuid;
int TYPE_xml;
+UserStats USRstats;
str WLCaction(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
str WLCappend(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
str WLCcatalog(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
@@ -1837,6 +1839,7 @@ str updateRef;
void updateScenario(str scen, str nme, MALfcn fcn);
Module userModule(void);
str userRef;
+size_t usrstatscnt;
str vectorRef;
str window_boundRef;
str wlcRef;
diff --git a/monetdb5/mal/mal_runtime.c b/monetdb5/mal/mal_runtime.c
--- a/monetdb5/mal/mal_runtime.c
+++ b/monetdb5/mal/mal_runtime.c
@@ -30,6 +30,97 @@ QueryQueue QRYqueue = NULL;
size_t qsize = 0, qhead = 0, qtail = 0;
static oid qtag= 1; // A unique query identifier
+UserStats USRstats = NULL;
+size_t usrstatscnt = 0;
+
+static void
+clearUSRstats(size_t idx)
+{
+ USRstats[idx].user= 0;
+ USRstats[idx].username = 0;
+ USRstats[idx].querycount = 0;
+ USRstats[idx].totalticks = 0;
+ USRstats[idx].started = 0;
+ USRstats[idx].finished = 0;
+ USRstats[idx].maxticks = 0;
+ USRstats[idx].maxquery = 0;
+}
+
+/*
+ * Find the index of the given 'user' in USRstats.
+ * For a new 'user' return a new free slot.
+ * If USRstats is full, extend it.
+ */
+static
+size_t
+getUSRstatsIdx(MalBlkPtr mb, oid user)
+{
+ size_t i = 0;
+ UserStats tmp = NULL;
+
+ for (i = 0; i < usrstatscnt; i++)
+ /* The array is dense, so we either find the user or an empty
slot. */
+ if (USRstats[i].user == user || USRstats[i].username == NULL)
+ return i;
+
+ /* expand USRstats */
+ tmp = (UserStats) GDKrealloc(USRstats, sizeof (struct USERSTAT) *
(size_t) (usrstatscnt += MAL_MAXCLIENTS));
+ if (tmp == NULL) {
+ /* It's not a fatal error if we can't extend USRstats.
+ * We don't want to affect existing USRstats. */
+ addMalException(mb,"getUSRstatsIdx" MAL_MALLOC_FAIL);
+ return (size_t) -1;
+ }
+ USRstats = tmp;
+ for ( ; i < usrstatscnt; i++)
+ clearUSRstats(i);
+ return usrstatscnt - MAL_MAXCLIENTS;
+}
+
+static
+void
+updateUserStats(Client cntxt, MalBlkPtr mb, lng ticks, time_t started, time_t
finished, str query)
+{
+ size_t idx = getUSRstatsIdx(mb, cntxt->user);
+
+ if (idx == (size_t) -1) {
+ addMalException(mb, "updateUserStats" "Failed to get an entry
in user statistics");
+ return;
+ }
+
+ if (USRstats[idx].username == NULL) {
+ USRstats[idx].user = cntxt->user;
+ USRstats[idx].username = GDKstrdup(cntxt->username);
+ }
+ USRstats[idx].querycount++;
+ USRstats[idx].totalticks += ticks;
+ if( ticks >= USRstats[idx].maxticks && query){
+ USRstats[idx].started = started;
+ USRstats[idx].finished = finished;
+ USRstats[idx].maxticks = ticks;
+ GDKfree(USRstats[idx].maxquery);
+ USRstats[idx].maxquery= GDKstrdup(query);
+ }
+}
+
+/*
+ * Free up the whole USRstats before mserver5 exits.
+ */
+static void
+dropUSRstats(void)
+{
+ size_t i;
+ MT_lock_set(&mal_delayLock);
+ for(i = 0; i < usrstatscnt; i++){
+ GDKfree(USRstats[i].username);
+ GDKfree(USRstats[i].maxquery);
+ clearUSRstats(i);
+ }
+ GDKfree(USRstats);
+ USRstats = NULL;
+ MT_lock_unset(&mal_delayLock);
+}
+
static str
isaSQLquery(MalBlkPtr mb){
int i;
@@ -116,6 +207,18 @@ runtimeProfileInit(Client cntxt, MalBlkP
QueryQueue tmp = NULL;
MT_lock_set(&mal_delayLock);
+
+ if(USRstats == NULL){
+ usrstatscnt = MAL_MAXCLIENTS;
+ USRstats = (UserStats) GDKzalloc( sizeof (struct USERSTAT) *
usrstatscnt);
+ if(USRstats == NULL) {
+ addMalException(mb,"runtimeProfileInit"
MAL_MALLOC_FAIL);
+ MT_lock_unset(&mal_delayLock);
+ return;
+ }
+ }
+
+ tmp = QRYqueue;
if ( QRYqueue == NULL) {
QRYqueue = (QueryQueue) GDKzalloc( sizeof (struct QRYQUEUE) *
(qsize= MAL_MAXCLIENTS));
@@ -141,7 +244,7 @@ runtimeProfileInit(Client cntxt, MalBlkP
i = 0;
}
assert(qhead < qsize);
- if( (int) (qsize - paused) < MAL_MAXCLIENTS){
+ if( qsize - paused < (size_t) MAL_MAXCLIENTS){
qsize += MAL_MAXCLIENTS;
tmp = (QueryQueue) GDKrealloc( QRYqueue, sizeof (struct
QRYQUEUE) * qsize);
if ( tmp == NULL){
@@ -171,6 +274,7 @@ runtimeProfileInit(Client cntxt, MalBlkP
QRYqueue[qhead].workers = (int) stk->workers;
QRYqueue[qhead].status = "running";
QRYqueue[qhead].cntxt = cntxt;
+ QRYqueue[qhead].ticks = GDKusec();
stk->tag = mb->tag = QRYqueue[qhead].tag;
advanceQRYqueue();
MT_lock_unset(&mal_delayLock);
@@ -186,8 +290,6 @@ runtimeProfileFinish(Client cntxt, MalBl
{
size_t i;
- (void) cntxt;
-
MT_lock_set(&mal_delayLock);
i=qtail;
while (i != qhead){
@@ -203,6 +305,8 @@ runtimeProfileFinish(Client cntxt, MalBl
QRYqueue[i].cntxt = 0;
QRYqueue[i].stk = 0;
QRYqueue[i].mb = 0;
+ QRYqueue[i].ticks = GDKusec() - QRYqueue[i].ticks;
+ updateUserStats(cntxt, mb, QRYqueue[i].ticks,
QRYqueue[i].start, QRYqueue[i].finished, QRYqueue[i].query);
// assume that the user is now idle
cntxt->idle = time(0);
break;
@@ -223,6 +327,9 @@ mal_runtime_reset(void)
qtag= 1;
qhead = 0;
qtail = 0;
+
+ dropUSRstats();
+ usrstatscnt = 0;
}
/*
diff --git a/monetdb5/mal/mal_runtime.h b/monetdb5/mal/mal_runtime.h
--- a/monetdb5/mal/mal_runtime.h
+++ b/monetdb5/mal/mal_runtime.h
@@ -35,16 +35,30 @@ typedef struct QRYQUEUE{
int idx;
int workers;
int memory;
+ lng ticks;
time_t start;
time_t finished;
} *QueryQueue;
mal_export size_t qhead, qtail, qsize;
+/* We keep a few statistics per user to identify unexpected behavior */
+typedef struct USERSTAT{
+ oid user; /* user id in the auth administration */
+ str username;
+ lng querycount;
+ lng totalticks;
+ time_t started;
+ time_t finished;
+ lng maxticks;
+ str maxquery;
+} *UserStats;
+mal_export size_t usrstatscnt;
+
typedef struct WORKINGSET{
Client cntxt;
- MalBlkPtr mb;
- MalStkPtr stk;
- InstrPtr pci;
+ MalBlkPtr mb;
+ MalStkPtr stk;
+ InstrPtr pci;
} Workingset;
mal_export Workingset workingset[THREADS];
@@ -57,4 +71,5 @@ mal_export lng getVolume(MalStkPtr stk,
mal_export lng getBatSpace(BAT *b);
mal_export QueryQueue QRYqueue;
+mal_export UserStats USRstats;
#endif
diff --git a/monetdb5/modules/mal/sysmon.c b/monetdb5/modules/mal/sysmon.c
--- a/monetdb5/modules/mal/sysmon.c
+++ b/monetdb5/modules/mal/sysmon.c
@@ -18,6 +18,129 @@
*/
str
+SYSMONstatistics(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
+{
+ BAT *user, *querycount, *totalticks, *started, *finished, *maxquery,
*maxticks;
+ bat *u = getArgReference_bat(stk,pci,0);
+ bat *c = getArgReference_bat(stk,pci,1);
+ bat *t = getArgReference_bat(stk,pci,2);
+ bat *s = getArgReference_bat(stk,pci,3);
+ bat *f = getArgReference_bat(stk,pci,4);
+ bat *m = getArgReference_bat(stk,pci,5);
+ bat *q = getArgReference_bat(stk,pci,6);
+ size_t i;
+ timestamp tsn = timestamp_nil;
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
