Changeset: 99f7ab4f895f for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=99f7ab4f895f Added Files: monetdb5/modules/mosaic/Tests/mosaic_calendar.malC monetdb5/modules/mosaic/Tests/mosaic_calendar.stable.err monetdb5/modules/mosaic/Tests/mosaic_calendar.stable.out Modified Files: monetdb5/modules/mosaic/Tests/All monetdb5/modules/mosaic/mosaic_calendar.c Branch: mosaic Log Message:
Minor fixes to calendar diffs (truncated from 369 to 300 lines): diff --git a/monetdb5/modules/mosaic/Tests/All b/monetdb5/modules/mosaic/Tests/All --- a/monetdb5/modules/mosaic/Tests/All +++ b/monetdb5/modules/mosaic/Tests/All @@ -8,6 +8,7 @@ mosaic_delta mosaic_linear mosaic_frame mosaic_prefix +mosaic_calendar mosaic_literal_double diff --git a/monetdb5/modules/mosaic/Tests/mosaic_calendar.malC b/monetdb5/modules/mosaic/Tests/mosaic_calendar.malC new file mode 100644 --- /dev/null +++ b/monetdb5/modules/mosaic/Tests/mosaic_calendar.malC @@ -0,0 +1,35 @@ +# Calendar compressions + + +d:= bat.new(:date); +bat.append(d,"2014-08-14":date); +bat.append(d,"2014-08-14":date); +bat.append(d,"2014-08-14":date); +bat.append(d,"2014-08-14":date); +bat.append(d,"2014-08-15":date); +bat.append(d,"2015-07-11":date); +bat.append(d,"2015-07-12":date); +bat.append(d,"2016-12-31":date); +d:= bat.append(d,d); + +io.print(d); +mosaic.compress(d,"calendar"); +#mosaic.dump(xd); +mosaic.decompress(d); +io.print(d); + +bs:= bat.new(:timestamp); +bat.append(bs,"2014-08-14 12:00:00":timestamp); +bat.append(bs,"2014-08-14 12:00:00":timestamp); +bat.append(bs,"2014-08-14 12:00:00":timestamp); +bat.append(bs,"2014-08-14 12:00:00":timestamp); +bat.append(bs,"2014-08-14 12:00:00":timestamp); +bat.append(bs,"2014-08-14 12:01:00":timestamp); +bat.append(bs,"2014-08-14 12:02:00":timestamp); +bs:= bat.append(bs,bs); + +io.print(bs); +mosaic.compress(bs,"calendar"); +#mosaic.dump(bs); +mosaic.decompress(bs); +io.print(bs); diff --git a/monetdb5/modules/mosaic/Tests/mosaic_calendar.stable.err b/monetdb5/modules/mosaic/Tests/mosaic_calendar.stable.err new file mode 100644 --- /dev/null +++ b/monetdb5/modules/mosaic/Tests/mosaic_calendar.stable.err @@ -0,0 +1,36 @@ +stderr of test 'mosaic_calendar` in directory 'monetdb5/modules/mosaic` itself: + + +# 10:49:14 > +# 10:49:14 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=33013" "--set" "mapi_usock=/var/tmp/mtest-2985/.s.monetdb.33013" "--set" "monet_prompt=" "--forcemito" "--dbpath=/export/scratch1/mk/mosaic//Linux/var/MonetDB/mTests_monetdb5_modules_mosaic" "--set" "embedded_r=yes" "--set" "embedded_py=true" +# 10:49:14 > + +# builtin opt gdk_dbpath = /export/scratch1/mk/mosaic//Linux/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = no +# builtin opt monet_prompt = > +# builtin opt monet_daemon = no +# builtin opt mapi_port = 50000 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 33013 +# cmdline opt mapi_usock = /var/tmp/mtest-2985/.s.monetdb.33013 +# cmdline opt monet_prompt = +# cmdline opt gdk_dbpath = /export/scratch1/mk/mosaic//Linux/var/MonetDB/mTests_monetdb5_modules_mosaic +# cmdline opt embedded_r = yes +# cmdline opt embedded_py = true +# cmdline opt gdk_debug = 536870922 + +# 10:49:15 > +# 10:49:15 > "mclient" "-lmal" "-ftest" "-Eutf-8" "--host=/var/tmp/mtest-2985" "--port=33013" +# 10:49:15 > + + +# 10:49:15 > +# 10:49:15 > "Done." +# 10:49:15 > + diff --git a/monetdb5/modules/mosaic/Tests/mosaic_calendar.stable.out b/monetdb5/modules/mosaic/Tests/mosaic_calendar.stable.out new file mode 100644 --- /dev/null +++ b/monetdb5/modules/mosaic/Tests/mosaic_calendar.stable.out @@ -0,0 +1,108 @@ +stdout of test 'mosaic_calendar` in directory 'monetdb5/modules/mosaic` itself: + + +# 10:49:14 > +# 10:49:14 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=33013" "--set" "mapi_usock=/var/tmp/mtest-2985/.s.monetdb.33013" "--set" "monet_prompt=" "--forcemito" "--dbpath=/export/scratch1/mk/mosaic//Linux/var/MonetDB/mTests_monetdb5_modules_mosaic" "--set" "embedded_r=yes" "--set" "embedded_py=true" +# 10:49:14 > + +# MonetDB 5 server v11.26.0 +# This is an unreleased version +# Serving database 'mTests_monetdb5_modules_mosaic', using 8 threads +# Compiled for x86_64-unknown-linux-gnu/64bit with 128bit integers +# Found 15.589 GiB available main-memory. +# Copyright (c) 1993-July 2008 CWI. +# Copyright (c) August 2008-2016 MonetDB B.V., all rights reserved +# Visit http://www.monetdb.org/ for further information +# Listening for connection requests on mapi:monetdb://vienna.da.cwi.nl:33013/ +# Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-2985/.s.monetdb.33013 +# MonetDB/SQL module loaded +# MonetDB/Python module loaded +# MonetDB/R module loaded + +Ready. + +# 10:49:15 > +# 10:49:15 > "mclient" "-lmal" "-ftest" "-Eutf-8" "--host=/var/tmp/mtest-2985" "--port=33013" +# 10:49:15 > + +#--------------------------# +# h t # name +# void date # type +#--------------------------# +[ 0@0, 2014-08-14 ] +[ 1@0, 2014-08-14 ] +[ 2@0, 2014-08-14 ] +[ 3@0, 2014-08-14 ] +[ 4@0, 2014-08-15 ] +[ 5@0, 2015-07-11 ] +[ 6@0, 2015-07-12 ] +[ 7@0, 2016-12-31 ] +[ 8@0, 2014-08-14 ] +[ 9@0, 2014-08-14 ] +[ 10@0, 2014-08-14 ] +[ 11@0, 2014-08-14 ] +[ 12@0, 2014-08-15 ] +[ 13@0, 2015-07-11 ] +[ 14@0, 2015-07-12 ] +[ 15@0, 2016-12-31 ] +#--------------------------# +# h t # name +# void date # type +#--------------------------# +[ 0@0, 2014-08-14 ] +[ 1@0, 2014-08-14 ] +[ 2@0, 2014-08-14 ] +[ 3@0, 2014-08-14 ] +[ 4@0, 2014-08-15 ] +[ 5@0, 2015-07-11 ] +[ 6@0, 2015-07-12 ] +[ 7@0, 2016-12-31 ] +[ 8@0, 2014-08-14 ] +[ 9@0, 2014-08-14 ] +[ 10@0, 2014-08-14 ] +[ 11@0, 2014-08-14 ] +[ 12@0, 2014-08-15 ] +[ 13@0, 2015-07-11 ] +[ 14@0, 2015-07-12 ] +[ 15@0, 2016-12-31 ] +#--------------------------# +# h t # name +# void timestamp # type +#--------------------------# +[ 0@0, 2014-08-14 12:00:00.000 ] +[ 1@0, 2014-08-14 12:00:00.000 ] +[ 2@0, 2014-08-14 12:00:00.000 ] +[ 3@0, 2014-08-14 12:00:00.000 ] +[ 4@0, 2014-08-14 12:00:00.000 ] +[ 5@0, 2014-08-14 12:01:00.000 ] +[ 6@0, 2014-08-14 12:02:00.000 ] +[ 7@0, 2014-08-14 12:00:00.000 ] +[ 8@0, 2014-08-14 12:00:00.000 ] +[ 9@0, 2014-08-14 12:00:00.000 ] +[ 10@0, 2014-08-14 12:00:00.000 ] +[ 11@0, 2014-08-14 12:00:00.000 ] +[ 12@0, 2014-08-14 12:01:00.000 ] +[ 13@0, 2014-08-14 12:02:00.000 ] +#--------------------------# +# h t # name +# void timestamp # type +#--------------------------# +[ 0@0, 2014-08-14 12:00:00.000 ] +[ 1@0, 2014-08-14 12:00:00.000 ] +[ 2@0, 2014-08-14 12:00:00.000 ] +[ 3@0, 2014-08-14 12:00:00.000 ] +[ 4@0, 2014-08-14 12:00:00.000 ] +[ 5@0, 2014-08-14 12:01:00.000 ] +[ 6@0, 2014-08-14 12:02:00.000 ] +[ 7@0, 2014-08-14 12:00:00.000 ] +[ 8@0, 2014-08-14 12:00:00.000 ] +[ 9@0, 2014-08-14 12:00:00.000 ] +[ 10@0, 2014-08-14 12:00:00.000 ] +[ 11@0, 2014-08-14 12:00:00.000 ] +[ 12@0, 2014-08-14 12:01:00.000 ] +[ 13@0, 2014-08-14 12:02:00.000 ] + +# 10:49:15 > +# 10:49:15 > "Done." +# 10:49:15 > + diff --git a/monetdb5/modules/mosaic/mosaic_calendar.c b/monetdb5/modules/mosaic/mosaic_calendar.c --- a/monetdb5/modules/mosaic/mosaic_calendar.c +++ b/monetdb5/modules/mosaic/mosaic_calendar.c @@ -137,9 +137,37 @@ MOSskip_calendar(Client cntxt, MOStask t // Create a larger calendar buffer then we allow for in the mosaic header first // Store the most frequent ones in the compressed heap header directly based on estimated savings -// Improve by using binary search rather then linear scan #define TMPDICT 16*256 +#define makeCalendar(TPE, MASK)\ +{ TPE *val = ((TPE*)task->src) + task->start,v,w;\ + BUN limit = task->stop - task->start > MOSlimit()? MOSlimit(): task->stop - task->start;\ + lng cw,cv;\ + for(i = 0; i< limit; i++, val++){\ + v= *val & DAYMASK; \ + MOSfind(j,dict.val##TPE,v,0,dictsize);\ + if(j == dictsize && dictsize == 0 ){\ + dict.val##TPE[j]= v;\ + cnt[j] = 1;\ + dictsize++;\ + } else \ + if( dictsize < TMPDICT && dict.val##TPE[j] != v){\ + w= v; cw= 1;\ + for( ; j< dictsize; j++)\ + if (dict.val##TPE[j] > w){\ + v =dict.val##TPE[j];\ + dict.val##TPE[j]= w;\ + w = v;\ + cv = cnt[j];\ + cnt[j]= cw;\ + cw = cv;\ + }\ + dictsize++;\ + dict.val##TPE[j]= w;\ + cnt[j] = 1;\ + } else if (dictsize < TMPDICT) cnt[j]++;\ +} } + /* there are only three kinds of calendardictionaries */ void MOScreatecalendar(Client cntxt, MOStask task) @@ -162,34 +190,7 @@ MOScreatecalendar(Client cntxt, MOStask // compress by factoring out the year-month, keeping 5 bits for the day if( task->type == TYPE_date) - { int *val = ((int*)task->src) + task->start,v,w; - BUN limit = task->stop - task->start > MOSlimit()? MOSlimit(): task->stop - task->start; - lng cw,cv; - for(i = 0; i< limit; i++, val++){ - v= *val & ~DAYMASK; - MOSfind(j,dict.valint,v,0,dictsize); - if(j == dictsize && dictsize == 0 ){ - dict.valint[j]= v; - cnt[j] = 1; - dictsize++; - } else - if(dictsize < TMPDICT && dict.valint[j] != v){ - w= v; cw= 1; - for( ; j< dictsize; j++) - if (dict.valint[j] > w){ - v =dict.valint[j]; - dict.valint[j]= w; - w = v; - cv = cnt[j]; - cnt[j]= cw; - cw = cv; - } - dictsize++; - dict.valint[j]= w; - cnt[j] = cw; - } else cnt[j]++; - } - } + makeCalendar(int,DAYMASK) if( task->type == TYPE_daytime){ } if( task->type == TYPE_timestamp){ @@ -282,6 +283,25 @@ MOSestimate_calendar(Client cntxt, MOSta } // insert a series of values into the compressor block using calendar +#define CALcompress(TPE, MASK)\ +{ TPE *val = ((TPE*)task->src) + task->start, v;\ + BitVector base = (BitVector) MOScodevector(task);\ + BUN limit = task->stop - task->start > MOSlimit()? MOSlimit(): task->stop - task->start;\ + for(i =0; i<limit; i++, val++){\ + v = *val & ~MASK;\ + MOSfind(j,task->hdr->dict.val##TPE,v,0,hdr->dictsize);\ + if(j == hdr->dictsize || task->hdr->dict.val##TPE[j] != v) \ + break;\ + else {\ + hdr->checksum.sum##TPE += v;\ + hdr->dictfreq[j]++;\ + MOSincCnt(blk,1);\ _______________________________________________ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
