Changeset: 4b50cc9b6556 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=4b50cc9b6556 Added Files: monetdb5/modules/mal/Tests/mosaic_literal_inplace.mal monetdb5/modules/mal/Tests/mosaic_literal_inplace.stable.err monetdb5/modules/mal/Tests/mosaic_literal_inplace.stable.out Modified Files: monetdb5/modules/mal/mosaic.c monetdb5/modules/mal/mosaic_prefix.c monetdb5/modules/mal/mosaic_runlength.c Branch: mosaic Log Message:
Cutoff expensive compressions e.g. prefix tests on each element is too expensive. Cutoff if you haven't found an opportunity quickly. diffs (truncated from 301 to 300 lines): diff --git a/monetdb5/modules/mal/Tests/mosaic_literal_inplace.mal b/monetdb5/modules/mal/Tests/mosaic_literal_inplace.mal new file mode 100644 --- /dev/null +++ b/monetdb5/modules/mal/Tests/mosaic_literal_inplace.mal @@ -0,0 +1,18 @@ +b:= bat.new(:oid,:int); +bat.append(b,50); +bat.append(b,19531015); +bat.append(b,9); +b:= bat.append(b,b); +base:= algebra.copy(b); + +io.print(b); + +#inplace variant, local decompress +y:= mosaic.compressStorage(b,"literal"); +z:= mosaic.decompress(y); +io.print(z); + +#inplace variant +y:= mosaic.compressStorage(b,"literal"); +z:= mosaic.decompressStorage(y); +io.print(z); diff --git a/monetdb5/modules/mal/Tests/mosaic_literal_inplace.stable.err b/monetdb5/modules/mal/Tests/mosaic_literal_inplace.stable.err new file mode 100644 --- /dev/null +++ b/monetdb5/modules/mal/Tests/mosaic_literal_inplace.stable.err @@ -0,0 +1,30 @@ +stderr of test 'mosaic_literal_inplace` in directory 'monetdb5/modules/mal` itself: + + +# 13:43:55 > +# 13:43:55 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=32870" "--set" "mapi_usock=/var/tmp/mtest-5579/.s.monetdb.32870" "--set" "monet_prompt=" "--forcemito" "--set" "mal_listing=2" "--dbpath=/export/scratch1/mk/mosaic//Linux/var/MonetDB/mTests_monetdb5_modules_mal" "mosaic_literal_inplace.mal" +# 13:43:55 > + +# builtin opt gdk_dbpath = /export/scratch1/mk/mosaic//Linux/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = no +# builtin opt monet_prompt = > +# builtin opt monet_daemon = no +# builtin opt mapi_port = 50000 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 32870 +# cmdline opt mapi_usock = /var/tmp/mtest-5579/.s.monetdb.32870 +# cmdline opt monet_prompt = +# cmdline opt mal_listing = 2 +# cmdline opt gdk_dbpath = /export/scratch1/mk/mosaic//Linux/var/MonetDB/mTests_monetdb5_modules_mal +# cmdline opt gdk_debug = 536870922 + +# 13:43:57 > +# 13:43:57 > "Done." +# 13:43:57 > + diff --git a/monetdb5/modules/mal/Tests/mosaic_literal_inplace.stable.out b/monetdb5/modules/mal/Tests/mosaic_literal_inplace.stable.out new file mode 100644 --- /dev/null +++ b/monetdb5/modules/mal/Tests/mosaic_literal_inplace.stable.out @@ -0,0 +1,71 @@ +stdout of test 'mosaic_literal_inplace` in directory 'monetdb5/modules/mal` itself: + + +# 13:43:55 > +# 13:43:55 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=32870" "--set" "mapi_usock=/var/tmp/mtest-5579/.s.monetdb.32870" "--set" "monet_prompt=" "--forcemito" "--set" "mal_listing=2" "--dbpath=/export/scratch1/mk/mosaic//Linux/var/MonetDB/mTests_monetdb5_modules_mal" "mosaic_literal_inplace.mal" +# 13:43:55 > + +# MonetDB 5 server v11.20.0 +# This is an unreleased version +# Serving database 'mTests_monetdb5_modules_mal', using 8 threads +# Compiled for x86_64-unknown-linux-gnu/64bit with 64bit OIDs and 128bit integers dynamically linked +# Found 15.590 GiB available main-memory. +# Copyright (c) 1993-July 2008 CWI. +# Copyright (c) August 2008-2014 MonetDB B.V., all rights reserved +# Visit http://www.monetdb.org/ for further information +# Listening for connection requests on mapi:monetdb://vienna.ins.cwi.nl:32870/ +# Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-5579/.s.monetdb.32870 +# MonetDB/GIS module loaded +# MonetDB/SQL module loaded +function user.main():void; + b := bat.new(:oid,:int); + bat.append(b,50); + bat.append(b,19531015); + bat.append(b,9); + b := bat.append(b,b); + base := algebra.copy(b); + io.print(b); +#inplace variant, local decompress + y := mosaic.compressStorage(b,"literal"); + z := mosaic.decompress(y); + io.print(z); +#inplace variant + y := mosaic.compressStorage(b,"literal"); + z := mosaic.decompressStorage(y); + io.print(z); +end main; +#--------------------------# +# h t # name +# void int # type +#--------------------------# +[ 0@0, 50 ] +[ 1@0, 19531015 ] +[ 2@0, 9 ] +[ 3@0, 50 ] +[ 4@0, 19531015 ] +[ 5@0, 9 ] +#--------------------------# +# h t # name +# void int # type +#--------------------------# +[ 0@0, 50 ] +[ 1@0, 19531015 ] +[ 2@0, 9 ] +[ 3@0, 50 ] +[ 4@0, 19531015 ] +[ 5@0, 9 ] +#--------------------------# +# h t # name +# void int # type +#--------------------------# +[ 0@0, 50 ] +[ 1@0, 19531015 ] +[ 2@0, 9 ] +[ 3@0, 50 ] +[ 4@0, 19531015 ] +[ 5@0, 9 ] + +# 13:43:57 > +# 13:43:57 > "Done." +# 13:43:57 > + diff --git a/monetdb5/modules/mal/mosaic.c b/monetdb5/modules/mal/mosaic.c --- a/monetdb5/modules/mal/mosaic.c +++ b/monetdb5/modules/mal/mosaic.c @@ -326,6 +326,8 @@ MOScompressInternal(Client cntxt, int *r // cutoff the filters, especially dictionary tests are expensive if( cutoff && cutoff > task->elm){ + if( task->blks[MOSAIC_PREFIX] == 0) + filter[MOSAIC_PREFIX] = 0; if( task->blks[MOSAIC_DICT] == 0) filter[MOSAIC_DICT] = 0; cutoff = 0; diff --git a/monetdb5/modules/mal/mosaic_prefix.c b/monetdb5/modules/mal/mosaic_prefix.c --- a/monetdb5/modules/mal/mosaic_prefix.c +++ b/monetdb5/modules/mal/mosaic_prefix.c @@ -81,7 +81,7 @@ MOSadvance_prefix(Client cntxt, MOStask task->start += MOSgetCnt(task->blk); switch(size){ case 1: - { bte *dst = (sht*) (((char*) task->blk) + MosaicBlkSize); + { bte *dst = (bte*) (((char*) task->blk) + MosaicBlkSize); bte mask = *dst++; bte val = *dst++; bits = val & (~mask); @@ -229,7 +229,7 @@ MOScompress_prefix(Client cntxt, MOStask switch(size){ case 1: { bte *v = (bte*) task->src, *w= v+1, val = *v, val2 = *w, mask; - bte *dst = (sht*) (((char*) blk) + MosaicBlkSize); + bte *dst = (bte*) (((char*) blk) + MosaicBlkSize); BUN limit = task->elm > MOSlimit()? MOSlimit(): task->elm; unsigned int m; @@ -247,19 +247,19 @@ MOScompress_prefix(Client cntxt, MOStask if ( val != (*w & mask) ) break; m = (unsigned int)( *w & (~mask)); // residu -{ cell = (i * rbits)/32; - lshift= 32 -((i * rbits) % 32) ; - if ( lshift > rbits){ - base[cell]= base[cell] | (m << (lshift-rbits)); - }else{ - rshift= 32 - ((i+1) * rbits) % 32; - base[cell]= base[cell] | (m >> (rbits-lshift)); - base[cell+1]= 0 | (m << rshift); - } -} + cell = (i * rbits)/32; + lshift= 32 -((i * rbits) % 32) ; + if ( lshift > rbits){ + base[cell]= base[cell] | (m << (lshift-rbits)); + }else{ + rshift= 32 - ((i+1) * rbits) % 32; + base[cell]= base[cell] | (m >> (rbits-lshift)); + base[cell+1]= 0 | (m << rshift); + } } MOSincCnt(blk,i); task->src += i * sizeof(int); + task->dst += (i * rbits)/8 + (i %8) != 0; } break; case 2: @@ -282,19 +282,19 @@ MOScompress_prefix(Client cntxt, MOStask if ( val != (*w & mask) ) break; m = (unsigned int)( *w & (~mask)); // residu -{ cell = (i * rbits)/32; - lshift= 32 -((i * rbits) % 32) ; - if ( lshift > rbits){ - base[cell]= base[cell] | (m << (lshift-rbits)); - }else{ - rshift= 32 - ((i+1) * rbits) % 32; - base[cell]= base[cell] | (m >> (rbits-lshift)); - base[cell+1]= 0 | (m << rshift); - } -} + cell = (i * rbits)/32; + lshift= 32 -((i * rbits) % 32) ; + if ( lshift > rbits){ + base[cell]= base[cell] | (m << (lshift-rbits)); + }else{ + rshift= 32 - ((i+1) * rbits) % 32; + base[cell]= base[cell] | (m >> (rbits-lshift)); + base[cell+1]= 0 | (m << rshift); + } } MOSincCnt(blk,i); task->src += i * sizeof(int); + task->dst += (i * rbits)/8 + (i %8) != 0; } break; case 4: @@ -312,28 +312,28 @@ MOScompress_prefix(Client cntxt, MOStask *base = 0; val = *v & mask; //reference value -//mnstr_printf(cntxt->fdout,"compress %o %o val %d bits %d, %d mask %o\n",*v,*w,val,bits, rbits,mask); + //mnstr_printf(cntxt->fdout,"compress %o %o val %d bits %d, %d mask %o\n",*v,*w,val,bits, rbits,mask); for(w = v, i = 0; i < limit; w++, i++){ if ( val != (*w & mask) ) break; m = *w & (~mask); // residu -//mnstr_printf(cntxt->fdout,"compress %d residu %d %o\n",*w,m,m); -{ cell = (i * rbits)/32; - lshift= 32 -((i * rbits) % 32) ; - if ( lshift > rbits){ - base[cell]= base[cell] | (m << (lshift-rbits)); - //mnstr_printf(cntxt->fdout,"[%d] shift %d rbits %d cell %o\n",cell, lshift, rbits, base[cell]); - }else{ - rshift= 32 - ((i+1) * rbits) % 32; - base[cell]= base[cell] | (m >> (rbits-lshift)); - base[cell+1]= 0 | (m << rshift); - //mnstr_printf(cntxt->fdout,"[%d] shift %d %d cell %o %o val %o %o\n", cell, lshift, rshift, - //base[cell],base[cell+1], (m >> (rbits-lshift)), (m <<rshift)); - } -} + //mnstr_printf(cntxt->fdout,"compress %d residu %d %o\n",*w,m,m); + cell = (i * rbits)/32; + lshift= 32 -((i * rbits) % 32) ; + if ( lshift > rbits){ + base[cell]= base[cell] | (m << (lshift-rbits)); + //mnstr_printf(cntxt->fdout,"[%d] shift %d rbits %d cell %o\n",cell, lshift, rbits, base[cell]); + }else{ + rshift= 32 - ((i+1) * rbits) % 32; + base[cell]= base[cell] | (m >> (rbits-lshift)); + base[cell+1]= 0 | (m << rshift); + //mnstr_printf(cntxt->fdout,"[%d] shift %d %d cell %o %o val %o %o\n", cell, lshift, rshift, + //base[cell],base[cell+1], (m >> (rbits-lshift)), (m <<rshift)); + } } MOSincCnt(blk,i); task->src += i * sizeof(int); + task->dst += (i * rbits)/8 + (i %8) != 0; } } #ifdef _DEBUG_MOSAIC_ @@ -371,7 +371,7 @@ MOSdecompress_prefix(Client cntxt, MOSta size =task->b->T->width; switch(size){ case 1: - { bte *dst = (sht*) (((char*) blk) + MosaicBlkSize); + { bte *dst = (bte*) (((char*) blk) + MosaicBlkSize); bte mask = *dst++, val = *dst++, v; bte m,m1,m2; BUN lim= MOSgetCnt(blk); diff --git a/monetdb5/modules/mal/mosaic_runlength.c b/monetdb5/modules/mal/mosaic_runlength.c --- a/monetdb5/modules/mal/mosaic_runlength.c +++ b/monetdb5/modules/mal/mosaic_runlength.c @@ -113,7 +113,7 @@ MOSskip_runlength(Client cntxt, MOStask if ( ((TYPE*)task->src)[i] != val)\ break;\ if ( i > MOSlimit() ) i = MOSlimit();\ - factor = ( (flt)i * sizeof(TYPE))/ (2 * MosaicBlkSize + sizeof(TYPE));\ + factor = ( (flt)i * sizeof(TYPE))/ (MosaicBlkSize + sizeof(TYPE));\ } // calculate the expected reduction using RLE in terms of elements compressed @@ -141,7 +141,7 @@ MOSestimate_runlength(Client cntxt, MOSt if ( ((int*)task->src)[i] != val) break; if ( i > MOSlimit() ) i = MOSlimit(); - factor = ( (flt)i * sizeof(int))/ (2 * MosaicBlkSize + sizeof(int)); + factor = ( (flt)i * sizeof(int))/ (MosaicBlkSize + sizeof(int)); } break; _______________________________________________ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
