Changeset: abd92a970269 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=abd92a970269
Modified Files:
gdk/gdk_bitvector.c
monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out
monetdb5/modules/mosaic/mosaic.c
monetdb5/modules/mosaic/mosaic.h
monetdb5/modules/mosaic/mosaic.mal
monetdb5/modules/mosaic/mosaic_prefix.c
sql/backends/monet5/UDF/cudf/Tests/udf-fuse.stable.out
sql/backends/monet5/UDF/cudf/Tests/udf-reverse.stable.out
sql/backends/monet5/sql_upgrades.c
sql/benchmarks/ssbm/Tests/01-explain.stable.out.int128
sql/benchmarks/ssbm/Tests/02-explain.stable.out.int128
sql/benchmarks/ssbm/Tests/03-explain.stable.out.int128
sql/benchmarks/ssbm/Tests/04-explain.stable.out.int128
sql/benchmarks/ssbm/Tests/05-explain.stable.out.int128
sql/benchmarks/ssbm/Tests/06-explain.stable.out.int128
sql/benchmarks/ssbm/Tests/07-explain.stable.out.int128
sql/benchmarks/ssbm/Tests/08-explain.stable.out.int128
sql/benchmarks/ssbm/Tests/09-explain.stable.out.int128
sql/benchmarks/ssbm/Tests/10-explain.stable.out.int128
sql/benchmarks/ssbm/Tests/11-explain.stable.out.int128
sql/benchmarks/ssbm/Tests/12-explain.stable.out
sql/benchmarks/ssbm/Tests/12-explain.stable.out.int128
sql/benchmarks/ssbm/Tests/13-explain.stable.out.int128
sql/benchmarks/tpch/Tests/01-explain.stable.out.int128
sql/benchmarks/tpch/Tests/02-explain.stable.out
sql/benchmarks/tpch/Tests/03-explain.stable.out
sql/benchmarks/tpch/Tests/03-explain.stable.out.int128
sql/benchmarks/tpch/Tests/04-explain.stable.out
sql/benchmarks/tpch/Tests/05-explain.stable.out.int128
sql/benchmarks/tpch/Tests/06-explain.stable.out.int128
sql/benchmarks/tpch/Tests/07-explain.stable.out.int128
sql/benchmarks/tpch/Tests/08-explain.stable.out.int128
sql/benchmarks/tpch/Tests/09-explain.stable.out.int128
sql/benchmarks/tpch/Tests/10-explain.stable.out.int128
sql/benchmarks/tpch/Tests/11-explain.stable.out.int128
sql/benchmarks/tpch/Tests/12-explain.stable.out.int128
sql/benchmarks/tpch/Tests/13-explain.stable.out
sql/benchmarks/tpch/Tests/14-explain.stable.out.int128
sql/benchmarks/tpch/Tests/15-explain.stable.out.int128
sql/benchmarks/tpch/Tests/16-explain.stable.out
sql/benchmarks/tpch/Tests/17-explain.stable.out.int128
sql/benchmarks/tpch/Tests/18-explain.stable.out.int128
sql/benchmarks/tpch/Tests/19-explain.stable.out.int128
sql/benchmarks/tpch/Tests/20-explain.stable.out.int128
sql/benchmarks/tpch/Tests/21-explain.stable.out
sql/benchmarks/tpch/Tests/22-explain.stable.out.int128
sql/test/BugTracker-2010/Tests/LIMIT_OFFSET_big-endian.Bug-2622.stable.out
sql/test/BugTracker-2010/Tests/group-by_ordered_column.Bug-2564.stable.out
sql/test/BugTracker-2012/Tests/rewrite_like_into_likesubselect.Bug-3179.stable.out
sql/test/BugTracker-2013/Tests/between.Bug-3259.stable.out.int128
sql/test/BugTracker-2013/Tests/between.Bug-3259.stable.out.int128.single
sql/test/BugTracker-2015/Tests/useless_casts.Bug-3756.stable.out
sql/test/BugTracker-2016/Tests/storagemodel.stable.out
sql/test/Tests/setoptimizer.stable.out
sql/test/Tests/setoptimizer.stable.out.Windows
sql/test/mergetables/Tests/mergequery.stable.out
sql/test/mosaic/Tests/compression.stable.out
sql/test/mosaic/Tests/compressionRLE.stable.out
sql/test/mosaic/Tests/compressionRLE2.stable.out
sql/test/mosaic/Tests/mix.sql
sql/test/mosaic/Tests/session_exit.stable.out
sql/test/mosaic/Tests/x
sql/test/mosaic/Tests/xqueries.stable.out
sql/test/mosaic/Tests/xqueries_delta.stable.out
sql/test/mosaic/Tests/xqueries_rle.stable.out
Branch: mosaic
Log Message:
Merge with default
And fixing prefix encoding
diffs (truncated from 4823 to 300 lines):
diff --git a/gdk/gdk_bitvector.c b/gdk/gdk_bitvector.c
--- a/gdk/gdk_bitvector.c
+++ b/gdk/gdk_bitvector.c
@@ -80,7 +80,7 @@ getBitVector(BitVector vector, BUN i, in
// spread over two cells
m1 = BITS - shift;
value = ((vector[cid] & (masks[m1]<<shift)) >> shift) |
((vector[cid+1] & masks[bits - m1]) << m1);
- //printf("#setBitVector %ld i "BUNFMT" bits %d value %3d cell
%10d %10d cid "BUNFMT" shift %d m1 %d\n",(long)vector,i,bits, value,
vector[cid], vector[cid+1],cid,shift,m1);
+ //printf("#getBitVector %ld i "BUNFMT" bits %d value %3d cell
%10d %10d cid "BUNFMT" shift %d m1 %d\n",(long)vector,i,bits, value,
vector[cid], vector[cid+1],cid,shift,m1);
}
return value;
}
@@ -109,7 +109,7 @@ setBitVector(BitVector vector, const BUN
m1 = BITS - shift;
vector[cid]= (vector[cid] & ~( masks[m1] << shift)) | ( (value &
masks[m1]) << shift);
vector[cid+1]= 0 | ( ((value>>m1) & masks[bits-m1]));
- //printf("#setBitVector %ld i "BUNFMT" bits %d value %3d cell
%10d cid "BUNFMT" "BUNFMT" shift %d m1 %d\n",(long)vector,i,bits, value,
vector[cid], vector[cid+1],cid,shift,m1);
+ //printf("#setBitVector %ld i "BUNFMT" bits %d value %3d cell
%10d %10d cid "BUNFMT" shift %d m1 %d\n",(long)vector,i,bits, value,
vector[cid], vector[cid+1],cid,shift,m1);
}
//printf("#get it back %d\n",getBitVector(vector,i,bits));
}
diff --git
a/monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out
b/monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out
--- a/monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out
+++ b/monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out
@@ -55,8 +55,8 @@ Ready.
% .L # table_name
% def # name
% clob # type
-% 579 # length
-[
"optimizer.inline();optimizer.remap();optimizer.costModel();optimizer.coercions();optimizer.evaluate();optimizer.emptybind();optimizer.pushselect();optimizer.aliases();optimizer.mitosis();optimizer.mergetable();optimizer.deadcode();optimizer.aliases();optimizer.constants();optimizer.commonTerms();optimizer.projectionpath();optimizer.deadcode();optimizer.reorder();optimizer.reduce();optimizer.matpack();optimizer.dataflow();optimizer.querylog();optimizer.multiplex();optimizer.generator();optimizer.profiler();optimizer.candidates();optimizer.jit();optimizer.garbageCollector();"
]
+% 598 # length
+[
"optimizer.inline();optimizer.remap();optimizer.costModel();optimizer.coercions();optimizer.evaluate();optimizer.emptybind();optimizer.pushselect();optimizer.aliases();optimizer.mitosis();optimizer.mergetable();optimizer.deadcode();optimizer.aliases();optimizer.constants();optimizer.commonTerms();optimizer.projectionpath();optimizer.deadcode();optimizer.reorder();optimizer.reduce();optimizer.matpack();optimizer.dataflow();optimizer.querylog();optimizer.multiplex();optimizer.generator();optimizer.mosaic();optimizer.profiler();optimizer.candidates();optimizer.jit();optimizer.garbageCollector();"
]
#explain copy into ttt from '/tmp/xyz';
% .explain # table_name
% mal # name
@@ -116,8 +116,8 @@ end user.s8_1;
% .L # table_name
% def # name
% clob # type
-% 602 # length
-[
"optimizer.inline();optimizer.remap();optimizer.costModel();optimizer.coercions();optimizer.evaluate();optimizer.emptybind();optimizer.pushselect();optimizer.aliases();optimizer.mitosis();optimizer.mergetable();optimizer.deadcode();optimizer.aliases();optimizer.constants();optimizer.commonTerms();optimizer.projectionpath();optimizer.deadcode();optimizer.reorder();optimizer.reduce();optimizer.matpack();optimizer.dataflow();optimizer.querylog();optimizer.multiplex();optimizer.generator();optimizer.profiler();optimizer.candidates();optimizer.jit();optimizer.sql_append();optimizer.garbageCollector();"
]
+% 621 # length
+[
"optimizer.inline();optimizer.remap();optimizer.costModel();optimizer.coercions();optimizer.evaluate();optimizer.emptybind();optimizer.pushselect();optimizer.aliases();optimizer.mitosis();optimizer.mergetable();optimizer.deadcode();optimizer.aliases();optimizer.constants();optimizer.commonTerms();optimizer.projectionpath();optimizer.deadcode();optimizer.reorder();optimizer.reduce();optimizer.matpack();optimizer.dataflow();optimizer.querylog();optimizer.multiplex();optimizer.generator();optimizer.mosaic();optimizer.profiler();optimizer.candidates();optimizer.jit();optimizer.sql_append();optimizer.garbageCollector();"
]
#explain copy into ttt from '/tmp/xyz';
% .explain # table_name
% mal # name
diff --git a/monetdb5/modules/mosaic/mosaic.c b/monetdb5/modules/mosaic/mosaic.c
--- a/monetdb5/modules/mosaic/mosaic.c
+++ b/monetdb5/modules/mosaic/mosaic.c
@@ -538,16 +538,24 @@ MOScompress(Client cntxt, MalBlkPtr mb,
str msg = MAL_SUCCEED;
int i;
MOStask task;
+ BAT *b;
+ bat *bid =getArgReference_bat(stk,pci,1);
+ bat *ret =getArgReference_bat(stk,pci,0);
#ifdef _DEBUG_MOSAIC_
int flg = 1;
#else
int flg = 0;
#endif
+ if ((b = BATdescriptor(*bid)) == NULL)
+ throw(MAL, "mosaic.decompress", INTERNAL_BAT_ACCESS);
+
(void) mb;
task= (MOStask) GDKzalloc(sizeof(*task));
- if( task == NULL)
+ if( task == NULL){
+ BBPunfix(b->batCacheid);
throw(MAL, "mosaic.compress", MAL_MALLOC_FAIL);
+ }
if( pci->argc == 3)
msg = *getArgReference_str(stk,pci,2);
@@ -559,6 +567,7 @@ MOScompress(Client cntxt, MalBlkPtr mb,
task->filter[i]= 1;
msg= MOScompressInternal(cntxt, getArgReference_bat(stk,pci,1), task,
flg);
+ BBPkeepref(*ret = b->batCacheid);
GDKfree(task);
return msg;
}
@@ -696,10 +705,19 @@ MOSdecompressInternal(Client cntxt, bat
return MAL_SUCCEED;
}
+// decompression does not change the BAT id
str
MOSdecompress(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
+ bat *bid = getArgReference_bat(stk,pci,1);
+ bat *ret = getArgReference_bat(stk,pci,0);
+ BAT *b;
+
(void) mb;
+
+ if ((b = BATdescriptor(*bid)) == NULL)
+ throw(MAL, "mosaic.decompress", INTERNAL_BAT_ACCESS);
+ BBPkeepref(*ret = b->batCacheid);
return MOSdecompressInternal(cntxt, getArgReference_bat(stk,pci,1));
}
@@ -1467,7 +1485,7 @@ MOSanalyseReport(Client cntxt, BAT *b, B
pat[i].clk1 = GDKms()- pat[i].clk1;
#ifdef _DEBUG_MOSAIC_
- mnstr_printf(cntxt->fdout,"#run experiment %d ratio %6.4f
"LLFMT" ms %s\n",i, task->ratio, clk, technique);
+ mnstr_printf(cntxt->fdout,"#run experiment %d ratio %6.4f
"LLFMT" ms %s\n",i, task->ratio, pat[i].clk1, pat[i].technique);
#endif
if( task->hdr == NULL){
// aborted compression experiment
diff --git a/monetdb5/modules/mosaic/mosaic.h b/monetdb5/modules/mosaic/mosaic.h
--- a/monetdb5/modules/mosaic/mosaic.h
+++ b/monetdb5/modules/mosaic/mosaic.h
@@ -32,6 +32,7 @@
#include "algebra.h"
//#define _DEBUG_MOSAIC_
+//#define _DEBUG_PREFIX_
/* do not invest in compressing BATs smaller than this */
#define MOSAIC_THRESHOLD 1
diff --git a/monetdb5/modules/mosaic/mosaic.mal
b/monetdb5/modules/mosaic/mosaic.mal
--- a/monetdb5/modules/mosaic/mosaic.mal
+++ b/monetdb5/modules/mosaic/mosaic.mal
@@ -1,18 +1,18 @@
module mosaic;
-pattern compress(b:bat[:any_1], prop:str)
+pattern compress(b:bat[:any_1], prop:str):bat[:any_1]
address MOScompress
comment "Apply dynamic compression over chunks";
-pattern compress(b:bat[:any_1])
+pattern compress(b:bat[:any_1]) :bat[:any_1]
address MOScompress
comment "Apply dynamic compression producing a temporary";
-pattern decompress(b:bat[:any_1])
+pattern decompress(b:bat[:any_1]) :bat[:any_1]
address MOSdecompress
comment "Apply dynamic decompression over chunks producing a temporary
version";
-pattern decompressStorage(b:bat[:any_1])
+pattern decompressStorage(b:bat[:any_1]) :bat[:any_1]
address MOSdecompressStorage
comment "Apply dynamic decompression, changing the storage layout permanently";
diff --git a/monetdb5/modules/mosaic/mosaic_prefix.c
b/monetdb5/modules/mosaic/mosaic_prefix.c
--- a/monetdb5/modules/mosaic/mosaic_prefix.c
+++ b/monetdb5/modules/mosaic/mosaic_prefix.c
@@ -39,7 +39,7 @@ MOSdump_prefix(Client cntxt, MOStask tas
MosaicBlk blk= task->blk;
void *val = (void*)(((char*) blk) + MosaicBlkSize);
- mnstr_printf(cntxt->fdout,"#rle "BUNFMT" ", MOSgetCnt(blk));
+ mnstr_printf(cntxt->fdout,"#prefix "BUNFMT" ", MOSgetCnt(blk));
switch(task->type){
case TYPE_bte:
mnstr_printf(cntxt->fdout,"bte %hhd", *(bte*) val); break;
@@ -141,7 +141,8 @@ MOSadvance_prefix(Client cntxt, MOStask
unsigned char mask = *dst++;
unsigned char val = *dst++;
bits = (int)(val & (~mask));
- bytes = getBitVectorSize(MOSgetCnt(task->blk),bits) *
sizeof(int) + 2 * size;
+ bytes = wordaligned(2 * sizeof(unsigned char),int);
+ bytes +=
wordaligned(getBitVectorSize(MOSgetCnt(task->blk),bits) * sizeof(int), int);
task->blk = (MosaicBlk) (((char*) dst) +
wordaligned(bytes, int));
}
break;
@@ -150,7 +151,8 @@ MOSadvance_prefix(Client cntxt, MOStask
unsigned short mask = *dst++;
unsigned short val = *dst++;
bits = (int)(val & (~mask));
- bytes = getBitVectorSize(MOSgetCnt(task->blk),bits) *
sizeof(int) + 2 * size;
+ bytes = wordaligned(2 * sizeof(unsigned short),int);
+ bytes +=
wordaligned(getBitVectorSize(MOSgetCnt(task->blk),bits) * sizeof(int), int);
task->blk = (MosaicBlk) (((char*) dst) +
wordaligned(bytes, int));
}
break;
@@ -159,8 +161,9 @@ MOSadvance_prefix(Client cntxt, MOStask
unsigned int mask = *dst++;
unsigned int val = *dst++;
bits = (int)(val & (~mask));
- bytes = getBitVectorSize(MOSgetCnt(task->blk),bits) *
sizeof(int) + 2 * size;
- task->blk = (MosaicBlk) (((char*) dst) +
wordaligned(bytes, int));
+ bytes = wordaligned(2 * sizeof(unsigned int),int);
+ bytes +=
wordaligned(getBitVectorSize(MOSgetCnt(task->blk),bits) * sizeof(int), int);
+ task->blk = (MosaicBlk) (((char*) dst) + bytes);
}
break;
case 8:
@@ -168,12 +171,13 @@ MOSadvance_prefix(Client cntxt, MOStask
ulng mask = *dst++;
ulng val = *dst++;
bits = (int)(val & (~mask));
- bytes = getBitVectorSize(MOSgetCnt(task->blk),bits) *
sizeof(int) + 2 * size;
+ bytes = wordaligned(2 * sizeof(ulng),int);
+ bytes +=
wordaligned(getBitVectorSize(MOSgetCnt(task->blk),bits) * sizeof(int), int);
task->blk = (MosaicBlk) (((char*) dst) +
wordaligned(bytes, int));
}
}
#ifdef _DEBUG_MOSAIC_
- mnstr_printf(cntxt->fdout,"advance mask width %d bytes %d %d
\n",bits,(int)bytes,(int)wordaligned(bytes,int));
+ mnstr_printf(cntxt->fdout,"#advance mask width %d bytes %d %d
\n",bits,(int)bytes,(int)wordaligned(bytes,int));
#endif
}
@@ -188,12 +192,12 @@ MOSskip_prefix(Client cntxt, MOStask tas
// Find common prefix
#define Prefix(Prefix,Mask,X,Y,N) \
{ int k, m = 1; \
- for(k=0; k<N; k++, X>>=1, Y>>=1){\
+ for(k=0; k<N; k+=1, X>>=1, Y>>=1){\
if( X == Y) break;\
m= (m<<1)|1; \
}\
Prefix = N-k;\
- Mask = ~(m >>1);\
+ Mask = ~(m >> 1);\
}
@@ -229,8 +233,10 @@ MOSestimate_prefix(Client cntxt, MOStask
if( task->range[MOSAIC_PREFIX] > task->start +1 /* need
at least two*/){
bits = (task->range[MOSAIC_PREFIX] -
task->start) * (8-prefixbits);
- store = bits/8 + ((bits % 8) >0);
- store = wordaligned( MosaicBlkSize + 2 *
sizeof(bte) + store,bte);
+ store = wordaligned(2 * sizeof(unsigned
char),int);
+ store += wordaligned(bits/8 + ((bits % 8)
>0),int);
+ store = wordaligned( MosaicBlkSize + store,int);
+
if( store >= (flt)i * sizeof(bte))
return 0.0;
return task->factor[MOSAIC_PREFIX] = ( (flt)i *
sizeof(bte))/ store;
@@ -243,8 +249,9 @@ MOSestimate_prefix(Client cntxt, MOStask
break;
}
bits = i * (8-prefixbits);
- store = bits/8 + ((bits % 8) >0);
- store = wordaligned( MosaicBlkSize + 2 * sizeof(bte) +
store,bte);
+ store = wordaligned(2 * sizeof(unsigned char),int);
+ store += wordaligned(bits/8 + ((bits % 8) >0),int);
+ store = wordaligned( MosaicBlkSize + store,int);
if( store >= (flt)i * sizeof(bte))
return 0.0;
if( task->dst + store >= task->bsrc->tmosaic->base +
task->bsrc->tmosaic->size)
@@ -268,8 +275,9 @@ MOSestimate_prefix(Client cntxt, MOStask
if( task->range[MOSAIC_PREFIX] > task->start + 1){
bits = (task->range[MOSAIC_PREFIX] -
task->start) * (16-prefixbits);
- store = bits/8 + ((bits % 8) >0);
- store = wordaligned( MosaicBlkSize + 2 *
sizeof(sht) + store,sht);
+ store = wordaligned(2 * sizeof(unsigned
short),int);
+ store += wordaligned(bits/8 + ((bits % 8)
>0),int);
+ store = wordaligned( MosaicBlkSize + store,int);
if( store >= (flt)i * sizeof(sht))
return 0.0;
return task->factor[MOSAIC_PREFIX] = ( (flt)i *
sizeof(sht))/ store;
@@ -282,8 +290,9 @@ MOSestimate_prefix(Client cntxt, MOStask
break;
}
bits = i * (16-prefixbits);
- store = bits/8 + ((bits % 8) >0);
- store = wordaligned( MosaicBlkSize + 2 * sizeof(sht) +
store,lng);
+ store = wordaligned(2 * sizeof(unsigned short),int);
+ store += wordaligned(bits/8 + ((bits % 8) >0),int);
+ store = wordaligned( MosaicBlkSize + store,int);
if( store >= (flt)i * sizeof(sht))
return 0.0;
if( task->dst + store >= task->bsrc->tmosaic->base +
task->bsrc->tmosaic->size)
@@ -307,8 +316,9 @@ MOSestimate_prefix(Client cntxt, MOStask
if( task->range[MOSAIC_PREFIX] > task->start + 1){
bits = (task->range[MOSAIC_PREFIX] -
task->start) * (32-prefixbits);
- store = bits/8 + ((bits % 8) >0);
- store = wordaligned( MosaicBlkSize + 2 *
sizeof(int) + store,lng);
+ store = wordaligned(2 * sizeof(unsigned
int),int);
+ store += wordaligned(bits/8 + ((bits % 8)
>0),int);
+ store = wordaligned( MosaicBlkSize + store,int);
if( store > (flt)i * sizeof(int))
return 0.0;
return task->factor[MOSAIC_PREFIX] = ( (flt)i *
sizeof(int))/ store;
@@ -321,8 +331,10 @@ MOSestimate_prefix(Client cntxt, MOStask
break;
}
bits = i * (32-prefixbits);
- store = bits/8 + ((bits % 8) >0);
- store = wordaligned( MosaicBlkSize + 2 * sizeof(int) +
store,lng);
+ // calculate the bitvector size
+ store = wordaligned(2 * sizeof(unsigned int),int);
+ store += wordaligned(bits/8 + ((bits % 8) >0),int);
+ store = wordaligned( MosaicBlkSize + store,int);
if( store >= (flt)i * sizeof(int))
return 0.0;
if( task->dst + store >= task->bsrc->tmosaic->base +
task->bsrc->tmosaic->size)
@@ -346,8 +358,9 @@ MOSestimate_prefix(Client cntxt, MOStask
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
