Changeset: 33822db8c22e for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=33822db8c22e
Modified Files:
monetdb5/modules/mosaic/mosaic_capped.c
monetdb5/modules/mosaic/mosaic_capped.h
monetdb5/modules/mosaic/mosaic_dictionary.h
monetdb5/modules/mosaic/mosaic_var.c
monetdb5/modules/mosaic/mosaic_var.h
Branch: mosaic
Log Message:
Refactor dictionary code a bit.
diffs (truncated from 449 to 300 lines):
diff --git a/monetdb5/modules/mosaic/mosaic_capped.c
b/monetdb5/modules/mosaic/mosaic_capped.c
--- a/monetdb5/modules/mosaic/mosaic_capped.c
+++ b/monetdb5/modules/mosaic/mosaic_capped.c
@@ -22,7 +22,6 @@
#include "gdk_bitvector.h"
#include "mosaic.h"
#include "mosaic_capped.h"
-#include "mosaic_dictionary.h"
#include "mosaic_private.h"
#include "group.h"
@@ -78,19 +77,6 @@ typedef struct _GlobalCappedInfo {
EstimationParameters parameters;
} GlobalCappedInfo;
-#define GET_BASE(INFO, TPE) ((TPE*) Tloc((INFO)->dict, 0))
-#define GET_COUNT(INFO) (BATcount((INFO)->dict))
-#define GET_CAP(INFO) (BATcapacity((INFO)->dict))
-#define GET_DELTA_COUNT(INFO) ((INFO)->parameters.delta_count)
-#define GET_BITS(INFO) ((INFO)->parameters.bits)
-#define GET_BITS_EXTENDED(INFO)
((INFO)->parameters.bits_extended)
-#define EXTEND(INFO, new_capacity) (BATextend((INFO)->dict, new_capacity)
== GDK_SUCCEED)
-
-// task dependent macro's
-#define GET_FINAL_DICT(TASK, TPE) (((TPE*) (TASK)->bsrc->tvmosaic->base) +
(TASK)->hdr->pos_capped)
-#define GET_FINAL_BITS(TASK) ((TASK)->hdr->bits_capped)
-#define GET_FINAL_DICT_COUNT(TASK) ((TASK)->hdr->length_capped);\
-
#define PresentInTempDictFuncDef(TPE) \
static inline \
bool presentInTempDict##TPE(GlobalCappedInfo* info, TPE val) {\
@@ -100,13 +86,13 @@ bool presentInTempDict##TPE(GlobalCapped
return key < dict_count && ((dict[key] == val) || (IS_NIL(TPE,
dict[key]) && IS_NIL(TPE, dict[key])) ) ;\
}
-#define CONDITIONAL_INSERT(INFO, VAL, TPE) presentInTempDict##TPE((INFO),
(VAL))
+#define CONDITIONAL_INSERT_capped(INFO, VAL, TPE)
presentInTempDict##TPE((INFO), (VAL))
#define DictionaryClass(TPE) \
find_value_DEF(TPE)\
PresentInTempDictFuncDef(TPE)\
insert_into_dict_DEF(TPE)\
-extend_delta_DEF(TPE, GlobalCappedInfo)\
+extend_delta_DEF(capped, TPE, GlobalCappedInfo)\
merge_delta_Into_dictionary_DEF(TPE, GlobalCappedInfo)\
compress_dictionary_DEF(TPE)\
decompress_dictionary_DEF(TPE)
@@ -134,7 +120,7 @@ MOSadvance_capped(MOStask task)
assert(cnt > 0);
task->start += (oid) cnt;
- bytes = (cnt * GET_FINAL_BITS(task))/8 + (((cnt *
GET_FINAL_BITS(task)) %8) != 0);
+ bytes = (cnt * GET_FINAL_BITS(task, capped))/8 + (((cnt *
GET_FINAL_BITS(task, capped)) %8) != 0);
task->blk = (MosaicBlk) (((char*) dst) + wordaligned(bytes,
BitVectorChunk));
}
@@ -167,7 +153,7 @@ MOSlayout_capped(MOStask task, BAT *btec
BUN cnt = MOSgetCnt(blk), input=0, output= 0;
input = cnt * ATOMsize(task->type);
- output = MosaicBlkSize + (cnt * GET_FINAL_BITS(task))/8 + (((cnt *
GET_FINAL_BITS(task)) %8) != 0);
+ output = MosaicBlkSize + (cnt * GET_FINAL_BITS(task, capped))/8 +
(((cnt * GET_FINAL_BITS(task, capped)) %8) != 0);
if( BUNappend(btech, "capped blk", false) != GDK_SUCCEED ||
BUNappend(bcount, &cnt, false) != GDK_SUCCEED ||
BUNappend(binput, &input, false) != GDK_SUCCEED ||
@@ -350,7 +336,7 @@ static str
vmh->dirty = true;
*length_dict = GET_COUNT(info);
- *bits_dict = calculateBits(*length_dict);
+ calculateBits(*bits_dict, *length_dict);
BBPreclaim(info->dict);
BBPreclaim(info->temp_dict);
@@ -367,7 +353,7 @@ finalizeDictionary_capped(MOStask task)
task->capped_info,
&task->hdr->pos_capped,
&task->hdr->length_capped,
- &GET_FINAL_BITS(task));
+ &GET_FINAL_BITS(task, capped));
}
void
@@ -381,15 +367,15 @@ MOScompress_capped(MOStask task, MosaicB
MOSsetCnt(blk,0);
switch(ATOMbasetype(task->type)){
- case TYPE_bte: DICTcompress(task, bte); break;
- case TYPE_sht: DICTcompress(task, sht); break;
- case TYPE_int: DICTcompress(task, int); break;
- case TYPE_lng: DICTcompress(task, lng); break;
- case TYPE_oid: DICTcompress(task, oid); break;
- case TYPE_flt: DICTcompress(task, flt); break;
- case TYPE_dbl: DICTcompress(task, dbl); break;
+ case TYPE_bte: DICTcompress(capped, bte); break;
+ case TYPE_sht: DICTcompress(capped, sht); break;
+ case TYPE_int: DICTcompress(capped, int); break;
+ case TYPE_lng: DICTcompress(capped, lng); break;
+ case TYPE_oid: DICTcompress(capped, oid); break;
+ case TYPE_flt: DICTcompress(capped, flt); break;
+ case TYPE_dbl: DICTcompress(capped, dbl); break;
#ifdef HAVE_HGE
- case TYPE_hge: DICTcompress(task, hge); break;
+ case TYPE_hge: DICTcompress(capped, hge); break;
#endif
}
}
@@ -398,21 +384,21 @@ void
MOSdecompress_capped(MOStask task)
{
switch(ATOMbasetype(task->type)){
- case TYPE_bte: DICTdecompress(task, bte); break;
- case TYPE_sht: DICTdecompress(task, sht); break;
- case TYPE_int: DICTdecompress(task, int); break;
- case TYPE_lng: DICTdecompress(task, lng); break;
- case TYPE_oid: DICTdecompress(task, oid); break;
- case TYPE_flt: DICTdecompress(task, flt); break;
- case TYPE_dbl: DICTdecompress(task, dbl); break;
+ case TYPE_bte: DICTdecompress(capped, bte); break;
+ case TYPE_sht: DICTdecompress(capped, sht); break;
+ case TYPE_int: DICTdecompress(capped, int); break;
+ case TYPE_lng: DICTdecompress(capped, lng); break;
+ case TYPE_oid: DICTdecompress(capped, oid); break;
+ case TYPE_flt: DICTdecompress(capped, flt); break;
+ case TYPE_dbl: DICTdecompress(capped, dbl); break;
#ifdef HAVE_HGE
- case TYPE_hge: DICTdecompress(task, hge); break;
+ case TYPE_hge: DICTdecompress(capped, hge); break;
#endif
}
}
#define scan_loop_capped(TPE, CI_NEXT, TEST) \
- scan_loop_dictionary(TPE, CI_NEXT, TEST)
+ scan_loop_dictionary(capped, TPE, CI_NEXT, TEST)
MOSselect_DEF(capped, bte)
MOSselect_DEF(capped, sht)
@@ -425,7 +411,7 @@ MOSselect_DEF(capped, hge)
#endif
#define projection_loop_capped(TPE, CI_NEXT) \
- projection_loop_dictionary(TPE, CI_NEXT)
+ projection_loop_dictionary(capped, TPE, CI_NEXT)
MOSprojection_DEF(capped, bte)
MOSprojection_DEF(capped, sht)
@@ -438,7 +424,7 @@ MOSprojection_DEF(capped, hge)
#endif
#define outer_loop_capped(HAS_NIL, NIL_MATCHES, TPE, LEFT_CI_NEXT,
RIGHT_CI_NEXT) \
- outer_loop_dictionary(HAS_NIL, NIL_MATCHES, TPE, LEFT_CI_NEXT,
RIGHT_CI_NEXT)
+ outer_loop_dictionary(HAS_NIL, NIL_MATCHES, capped, TPE, LEFT_CI_NEXT,
RIGHT_CI_NEXT)
MOSjoin_COUI_DEF(capped, bte)
MOSjoin_COUI_DEF(capped, sht)
diff --git a/monetdb5/modules/mosaic/mosaic_capped.h
b/monetdb5/modules/mosaic/mosaic_capped.h
--- a/monetdb5/modules/mosaic/mosaic_capped.h
+++ b/monetdb5/modules/mosaic/mosaic_capped.h
@@ -17,6 +17,7 @@
#include "mal_interpreter.h"
#include "mal_client.h"
#include "mosaic_utility.h"
+#include "mosaic_dictionary.h"
#define DICTTHRESHOLD 4192
#define DICTSIZE 256
diff --git a/monetdb5/modules/mosaic/mosaic_dictionary.h
b/monetdb5/modules/mosaic/mosaic_dictionary.h
--- a/monetdb5/modules/mosaic/mosaic_dictionary.h
+++ b/monetdb5/modules/mosaic/mosaic_dictionary.h
@@ -8,13 +8,13 @@
/*TODO: assuming (for now) that bats have nils during compression*/
static const bool nil = true;
-static unsigned char
-calculateBits(BUN count) {
- unsigned char bits = 0;
- while (count >> bits) {
- bits++;
- }
- return bits;
+#define calculateBits(RES, COUNT)\
+{\
+ unsigned char bits = 0;\
+ while ((COUNT) >> bits) {\
+ bits++;\
+ }\
+ (RES) = bits;\
}
typedef struct _EstimationParameters {
@@ -24,6 +24,19 @@ typedef struct _EstimationParameters {
unsigned char bits_extended; // number of bits required to index the
info after the delta would have been merged.
} EstimationParameters;
+#define GET_BASE(INFO, TPE) ((TPE*) Tloc((INFO)->dict, 0))
+#define GET_COUNT(INFO) (BATcount((INFO)->dict))
+#define GET_CAP(INFO) (BATcapacity((INFO)->dict))
+#define GET_DELTA_COUNT(INFO) ((INFO)->parameters.delta_count)
+#define GET_BITS(INFO) ((INFO)->parameters.bits)
+#define GET_BITS_EXTENDED(INFO)
((INFO)->parameters.bits_extended)
+#define EXTEND(INFO, new_capacity) (BATextend((INFO)->dict, new_capacity)
== GDK_SUCCEED)
+
+// task dependent macro's
+#define GET_FINAL_DICT(task, NAME, TPE) (((TPE*) (task)->bsrc->tvmosaic->base)
+ (task)->hdr->pos_##NAME)
+#define GET_FINAL_BITS(task, NAME) ((task)->hdr->bits_##NAME)
+#define GET_FINAL_DICT_COUNT(task, NAME) ((task)->hdr->length_##NAME);\
+
#define find_value_DEF(TPE) \
static inline \
BUN find_value_##TPE(TPE* dict, BUN dict_count, TPE val) {\
@@ -65,7 +78,7 @@ void insert_into_dict_##TPE(TPE* dict, B
(*dict_count)++;\
dict[key] = w;\
}
-#define extend_delta_DEF(TPE, DICTIONARY_TYPE) \
+#define extend_delta_DEF(NAME, TPE, DICTIONARY_TYPE) \
static str \
extend_delta_##TPE(BUN* nr_compressed, BUN* delta_count, BUN limit,
DICTIONARY_TYPE* info, TPE* val) {\
BUN buffer_size = 256;\
@@ -77,7 +90,7 @@ extend_delta_##TPE(BUN* nr_compressed, B
BUN pos = find_value_##TPE(dict, dict_count, *val);\
if (pos == dict_count || !ARE_EQUAL(delta[pos], *val, nil,
TPE)) {\
/*This value is not in the base dictionary. See if we
can add it to the delta dictionary.*/;\
- if (CONDITIONAL_INSERT(info, *val, TPE)) {\
+ if (CONDITIONAL_INSERT_##NAME(info, *val, TPE)) {\
BUN key = find_value_##TPE(delta,
(*delta_count), *val);\
if (key < *delta_count && ARE_EQUAL(delta[key],
*val, nil, TPE)) {\
/*This delta value is already in the
dictionary hence we can skip it.*/\
@@ -95,7 +108,7 @@ extend_delta_##TPE(BUN* nr_compressed, B
}\
GET_DELTA_COUNT(info) = (*delta_count);\
BUN new_count = dict_count + GET_DELTA_COUNT(info);\
- GET_BITS_EXTENDED(info) = calculateBits(new_count);\
+ calculateBits(GET_BITS_EXTENDED(info), new_count);\
return MAL_SUCCEED;\
}
#define merge_delta_Into_dictionary_DEF(TPE, DICTIONARY_TYPE) \
@@ -154,33 +167,33 @@ typedef struct {
#define MOScodevectorDict(Task) (((char*) (Task)->blk) +
wordaligned(sizeof(MOSBlkHdr_dictionary_t), BitVectorChunk))
// insert a series of values into the compressor block using dictionary
-#define DICTcompress(TASK, TPE) {\
- TPE *val = getSrc(TPE, (TASK));\
+#define DICTcompress(NAME, TPE) {\
+ TPE *val = getSrc(TPE, (task));\
BUN cnt = estimate->cnt;\
- (TASK)->dst = MOScodevectorDict(TASK);\
- BitVector base = (BitVector) ((TASK)->dst);\
+ (task)->dst = MOScodevectorDict(task);\
+ BitVector base = (BitVector) ((task)->dst);\
BUN i;\
- TPE* dict = GET_FINAL_DICT(TASK, TPE);\
- BUN dict_size = GET_FINAL_DICT_COUNT(TASK);\
- bte bits = GET_FINAL_BITS(task);\
+ TPE* dict = GET_FINAL_DICT(task, NAME, TPE);\
+ BUN dict_size = GET_FINAL_DICT_COUNT(task, NAME);\
+ bte bits = GET_FINAL_BITS(task, NAME);\
compress_dictionary_##TPE(dict, dict_size, &i, val, cnt, base, bits);\
- MOSsetCnt((TASK)->blk, i);\
+ MOSsetCnt((task)->blk, i);\
}
// the inverse operator, extend the src
-#define DICTdecompress(TASK, TPE)\
-{ BUN cnt = MOSgetCnt((TASK)->blk);\
- BitVector base = (BitVector) MOScodevectorDict(TASK);\
- bte bits = GET_FINAL_BITS(task);\
- TPE* dict = GET_FINAL_DICT(TASK, TPE);\
- TPE* dest = (TPE*) (TASK)->src;\
+#define DICTdecompress(NAME, TPE)\
+{ BUN cnt = MOSgetCnt((task)->blk);\
+ BitVector base = (BitVector) MOScodevectorDict(task);\
+ bte bits = GET_FINAL_BITS(task, NAME);\
+ TPE* dict = GET_FINAL_DICT(task, NAME, TPE);\
+ TPE* dest = (TPE*) (task)->src;\
decompress_dictionary_##TPE(dict, bits, base, cnt, &dest);\
}
-#define scan_loop_dictionary(TPE, CANDITER_NEXT, TEST) {\
- TPE* dict = GET_FINAL_DICT(task, TPE);\
+#define scan_loop_dictionary(NAME, TPE, CANDITER_NEXT, TEST) {\
+ TPE* dict = GET_FINAL_DICT(task, NAME, TPE);\
BitVector base = (BitVector) MOScodevectorDict(task);\
- bte bits = GET_FINAL_BITS(task);\
+ bte bits = GET_FINAL_BITS(task, NAME);\
for (oid c = canditer_peekprev(task->ci); !is_oid_nil(c) && c < last; c =
CANDITER_NEXT(task->ci)) {\
BUN i = (BUN) (c - first);\
BitVectorChunk j = getBitVector(base,i,bits); \
@@ -191,11 +204,11 @@ typedef struct {
}\
}
-#define projection_loop_dictionary(TPE, CANDITER_NEXT)\
+#define projection_loop_dictionary(NAME, TPE, CANDITER_NEXT)\
{\
- TPE* dict = GET_FINAL_DICT(task, TPE);\
+ TPE* dict = GET_FINAL_DICT(task, NAME, TPE);\
BitVector base = (BitVector) MOScodevectorDict(task);\
- bte bits = GET_FINAL_BITS(task);\
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
