Changeset: 1c79d78a1df8 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/1c79d78a1df8
Branch: default
Log Message:
Merge branch string_imprints into default.
diffs (truncated from 2851 to 300 lines):
diff --git a/clients/Tests/MAL-signatures.stable.out
b/clients/Tests/MAL-signatures.stable.out
--- a/clients/Tests/MAL-signatures.stable.out
+++ b/clients/Tests/MAL-signatures.stable.out
@@ -9167,6 +9167,8 @@
[ "optimizer", "reorder", "pattern optimizer.reorder():str ",
"OPTwrapper;", "" ]
[ "optimizer", "reorder", "pattern optimizer.reorder(X_0:str,
X_1:str):str ", "OPTwrapper;", "Reorder by dataflow dependencies" ]
[ "optimizer", "sequential_pipe", "function
optimizer.sequential_pipe():void;", "", "" ]
+[ "optimizer", "strimps", "pattern optimizer.strimps():str ",
"OPTwrapper;", "" ]
+[ "optimizer", "strimps", "pattern optimizer.strimps(X_0:str,
X_1:str):str ", "OPTwrapper;", "Use strimps index if appropriate" ]
[ "optimizer", "volcano", "pattern optimizer.volcano():str ",
"OPTwrapper;", "" ]
[ "optimizer", "volcano", "pattern optimizer.volcano(X_0:str,
X_1:str):str ", "OPTwrapper;", "Simulate volcano style execution" ]
[ "optimizer", "volcano_pipe", "function optimizer.volcano_pipe():void;",
"", "" ]
@@ -9316,6 +9318,7 @@
[ "sql", "covariancep", "pattern sql.covariancep(X_0:lng, X_1:lng,
X_2:bit, X_3:bit, X_4:int, X_5:oid, X_6:oid):dbl ", "SQLcovar_pop;",
"return the covariance population value of groups" ]
[ "sql", "covariancep", "pattern sql.covariancep(X_0:sht, X_1:sht,
X_2:bit, X_3:bit, X_4:int, X_5:oid, X_6:oid):dbl ", "SQLcovar_pop;",
"return the covariance population value of groups" ]
[ "sql", "createorderindex", "unsafe pattern
sql.createorderindex(X_0:str, X_1:str, X_2:str):void ",
"sql_createorderindex;", "Instantiate the order index on a column"
]
+[ "sql", "createstrimps", "unsafe pattern
sql.createstrimps(X_0:str, X_1:str, X_2:str):void ", "sql_createstrimps;",
"Instantiate the strimps index on a column" ]
[ "sql", "cume_dist", "pattern sql.cume_dist(X_0:any_1, X_1:bit,
X_2:bit):dbl ", "SQLcume_dist;", "return the accumulated
distribution of the number of rows per group to the total number of partition
rows" ]
[ "sql", "current_time", "pattern sql.current_time():daytime ",
"SQLcurrent_daytime;", "Get the clients current daytime" ]
[ "sql", "current_timestamp", "pattern
sql.current_timestamp():timestamp ", "SQLcurrent_timestamp;", "Get the
clients current timestamp" ]
@@ -9607,6 +9610,10 @@
[ "streams", "readStr", "unsafe command
streams.readStr(X_0:streams):str ", "mnstr_read_stringwrap;", "read
string data from the stream" ]
[ "streams", "writeInt", "unsafe command streams.writeInt(X_0:streams,
X_1:int):void ", "mnstr_writeIntwrap;", "write data on the stream" ]
[ "streams", "writeStr", "unsafe command streams.writeStr(X_0:streams,
X_1:str):void ", "mnstr_write_stringwrap;", "write data on the stream"
]
+[ "strimps", "mkstrimp", "unsafe pattern strimps.mkstrimp(X_0:bat[:str],
X_1:bat[:oid]):void ", "PATstrimpCreate;", "construct the strimp a BAT"
]
+[ "strimps", "strimpfilter", "pattern strimps.strimpfilter(X_0:str,
X_1:str):bit ", "PATstrimpFilter;", "" ]
+[ "strimps", "strimpfilterjoin", "pattern
strimps.strimpfilterjoin(X_0:str, X_1:any, X_2:any, X_3:any, X_4:any, X_5:any)
(X_6:bat[:oid], X_7:bat[:str]) ", "PATstrimpFilter;", "" ]
+[ "strimps", "strimpfilterselect", "pattern
strimps.strimpfilterselect(X_0:bat[:str], X_1:bat[:oid], X_2:str,
X_3:bit):bat[:oid] ", "PATstrimpFilterSelect;", "" ]
[ "sysmon", "pause", "unsafe pattern sysmon.pause(X_0:int):void ",
"SYSMONpause;", "Suspend a running query" ]
[ "sysmon", "pause", "unsafe pattern sysmon.pause(X_0:lng):void ",
"SYSMONpause;", "Suspend a running query" ]
[ "sysmon", "pause", "unsafe pattern sysmon.pause(X_0:sht):void ",
"SYSMONpause;", "Suspend a running query" ]
diff --git a/clients/Tests/MAL-signatures.stable.out.int128
b/clients/Tests/MAL-signatures.stable.out.int128
--- a/clients/Tests/MAL-signatures.stable.out.int128
+++ b/clients/Tests/MAL-signatures.stable.out.int128
@@ -12467,6 +12467,8 @@
[ "optimizer", "reorder", "pattern optimizer.reorder():str ",
"OPTwrapper;", "" ]
[ "optimizer", "reorder", "pattern optimizer.reorder(X_0:str,
X_1:str):str ", "OPTwrapper;", "Reorder by dataflow dependencies" ]
[ "optimizer", "sequential_pipe", "function
optimizer.sequential_pipe():void;", "", "" ]
+[ "optimizer", "strimps", "pattern optimizer.strimps():str ",
"OPTwrapper;", "" ]
+[ "optimizer", "strimps", "pattern optimizer.strimps(X_0:str,
X_1:str):str ", "OPTwrapper;", "Use strimps index if appropriate" ]
[ "optimizer", "volcano", "pattern optimizer.volcano():str ",
"OPTwrapper;", "" ]
[ "optimizer", "volcano", "pattern optimizer.volcano(X_0:str,
X_1:str):str ", "OPTwrapper;", "Simulate volcano style execution" ]
[ "optimizer", "volcano_pipe", "function optimizer.volcano_pipe():void;",
"", "" ]
@@ -12621,6 +12623,7 @@
[ "sql", "covariancep", "pattern sql.covariancep(X_0:lng, X_1:lng,
X_2:bit, X_3:bit, X_4:int, X_5:oid, X_6:oid):dbl ", "SQLcovar_pop;",
"return the covariance population value of groups" ]
[ "sql", "covariancep", "pattern sql.covariancep(X_0:sht, X_1:sht,
X_2:bit, X_3:bit, X_4:int, X_5:oid, X_6:oid):dbl ", "SQLcovar_pop;",
"return the covariance population value of groups" ]
[ "sql", "createorderindex", "unsafe pattern
sql.createorderindex(X_0:str, X_1:str, X_2:str):void ",
"sql_createorderindex;", "Instantiate the order index on a column"
]
+[ "sql", "createstrimps", "unsafe pattern
sql.createstrimps(X_0:str, X_1:str, X_2:str):void ", "sql_createstrimps;",
"Instantiate the strimps index on a column" ]
[ "sql", "cume_dist", "pattern sql.cume_dist(X_0:any_1, X_1:bit,
X_2:bit):dbl ", "SQLcume_dist;", "return the accumulated
distribution of the number of rows per group to the total number of partition
rows" ]
[ "sql", "current_time", "pattern sql.current_time():daytime ",
"SQLcurrent_daytime;", "Get the clients current daytime" ]
[ "sql", "current_timestamp", "pattern
sql.current_timestamp():timestamp ", "SQLcurrent_timestamp;", "Get the
clients current timestamp" ]
@@ -12928,6 +12931,10 @@
[ "streams", "readStr", "unsafe command
streams.readStr(X_0:streams):str ", "mnstr_read_stringwrap;", "read
string data from the stream" ]
[ "streams", "writeInt", "unsafe command streams.writeInt(X_0:streams,
X_1:int):void ", "mnstr_writeIntwrap;", "write data on the stream" ]
[ "streams", "writeStr", "unsafe command streams.writeStr(X_0:streams,
X_1:str):void ", "mnstr_write_stringwrap;", "write data on the stream"
]
+[ "strimps", "mkstrimp", "unsafe pattern strimps.mkstrimp(X_0:bat[:str],
X_1:bat[:oid]):void ", "PATstrimpCreate;", "construct the strimp a BAT"
]
+[ "strimps", "strimpfilter", "pattern strimps.strimpfilter(X_0:str,
X_1:str):bit ", "PATstrimpFilter;", "" ]
+[ "strimps", "strimpfilterjoin", "pattern
strimps.strimpfilterjoin(X_0:str, X_1:any, X_2:any, X_3:any, X_4:any, X_5:any)
(X_6:bat[:oid], X_7:bat[:str]) ", "PATstrimpFilter;", "" ]
+[ "strimps", "strimpfilterselect", "pattern
strimps.strimpfilterselect(X_0:bat[:str], X_1:bat[:oid], X_2:str,
X_3:bit):bat[:oid] ", "PATstrimpFilterSelect;", "" ]
[ "sysmon", "pause", "unsafe pattern sysmon.pause(X_0:int):void ",
"SYSMONpause;", "Suspend a running query" ]
[ "sysmon", "pause", "unsafe pattern sysmon.pause(X_0:lng):void ",
"SYSMONpause;", "Suspend a running query" ]
[ "sysmon", "pause", "unsafe pattern sysmon.pause(X_0:sht):void ",
"SYSMONpause;", "Suspend a running query" ]
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -393,6 +393,9 @@ BUN ORDERfndlast(BAT *b, Heap *oidxh, co
BUN SORTfnd(BAT *b, const void *v);
BUN SORTfndfirst(BAT *b, const void *v);
BUN SORTfndlast(BAT *b, const void *v);
+gdk_return STRMPappendBitstring(BAT *b, const str s);
+gdk_return STRMPcreate(BAT *b, BAT *s);
+BAT *STRMPfilter(BAT *b, BAT *s, const str q);
MT_Id THRcreate(void (*f)(void *), void *arg, enum MT_thr_detach d, const char
*name);
void *THRdata[THREADDATA];
void THRdel(Thread t);
@@ -1152,6 +1155,7 @@ const char *minusRef;
const char *mirrorRef;
const char *mitosisRef;
const char *mkeyRef;
+const char *mkstrimpsRef;
const char *mmathRef;
const char *modRef;
char monet_characteristics[4096];
@@ -1374,6 +1378,8 @@ void strAfterCall(ValPtr v, ValPtr bak);
void strBeforeCall(ValPtr v, ValPtr bak);
const char *strRef;
const char *streamsRef;
+const char *strimpFilterSelectRef;
+const char *strimpsRef;
const char *subavgRef;
const char *subcountRef;
const char *subdeltaRef;
diff --git a/gdk/CMakeLists.txt b/gdk/CMakeLists.txt
--- a/gdk/CMakeLists.txt
+++ b/gdk/CMakeLists.txt
@@ -81,6 +81,7 @@ target_sources(bat
gdk_analytic_func.c
gdk_analytic.h
gdk_tracer.c gdk_tracer.h
+ gdk_strimps.c gdk_strimps.h
PUBLIC
${gdk_public_headers})
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -570,6 +570,7 @@ typedef struct {
typedef struct Hash Hash;
typedef struct Imprints Imprints;
+typedef struct Strimps Strimps;
/*
* @+ Binary Association Tables
@@ -736,6 +737,7 @@ typedef struct {
Hash *hash; /* hash table */
Imprints *imprints; /* column imprints index */
Heap *orderidx; /* order oid index */
+ Strimps *strimps; /* string imprint index */
PROPrec *props; /* list of dynamic properties stored in the bat
descriptor */
} COLrec;
@@ -806,6 +808,7 @@ typedef struct BAT {
#define thash T.hash
#define timprints T.imprints
#define tprops T.props
+#define tstrimps T.strimps
/* some access functions for the bitmask type */
@@ -2384,6 +2387,11 @@ gdk_export BAT *BATsample_with_seed(BAT
CALLBACK; \
} while (0)
+/*
+ * String Imprints Development/Testing. TODO: remove the following.
+ */
+
+#include "gdk_strimps.h"
typedef struct gdk_callback {
char *name;
int argc;
diff --git a/gdk/gdk_align.c b/gdk/gdk_align.c
--- a/gdk/gdk_align.c
+++ b/gdk/gdk_align.c
@@ -133,6 +133,8 @@ VIEWcreate(oid seq, BAT *b)
bn->timprints = NULL;
/* Order OID index */
bn->torderidx = NULL;
+ /* Only the parent should have a pointer to the strimp */
+ bn->tstrimps = NULL;
if (BBPcacheit(bn, true) != GDK_SUCCEED) { /* enter in BBP */
if (tp) {
BBPunshare(tp);
@@ -377,6 +379,7 @@ VIEWdestroy(BAT *b)
HASHdestroy(b);
IMPSdestroy(b);
OIDXdestroy(b);
+ STRMPdestroy(b);
PROPdestroy(b);
VIEWunlink(b);
diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c
--- a/gdk/gdk_bat.c
+++ b/gdk/gdk_bat.c
@@ -583,6 +583,7 @@ BATclear(BAT *b, bool force)
HASHdestroy(b);
IMPSdestroy(b);
OIDXdestroy(b);
+ STRMPdestroy(b);
PROPdestroy(b);
/* we must dispose of all inserted atoms */
@@ -669,6 +670,7 @@ BATfree(BAT *b)
HASHfree(b);
IMPSfree(b);
OIDXfree(b);
+ STRMPfree(b);
MT_lock_set(&b->theaplock);
if (nunique != BUN_NONE) {
b->tunique_est = (double) nunique;
@@ -1268,6 +1270,7 @@ BUNappendmulti(BAT *b, const void *value
IMPSdestroy(b); /* no support for inserts in imprints yet */
OIDXdestroy(b);
+ STRMPdestroy(b); /* TODO: use STRMPappendBitstring */
return GDK_SUCCEED;
}
@@ -1494,6 +1497,7 @@ BUNinplacemulti(BAT *b, const oid *posit
}
OIDXdestroy(b);
IMPSdestroy(b);
+ STRMPdestroy(b);
if (b->tvarsized && b->ttype) {
var_t _d;
diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c
--- a/gdk/gdk_batop.c
+++ b/gdk/gdk_batop.c
@@ -693,6 +693,7 @@ BATappend2(BAT *b, BAT *n, BAT *s, bool
IMPSdestroy(b); /* imprints do not support updates yet */
OIDXdestroy(b);
+ STRMPdestroy(b); /* TODO: use STRMPappendBitString */
MT_lock_set(&b->theaplock);
if (BATcount(b) == 0 || b->tmaxpos != BUN_NONE) {
if (ni.maxpos != BUN_NONE) {
@@ -906,6 +907,7 @@ BATdel(BAT *b, BAT *d)
OIDXdestroy(b);
HASHdestroy(b);
PROPdestroy(b);
+ STRMPdestroy(b);
if (BATtdense(d)) {
oid o = d->tseqbase;
BUN c = BATcount(d);
@@ -1107,6 +1109,7 @@ BATappend_or_update(BAT *b, BAT *p, cons
OIDXdestroy(b);
IMPSdestroy(b);
+ STRMPdestroy(b);
MT_lock_set(&b->theaplock);
if (ni.count > BATcount(b) / GDK_UNIQUE_ESTIMATE_KEEP_FRACTION) {
b->tunique_est = 0;
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -1113,6 +1113,7 @@ fixhashashbat(BAT *b)
IMPSdestroy(b);
OIDXdestroy(b);
PROPdestroy(b);
+ STRMPdestroy(b);
/* make backup of heaps */
const char *t;
@@ -4447,6 +4448,11 @@ BBPdiskscan(const char *parent, size_t b
#else
delete = true;
#endif
+ } else if (strncmp(p + 1, "tstrimps", 8) == 0) {
+ BAT *b = getdesc(bid);
+ delete = b == NULL;
+ if (!delete)
+ b->tstrimps = (Strimps *)1;
} else if (strncmp(p + 1, "new", 3) != 0) {
ok = false;
}
diff --git a/gdk/gdk_private.h b/gdk/gdk_private.h
--- a/gdk/gdk_private.h
+++ b/gdk/gdk_private.h
@@ -18,6 +18,9 @@
/* persist order index heaps for persistent BATs */
#define PERSISTENTIDX 1
+/* persist strimp heaps for persistent BATs */
+#define PERSISTENTSTRIMP 1
+
#include "gdk_system_private.h"
enum heaptype {
@@ -25,7 +28,8 @@ enum heaptype {
varheap,
hashheap,
imprintsheap,
- orderidxheap
+ orderidxheap,
+ strimpheap
};
gdk_return ATOMheap(int id, Heap *hp, size_t cap)
@@ -225,6 +229,14 @@ void IMPSincref(Imprints *imprints)
void IMPSprint(BAT *b) /* never called: for debugging only */
__attribute__((__cold__));
#endif
+void STRMPincref(Strimps *strimps)
+ __attribute__((__visibility__("hidden")));
+void STRMPdecref(Strimps *strimps, bool remove)
+ __attribute__((__visibility__("hidden")));
+void STRMPdestroy(BAT *b)
+ __attribute__((__visibility__("hidden")));
+void STRMPfree(BAT *b)
+ __attribute__((__visibility__("hidden")));
void MT_init_posix(void)
__attribute__((__visibility__("hidden")));
void *MT_mmap(const char *path, int mode, size_t len)
@@ -406,6 +418,17 @@ struct Imprints {
BUN dictcnt; /* counter for cache dictionary */
};
+struct Strimps {
+ Heap strimps;
+ uint8_t *sizes_base; /* pointer into strimps heap (pair sizes) */
+ uint8_t *pairs_base; /* pointer into strimps heap (pairs start) */
+ void *bitstrings_base; /* pointer into strimps heap (bitstrings start)
*/
+ size_t rec_cnt; /* reconstruction counter: how many
+ bitstrings were added after header
+ construction */
+ /* bitstrings_base is a pointer to uint64_t */
+};
+
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
