Changeset: b25d3219618c for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/b25d3219618c
Modified Files:
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
clients/Tests/exports.stable.out
gdk/gdk.h
gdk/gdk_aggr.c
gdk/gdk_bat.c
gdk/gdk_batop.c
gdk/gdk_bbp.c
gdk/gdk_calc.c
gdk/gdk_calc_compare.h
gdk/gdk_group.c
gdk/gdk_hash.c
gdk/gdk_hash.h
gdk/gdk_join.c
gdk/gdk_private.h
gdk/gdk_select.c
gdk/gdk_string.c
gdk/gdk_system.h
gdk/gdk_unique.c
monetdb5/modules/kernel/alarm.c
monetdb5/modules/mal/iterator.c
monetdb5/modules/mal/mkey.c
monetdb5/modules/mal/pcre.c
monetdb5/modules/mal/tokenizer.c
monetdb5/modules/mal/wlc.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_rank.c
sql/backends/monet5/sql_result.c
sql/backends/monet5/sql_upgrades.c
sql/common/sql_types.c
sql/storage/bat/bat_logger.c
tools/monetdbe/monetdbe.c
Branch: default
Log Message:
Merge with Jul2021 branch.
diffs (truncated from 30149 to 300 lines):
diff --git a/clients/Tests/MAL-signatures.stable.out
b/clients/Tests/MAL-signatures.stable.out
--- a/clients/Tests/MAL-signatures.stable.out
+++ b/clients/Tests/MAL-signatures.stable.out
@@ -5753,6 +5753,7 @@ stdout of test 'MAL-signatures` in direc
[ "batcalc", "uuid", "command batcalc.uuid(X_0:bat[:str],
X_1:bat[:oid]):bat[:uuid] ", "UUIDstr2uuid_bulk;", "" ]
[ "batcalc", "uuid", "command batcalc.uuid(X_0:bat[:uuid],
X_1:bat[:oid]):bat[:uuid] ", "UUIDuuid2uuid_bulk;", "" ]
[ "batcalc", "wkb", "command batcalc.wkb(X_0:bat[:wkb], X_1:bat[:oid],
X_2:int, X_3:int):bat[:wkb] ", "geom_2_geom_bat;", "" ]
+[ "batcalc", "wkb", "command batcalc.wkb(X_0:bat[:str], X_1:bat[:oid],
X_2:int, X_3:int):bat[:wkb] ", "wkbFromText_bat_cand;", "" ]
[ "batcalc", "xml", "command batcalc.xml(X_0:bat[:str]):bat[:xml] ",
"BATXMLstr2xml;", "" ]
[ "batcalc", "xor", "pattern batcalc.xor(X_0:bat[:bit],
X_1:bat[:bit]):bat[:bit] ", "CMDbatXOR;", "" ]
[ "batcalc", "xor", "pattern batcalc.xor(X_0:bat[:bit], X_1:bat[:bit],
X_2:bat[:oid], X_3:bat[:oid]):bat[:bit] ", "CMDbatXOR;", "" ]
diff --git a/clients/Tests/MAL-signatures.stable.out.int128
b/clients/Tests/MAL-signatures.stable.out.int128
--- a/clients/Tests/MAL-signatures.stable.out.int128
+++ b/clients/Tests/MAL-signatures.stable.out.int128
@@ -8277,6 +8277,7 @@ stdout of test 'MAL-signatures` in direc
[ "batcalc", "uuid", "command batcalc.uuid(X_0:bat[:str],
X_1:bat[:oid]):bat[:uuid] ", "UUIDstr2uuid_bulk;", "" ]
[ "batcalc", "uuid", "command batcalc.uuid(X_0:bat[:uuid],
X_1:bat[:oid]):bat[:uuid] ", "UUIDuuid2uuid_bulk;", "" ]
[ "batcalc", "wkb", "command batcalc.wkb(X_0:bat[:wkb], X_1:bat[:oid],
X_2:int, X_3:int):bat[:wkb] ", "geom_2_geom_bat;", "" ]
+[ "batcalc", "wkb", "command batcalc.wkb(X_0:bat[:str], X_1:bat[:oid],
X_2:int, X_3:int):bat[:wkb] ", "wkbFromText_bat_cand;", "" ]
[ "batcalc", "xml", "command batcalc.xml(X_0:bat[:str]):bat[:xml] ",
"BATXMLstr2xml;", "" ]
[ "batcalc", "xor", "pattern batcalc.xor(X_0:bat[:bit],
X_1:bat[:bit]):bat[:bit] ", "CMDbatXOR;", "" ]
[ "batcalc", "xor", "pattern batcalc.xor(X_0:bat[:bit], X_1:bat[:bit],
X_2:bat[:oid], X_3:bat[:oid]):bat[:bit] ", "CMDbatXOR;", "" ]
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -187,7 +187,7 @@ BAT *BATsample_with_seed(BAT *b, BUN n,
gdk_return BATsave(BAT *b) __attribute__((__warn_unused_result__));
BAT *BATselect(BAT *b, BAT *s, const void *tl, const void *th, bool li, bool
hi, bool anti);
gdk_return BATsemijoin(BAT **r1p, BAT **r2p, BAT *l, BAT *r, BAT *sl, BAT *sr,
bool nil_matches, bool max_one, BUN estimate)
__attribute__((__warn_unused_result__));
-gdk_return BATsetaccess(BAT *b, restrict_t mode);
+BAT *BATsetaccess(BAT *b, restrict_t mode)
__attribute__((__warn_unused_result__));
void BATsetcapacity(BAT *b, BUN cnt);
void BATsetcount(BAT *b, BUN cnt);
BAT *BATslice(BAT *b, BUN low, BUN high);
diff --git a/common/stream/stdio_stream.c b/common/stream/stdio_stream.c
--- a/common/stream/stdio_stream.c
+++ b/common/stream/stdio_stream.c
@@ -31,9 +31,8 @@ file_read(stream *restrict s, void *rest
return -1;
}
- if (elmsize && cnt && !feof(fp)) {
- if (ferror(fp) ||
- ((rc = fread(buf, elmsize, cnt, fp)) == 0 && ferror(fp))) {
+ if (elmsize && cnt) {
+ if ((rc = fread(buf, elmsize, cnt, fp)) == 0 && ferror(fp)) {
mnstr_set_error_errno(s, MNSTR_READ_ERROR, "read
error");
return -1;
}
@@ -55,7 +54,7 @@ file_write(stream *restrict s, const voi
if (elmsize && cnt) {
size_t rc = fwrite(buf, elmsize, cnt, fp);
- if (ferror(fp)) {
+ if (!rc && ferror(fp)) {
mnstr_set_error_errno(s, MNSTR_WRITE_ERROR, "write
error");
return -1;
}
diff --git a/common/utils/mcrypt.c b/common/utils/mcrypt.c
--- a/common/utils/mcrypt.c
+++ b/common/utils/mcrypt.c
@@ -505,6 +505,8 @@ mcrypt_hashPassword(
} else
#endif
{
+ (void) len;
+ (void) ret;
(void) algo;
(void) password;
(void) challenge;
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -331,6 +331,8 @@
#include <limits.h> /* for *_MIN and *_MAX */
#include <float.h> /* for FLT_MAX and DBL_MAX */
+typedef enum { GDK_FAIL, GDK_SUCCEED } gdk_return;
+
#include "gdk_system.h"
#include "gdk_posix.h"
#include "stream.h"
@@ -524,8 +526,6 @@ typedef size_t BUN;
/*
* @- Checking and Error definitions:
*/
-typedef enum { GDK_FAIL, GDK_SUCCEED } gdk_return;
-
#define ATOMextern(t) (ATOMstorage(t) >= TYPE_str)
typedef enum {
@@ -776,14 +776,6 @@ typedef struct BAT {
MT_Lock batIdxLock; /* lock to manipulate other indexes/properties
*/
} BAT;
-typedef struct BATiter {
- BAT *b;
- union {
- oid tvid;
- bool tmsk;
- };
-} BATiter;
-
/* macros to hide complexity of the BAT structure */
#define ttype T.type
#define tkey T.key
@@ -879,6 +871,98 @@ gdk_export size_t HEAPmemsize(Heap *h);
gdk_export void HEAPdecref(Heap *h, bool remove);
gdk_export void HEAPincref(Heap *h);
+/* BAT iterator, also protects use of BAT heaps with reference counts */
+typedef struct BATiter {
+ BAT *b;
+ Heap *h;
+ void *base;
+ Heap *vh;
+ BUN count;
+ uint16_t width;
+ uint8_t shift;
+ int8_t type;
+ oid tseq;
+ union {
+ oid tvid;
+ bool tmsk;
+ };
+#ifndef NDEBUG
+ bool locked;
+#endif
+} BATiter;
+
+static inline BATiter
+bat_iterator(BAT *b)
+{
+ /* needs matching bat_iterator_end */
+ BATiter bi;
+ if (b) {
+ MT_lock_set(&b->theaplock);
+ bi = (BATiter) {
+ .b = b,
+ .h = b->theap,
+ .base = b->theap->base ? b->theap->base + (b->tbaseoff
<< b->tshift) : NULL,
+ .vh = b->tvheap,
+ .count = b->batCount,
+ .width = b->twidth,
+ .shift = b->tshift,
+ .type = b->ttype,
+ .tseq = b->tseqbase,
+#ifndef NDEBUG
+ .locked = true,
+#endif
+ };
+ HEAPincref(bi.h);
+ if (bi.vh)
+ HEAPincref(bi.vh);
+ MT_lock_unset(&b->theaplock);
+ } else {
+ bi = (BATiter) {
+ .b = NULL,
+#ifndef NDEBUG
+ .locked = true,
+#endif
+ };
+ }
+ return bi;
+}
+
+static inline void
+bat_iterator_end(BATiter *bip)
+{
+ /* matches bat_iterator */
+ assert(bip);
+ assert(bip->locked);
+ if (bip->h)
+ HEAPdecref(bip->h, false);
+ if (bip->vh)
+ HEAPdecref(bip->vh, false);
+ *bip = (BATiter) {0};
+}
+
+static inline BATiter
+bat_iterator_nolock(BAT *b)
+{
+ /* does not get matched by bat_iterator_end */
+ if (b) {
+ return (BATiter) {
+ .b = b,
+ .h = b->theap,
+ .base = b->theap->base ? b->theap->base + (b->tbaseoff
<< b->tshift) : NULL,
+ .vh = b->tvheap,
+ .count = b->batCount,
+ .width = b->twidth,
+ .shift = b->tshift,
+ .type = b->ttype,
+ .tseq = b->tseqbase,
+#ifndef NDEBUG
+ .locked = false,
+#endif
+ };
+ }
+ return (BATiter) {0};
+}
+
/*
* @- Internal HEAP Chunk Management
* Heaps are used in BATs to store data for variable-size atoms. The
@@ -995,7 +1079,6 @@ gdk_export BUN BUNfnd(BAT *b, const void
(BUN) (*(const oid*)(v) - (b)->tseqbase))
#define BATttype(b) (BATtdense(b) ? TYPE_oid : (b)->ttype)
-#define Tbase(b) ((b)->tvheap->base)
#define Tsize(b) ((b)->twidth)
@@ -1011,12 +1094,12 @@ typedef var_t stridx_t;
#define SIZEOF_STRIDX_T SIZEOF_VAR_T
#define GDK_VARALIGN SIZEOF_STRIDX_T
-#define BUNtvaroff(bi,p) VarHeapVal(Tloc((bi).b, 0), (p), (bi).b->twidth)
+#define BUNtvaroff(bi,p) VarHeapVal((bi).base, (p), (bi).width)
-#define BUNtloc(bi,p) (ATOMstorage((bi).b->ttype) == TYPE_msk ? Tmsk(&(bi),
p) : Tloc((bi).b,p))
+#define BUNtloc(bi,p) (ATOMstorage((bi).type) == TYPE_msk ? Tmsk(&(bi), p) :
(void *) ((char *) (bi).base + ((p) << (bi).shift)))
#define BUNtpos(bi,p) Tpos(&(bi),p)
-#define BUNtvar(bi,p) (assert((bi).b->ttype && (bi).b->tvarsized), (void *)
(Tbase((bi).b)+BUNtvaroff(bi,p)))
-#define BUNtail(bi,p)
((bi).b->ttype?(bi).b->tvarsized?BUNtvar(bi,p):BUNtloc(bi,p):BUNtpos(bi,p))
+#define BUNtvar(bi,p) (assert((bi).type && (bi).b->tvarsized), (void *)
((bi).vh->base+BUNtvaroff(bi,p)))
+#define BUNtail(bi,p)
((bi).type?(bi).b->tvarsized?BUNtvar(bi,p):BUNtloc(bi,p):BUNtpos(bi,p))
#define BUNlast(b) (assert((b)->batCount <= BUN_MAX), (b)->batCount)
@@ -1031,6 +1114,8 @@ typedef var_t stridx_t;
static inline oid
BUNtoid(BAT *b, BUN p)
{
+ oid o;
+
assert(ATOMtype(b->ttype) == TYPE_oid);
/* BATcount is the number of valid entries, so with
* exceptions, the last value can well be larger than
@@ -1039,13 +1124,18 @@ BUNtoid(BAT *b, BUN p)
assert(b->ttype == TYPE_void || b->tvheap == NULL);
if (is_oid_nil(b->tseqbase)) {
if (b->ttype == TYPE_void)
- return b->tseqbase;
- return ((const oid *) b->theap->base)[p + b->tbaseoff];
+ return b->tseqbase; /* i.e. oid_nil */
+ MT_lock_set(&b->theaplock);
+ o = ((const oid *) b->theap->base)[p + b->tbaseoff];
+ MT_lock_unset(&b->theaplock);
+ return o;
}
- oid o = b->tseqbase + p;
+ o = b->tseqbase + p;
if (b->ttype == TYPE_oid || b->tvheap == NULL) {
return o;
}
+ /* b->tvheap != NULL, so we know there will be no parallel
+ * modifications (so no locking) */
assert(!mask_cand(b));
/* exceptions only allowed on transient BATs */
assert(b->batRole == TRANSIENT);
@@ -1072,8 +1162,6 @@ BUNtoid(BAT *b, BUN p)
return o + hi;
}
-#define bat_iterator(_b) ((BATiter) {.b = (_b), .tvid = 0})
-
/*
* @- BAT properties
* @multitable @columnfractions 0.08 0.7
@@ -1144,7 +1232,8 @@ typedef enum {
BAT_APPEND, /* only reads and appends allowed */
} restrict_t;
-gdk_export gdk_return BATsetaccess(BAT *b, restrict_t mode);
+gdk_export BAT *BATsetaccess(BAT *b, restrict_t mode)
+ __attribute__((__warn_unused_result__));
gdk_export restrict_t BATgetaccess(BAT *b);
@@ -1320,8 +1409,8 @@ BATsettrivprop(BAT *b)
int c;
if (b->tvarsized)
c = ATOMcmp(b->ttype,
- Tbase(b) + VarHeapVal(Tloc(b, 0), 0,
b->twidth),
- Tbase(b) + VarHeapVal(Tloc(b, 0), 1,
b->twidth));
+ b->tvheap->base + VarHeapVal(Tloc(b, 0), 0,
b->twidth),
+ b->tvheap->base + VarHeapVal(Tloc(b, 0), 1,
b->twidth));
else
c = ATOMcmp(b->ttype, Tloc(b, 0), Tloc(b, 1));
b->tsorted = c <= 0;
@@ -1502,6 +1591,8 @@ gdk_export BAT *BBPquickdesc(bat b, bool
#include "gdk_tracer.h"
+gdk_export gdk_return GDKtracer_fill_comp_info(BAT *id, BAT *component, BAT
*log_level);
+
#define GDKerror(format, ...) \
GDKtracer_log(__FILE__, __func__, __LINE__, M_ERROR, \
GDK, NULL, format, ##__VA_ARGS__)
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
