Changeset: 361512dd2813 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=361512dd2813
Added Files:
sql/test/BugTracker-2019/Tests/double-free.Bug-6757.sql
sql/test/BugTracker-2019/Tests/double-free.Bug-6757.stable.err
sql/test/BugTracker-2019/Tests/double-free.Bug-6757.stable.out
Modified Files:
gdk/gdk_bbp.c
gdk/gdk_heap.c
gdk/gdk_interprocess.c
gdk/gdk_storage.c
sql/backends/monet5/UDF/pyapi/pyapi.c
sql/server/rel_optimizer.c
sql/server/rel_semantic.c
sql/storage/store.c
sql/test/BugTracker-2019/Tests/All
sql/test/BugTracker-2019/Tests/cte-union.Bug-6755.stable.err
sql/test/BugTracker-2019/Tests/cte-union.Bug-6755.stable.out
sql/test/BugTracker-2019/Tests/sequence-first-next-value.Bug-6743.stable.out
sql/test/subquery/Tests/subquery2.sql
sql/test/testdb-reload/Tests/reload.py
testing/process.py
tools/merovingian/daemon/controlrunner.c
Branch: grouping-analytics
Log Message:
Merge with default branch
diffs (truncated from 2417 to 300 lines):
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -1737,7 +1737,7 @@ BBPdir_subcommit(int cnt, bat *subcommit
assert(subcommit[n - 1] < subcommit[n]);
#endif
- if ((nbbpf = GDKfilelocate(0, "BBP", "w", "dir")) == NULL)
+ if ((nbbpf = GDKfilelocate(0, "BBP", "wx", "dir")) == NULL)
return GDK_FAIL;
n = (bat) ATOMIC_GET(&BBPsize);
@@ -1849,7 +1849,7 @@ BBPdir(int cnt, bat *subcommit)
return BBPdir_subcommit(cnt, subcommit);
IODEBUG fprintf(stderr, "#BBPdir: writing BBP.dir (%d bats).\n", (int)
(bat) ATOMIC_GET(&BBPsize));
- if ((fp = GDKfilelocate(0, "BBP", "w", "dir")) == NULL) {
+ if ((fp = GDKfilelocate(0, "BBP", "wx", "dir")) == NULL) {
goto bailout;
}
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -61,7 +61,7 @@ HEAPcreatefile(int farmid, size_t *maxsz
fn = path;
}
/* round up to mulitple of GDK_mmap_pagesize */
- fd = GDKfdlocate(NOFARM, fn, "wb", NULL);
+ fd = GDKfdlocate(NOFARM, fn, "wxb", NULL);
if (fd >= 0) {
close(fd);
base = GDKload(NOFARM, fn, NULL, *maxsz, maxsz, STORE_MMAP);
@@ -119,36 +119,12 @@ HEAPalloc(Heap *h, size_t nitems, size_t
}
if (!GDKinmemory() && h->base == NULL) {
char *nme;
- struct stat st;
- if(!(nme = GDKfilepath(h->farmid, BATDIR, h->filename, NULL))) {
- GDKerror("HEAPalloc: malloc failure");
+ nme = GDKfilepath(h->farmid, BATDIR, h->filename, NULL);
+ if (nme == NULL)
return GDK_FAIL;
- }
- if (stat(nme, &st) < 0) {
- h->storage = STORE_MMAP;
- h->base = HEAPcreatefile(NOFARM, &h->size, nme);
- } else {
- int fd;
-
- fd = GDKfdlocate(NOFARM, nme, "wb", NULL);
- if (fd >= 0) {
- char of[sizeof(h->filename)];
- char *ext;
- close(fd);
- strncpy(of, h->filename, sizeof(of));
-#ifdef STATIC_CODE_ANALYSIS
- /* help coverity */
- of[sizeof(h->filename) - 1] = 0;
-#endif
- ext = decompose_filename(of);
- h->newstorage = STORE_MMAP;
- if (HEAPload(h, of, ext, false) != GDK_SUCCEED)
- h->base = NULL; /* superfluous */
- /* success checked by looking at
- * h->base below */
- }
- }
+ h->storage = STORE_MMAP;
+ h->base = HEAPcreatefile(NOFARM, &h->size, nme);
GDKfree(nme);
}
if (h->base == NULL) {
@@ -270,6 +246,7 @@ HEAPextend(Heap *h, size_t size, bool ma
HEAPfree(&bak, false);
return GDK_SUCCEED;
}
+ GDKclrerr();
}
fd = GDKfdlocate(h->farmid, nme, "wb", ext);
if (fd >= 0) {
@@ -445,7 +422,7 @@ GDKupgradevarheap(BAT *b, var_t v, bool
const char *base = b->theap.base;
/* first save heap in file with extra .tmp extension */
- if ((fd = GDKfdlocate(b->theap.farmid, b->theap.filename, "wb",
"tmp")) < 0)
+ if ((fd = GDKfdlocate(b->theap.farmid, b->theap.filename,
"wxb", "tmp")) < 0)
return GDK_FAIL;
while (size > 0) {
ret = write(fd, base, (unsigned) MIN(1 << 30, size));
diff --git a/gdk/gdk_interprocess.c b/gdk/gdk_interprocess.c
--- a/gdk/gdk_interprocess.c
+++ b/gdk/gdk_interprocess.c
@@ -28,7 +28,7 @@
#include <sys/sem.h>
#include <time.h>
-static size_t interprocess_unique_id = 1;
+static ATOMIC_TYPE interprocess_unique_id = ATOMIC_VAR_INIT(1);
static key_t base_key = 800000000;
// Regular ftok produces too many collisions
@@ -46,12 +46,7 @@ ftok_enhanced(int id, key_t * return_key
size_t
GDKuniqueid(size_t offset)
{
- // TODO: lock this here instead of in pyapi
- size_t id;
-
- id = interprocess_unique_id;
- interprocess_unique_id += offset;
- return id;
+ return (size_t) ATOMIC_ADD(&interprocess_unique_id, (ATOMIC_BASE_TYPE)
offset);
}
//! Create a memory mapped file if it does not exist and open it
@@ -75,19 +70,21 @@ GDKinitmmap(size_t id, size_t size, size
size = (maxsize + GDK_mmap_pagesize - 1) & ~(GDK_mmap_pagesize - 1);
if (size == 0)
size = GDK_mmap_pagesize; */
- fd = GDKfdlocate(0, address, "wb", "tmp");
- if (fd < 0) {
- return NULL;
- }
path = GDKfilepath(0, BATDIR, address, "tmp");
if (path == NULL) {
return NULL;
}
- close(fd);
- if (GDKextend(path, size) != GDK_SUCCEED) {
+ fd = GDKfdlocate(NOFARM, path, "wb", NULL);
+ if (fd < 0) {
GDKfree(path);
return NULL;
}
+ if (GDKextendf(fd, size, path) != GDK_SUCCEED) {
+ close(fd);
+ GDKfree(path);
+ return NULL;
+ }
+ close(fd);
ptr = GDKmmap(path, mod, size);
GDKfree(path);
if (ptr == NULL) {
diff --git a/gdk/gdk_storage.c b/gdk/gdk_storage.c
--- a/gdk/gdk_storage.c
+++ b/gdk/gdk_storage.c
@@ -194,7 +194,7 @@ int
GDKfdlocate(int farmid, const char *nme, const char *mode, const char
*extension)
{
char *path = NULL;
- int fd, flags = 0;
+ int fd, flags = O_CLOEXEC;
assert(!GDKinmemory());
if (nme == NULL || *nme == 0)
@@ -212,12 +212,14 @@ GDKfdlocate(int farmid, const char *nme,
mode++;
#ifdef _CYGNUS_H_
} else {
- flags = _FRDSEQ; /* WIN32
CreateFile(FILE_FLAG_SEQUENTIAL_SCAN) */
+ flags |= _FRDSEQ; /* WIN32
CreateFile(FILE_FLAG_SEQUENTIAL_SCAN) */
#endif
}
if (strchr(mode, 'w')) {
flags |= O_WRONLY | O_CREAT;
+ if (strchr(mode, 'x'))
+ flags |= O_EXCL;
} else if (!strchr(mode, '+')) {
flags |= O_RDONLY;
} else {
@@ -226,11 +228,11 @@ GDKfdlocate(int farmid, const char *nme,
#ifdef WIN32
flags |= strchr(mode, 'b') ? O_BINARY : O_TEXT;
#endif
- fd = open(nme, flags | O_CLOEXEC, MONETDB_MODE);
+ fd = open(nme, flags, MONETDB_MODE);
if (fd < 0 && *mode == 'w') {
/* try to create the directory, in case that was the problem */
if (GDKcreatedir(nme) == GDK_SUCCEED) {
- fd = open(nme, flags | O_CLOEXEC, MONETDB_MODE);
+ fd = open(nme, flags, MONETDB_MODE);
if (fd < 0)
GDKsyserror("GDKfdlocate: cannot open file
%s\n", nme);
}
@@ -584,10 +586,12 @@ GDKload(int farmid, const char *nme, con
nme = path;
}
if (nme != NULL && GDKextend(nme, size) == GDK_SUCCEED) {
- int mod = MMAP_READ | MMAP_WRITE | MMAP_SEQUENTIAL |
MMAP_SYNC;
+ int mod = MMAP_READ | MMAP_WRITE | MMAP_SEQUENTIAL;
if (mode == STORE_PRIV)
mod |= MMAP_COPY;
+ else
+ mod |= MMAP_SYNC;
ret = GDKmmap(nme, mod, size);
if (ret != NULL) {
/* success: update allocated size */
diff --git a/sql/backends/monet5/UDF/pyapi/pyapi.c
b/sql/backends/monet5/UDF/pyapi/pyapi.c
--- a/sql/backends/monet5/UDF/pyapi/pyapi.c
+++ b/sql/backends/monet5/UDF/pyapi/pyapi.c
@@ -329,9 +329,7 @@ static str PyAPIeval(Client cntxt, MalBl
int mmap_count = 4 + pci->retc * 2;
// create initial shared memory
- MT_lock_set(&pyapiLock);
mmap_id = GDKuniqueid(mmap_count);
- MT_lock_unset(&pyapiLock);
mmap_ptrs = GDKzalloc(mmap_count * sizeof(void *));
mmap_sizes = GDKzalloc(mmap_count * sizeof(size_t));
diff --git a/sql/server/rel_semantic.c b/sql/server/rel_semantic.c
--- a/sql/server/rel_semantic.c
+++ b/sql/server/rel_semantic.c
@@ -88,6 +88,8 @@ rel_parse(mvc *m, sql_schema *s, char *q
bstream_destroy(m->scanner.rs);
m->sym = NULL;
+ o.vars = m->vars; /* may have been realloc'ed */
+ o.sizevars = m->sizevars;
if (m->session->status || m->errstr[0]) {
int status = m->session->status;
char errstr[ERRSIZE];
diff --git a/sql/storage/store.c b/sql/storage/store.c
--- a/sql/storage/store.c
+++ b/sql/storage/store.c
@@ -2122,13 +2122,16 @@ flusher_should_run(void)
{
// We will flush if we have a reason to and no reason not to.
char *reason_to = NULL, *reason_not_to = NULL;
+ int changes;
if (flusher.countdown_ms <= 0)
reason_to = "timer expired";
int many_changes = GDKdebug & FORCEMITOMASK ? 100 : 1000000;
- if (logger_funcs.changes() >= many_changes)
+ if ((changes = logger_funcs.changes()) >= many_changes)
reason_to = "many changes";
+ else if (changes == 0)
+ reason_not_to = "no changes";
// Read and clear flush_now. If we decide not to flush
// we'll put it back.
diff --git a/sql/test/BugTracker-2019/Tests/All
b/sql/test/BugTracker-2019/Tests/All
--- a/sql/test/BugTracker-2019/Tests/All
+++ b/sql/test/BugTracker-2019/Tests/All
@@ -33,4 +33,5 @@ alter_table_drop_column.Bug-6749
HAVE_PYMONETDB?remote-table-non-existent-column.Bug-6750
cte-union.Bug-6755
merge-table-limit.Bug-6756
+double-free.Bug-6757
HAVE_LIBPY3?python-loader-string.Bug-6759
diff --git a/sql/test/BugTracker-2019/Tests/double-free.Bug-6757.sql
b/sql/test/BugTracker-2019/Tests/double-free.Bug-6757.sql
new file mode 100644
--- /dev/null
+++ b/sql/test/BugTracker-2019/Tests/double-free.Bug-6757.sql
@@ -0,0 +1,976 @@
+start transaction;
+
+CREATE TABLE "sys"."params_str" (
+ "paramname" CHARACTER LARGE OBJECT,
+ "value" CHARACTER LARGE OBJECT,
+ "prob" DOUBLE
+);
+CREATE TABLE "sys"."bm_0_obj_dict" (
+ "id" INTEGER NOT NULL,
+ "idstr" CHARACTER LARGE OBJECT NOT NULL,
+ "prob" FLOAT(51) NOT NULL,
+ CONSTRAINT "bm_0_obj_dict_id_pkey" PRIMARY KEY ("id"),
+ CONSTRAINT "bm_0_obj_dict_idstr_unique" UNIQUE ("idstr")
+);
+CREATE TABLE "sys"."bm_0_obj_type" (
+ "id" INTEGER NOT NULL,
+ "type" INTEGER NOT NULL,
+ "typestr" CHARACTER LARGE OBJECT NOT NULL,
+ "prob" DOUBLE NOT NULL,
+ CONSTRAINT "bm_0_obj_type_id_fkey" FOREIGN KEY ("id") REFERENCES
"sys"."bm_0_obj_dict" ("id"),
+ CONSTRAINT "bm_0_obj_type_type_fkey" FOREIGN KEY ("type") REFERENCES
"sys"."bm_0_obj_dict" ("id")
+);
+CREATE TABLE "sys"."tr_0_obj_dict" (
+ "id" INTEGER NOT NULL,
+ "idstr" CHARACTER LARGE OBJECT NOT NULL,
+ "prob" FLOAT(51) NOT NULL,
+ CONSTRAINT "tr_0_obj_dict_id_pkey" PRIMARY KEY ("id"),
+ CONSTRAINT "tr_0_obj_dict_idstr_unique" UNIQUE ("idstr")
+);
+CREATE TABLE "sys"."tr_0_obj_type" (
+ "id" INTEGER NOT NULL,
+ "type" INTEGER NOT NULL,
+ "typestr" CHARACTER LARGE OBJECT NOT NULL,
+ "prob" DOUBLE NOT NULL,
+ CONSTRAINT "tr_0_obj_type_id_fkey" FOREIGN KEY ("id") REFERENCES
"sys"."tr_0_obj_dict" ("id"),
+ CONSTRAINT "tr_0_obj_type_type_fkey" FOREIGN KEY ("type") REFERENCES
"sys"."tr_0_obj_dict" ("id")
+);
+
+CREATE TABLE "sys"."_cachedrel_4" (
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
