Changeset: 633b99a61e7e for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/633b99a61e7e
Modified Files:
gdk/CMakeLists.txt
gdk/gdk_bat.c
gdk/gdk_bbp.c
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql_statement.c
sql/common/sql_list.c
sql/server/rel_select.c
Branch: default
Log Message:
merged with jun2023
diffs (truncated from 1721 to 300 lines):
diff --git a/ChangeLog.Sep2022 b/ChangeLog.Sep2022
deleted file mode 100644
--- a/ChangeLog.Sep2022
+++ /dev/null
@@ -1,3 +0,0 @@
-# ChangeLog file for devel
-# This file is updated with Maddlog
-
diff --git a/clients/ChangeLog.Sep2022 b/clients/ChangeLog.Sep2022
deleted file mode 100644
--- a/clients/ChangeLog.Sep2022
+++ /dev/null
@@ -1,3 +0,0 @@
-# ChangeLog file for clients
-# This file is updated with Maddlog
-
diff --git a/clients/mapilib/ChangeLog.Sep2022
b/clients/mapilib/ChangeLog.Sep2022
deleted file mode 100644
--- a/clients/mapilib/ChangeLog.Sep2022
+++ /dev/null
@@ -1,3 +0,0 @@
-# ChangeLog file for mapilib
-# This file is updated with Maddlog
-
diff --git a/clients/odbc/ChangeLog.Sep2022 b/clients/odbc/ChangeLog.Sep2022
deleted file mode 100644
--- a/clients/odbc/ChangeLog.Sep2022
+++ /dev/null
@@ -1,3 +0,0 @@
-# ChangeLog file for odbc
-# This file is updated with Maddlog
-
diff --git a/common/stream/ChangeLog.Sep2022 b/common/stream/ChangeLog.Sep2022
deleted file mode 100644
--- a/common/stream/ChangeLog.Sep2022
+++ /dev/null
@@ -1,3 +0,0 @@
-# ChangeLog file for stream
-# This file is updated with Maddlog
-
diff --git a/gdk/CMakeLists.txt b/gdk/CMakeLists.txt
--- a/gdk/CMakeLists.txt
+++ b/gdk/CMakeLists.txt
@@ -145,6 +145,7 @@ install(TARGETS
EXPORT gdkTargets
RUNTIME
DESTINATION ${CMAKE_INSTALL_BINDIR}
+ COMPONENT monetdb
LIBRARY
DESTINATION ${CMAKE_INSTALL_LIBDIR}
COMPONENT monetdb
@@ -155,8 +156,7 @@ install(TARGETS
PUBLIC_HEADER
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/monetdb
COMPONENT monetdbdev
- INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
- COMPONENT monetdbdev)
+ INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
install(EXPORT gdkTargets
FILE gdkTargets.cmake
diff --git a/gdk/ChangeLog.Sep2022 b/gdk/ChangeLog.Sep2022
deleted file mode 100644
--- a/gdk/ChangeLog.Sep2022
+++ /dev/null
@@ -1,3 +0,0 @@
-# ChangeLog file for GDK
-# This file is updated with Maddlog
-
diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c
--- a/gdk/gdk_bat.c
+++ b/gdk/gdk_bat.c
@@ -283,10 +283,12 @@ COLnew2(oid hseq, int tt, BUN cap, role_
}
if (bn->tvheap && width == 0 && ATOMheap(tt, bn->tvheap, cap) !=
GDK_SUCCEED) {
+ HEAPfree(bn->theap, true);
goto bailout;
}
DELTAinit(bn);
if (BBPcacheit(bn, true) != GDK_SUCCEED) {
+ /* cannot happen, function always returns success */
goto bailout;
}
TRC_DEBUG(ALGO, "-> " ALGOBATFMT "\n", ALGOBATPAR(bn));
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -2207,8 +2207,7 @@ void
BBPdump(void)
{
size_t mem = 0, vm = 0;
- size_t cmem = 0, cvm = 0;
- int n = 0, nc = 0;
+ int n = 0;
for (bat i = 0; i < (bat) ATOMIC_GET(&BBPsize); i++) {
if (BBP_refs(i) == 0 && BBP_lrefs(i) == 0)
@@ -2240,15 +2239,9 @@ BBPdump(void)
b->theap->farmid,
b->theap->base == NULL ? "X" :
b->theap->storage == STORE_MMAP ? "M" : "",
status & BBPSWAPPED ? "(Swapped)" :
b->theap->dirty ? "(Dirty)" : "");
- if (BBP_logical(i) && BBP_logical(i)[0] == '.')
{
- cmem += HEAPmemsize(b->theap);
- cvm += HEAPvmsize(b->theap);
- nc++;
- } else {
- mem += HEAPmemsize(b->theap);
- vm += HEAPvmsize(b->theap);
- n++;
- }
+ mem += HEAPmemsize(b->theap);
+ vm += HEAPvmsize(b->theap);
+ n++;
}
}
if (b->tvheap) {
@@ -2264,13 +2257,8 @@ BBPdump(void)
b->tvheap->farmid,
b->tvheap->base == NULL ? "X" :
b->tvheap->storage == STORE_MMAP ? "M" : "",
b->tvheap->dirty ? "(Dirty)" : "");
- if (BBP_logical(i) && BBP_logical(i)[0] == '.')
{
- cmem += HEAPmemsize(b->tvheap);
- cvm += HEAPvmsize(b->tvheap);
- } else {
- mem += HEAPmemsize(b->tvheap);
- vm += HEAPvmsize(b->tvheap);
- }
+ mem += HEAPmemsize(b->tvheap);
+ vm += HEAPvmsize(b->tvheap);
}
}
if (MT_rwlock_rdtry(&b->thashlock)) {
@@ -2280,22 +2268,15 @@ BBPdump(void)
fprintf(stderr, " Thash=[%zu,%zu,f=%d/%d]", m,
v,
b->thash->heaplink.farmid,
b->thash->heapbckt.farmid);
- if (BBP_logical(i) && BBP_logical(i)[0] == '.')
{
- cmem += m;
- cvm += v;
- } else {
- mem += m;
- vm += v;
- }
+ mem += m;
+ vm += v;
}
MT_rwlock_rdunlock(&b->thashlock);
}
fprintf(stderr, " role: %s\n",
b->batRole == PERSISTENT ? "persistent" : "transient");
}
- fprintf(stderr,
- "# %d bats: mem=%zu, vm=%zu %d cached bats: mem=%zu, vm=%zu\n",
- n, mem, vm, nc, cmem, cvm);
+ fprintf(stderr, "# %d bats: mem=%zu, vm=%zu\n", n, mem, vm);
fflush(stderr);
}
@@ -2916,13 +2897,18 @@ decref(bat i, bool logical, bool lock, c
/* we destroy transients asap and unload persistent bats only
* if they have been made cold or are not dirty */
unsigned chkflag = BBPSYNCING;
- if (b && GDKvm_cursize() < GDK_vm_maxsize) {
- if (!locked) {
- MT_lock_set(&b->theaplock);
- locked = true;
- }
- if (((b->theap ? b->theap->size : 0) + (b->tvheap ?
b->tvheap->size : 0)) < (GDK_vm_maxsize - GDKvm_cursize()) / 32)
- chkflag |= BBPHOT;
+ size_t cursize;
+ bool swapdirty = false;
+ if (b) {
+ if ((cursize = GDKvm_cursize()) < (size_t) (GDK_vm_maxsize *
0.75)) {
+ if (!locked) {
+ MT_lock_set(&b->theaplock);
+ locked = true;
+ }
+ if (((b->theap ? b->theap->size : 0) + (b->tvheap ?
b->tvheap->size : 0)) < (GDK_vm_maxsize - cursize) / 32)
+ chkflag |= BBPHOT;
+ } else if (cursize > (size_t) (GDK_vm_maxsize * 0.85))
+ swapdirty = true;
}
/* only consider unloading if refs is 0; if, in addition, lrefs
* is 0, we can definitely unload, else only if some more
@@ -2930,7 +2916,7 @@ decref(bat i, bool logical, bool lock, c
if (BBP_refs(i) == 0 &&
(BBP_lrefs(i) == 0 ||
(b != NULL && b->theap != NULL
- ? (!BATdirty(b) &&
+ ? ((swapdirty || !BATdirty(b)) &&
!(BBP_status(i) & chkflag) &&
(BBP_status(i) & BBPPERSISTENT) &&
/* cannot unload in-memory data */
diff --git a/gdk/gdk_hash.c b/gdk/gdk_hash.c
--- a/gdk/gdk_hash.c
+++ b/gdk/gdk_hash.c
@@ -141,8 +141,14 @@ HASHnew(Hash *h, int tpe, BUN size, BUN
h->heaplink.dirty = true;
h->Link = h->heaplink.base;
}
- if (HEAPalloc(&h->heapbckt, mask + HASH_HEADER_SIZE * SIZEOF_SIZE_T /
h->width, h->width) != GDK_SUCCEED)
+ if (HEAPalloc(&h->heapbckt, mask + HASH_HEADER_SIZE * SIZEOF_SIZE_T /
h->width, h->width) != GDK_SUCCEED) {
+ if (!bcktonly) {
+ HEAPfree(&h->heaplink, true);
+ h->heaplink.free = 0;
+ h->Link = NULL;
+ }
return GDK_FAIL;
+ }
h->heapbckt.free = mask * h->width + HASH_HEADER_SIZE * SIZEOF_SIZE_T;
h->heapbckt.dirty = true;
h->nbucket = mask;
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -143,7 +143,7 @@ gdk_return
HEAPalloc(Heap *h, size_t nitems, size_t itemsize)
{
size_t size = 0;
- QryCtx *qc = NULL;
+ QryCtx *qc = h->farmid == 1 ? MT_thread_get_qry_ctx() : NULL;
h->base = NULL;
h->size = 1;
@@ -165,7 +165,7 @@ HEAPalloc(Heap *h, size_t nitems, size_t
h->size < ((GDK_mem_maxsize - allocated) >> 6))) {
h->storage = STORE_MEM;
size = h->size;
- if (h->farmid == 1 && (qc = MT_thread_get_qry_ctx()) != NULL) {
+ if (qc != NULL) {
ATOMIC_BASE_TYPE sz = ATOMIC_ADD(&qc->datasize, size);
sz += size;
if (qc->maxmem > 0 && sz > qc->maxmem) {
@@ -174,22 +174,22 @@ HEAPalloc(Heap *h, size_t nitems, size_t
return GDK_FAIL;
}
}
- h->base = GDKmalloc(h->size);
- TRC_DEBUG(HEAP, "%s %zu %p\n", h->filename, h->size, h->base);
+ h->base = GDKmalloc(size);
+ TRC_DEBUG(HEAP, "%s %zu %p\n", h->filename, size, h->base);
+ if (h->base == NULL && qc != NULL)
+ ATOMIC_SUB(&qc->datasize, size);
}
- char *nme = NULL;
- if (!GDKinmemory(h->farmid) && h->base == NULL) {
- nme = GDKfilepath(h->farmid, BATDIR, h->filename, NULL);
+ if (h->base == NULL && !GDKinmemory(h->farmid)) {
+ char *nme = GDKfilepath(h->farmid, BATDIR, h->filename, NULL);
if (nme == NULL)
return GDK_FAIL;
h->storage = STORE_MMAP;
h->size = (h->size + GDK_mmap_pagesize - 1) &
~(GDK_mmap_pagesize - 1);
- if (h->farmid == 1 && h->size > size &&
- (qc != NULL || (qc = MT_thread_get_qry_ctx()) != NULL)) {
- ATOMIC_BASE_TYPE sz = ATOMIC_ADD(&qc->datasize, h->size
- size);
- sz += h->size - size;
- size = h->size;
+ size = h->size;
+ if (qc != NULL) {
+ ATOMIC_BASE_TYPE sz = ATOMIC_ADD(&qc->datasize, size);
+ sz += size;
if (qc->maxmem > 0 && sz > qc->maxmem) {
ATOMIC_SUB(&qc->datasize, size);
GDKerror("Query using too much memory.\n");
@@ -198,22 +198,20 @@ HEAPalloc(Heap *h, size_t nitems, size_t
}
h->base = HEAPcreatefile(NOFARM, &h->size, nme);
h->hasfile = true;
- }
- if (h->base == NULL) {
- /* remove file we may just have created */
- if (nme != NULL) {
- /* file may or may not exist, depending on what
+ if (h->base == NULL) {
+ if (qc != NULL)
+ ATOMIC_SUB(&qc->datasize, size);
+ /* remove file we may just have created
+ * it may or may not exist, depending on what
* failed */
(void) MT_remove(nme);
GDKfree(nme);
h->hasfile = false; /* just removed it */
+ GDKerror("Insufficient space for HEAP of %zu bytes.",
h->size);
+ return GDK_FAIL;
}
- if (qc != NULL && size > 0)
- ATOMIC_SUB(&qc->datasize, size);
- GDKerror("Insufficient space for HEAP of %zu bytes.", h->size);
- return GDK_FAIL;
+ GDKfree(nme);
}
- GDKfree(nme);
h->newstorage = h->storage;
return GDK_SUCCEED;
}
@@ -240,7 +238,8 @@ gdk_return
HEAPextend(Heap *h, size_t size, bool mayshare)
_______________________________________________
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org
