Changeset: 0aacfb64b8d2 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/0aacfb64b8d2
Modified Files:
gdk/gdk_bbp.c
gdk/gdk_heap.c
gdk/gdk_private.h
gdk/gdk_utils.c
tools/mserver/mserver5.c
Branch: default
Log Message:
Merge with Jun2023 branch.
diffs (truncated from 326 to 300 lines):
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -1700,6 +1700,15 @@ BBPinit(void)
fclose(fp);
}
+ /* remove trailing free bats from potential free list (they will
+ * get added when needed) */
+ for (bat i = (bat) ATOMIC_GET(&BBPsize) - 1; i > 0; i--) {
+ if (BBP_desc(i) != NULL)
+ break;
+ bbpsize--;
+ }
+ ATOMIC_SET(&BBPsize, bbpsize);
+
/* add free bats to free list in such a way that low numbered
* ones are at the head of the list */
for (bat i = (bat) ATOMIC_GET(&BBPsize) - 1; i > 0; i--) {
@@ -2554,36 +2563,51 @@ BBPuncacheit(bat i, bool unloaddesc)
* BBPclear removes a BAT from the BBP directory forever.
*/
static inline void
-BBPhandover(Thread t, int n)
+BBPhandover(Thread t, uint32_t n)
{
+ bat *p, bid;
/* take one bat from our private free list and hand it over to
* the global free list */
- bat i = t->freebats;
- bat bid = i;
- if (i == 0)
- return;
- for (int j = 1; j < n; j++) {
- if (BBP_next(i) == 0) {
- n = j;
- break;
- }
- i = BBP_next(i);
+ if (n >= t->nfreebats) {
+ bid = t->freebats;
+ t->freebats = 0;
+ t->nfreebats = 0;
+ } else {
+ p = &t->freebats;
+ for (uint32_t i = n; i < t->nfreebats; i++)
+ p = &BBP_next(*p);
+ bid = *p;
+ *p = 0;
+ t->nfreebats -= n;
}
- t->freebats = BBP_next(i);
- t->nfreebats -= n;
- BBP_next(i) = 0;
- bat *p = &BBP_free;
- while (n > 0) {
+ p = &BBP_free;
+ while (bid != 0) {
while (*p && *p < bid)
p = &BBP_next(*p);
- i = BBP_next(bid);
+ bat i = BBP_next(bid);
BBP_next(bid) = *p;
*p = bid;
bid = i;
- n--;
}
}
+#ifndef NDEBUG
+extern void printlist(bat bid) __attribute__((__cold__));
+/* print a bat free list, pass start of free list as argument
+ * to be used from the debugger */
+void
+printlist(bat bid)
+{
+ int n = 0;
+ while (bid) {
+ printf("%d ", bid);
+ bid = BBP_next(bid);
+ n++;
+ }
+ printf("(%d)\n", n);
+}
+#endif
+
static inline void
bbpclear(bat i, bool lock)
{
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -157,6 +157,14 @@ HEAPalloc(Heap *h, size_t nitems, size_t
h->free = 0;
h->cleanhash = false;
+#ifdef SIZE_CHECK_IN_HEAPS_ONLY
+ if (GDKvm_cursize() + h->size >= GDK_vm_maxsize &&
+ !MT_thread_override_limits()) {
+ GDKerror("allocating too much memory (current: %zu, requested:
%zu, limit: %zu)\n", GDKvm_cursize(), h->size, GDK_vm_maxsize);
+ return GDK_FAIL;
+ }
+#endif
+
size_t allocated;
if (GDKinmemory(h->farmid) ||
((allocated = GDKmem_cursize()) + h->size < GDK_mem_maxsize &&
@@ -255,6 +263,14 @@ HEAPextend(Heap *h, size_t size, bool ma
}
failure = "size > h->size";
+#ifdef SIZE_CHECK_IN_HEAPS_ONLY
+ if (GDKvm_cursize() + size - h->size >= GDK_vm_maxsize &&
+ !MT_thread_override_limits()) {
+ GDKerror("allocating too much memory (current: %zu, requested:
%zu, limit: %zu)\n", GDKvm_cursize(), size - h->size, GDK_vm_maxsize);
+ return GDK_FAIL;
+ }
+#endif
+
if (h->storage != STORE_MEM) {
char *p;
char *path;
@@ -826,6 +842,14 @@ HEAPload_intern(Heap *h, const char *nme
GDKfree(srcpath);
GDKfree(dstpath);
+#ifdef SIZE_CHECK_IN_HEAPS_ONLY
+ if (GDKvm_cursize() + h->size >= GDK_vm_maxsize &&
+ !MT_thread_override_limits()) {
+ GDKerror("allocating too much memory (current: %zu, requested:
%zu, limit: %zu)\n", GDKvm_cursize(), h->size, GDK_vm_maxsize);
+ return GDK_FAIL;
+ }
+#endif
+
size_t size = h->size;
QryCtx *qc = NULL;
if (h->storage != STORE_MEM)
diff --git a/gdk/gdk_private.h b/gdk/gdk_private.h
--- a/gdk/gdk_private.h
+++ b/gdk/gdk_private.h
@@ -23,6 +23,9 @@
/* persist strimp heaps for persistent BATs */
#define PERSISTENTSTRIMP 1
+/* only check whether we exceed gdk_vm_maxsize when allocating heaps */
+/* #define SIZE_CHECK_IN_HEAPS_ONLY 1 */
+
#include "gdk_system_private.h"
enum heaptype {
diff --git a/gdk/gdk_utils.c b/gdk/gdk_utils.c
--- a/gdk/gdk_utils.c
+++ b/gdk/gdk_utils.c
@@ -1927,12 +1927,14 @@ GDKmalloc_internal(size_t size, bool cle
return NULL;
}
#endif
+#ifndef SIZE_CHECK_IN_HEAPS_ONLY
if (size > SMALL_MALLOC &&
GDKvm_cursize() + size >= GDK_vm_maxsize &&
!MT_thread_override_limits()) {
GDKerror("allocating too much memory\n");
return NULL;
}
+#endif
/* pad to multiple of eight bytes and add some extra space to
* write real size in front; when debugging, also allocate
@@ -2030,11 +2032,14 @@ GDKfree(void *s)
asize = ((size_t *) s)[-1]; /* how much allocated last */
#if !defined(NDEBUG) && !defined(SANITIZER)
+ size_t *p = s;
assert((asize & 2) == 0); /* check against duplicate free */
+ size_t size = p[-2];
+ assert(((size + 7) & ~7) + MALLOC_EXTRA_SPACE + DEBUG_SPACE == asize);
/* check for out-of-bounds writes */
- for (size_t i = ((size_t *) s)[-2]; i < asize - MALLOC_EXTRA_SPACE; i++)
+ for (size_t i = size; i < asize - MALLOC_EXTRA_SPACE; i++)
assert(((char *) s)[i] == '\xBD');
- ((size_t *) s)[-1] |= 2; /* indicate area is freed */
+ p[-1] |= 2; /* indicate area is freed */
/* overwrite memory that is to be freed with a pattern that
* will help us recognize access to already freed memory in
@@ -2064,6 +2069,7 @@ GDKrealloc(void *s, size_t size)
nsize = (size + 7) & ~7;
asize = os[-1]; /* how much allocated last */
+#ifndef SIZE_CHECK_IN_HEAPS_ONLY
if (size > SMALL_MALLOC &&
nsize > asize &&
GDKvm_cursize() + nsize - asize >= GDK_vm_maxsize &&
@@ -2071,10 +2077,12 @@ GDKrealloc(void *s, size_t size)
GDKerror("allocating too much memory\n");
return NULL;
}
+#endif
#if !defined(NDEBUG) && !defined(SANITIZER)
assert((asize & 2) == 0); /* check against duplicate free */
/* check for out-of-bounds writes */
- osize = os[-2]; /* how much asked for last */
+ osize = os[-2]; /* how much asked for last */
+ assert(((osize + 7) & ~7) + MALLOC_EXTRA_SPACE + DEBUG_SPACE == asize);
for (size_t i = osize; i < asize - MALLOC_EXTRA_SPACE; i++)
assert(((char *) s)[i] == '\xBD');
/* if shrinking, write debug pattern into to-be-freed memory */
@@ -2087,6 +2095,8 @@ GDKrealloc(void *s, size_t size)
if (s == NULL) {
#if !defined(NDEBUG) && !defined(SANITIZER)
os[-1] &= ~2; /* not freed after all */
+ assert(os[-1] == asize);
+ assert(os[-2] == osize);
#endif
GDKsyserror("realloc failed; memory requested: %zu, memory in
use: %zu, virtual memory in use: %zu\n", size, GDKmem_cursize(),
GDKvm_cursize());;
return NULL;
@@ -2139,11 +2149,13 @@ GDKmmap(const char *path, int mode, size
{
void *ret;
+#ifndef SIZE_CHECK_IN_HEAPS_ONLY
if (GDKvm_cursize() + len >= GDK_vm_maxsize &&
!MT_thread_override_limits()) {
GDKerror("requested too much virtual memory; memory requested:
%zu, memory in use: %zu, virtual memory in use: %zu\n", len, GDKmem_cursize(),
GDKvm_cursize());
return NULL;
}
+#endif
ret = MT_mmap(path, mode, len);
if (ret != NULL)
meminc(len);
@@ -2170,12 +2182,14 @@ GDKmremap(const char *path, int mode, vo
{
void *ret;
+#ifndef SIZE_CHECK_IN_HEAPS_ONLY
if (*new_size > old_size &&
GDKvm_cursize() + *new_size - old_size >= GDK_vm_maxsize &&
!MT_thread_override_limits()) {
GDKerror("requested too much virtual memory; memory requested:
%zu, memory in use: %zu, virtual memory in use: %zu\n", *new_size,
GDKmem_cursize(), GDKvm_cursize());
return NULL;
}
+#endif
ret = MT_mremap(path, mode, old_address, old_size, new_size);
if (ret != NULL) {
memdec(old_size);
diff --git a/monetdb5/mal/mal_session.c b/monetdb5/mal/mal_session.c
--- a/monetdb5/mal/mal_session.c
+++ b/monetdb5/mal/mal_session.c
@@ -916,6 +916,11 @@ optimizeMALBlock(Client cntxt, MalBlkPtr
if (getModuleId(p) == optimizerRef && p->fcn && p->token !=
REMsymbol) {
actions++;
msg = (*(str (*)(Client, MalBlkPtr, MalStkPtr,
InstrPtr))p->fcn)(cntxt, mb, 0, p);
+ if (mb->errors) {
+ freeException(msg);
+ msg = mb->errors;
+ mb->errors = NULL;
+ }
if (msg) {
str place = getExceptionPlace(msg);
str nmsg = NULL;
diff --git a/monetdb5/optimizer/opt_wrapper.c b/monetdb5/optimizer/opt_wrapper.c
--- a/monetdb5/optimizer/opt_wrapper.c
+++ b/monetdb5/optimizer/opt_wrapper.c
@@ -111,8 +111,11 @@ OPTwrapper(Client cntxt, MalBlkPtr mb, M
if( p == NULL)
throw(MAL, "opt_wrapper", SQLSTATE(HY002) "missing optimizer
statement");
- if( mb->errors)
- throw(MAL, "opt_wrapper", SQLSTATE(42000) "MAL block contains
errors");
+ if (mb->errors) {
+ msg = mb->errors;
+ mb->errors = NULL;
+ return msg;
+ }
fcnnme = getFunctionId(p);
if( p && p->argc > 1 ){
@@ -147,6 +150,10 @@ OPTwrapper(Client cntxt, MalBlkPtr mb, M
for (i = 0; codes[i].nme != NULL; i++) {
if (strcmp(codes[i].nme, id) == 0) {
msg = (str)(*codes[i].fcn)(cntxt, mb, stk, p);
+ if (mb->errors) {
+ msg = mb->errors;
+ mb->errors = NULL;
+ }
clk = GDKusec() - clk;
MT_lock_set(&codeslock);
codes[i].timing += clk;
diff --git a/tools/mserver/mserver5.c b/tools/mserver/mserver5.c
--- a/tools/mserver/mserver5.c
+++ b/tools/mserver/mserver5.c
@@ -436,7 +436,7 @@ main(int argc, char **av)
optarg[optarglen - 1] == '\\'))
optarg[--optarglen] = '\0';
dbpath = absolute_path(optarg);
- if( dbpath == NULL)
+ if (dbpath == NULL)
fprintf(stderr, "#error: can not
allocate memory for dbpath\n");
else
setlen = mo_add_option(&set, setlen,
opt_cmdline, "gdk_dbpath", dbpath);
_______________________________________________
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org
