Changeset: 7145f5afe4d8 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/7145f5afe4d8
Modified Files:
buildtools/coverity_model.c
clients/Tests/exports.stable.out
gdk/gdk_heap.c
gdk/gdk_posix.h
gdk/gdk_utils.c
gdk/gdk_utils.h
monetdb5/modules/mal/mal_io.c
Branch: default
Log Message:
Count copy-on-write memory maps towards malloced memory.
This means that GDK_vm_cursize is exclusively file-backed memory maps.
diffs (281 lines):
diff --git a/buildtools/coverity_model.c b/buildtools/coverity_model.c
--- a/buildtools/coverity_model.c
+++ b/buildtools/coverity_model.c
@@ -126,7 +126,7 @@ GDKmmap(const char *path, int mode, size
}
gdk_return
-GDKmunmap(void *p, size_t size)
+GDKmunmap(void *p, int mode, size_t size)
{
int failed;
__coverity_free__(p);
@@ -139,7 +139,7 @@ GDKmremap(const char *path, int mode, vo
{
void *p = GDKmmap(path, mode, new_size);
if (p) {
- (void) GDKmunmap(old_address, old_size);
+ (void) GDKmunmap(old_address, mode, old_size);
}
return p;
}
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -299,7 +299,7 @@ size_t GDKmem_cursize(void);
gdk_return GDKmergeidx(BAT *b, BAT **a, int n_ar);
void *GDKmmap(const char *path, int mode, size_t len)
__attribute__((__warn_unused_result__));
int GDKms(void);
-gdk_return GDKmunmap(void *addr, size_t len);
+gdk_return GDKmunmap(void *addr, int mode, size_t len);
int GDKnr_threads;
void GDKprepareExit(void);
void GDKprintinfo(void);
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -687,7 +687,11 @@ HEAPfree(Heap *h, bool rmheap)
//heap is stored in regular C memory rather than GDK
memory,so we call free()
free(h->base);
} else if (h->storage != STORE_NOWN) { /* mapped file, or
STORE_PRIV */
- gdk_return ret = GDKmunmap(h->base, h->size);
+ gdk_return ret = GDKmunmap(h->base,
+ h->storage == STORE_PRIV ?
+ MMAP_COPY | MMAP_READ |
MMAP_WRITE :
+ MMAP_READ | MMAP_WRITE,
+ h->size);
if (ret != GDK_SUCCEED) {
GDKsyserror("HEAPfree: %s was not mapped\n",
diff --git a/gdk/gdk_posix.h b/gdk/gdk_posix.h
--- a/gdk/gdk_posix.h
+++ b/gdk/gdk_posix.h
@@ -35,12 +35,6 @@
#include <direct.h>
#endif
-/*
- * @- virtual memory
- */
-#define MT_VMUNITLOG 16
-#define MT_VMUNITSIZE (1 << MT_VMUNITLOG)
-
/* make sure POSIX_MADV_* and posix_madvise() are defined somehow */
#ifdef HAVE_SYS_MMAN_H
# ifndef __USE_BSD
diff --git a/gdk/gdk_utils.c b/gdk/gdk_utils.c
--- a/gdk/gdk_utils.c
+++ b/gdk/gdk_utils.c
@@ -452,7 +452,7 @@ size_t GDK_mmap_pagesize = MMAP_PAGESIZE
size_t GDK_mem_maxsize = GDK_VM_MAXSIZE;
size_t GDK_vm_maxsize = GDK_VM_MAXSIZE;
-#define SEG_SIZE(x,y) ((x)+(((x)&((1<<(y))-1))?(1<<(y))-((x)&((1<<(y))-1)):0))
+#define SEG_SIZE(x) ((ssize_t) (((x) + _MT_pagesize - 1) & ~(_MT_pagesize -
1)))
/* This block is to provide atomic addition and subtraction to select
* variables. We use intrinsic functions (recognized and inlined by
@@ -1666,13 +1666,30 @@ GDKvm_cursize(void)
#define heapinc(_memdelta) \
ATOMIC_ADD(&GDK_mallocedbytes_estimate, _memdelta)
+#ifndef NDEBUG
+#define heapdec(_memdelta)
\
+ do { \
+ ATOMIC_BASE_TYPE old = ATOMIC_ADD(&GDK_mallocedbytes_estimate,
_memdelta); \
+ assert(old >= (ATOMIC_BASE_TYPE) _memdelta); \
+ } while (0)
+#else
#define heapdec(_memdelta) \
ATOMIC_SUB(&GDK_mallocedbytes_estimate, _memdelta)
+#endif
#define meminc(vmdelta)
\
- ATOMIC_ADD(&GDK_vm_cursize, (ssize_t) SEG_SIZE((vmdelta), MT_VMUNITLOG))
+ ATOMIC_ADD(&GDK_vm_cursize, SEG_SIZE(vmdelta))
+#ifndef NDEBUG
#define memdec(vmdelta)
\
- ATOMIC_SUB(&GDK_vm_cursize, (ssize_t) SEG_SIZE((vmdelta), MT_VMUNITLOG))
+ do { \
+ ssize_t diff = SEG_SIZE(vmdelta); \
+ ATOMIC_BASE_TYPE old = ATOMIC_SUB(&GDK_vm_cursize, diff); \
+ assert(old >= (ATOMIC_BASE_TYPE) diff); \
+ } while (0)
+#else
+#define memdec(vmdelta)
\
+ ATOMIC_SUB(&GDK_vm_cursize, SEG_SIZE(vmdelta))
+#endif
/* Memory allocation
*
@@ -1954,22 +1971,29 @@ GDKmmap(const char *path, int mode, size
}
#endif
ret = MT_mmap(path, mode, len);
- if (ret != NULL)
- meminc(len);
- else
+ if (ret != NULL) {
+ if (mode & MMAP_COPY)
+ heapinc(len);
+ else
+ meminc(len);
+ } else
GDKerror("requesting virtual memory failed; memory requested:
%zu, memory in use: %zu, virtual memory in use: %zu\n", len, GDKmem_cursize(),
GDKvm_cursize());
return ret;
}
#undef GDKmunmap
gdk_return
-GDKmunmap(void *addr, size_t size)
+GDKmunmap(void *addr, int mode, size_t size)
{
int ret;
ret = MT_munmap(addr, size);
- if (ret == 0)
- memdec(size);
+ if (ret == 0) {
+ if (mode & MMAP_COPY)
+ heapdec(size);
+ else
+ memdec(size);
+ }
return ret == 0 ? GDK_SUCCEED : GDK_FAIL;
}
@@ -1989,8 +2013,13 @@ GDKmremap(const char *path, int mode, vo
#endif
ret = MT_mremap(path, mode, old_address, old_size, new_size);
if (ret != NULL) {
- memdec(old_size);
- meminc(*new_size);
+ if (mode & MMAP_COPY) {
+ heapdec(old_size);
+ heapinc(*new_size);
+ } else {
+ memdec(old_size);
+ meminc(*new_size);
+ }
} else {
GDKerror("requesting virtual memory failed; memory requested:
%zu, memory in use: %zu, virtual memory in use: %zu\n", *new_size,
GDKmem_cursize(), GDKvm_cursize());
}
diff --git a/gdk/gdk_utils.h b/gdk/gdk_utils.h
--- a/gdk/gdk_utils.h
+++ b/gdk/gdk_utils.h
@@ -79,7 +79,7 @@ gdk_export size_t GDK_vm_maxsize; /* max
gdk_export void *GDKmmap(const char *path, int mode, size_t len)
__attribute__((__warn_unused_result__));
-gdk_export gdk_return GDKmunmap(void *addr, size_t len);
+gdk_export gdk_return GDKmunmap(void *addr, int mode, size_t len);
gdk_export size_t GDKmem_cursize(void); /* RAM/swapmem that MonetDB has
claimed from OS */
gdk_export size_t GDKvm_cursize(void); /* current MonetDB VM address space
usage */
@@ -213,13 +213,15 @@ gdk_export int GDKms(void);
_res); \
_res; \
})
-#define GDKmunmap(p, l) \
- ({ void *_ptr = (p); \
+#define GDKmunmap(p, m, l) \
+ ({ \
+ void *_ptr = (p); \
+ int _mode = (m); \
size_t _len = (l); \
- gdk_return _res = GDKmunmap(_ptr, _len); \
+ gdk_return _res = GDKmunmap(_ptr, _mode, _len); \
TRC_DEBUG(ALLOC, \
- "GDKmunmap(%p,%zu) -> %u\n", \
- _ptr, _len, _res); \
+ "GDKmunmap(%p,0x%x,%zu) -> %u\n", \
+ _ptr, _mode, _len, _res); \
_res; \
})
#define malloc(s) \
@@ -315,14 +317,14 @@ GDKmmap_debug(const char *path, int mode
}
#define GDKmmap(p, m, l) GDKmmap_debug((p), (m), (l))
static inline gdk_return
-GDKmunmap_debug(void *ptr, size_t len)
+GDKmunmap_debug(void *ptr, int mode, size_t len)
{
- gdk_return res = GDKmunmap(ptr, len);
- TRC_DEBUG(ALLOC, "GDKmunmap(%p,%zu) -> %d\n",
- ptr, len, (int) res);
+ gdk_return res = GDKmunmap(ptr, mode, len);
+ TRC_DEBUG(ALLOC, "GDKmunmap(%p,0x%x%zu) -> %d\n",
+ ptr, mode, len, (int) res);
return res;
}
-#define GDKmunmap(p, l) GDKmunmap_debug((p), (l))
+#define GDKmunmap(p, m, l) GDKmunmap_debug((p), (m), (l))
static inline void *
malloc_debug(size_t size)
{
diff --git a/monetdb5/modules/mal/mal_io.c b/monetdb5/modules/mal/mal_io.c
--- a/monetdb5/modules/mal/mal_io.c
+++ b/monetdb5/modules/mal/mal_io.c
@@ -750,7 +750,7 @@ IOimport(void *ret, bat *bid, str *fnme)
BBPunfix(b->batCacheid);
GDKfree(buf);
GDKfree(t);
- GDKmunmap(base, end - base);
+ GDKmunmap(base,
MMAP_SEQUENTIAL, end - base);
throw(MAL, "io.import",
SQLSTATE(HY013)
MAL_MALLOC_FAIL);
}
@@ -775,7 +775,7 @@ IOimport(void *ret, bat *bid, str *fnme)
BBPunfix(b->batCacheid);
GDKfree(buf);
GDKfree(t);
- GDKmunmap(base, end - base);
+ GDKmunmap(base, MMAP_SEQUENTIAL, end - base);
throw(MAL, "io.import", SQLSTATE(HY013)
MAL_MALLOC_FAIL);
}
buf = tmp;
@@ -796,7 +796,7 @@ IOimport(void *ret, bat *bid, str *fnme)
BBPunfix(b->batCacheid);
snprintf(msg, sizeof(msg), "error in input %s", buf);
GDKfree(buf);
- GDKmunmap(base, end - base);
+ GDKmunmap(base, MMAP_SEQUENTIAL, end - base);
GDKfree(t);
throw(MAL, "io.import", "%s", msg);
}
@@ -805,7 +805,7 @@ IOimport(void *ret, bat *bid, str *fnme)
BBPunfix(b->batCacheid);
snprintf(msg, sizeof(msg), "error in input %s", buf);
GDKfree(buf);
- GDKmunmap(base, end - base);
+ GDKmunmap(base, MMAP_SEQUENTIAL, end - base);
GDKfree(t);
throw(MAL, "io.import", "%s", msg);
}
@@ -814,7 +814,7 @@ IOimport(void *ret, bat *bid, str *fnme)
BBPunfix(b->batCacheid);
GDKfree(buf);
GDKfree(t);
- GDKmunmap(base, end - base);
+ GDKmunmap(base, MMAP_SEQUENTIAL, end - base);
throw(MAL, "io.import", "insert failed");
}
#if 0 /* why do this? any
measured effects? */
@@ -824,7 +824,7 @@ IOimport(void *ret, bat *bid, str *fnme)
#ifndef WIN32
#define MAXBUF 40*MT_pagesize()
if ((unsigned) (cur - base) > MAXBUF) {
- GDKmunmap(base, MAXBUF);
+ GDKmunmap(base, MMAP_SEQUENTIAL, MAXBUF);
base += MAXBUF;
}
#endif
@@ -834,7 +834,7 @@ IOimport(void *ret, bat *bid, str *fnme)
if (t)
GDKfree(t);
GDKfree(buf);
- GDKmunmap(base, end - base);
+ GDKmunmap(base, MMAP_SEQUENTIAL, end - base);
BBPunfix(b->batCacheid);
return MAL_SUCCEED;
}
_______________________________________________
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org
