Changeset: 3b5247b0b3d8 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=3b5247b0b3d8
Modified Files:
NT/monetdb_config.h.in
configure.ag
gdk/gdk_heap.c
gdk/gdk_posix.c
gdk/gdk_private.h
Branch: default
Log Message:
Merge with Feb2013 branch.
diffs (truncated from 339 to 300 lines):
diff --git a/NT/monetdb_config.h.in b/NT/monetdb_config.h.in
--- a/NT/monetdb_config.h.in
+++ b/NT/monetdb_config.h.in
@@ -301,6 +301,9 @@
/* Define to 1 if you have the <memory.h> header file. */
#define HAVE_MEMORY_H 1
+/* Define to 1 if you have the `mremap' function. */
+/* #undef HAVE_MREMAP */
+
/* Define if you have the mseed library */
/* #undef HAVE_MSEED */
diff --git a/configure.ag b/configure.ag
--- a/configure.ag
+++ b/configure.ag
@@ -2564,6 +2564,7 @@ AC_CHECK_FUNCS([\
localtime_r \
lockf \
madvise \
+ mremap \
nl_langinfo \
_NSGetExecutablePath \
pipe \
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -368,16 +368,24 @@ HEAPextend(Heap *h, size_t size)
if (size <= h->size)
return 0;
- if (h->storage != STORE_MEM) {
- HEAPDEBUG fprintf(stderr, "#HEAPextend: extending %s mmapped
heap\n", h->storage == STORE_MMAP ? "shared" : "privately");
- /* memory mapped files extend: save and remap */
- if (HEAPsave_intern(h, nme, ext, ".tmp") < 0)
- return -1;
- HEAPfree(h);
- h->maxsize = h->size = size;
- if (HEAPload_intern(h, nme, ext, ".tmp", FALSE) >= 0) {
- return 0;
- }
+ if (h->storage != STORE_MEM) {
+ char *p;
+ long_str path;
+
+ HEAPDEBUG fprintf(stderr, "#HEAPextend: extending %s mmapped
heap (%s)\n", h->storage == STORE_MMAP ? "shared" : "privately", h->filename);
+ /* extend memory mapped file */
+ GDKfilepath(path, BATDIR, nme, ext);
+ size = (1 + ((size - 1) >> REMAP_PAGE_MAXBITS)) <<
REMAP_PAGE_MAXBITS;
+ p = MT_mremap(path,
+ h->storage == STORE_PRIV ?
+ MMAP_COPY | MMAP_READ | MMAP_WRITE :
+ MMAP_READ | MMAP_WRITE,
+ h->base, h->size, size);
+ if (p) {
+ h->maxsize = h->size = size;
+ h->base = p;
+ return 0;
+ }
} else {
/* extend a malloced heap, possibly switching over to
* file-mapped storage */
diff --git a/gdk/gdk_posix.c b/gdk/gdk_posix.c
--- a/gdk/gdk_posix.c
+++ b/gdk/gdk_posix.c
@@ -57,10 +57,6 @@ extern char *sbrk(int);
# include <sys/user.h>
#endif
-#if defined(DEBUG_ALLOC) && SIZEOF_VOID_P > 4
-#undef DEBUG_ALLOC
-#endif
-
#ifdef WIN32
int GDK_mem_pagebits = 16; /* on windows, the mmap addresses can be set by
the 64KB */
#else
@@ -70,6 +66,9 @@ int GDK_mem_pagebits = 14; /* on linux,
#ifndef MAP_NORESERVE
# define MAP_NORESERVE MAP_PRIVATE
#endif
+#if defined(MAP_ANON) && !defined(MAP_ANONYMOUS)
+#define MAP_ANONYMOUS MAP_ANON
+#endif
#define MMAP_ADVISE 7
#define MMAP_WRITABLE (MMAP_WRITE|MMAP_COPY)
@@ -390,11 +389,188 @@ MT_munmap(void *p, size_t len)
int ret = munmap(p, len);
#ifdef MMAP_DEBUG
- mnstr_printf(GDKstdout, "#munmap(" LLFMT "," LLFMT ",%d) = %d\n", (long
long) p, (long long) len, ret);
+ fprintf(stderr, "#munmap(" PTRFMT "," SZFMT ",%d) = %d\n", PTRFMTCAST
p, len, ret);
#endif
return ret;
}
+/* expand or shrink a memory map (ala realloc).
+ * the address returned may be different from the address going in.
+ * in case of failure, the old address is still mapped and NULL is returned.
+ */
+void *
+MT_mremap(const char *path, int mode, void *old_address, size_t old_size,
size_t new_size)
+{
+ void *p;
+ int fd = -1;
+ int flags = mode & MMAP_COPY ? MAP_PRIVATE : MAP_SHARED;
+ int prot = PROT_WRITE | PROT_READ;
+
+ /* doesn't make sense for us to extend read-only memory map */
+ assert(mode & MMAP_WRITABLE);
+
+ if (new_size < old_size) {
+ /* shrink */
+ munmap((char *) old_address + new_size, old_size - new_size);
+#ifdef MMAP_DEBUG
+ fprintf(stderr, "MT_mremap(%s,"PTRFMT","SZFMT","SZFMT") ->
shrinking\n", path?path:"NULL", PTRFMTCAST old_address, old_size, new_size);
+#endif
+ return old_address;
+ }
+ if (new_size == old_size) {
+ /* do nothing */
+#ifdef MMAP_DEBUG
+ fprintf(stderr, "MT_mremap(%s,"PTRFMT","SZFMT","SZFMT") ->
unchanged\n", path?path:"NULL", PTRFMTCAST old_address, old_size, new_size);
+#endif
+ return old_address;
+ }
+
+ if (!(mode & MMAP_COPY) && path != NULL) {
+ /* "normal" memory map */
+ struct stat stb;
+
+ if ((fd = open(path, O_RDWR)) < 0)
+ return NULL;
+ if (fstat(fd, &stb) < 0) {
+ /* shouldn't happen */
+ close(fd);
+ return NULL;
+ }
+ /* if necessary, extend the underlying file */
+ if (stb.st_size < (off_t) new_size &&
+ (lseek(fd, new_size - 1, SEEK_SET) < 0 ||
+ write(fd, "\0", 1) < 0)) {
+ close(fd);
+ return NULL;
+ }
+#ifdef HAVE_MREMAP
+ /* on Linux it's easy */
+ p = mremap(old_address, old_size, new_size, MREMAP_MAYMOVE);
+#else
+ /* try to map extension at end of current map */
+ p = mmap((char *) old_address + old_size, new_size - old_size,
+ prot, flags, fd, old_size);
+ /* if it failed, there is no point trying a full mmap:
+ * that too won't fit */
+ if (p != MAP_FAILED) {
+ if (p == (char *) old_address + old_size) {
+ /* we got the requested address, make
+ * sure we return the correct (old)
+ * address */
+ p = old_address;
+ } else {
+ /* we got some other address: discard
+ * it and make full mmap */
+ munmap(p, new_size - old_size);
+#ifdef NO_MMAP_ALIASING
+ msync(old_address, old_size, MS_SYNC);
+#endif
+ /* first create full mmap, then, if
+ * successful, remove old mmap */
+ p = mmap(NULL, new_size, prot, flags, fd, 0);
+ if (p != MAP_FAILED)
+ munmap(old_address, old_size);
+ }
+ }
+#endif /* HAVE_MREMAP */
+ close(fd);
+ } else {
+ /* "copy-on-write" or "anonymous" memory map */
+#ifdef MAP_ANONYMOUS
+ flags |= MAP_ANONYMOUS;
+#else
+ if ((fd = open("/dev/zero", O_RDWR)) < 0)
+ return NULL;
+#endif
+ /* try to map an anonymous area as extent to the
+ * current map */
+ p = mmap((char *) old_address + old_size, new_size - old_size,
+ prot, flags, fd, 0);
+ /* no point trying a full map if this didn't work:
+ * there isn't enough space */
+ if (p != MAP_FAILED) {
+ if (p == (char *) old_address + old_size) {
+ /* we got the requested address, make
+ * sure we return the correct (old)
+ * address */
+ p = old_address;
+ } else {
+ /* we got some other address: discard
+ * it and make full mmap */
+ munmap(p, new_size - old_size);
+#ifdef HAVE_MREMAP
+ /* first get an area large enough for
+ * new_size */
+ p = mmap(NULL, new_size, prot, flags, fd, 0);
+ if (p != MAP_FAILED) {
+ /* then overlay old mmap over new */
+ void *q;
+
+ q = mremap(old_address, old_size,
+ old_size,
+ MREMAP_FIXED |
MREMAP_MAYMOVE,
+ p);
+ assert(q == p || q == MAP_FAILED);
+ if (q == MAP_FAILED) {
+ /* we didn't expect this... */
+ munmap(p, new_size);
+ p = MAP_FAILED;
+ }
+ }
+#else
+ p = MAP_FAILED;
+ if (path == NULL ||
+ new_size <= GDK_mmap_minsize) {
+ /* size not too big yet or
+ * anonymous, try to make new
+ * anonymous mmap and copy
+ * data over */
+ p = mmap(NULL, new_size, prot, flags,
+ fd, 0);
+ if (p != MAP_FAILED) {
+ memcpy(p, old_address,
+ old_size);
+ munmap(old_address, old_size);
+ }
+ /* if it failed, try alternative */
+ }
+ if (p == MAP_FAILED && path != NULL) {
+ /* write data to disk, then
+ * mmap it to new address */
+ if (fd >= 0)
+ close(fd);
+ p = malloc(strlen(path) + 5);
+ strcat(strcpy(p, path), ".tmp");
+ fd = open(p, O_RDWR | O_CREAT,
+ MONETDB_MODE);
+ free(p);
+ if (fd < 0)
+ return NULL;
+ if (write(fd, old_address,
+ old_size) < 0 ||
+ lseek(fd, new_size - 1,
+ SEEK_SET) < 0 ||
+ write(fd, "\0", 1) < 0) {
+ close(fd);
+ return NULL;
+ }
+ p = mmap(NULL, new_size, prot, flags,
+ fd, 0);
+ if (p != MAP_FAILED)
+ munmap(old_address, old_size);
+ }
+#endif /* HAVE_MREMAP */
+ }
+ }
+ if (fd >= 0)
+ close(fd);
+ }
+#ifdef MMAP_DEBUG
+ fprintf(stderr, "MT_mremap(%s,"PTRFMT","SZFMT","SZFMT") ->
"PTRFMT"%s\n", path?path:"NULL", PTRFMTCAST old_address, old_size, new_size,
PTRFMTCAST p, path && mode & MMAP_COPY ? " private" : "");
+#endif
+ return p == MAP_FAILED ? NULL : p;
+}
+
int
MT_msync(void *p, size_t off, size_t len, int mode)
{
@@ -403,9 +579,9 @@ MT_msync(void *p, size_t off, size_t len
((mode & MMAP_ASYNC) ? MS_ASYNC : MS_INVALIDATE));
#ifdef MMAP_DEBUG
- mnstr_printf(GDKstdout,
- "#msync(" LLFMT "," LLFMT ",%s) = %d\n",
- (long long) p, (long long) len,
+ fprintf(stderr,
+ "#msync(" PTRFMT "," SZFMT ",%s) = %d\n",
+ PTRFMTCAST p, len,
(mode & MMAP_SYNC) ? "MS_SYNC" :
((mode & MMAP_ASYNC) ? "MS_ASYNC" : "MS_INVALIDATE"),
ret);
@@ -588,6 +764,29 @@ MT_munmap(void *p, size_t dummy)
return -(UnmapViewOfFile(p) == 0);
}
+void *
+MT_mremap(const char *path, int mode, void *old_address, size_t old_size,
size_t new_size)
+{
+ void *p;
+
+ /* doesn't make sense for us to extend read-only memory map */
+ assert(mode & MMAP_WRITABLE);
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
http://mail.monetdb.org/mailman/listinfo/checkin-list
