Changeset: 4418cfd55eb8 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=4418cfd55eb8
Modified Files:
clients/Tests/exports.stable.out
gdk/gdk.h
gdk/gdk_bat.c
gdk/gdk_heap.c
gdk/gdk_posix.c
gdk/gdk_private.h
gdk/gdk_storage.c
monetdb5/extras/crackers/crackers_sortmerge.mx
monetdb5/modules/mal/tablet_sql.c
Branch: default
Log Message:
Massive cleanup of mmap and madvise related code.
We no longer use "remap" for memory maps. That was only for XQuery.
Also removed the already commented out code for madvise.
The calls to BATaccessBegin and BATaccessEnd are now dummies.
diffs (truncated from 1226 to 300 lines):
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -18,7 +18,6 @@ str ATOMname(int id);
ptr ATOMnil(int id);
int ATOMprint(int id, const void *val, stream *fd);
void ATOMproperty(char *nme, char *property, GDKfcn fcn, int val);
-size_t BATaccess(BAT *b, int what, int advice, int load);
BAT *BATalpha(BAT *b);
BAT *BATantijoin(BAT *l, BAT *r);
BAT *BATantiuselect_(BAT *b, const void *tl, const void *th, bit li, bit hi);
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -1595,9 +1595,8 @@ gdk_export int GDK_mem_pagebits; /* page
#define USE_THASH 8 /* hash index */
#define USE_ALL (USE_HEAD|USE_TAIL|USE_HHASH|USE_THASH)
-#define BATaccessBegin(b,what,advice) BATaccess(b,what,advice,1)
-#define BATaccessEnd(b,what,advice) BATaccess(b,what,advice,-1)
-gdk_export size_t BATaccess(BAT *b, int what, int advice, int load);
+#define BATaccessBegin(b,what,advice) ((void) 0)
+#define BATaccessEnd(b,what,advice) ((void) 0)
gdk_export BAT *BATsave(BAT *b);
gdk_export int BATmmap(BAT *b, int hb, int tb, int hh, int th, int force);
gdk_export int BATmadvise(BAT *b, int hb, int tb, int hh, int th);
diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c
--- a/gdk/gdk_bat.c
+++ b/gdk/gdk_bat.c
@@ -2385,55 +2385,18 @@ BATmmap(BAT *b, int hb, int tb, int hhp,
/*
* @- BATmadvise
+ * deprecated
*/
-#define madvise(adv, hp, len) \
- do { \
- if (adv >= 0 && (hp) && len > 0 && (hp)->base && \
- ((hp)->storage != STORE_MEM) && \
- MT_madvise((hp)->base, len, BUF_TO_MMAP[adv])) { \
- GDKsyserror("madvise(" PTRFMT ", " SZFMT ", %d) on " \
- #hp " " #adv " failed\n", \
- PTRFMTCAST (hp)->base, len, adv); \
- return -1; \
- } \
- } while (0)
-
-static int BUF_TO_MMAP[] = {
- /* BUF_NORMAL */ MMAP_NORMAL,
- /* BUF_RANDOM */ MMAP_RANDOM,
- /* BUF_SEQUENTIAL */ MMAP_SEQUENTIAL,
- /* BUF_WILLNEED */ MMAP_WILLNEED,
- /* BUF_DONTNEED */ MMAP_DONTNEED
-};
-
int
BATmadvise(BAT *b, int hb, int tb, int hhp, int thp)
{
+ (void) b;
+ (void) hb;
+ (void) tb;
+ (void) hhp;
+ (void) thp;
BATcheck(b, "BATmadvise");
- /* A varsized string heap never has sequential access, setting
- * it is no good. */
- assert(!(ATOMstorage(b->htype) == TYPE_str && b->H->vheap && hhp ==
BUF_SEQUENTIAL));
- assert(!(ATOMstorage(b->ttype) == TYPE_str && b->T->vheap && thp ==
BUF_SEQUENTIAL));
-
- /* If the BAT is read-only, set the madvice for the actually
- * used part of the BAT (e.g. till the free offset), else,
- * apply the advice to the entire BAT (the size), since
- * writing may extend to there */
- if (BAThrestricted(b) == BAT_READ) {
- madvise(hb, &b->H->heap, b->H->heap.free);
- madvise(hhp, b->H->vheap, b->H->vheap->free);
- } else {
- madvise(hb, &b->H->heap, b->H->heap.size);
- madvise(hhp, b->H->vheap, b->H->vheap->size);
- }
- if (BATtrestricted(b) == BAT_READ) {
- madvise(tb, &b->T->heap, b->T->heap.free);
- madvise(thp, b->T->vheap, b->T->vheap->free);
- } else {
- madvise(tb, &b->T->heap, b->T->heap.size);
- madvise(thp, b->T->vheap, b->T->vheap->size);
- }
return 0;
}
@@ -2597,9 +2560,6 @@ backup_new(Heap *hp, int lockbat)
static int
HEAPchangeaccess(Heap *hp, int dstmode, int existing)
{
- if (hp->storage == STORE_MMAP && hp->size >= MT_MMAP_TILE) {
- MT_mmap_inform(hp->base, hp->size, 0, MMAP_NORMAL, (dstmode ==
BAT_READ) ? -1 : 1); /* inform vmtrim of the new mode */
- }
if (hp->base == NULL || hp->newstorage == STORE_MEM || !existing ||
dstmode == -1)
return hp->newstorage; /* 0<=>2,1<=>3,a<=>b */
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -352,7 +352,6 @@ HEAPextend(Heap *h, size_t size)
HEAPfree(h);
h->maxsize = h->size = size;
if (HEAPload_intern(h, nme, ext, ".tmp", FALSE) >= 0) {
- MT_madvise(h->base, h->size, MMAP_SEQUENTIAL);
return 0;
}
} else {
@@ -416,7 +415,6 @@ HEAPextend(Heap *h, size_t size)
HEAPDEBUG mnstr_printf(GDKerr, "#HEAPextend:
converting malloced to %s mmapped heap\n", h->newstorage == STORE_MMAP ?
"shared" : "privately");
/* try to allocate a memory-mapped based heap */
if (HEAPload(h, nme, ext, FALSE) >= 0) {
- MT_madvise(h->base, h->size,
MMAP_SEQUENTIAL);
/* copy data to heap and free old
memory */
memcpy(h->base, bak.base, bak.free);
HEAPfree(&bak);
@@ -433,7 +431,6 @@ HEAPextend(Heap *h, size_t size)
/* and load heap back in via memory-mapped file
*/
if (HEAPload_intern(h, nme, ext, ".tmp", FALSE)
>= 0) {
/* success! */
- MT_madvise(h->base, h->size,
MMAP_SEQUENTIAL);
GDKclrerr(); /* don't leak errors
from e.g. HEAPload */
return 0;
}
diff --git a/gdk/gdk_posix.c b/gdk/gdk_posix.c
--- a/gdk/gdk_posix.c
+++ b/gdk/gdk_posix.c
@@ -37,7 +37,7 @@
#include <unistd.h> /* sbrk on Solaris */
#include <string.h> /* strncpy */
-#if defined(__hpux)
+#ifdef __hpux
extern char *sbrk(int);
#endif
@@ -269,470 +269,6 @@ char *MT_heapbase = NULL;
#include <semaphore.h>
#endif
-typedef struct {
- char path[128]; /* mapped file, retained for debugging */
- char *base; /* base address */
- size_t len; /* length of map */
- size_t first_tile; /* from here we started saving tiles */
- size_t save_tile; /* next tile to save */
- size_t unload_tile; /* next tile to unload */
- int wrap_tile; /* unloading has reached end of map? */
- int last_tile; /* saving has reached end of map? */
- int fd; /* open fd (==-1 for anon vm), retained to give
posix_fadvise */
- int usecnt; /* number of threads accessing the heap now */
- int random; /* number of threads accessing the heap
randomly now */
- int writable;
- int next;
-} MT_mmap_t;
-
-#ifdef HAVE_POSIX_FADVISE
-static int do_not_use_posix_fadvise = 0;
-#endif
-
-MT_mmap_t MT_mmap_tab[MT_MMAP_BUFSIZE];
-int MT_mmap_cur = -1, MT_mmap_busy = -1, MT_mmap_first = -1, MT_mmap_free = 0;
-
-pthread_mutex_t MT_mmap_lock, MT_mmap_relock;
-
-static void
-MT_mmap_empty(int i)
-{
- MT_mmap_tab[i].path[0] = 0;
- MT_mmap_tab[i].base = NULL;
- MT_mmap_tab[i].len = 0;
- MT_mmap_tab[i].writable = 0;
- MT_mmap_tab[i].fd = -1;
- MT_mmap_tab[i].usecnt = 0;
- MT_mmap_tab[i].random = 0;
-}
-
-static void
-MT_mmap_init(void)
-{
- int i;
-
- /* create lock */
- pthread_mutex_init(&MT_mmap_lock, 0);
- pthread_mutex_init(&MT_mmap_relock, 0);
-
- for (i = 0; i < MT_MMAP_BUFSIZE; i++) {
- MT_mmap_tab[i].next = i + 1;
- MT_mmap_empty(i);
- }
- MT_mmap_tab[i - 1].next = -1;
-}
-
-/* returns previous element (to facilitate deletion) */
-static int
-MT_mmap_find(void *base)
-{
- /* maybe consider a hash table iso linked list?? */
- int i, prev = MT_MMAP_BUFSIZE;
-
- for (i = MT_mmap_first; i >= 0; i = MT_mmap_tab[i].next) {
- if (MT_mmap_tab[i].base <= (char *) base && (char *) base <
MT_mmap_tab[i].base + MT_mmap_tab[i].len) {
- return prev;
- }
- prev = i;
- }
- return i;
-}
-
-static int
-MT_mmap_idx(void *base, size_t len)
-{
- if (len > MT_MMAP_TILE) {
- int i = MT_mmap_find(base);
-
- if (i >= 0) {
- if (i == MT_MMAP_BUFSIZE) {
- return MT_mmap_first;
- } else {
- return MT_mmap_tab[i].next;
- }
- }
- }
- return -1;
-}
-
-#ifndef NATIVE_WIN32
-static int
-MT_mmap_new(char *path, void *base, size_t len, int fd, int writable)
-{
- (void) pthread_mutex_lock(&MT_mmap_lock);
- if (len > MT_MMAP_TILE && MT_mmap_free >= 0) {
- int i = MT_mmap_free;
-
- MT_mmap_free = MT_mmap_tab[i].next;
- MT_mmap_tab[i].next = MT_mmap_first;
- MT_mmap_first = i;
- if (MT_mmap_cur == -1)
- MT_mmap_cur = i;
-#ifdef MMAP_DEBUG
- mnstr_printf(GDKstdout, "#MT_mmap_new: %s fd=%d\n", path, fd);
-#endif
- strncpy(MT_mmap_tab[i].path, path, 128);
- MT_mmap_tab[i].base = base;
- MT_mmap_tab[i].len = len;
- MT_mmap_tab[i].save_tile = 1;
- MT_mmap_tab[i].last_tile = 0;
- MT_mmap_tab[i].wrap_tile = 0;
- MT_mmap_tab[i].first_tile = 0;
- MT_mmap_tab[i].unload_tile = 0;
- MT_mmap_tab[i].writable = writable;
- MT_mmap_tab[i].fd = fd;
- MT_mmap_tab[i].usecnt = 0;
- MT_mmap_tab[i].random = 0;
- fd = -fd;
- }
- (void) pthread_mutex_unlock(&MT_mmap_lock);
- return fd;
-}
-
-static void
-MT_mmap_del(void *base, size_t len)
-{
- int relock = 0;
- while (len > MT_MMAP_TILE) {
- int prev;
- if (relock)
- (void) pthread_mutex_unlock(&MT_mmap_relock);
- (void) pthread_mutex_lock(&MT_mmap_lock);
- prev = MT_mmap_find(base);
- if (prev >= 0) {
- int ret, victim = (prev == MT_MMAP_BUFSIZE) ?
MT_mmap_first : MT_mmap_tab[prev].next;
- if (relock == 0 && MT_mmap_busy == victim) {
- /* OOPS, the vmtrim thread is saving a tile of
this heap... wait for it to finish with relock */
- (void) pthread_mutex_unlock(&MT_mmap_lock);
- relock = 1;
- continue;
- }
- if (prev == MT_MMAP_BUFSIZE) {
- MT_mmap_first = MT_mmap_tab[MT_mmap_first].next;
- } else {
- MT_mmap_tab[prev].next =
MT_mmap_tab[victim].next;
- }
- if (MT_mmap_cur == victim) {
- MT_mmap_cur = MT_mmap_first;
- }
-#ifdef HAVE_POSIX_FADVISE
- if (!do_not_use_posix_fadvise && MT_mmap_tab[victim].fd
>= 0) {
- /* tell the OS quite clearly that you want to
drop this */
- ret = posix_fadvise(MT_mmap_tab[victim].fd,
0LL, MT_mmap_tab[victim].len & ~(MT_pagesize() - 1), POSIX_FADV_DONTNEED);
-#ifdef MMAP_DEBUG
- mnstr_printf(GDKstdout,
- "#MT_mmap_del:
posix_fadvise(%s,fd=%d,%uMB,POSIX_FADV_DONTNEED) = %d\n",
- MT_mmap_tab[victim].path,
- MT_mmap_tab[victim].fd,
- (unsigned int)
(MT_mmap_tab[victim].len >> 20),
_______________________________________________
Checkin-list mailing list
Checkin-list@monetdb.org
http://mail.monetdb.org/mailman/listinfo/checkin-list
