Changeset: 8cb4b42b80de for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8cb4b42b80de
Modified Files:
gdk/gdk.h
gdk/gdk_bbp.c
gdk/gdk_bbp.h
gdk/gdk_group.c
gdk/gdk_hash.c
gdk/gdk_heap.c
gdk/gdk_imprints.c
gdk/gdk_join.c
gdk/gdk_logger.c
gdk/gdk_orderidx.c
gdk/gdk_private.h
gdk/gdk_storage.c
gdk/gdk_tm.c
gdk/gdk_utils.c
monetdb5/ChangeLog
monetdb5/mal/mal.c
monetdb5/mal/mal_authorize.c
monetdb5/mal/mal_embedded.c
monetdb5/mal/mal_profiler.c
monetdb5/mal/mal_session.c
monetdb5/modules/mal/mal_mapi.c
sql/backends/monet5/sql_gencode.c
sql/backends/monet5/sql_scenario.c
sql/storage/bat/bat_storage.c
testing/Mtest.py.in
tools/merovingian/daemon/controlrunner.c
tools/monetdbe/monetdbe.c
tools/mserver/mserver5.1.in
tools/mserver/mserver5.c
tools/mserver/shutdowntest.c
Branch: default
Log Message:
Implemented option --dbextra=:inmemory, causing transients to stay in memory.
Also an option --transient-inmemory for Mtest.py to test this.
diffs (truncated from 1097 to 300 lines):
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -1126,7 +1126,7 @@ gdk_export void BATmsync(BAT *b);
#define NOFARM (-1) /* indicate to GDKfilepath to create relative path */
gdk_export char *GDKfilepath(int farmid, const char *dir, const char *nme,
const char *ext);
-gdk_export bool GDKinmemory(void);
+gdk_export bool GDKinmemory(int farmid);
gdk_export bool GDKembedded(void);
gdk_export gdk_return GDKcreatedir(const char *nme);
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -316,7 +316,7 @@ BBPselectfarm(role_t role, int type, enu
(void) type; /* may use in future */
(void) hptype; /* may use in future */
- if (GDKinmemory())
+ if (GDKinmemory(0))
return 0;
#ifndef PERSISTENTHASH
@@ -328,7 +328,7 @@ BBPselectfarm(role_t role, int type, enu
role = TRANSIENT;
#endif
for (i = 0; i < MAXFARMS; i++)
- if (BBPfarms[i].dirname && BBPfarms[i].roles & (1 << (int)
role))
+ if (BBPfarms[i].roles & (1U << (int) role))
return i;
/* must be able to find farms for TRANSIENT and PERSISTENT */
assert(role != TRANSIENT && role != PERSISTENT);
@@ -1105,14 +1105,17 @@ BBPheader(FILE *fp, int *lineno)
}
bool
-GDKinmemory(void)
+GDKinmemory(int farmid)
{
- return BBPfarms[0].dirname == NULL;
+ if (farmid == NOFARM)
+ farmid = 0;
+ assert(farmid >= 0 && farmid < MAXFARMS);
+ return BBPfarms[farmid].dirname == NULL;
}
/* all errors are fatal */
gdk_return
-BBPaddfarm(const char *dirname, int rolemask, bool logerror)
+BBPaddfarm(const char *dirname, uint32_t rolemask, bool logerror)
{
struct stat st;
int i;
@@ -1134,7 +1137,9 @@ BBPaddfarm(const char *dirname, int role
GDKerror("bad rolemask\n");
return GDK_FAIL;
}
- if (mkdir(dirname, MONETDB_DIRMODE) < 0) {
+ if (strcmp(dirname, ":inmemory") == 0) {
+ dirname = NULL;
+ } else if (mkdir(dirname, MONETDB_DIRMODE) < 0) {
if (errno == EEXIST) {
if (stat(dirname, &st) == -1 || !S_ISDIR(st.st_mode)) {
if (logerror)
@@ -1148,17 +1153,20 @@ BBPaddfarm(const char *dirname, int role
}
}
for (i = 0; i < MAXFARMS; i++) {
- if (BBPfarms[i].dirname == NULL) {
- BBPfarms[i].dirname = GDKstrdup(dirname);
- if (BBPfarms[i].dirname == NULL)
- return GDK_FAIL;
+ if (BBPfarms[i].roles == 0) {
+ if (dirname) {
+ BBPfarms[i].dirname = GDKstrdup(dirname);
+ if (BBPfarms[i].dirname == NULL)
+ return GDK_FAIL;
+ }
BBPfarms[i].roles = rolemask;
- if ((rolemask & 1) == 0) {
+ if ((rolemask & 1) == 0 && dirname != NULL) {
char *bbpdir;
int j;
for (j = 0; j < i; j++)
- if (strcmp(BBPfarms[i].dirname,
+ if (BBPfarms[j].dirname != NULL &&
+ strcmp(BBPfarms[i].dirname,
BBPfarms[j].dirname) == 0)
return GDK_SUCCEED;
/* if an extra farm, make sure we
@@ -1214,7 +1222,7 @@ BBPinit(void)
* array */
static_assert((uint64_t) N_BBPINIT * BBPINIT < (UINT64_C(1) << (3 *
(sizeof(BBP[0][0].bak) - 5))), "\"bak\" array in BBPrec is too small");
- if (!GDKinmemory()) {
+ if (!GDKinmemory(0)) {
str bbpdirstr, backupbbpdirstr;
if (!(bbpdirstr = GDKfilepath(0, BATDIR, "BBP", "dir"))) {
@@ -1295,7 +1303,7 @@ BBPinit(void)
memset(BBP, 0, sizeof(BBP));
ATOMIC_SET(&BBPsize, 1);
- if (GDKinmemory()) {
+ if (GDKinmemory(0)) {
bbpversion = GDKLIBRARY;
} else {
bbpversion = BBPheader(fp, &lineno);
@@ -1305,7 +1313,7 @@ BBPinit(void)
BBPextend(0, false); /* allocate BBP records */
- if (!GDKinmemory()) {
+ if (!GDKinmemory(0)) {
ATOMIC_SET(&BBPsize, 1);
if (BBPreadEntries(fp, bbpversion, lineno) != GDK_SUCCEED)
return GDK_FAIL;
@@ -1318,7 +1326,7 @@ BBPinit(void)
}
/* will call BBPrecover if needed */
- if (!GDKinmemory() && BBPprepare(false) != GDK_SUCCEED) {
+ if (!GDKinmemory(0) && BBPprepare(false) != GDK_SUCCEED) {
TRC_CRITICAL(GDK, "cannot properly prepare process %s. Please
check whether your disk is full or write-protected", BAKDIR);
return GDK_FAIL;
}
@@ -1482,10 +1490,10 @@ new_bbpentry(FILE *fp, bat i)
assert(BBP_desc(i)->batCacheid == i);
assert(BBP_desc(i)->batRole == PERSISTENT);
assert(0 <= BBP_desc(i)->theap.farmid && BBP_desc(i)->theap.farmid <
MAXFARMS);
- assert(BBPfarms[BBP_desc(i)->theap.farmid].roles & (1 << PERSISTENT));
+ assert(BBPfarms[BBP_desc(i)->theap.farmid].roles & (1U << PERSISTENT));
if (BBP_desc(i)->tvheap) {
assert(0 <= BBP_desc(i)->tvheap->farmid &&
BBP_desc(i)->tvheap->farmid < MAXFARMS);
- assert(BBPfarms[BBP_desc(i)->tvheap->farmid].roles & (1 <<
PERSISTENT));
+ assert(BBPfarms[BBP_desc(i)->tvheap->farmid].roles & (1U <<
PERSISTENT));
}
#endif
@@ -2017,7 +2025,7 @@ BBPinsert(BAT *bn)
BBP_logical(i) = BBP_bak(i);
/* Keep the physical location around forever */
- if (!GDKinmemory() && *BBP_physical(i) == 0) {
+ if (!GDKinmemory(0) && *BBP_physical(i) == 0) {
BBPgetsubdir(dirname, i);
if (*dirname) /* i.e., i >= 0100 */
@@ -2444,7 +2452,7 @@ decref(bat i, bool logical, bool release
* if they have been made cold or are not dirty */
if (BBP_refs(i) > 0 ||
(BBP_lrefs(i) > 0 &&
- (b == NULL || BATdirty(b) || !(BBP_status(i) & BBPPERSISTENT) ||
GDKinmemory()))) {
+ (b == NULL || BATdirty(b) || !(BBP_status(i) & BBPPERSISTENT) ||
GDKinmemory(b->theap.farmid)))) {
/* bat cannot be swapped out */
} else if (b ? b->batSharecnt == 0 : (BBP_status(i) & BBPTMP)) {
/* bat will be unloaded now. set the UNLOADING bit
diff --git a/gdk/gdk_bbp.h b/gdk/gdk_bbp.h
--- a/gdk/gdk_bbp.h
+++ b/gdk/gdk_bbp.h
@@ -56,7 +56,7 @@
gdk_export bat getBBPsize(void); /* current occupied size of BBP array */
/* global calls */
-gdk_export gdk_return BBPaddfarm(const char *dirname, int rolemask, bool
logerror);
+gdk_export gdk_return BBPaddfarm(const char *dirname, uint32_t rolemask, bool
logerror);
/* update interface */
gdk_export void BBPclear(bat bid);
diff --git a/gdk/gdk_group.c b/gdk/gdk_group.c
--- a/gdk/gdk_group.c
+++ b/gdk/gdk_group.c
@@ -1048,7 +1048,7 @@ BATgroup_internal(BAT **groups, BAT **ex
* BATassertProps for similar code; we also exploit if
* g is clustered */
algomsg = "new partial hash -- ";
- nme = GDKinmemory() ? ":inmemory" : BBP_physical(b->batCacheid);
+ nme = GDKinmemory(b->theap.farmid) ? ":inmemory" :
BBP_physical(b->batCacheid);
if (grps && !gc) {
/* we manipulate the hash value after having
* calculated it, and when doing that, we
diff --git a/gdk/gdk_hash.c b/gdk/gdk_hash.c
--- a/gdk/gdk_hash.c
+++ b/gdk/gdk_hash.c
@@ -427,7 +427,7 @@ BATcheckhash(BAT *b)
Hash *h;
int fd;
- assert(!GDKinmemory());
+ assert(!GDKinmemory(b->theap.farmid));
b->thash = NULL;
if ((h = GDKzalloc(sizeof(*h))) != NULL &&
(h->heaplink.farmid = BBPselectfarm(b->batRole,
b->ttype, hashheap)) >= 0 &&
@@ -715,7 +715,7 @@ BAThash_impl(BAT *restrict b, struct can
oid o;
BUN hnil, hget, hb;
Hash *h = NULL;
- const char *nme = GDKinmemory() ? ":inmemory" :
BBP_physical(b->batCacheid);
+ const char *nme = GDKinmemory(b->theap.farmid) ? ":inmemory" :
BBP_physical(b->batCacheid);
BATiter bi = bat_iterator(b);
PROPrec *prop;
bool hascand = ci->tpe != cand_dense || ci->ncand != BATcount(b);
@@ -972,7 +972,7 @@ BAThash(BAT *b)
return GDK_FAIL;
}
#ifdef PERSISTENTHASH
- if (BBP_status(b->batCacheid) & BBPEXISTING && !b->theap.dirty
&& !GDKinmemory()) {
+ if (BBP_status(b->batCacheid) & BBPEXISTING && !b->theap.dirty
&& !GDKinmemory(b->theap.farmid)) {
MT_Id tid;
BBPfix(b->batCacheid);
char name[MT_NAME_LEN];
@@ -1114,7 +1114,7 @@ HASHfree(BAT *b)
Hash *h;
MT_lock_set(&b->batIdxLock);
if ((h = b->thash) != NULL && h != (Hash *) 1) {
- bool rmheap = GDKinmemory() || h->heaplink.dirty ||
h->heapbckt.dirty;
+ bool rmheap = h->heaplink.dirty || h->heapbckt.dirty;
TRC_DEBUG(ACCELERATOR, ALGOBATFMT " free hash %s\n",
ALGOBATPAR(b),
rmheap ? "removing" : "keeping");
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -109,14 +109,14 @@ HEAPalloc(Heap *h, size_t nitems, size_t
GDKerror("allocating more than heap can accomodate\n");
return GDK_FAIL;
}
- if (GDKinmemory() ||
+ if (GDKinmemory(h->farmid) ||
(GDKmem_cursize() + h->size < GDK_mem_maxsize &&
h->size < (h->farmid == 0 ? GDK_mmap_minsize_persistent :
GDK_mmap_minsize_transient))) {
h->storage = STORE_MEM;
h->base = GDKmalloc(h->size);
TRC_DEBUG(HEAP, "HEAPalloc %zu %p\n", h->size, h->base);
}
- if (!GDKinmemory() && h->base == NULL) {
+ if (!GDKinmemory(h->farmid) && h->base == NULL) {
char *nme;
nme = GDKfilepath(h->farmid, BATDIR, h->filename, NULL);
@@ -158,7 +158,7 @@ HEAPextend(Heap *h, size_t size, bool ma
char nme[sizeof(h->filename)], *ext;
const char *failure = "None";
- if (GDKinmemory()) {
+ if (GDKinmemory(h->farmid)) {
strcpy_len(nme, ":inmemory", sizeof(nme));
ext = "ext";
} else {
@@ -200,7 +200,7 @@ HEAPextend(Heap *h, size_t size, bool ma
* file-mapped storage */
Heap bak = *h;
bool exceeds_swap = size + GDKmem_cursize() >= GDK_mem_maxsize;
- bool must_mmap = !GDKinmemory() && (exceeds_swap ||
h->newstorage != STORE_MEM || size >= (h->farmid == 0 ?
GDK_mmap_minsize_persistent : GDK_mmap_minsize_transient));
+ bool must_mmap = !GDKinmemory(h->farmid) && (exceeds_swap ||
h->newstorage != STORE_MEM || size >= (h->farmid == 0 ?
GDK_mmap_minsize_persistent : GDK_mmap_minsize_transient));
h->size = size;
@@ -217,7 +217,7 @@ HEAPextend(Heap *h, size_t size, bool ma
failure = "h->storage == STORE_MEM && !must_map &&
!h->base";
}
- if (!GDKinmemory()) {
+ if (!GDKinmemory(h->farmid)) {
/* too big: convert it to a disk-based temporary heap */
bool existing = false;
@@ -579,7 +579,7 @@ HEAPfree(Heap *h, bool rmheap)
}
} else
#endif
- if (rmheap) {
+ if (rmheap && !GDKinmemory(h->farmid)) {
char *path = GDKfilepath(h->farmid, BATDIR, h->filename, NULL);
if (path && remove(path) != 0 && errno != ENOENT)
perror(path);
diff --git a/gdk/gdk_imprints.c b/gdk/gdk_imprints.c
--- a/gdk/gdk_imprints.c
+++ b/gdk/gdk_imprints.c
@@ -208,7 +208,7 @@ BATcheckimprints(BAT *b)
Imprints *imprints;
const char *nme = BBP_physical(b->batCacheid);
- assert(!GDKinmemory());
+ assert(!GDKinmemory(b->theap.farmid));
b->timprints = NULL;
if ((imprints = GDKzalloc(sizeof(Imprints))) != NULL &&
(imprints->imprints.farmid =
BBPselectfarm(b->batRole, b->ttype, imprintsheap)) >= 0) {
@@ -377,7 +377,7 @@ BATimprints(BAT *b)
if (b->timprints == NULL) {
BUN cnt;
- const char *nme = GDKinmemory() ? ":inmemory" :
BBP_physical(b->batCacheid);
+ const char *nme = GDKinmemory(b->theap.farmid) ? ":inmemory" :
BBP_physical(b->batCacheid);
size_t pages;
MT_lock_unset(&b->batIdxLock);
@@ -532,7 +532,7 @@ BATimprints(BAT *b)
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
