Changeset: 3396814abd0e for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/3396814abd0e
Branch: transaction_layer_revamp
Log Message:
merge with default
diffs (truncated from 315 to 300 lines):
diff --git a/monetdb5/mal/mal_client.c b/monetdb5/mal/mal_client.c
--- a/monetdb5/mal/mal_client.c
+++ b/monetdb5/mal/mal_client.c
@@ -270,6 +270,7 @@ MCinitClientRecord(Client c, oid user, b
c->qryctx.starttime = 0;
ATOMIC_SET(&c->qryctx.datasize, 0);
c->qryctx.maxmem = 0;
+ c->maxmem = 0;
c->itrace = 0;
c->errbuf = 0;
@@ -372,6 +373,7 @@ MCinitClientThread(Client c)
Client
MCforkClient(Client father)
{
+ /* TO BE REMOVED: this function is not used anywhere */
Client son = NULL;
str prompt;
@@ -395,6 +397,7 @@ MCforkClient(Client father)
son->memorylimit = father->memorylimit;
son->qryctx.querytimeout = father->qryctx.querytimeout;
son->qryctx.maxmem = father->qryctx.maxmem;
+ son->maxmem = father->maxmem;
son->sessiontimeout = father->sessiontimeout;
if (son->prompt)
@@ -571,27 +574,10 @@ MCactiveClients(void)
size_t
MCmemoryClaim(void)
{
- size_t claim = 0;
- int active = 1;
-
- Client cntxt = mal_clients;
-
- MT_lock_set(&mal_contextLock);
- for(cntxt = mal_clients; cntxt<mal_clients+MAL_MAXCLIENTS; cntxt++) {
- if( cntxt->idle == 0 && cntxt->mode == RUNCLIENT){
- if(cntxt->memorylimit){
- claim += cntxt->memorylimit;
- active ++;
- } else {
- MT_lock_unset(&mal_contextLock);
- return GDK_mem_maxsize;
- }
- }
- }
- MT_lock_unset(&mal_contextLock);
- if(active == 0 || claim * LL_CONSTANT(1048576) >= GDK_mem_maxsize)
- return GDK_mem_maxsize;
- return claim * LL_CONSTANT(1048576);
+ /* TO BE REMOVED */
+ /* this function used to be more complex, but in the end it always
+ * returned GDK_mem_maxsize, so that's what we do now */
+ return GDK_mem_maxsize;
}
str
diff --git a/monetdb5/mal/mal_client.h b/monetdb5/mal/mal_client.h
--- a/monetdb5/mal/mal_client.h
+++ b/monetdb5/mal/mal_client.h
@@ -69,6 +69,7 @@ typedef struct CLIENT {
char optimizer[IDLENGTH];/* The optimizer pipe preferred for this
session */
int workerlimit; /* maximum number of workthreads
processing a query */
int memorylimit; /* Memory claim highwater mark,
0 = no limit */
+ lng maxmem; /* maximum memory from
db_user_info table */
lng sessiontimeout; /* session abort after x usec,
0 = no limit */
QryCtx qryctx; /* per query limitations */
diff --git a/monetdb5/modules/mal/clients.c b/monetdb5/modules/mal/clients.c
--- a/monetdb5/modules/mal/clients.c
+++ b/monetdb5/modules/mal/clients.c
@@ -334,14 +334,18 @@ CLTsetmemorylimit(Client cntxt, MalBlkPt
throw(MAL, "clients.setmemorylimit", "The memmory limit cannot
be NULL");
if( limit < 0)
throw(MAL, "clients.setmemorylimit", "The memmory limit cannot
be negative");
- if( (size_t) limit > GDK_mem_maxsize / 1048576)
- throw(MAL,"clients.setmemorylimit","Memory claim beyond
physical memory");
MT_lock_set(&mal_contextLock);
if (mal_clients[idx].mode == FREECLIENT)
msg = createException(MAL,"clients.setmemorylimit","Session not
active anymore");
- else
+ else if (cntxt->user != MAL_ADMIN &&
+ mal_clients[idx].maxmem > 0 &&
+ mal_clients[idx].maxmem < (lng) limit << 20)
+ msg = createException(MAL, "clients.setmemorylimit","Cannot
increase memory limit");
+ else {
mal_clients[idx].memorylimit = limit;
+ mal_clients[idx].qryctx.maxmem = (ATOMIC_BASE_TYPE) limit << 20;
+ }
MT_lock_unset(&mal_contextLock);
return msg;
}
diff --git a/monetdb5/optimizer/opt_mergetable.c
b/monetdb5/optimizer/opt_mergetable.c
--- a/monetdb5/optimizer/opt_mergetable.c
+++ b/monetdb5/optimizer/opt_mergetable.c
@@ -430,7 +430,8 @@ mat_apply1(MalBlkPtr mb, InstrPtr p, mat
is_assign = (n >= 0);
}
- if((r = newInstructionArgs(mb, matRef, packRef, mat[m].mi->argc)) ==
NULL)
+ if(m < 0 ||
+ (r = newInstructionArgs(mb, matRef, packRef, mat[m].mi->argc)) ==
NULL)
return -1;
getArg(r, 0) = getArg(p,0);
tpe = getArgType(mb,p,0);
@@ -501,6 +502,7 @@ mat_apply(MalBlkPtr mb, InstrPtr p, matl
return mat_apply1(mb, p, ml, is_a_mat(getArg(p,1),ml), 1);
assert(nrmats <= 8);
+ assert(p->retc < p->argc); /* i.e. matvar[0] gets initialized */
for(k=p->retc, l=0; k < p->argc; k++) {
int mv = is_a_mat(getArg(p,k), ml);
if (mv >=0) {
diff --git a/monetdb5/optimizer/opt_mitosis.c b/monetdb5/optimizer/opt_mitosis.c
--- a/monetdb5/optimizer/opt_mitosis.c
+++ b/monetdb5/optimizer/opt_mitosis.c
@@ -25,7 +25,7 @@ OPTmitosisImplementation(Client cntxt, M
str schema = 0, table = 0;
BUN r = 0, rowcnt = 0; /* table should be sizeable to consider
parallel execution*/
InstrPtr p, q, *old, target = 0;
- size_t argsize = 6 * sizeof(lng), m = 0, memclaim;
+ size_t argsize = 6 * sizeof(lng), m = 0;
/* estimate size per operator estimate: 4 args + 2 res*/
int threads = GDKnr_threads ? GDKnr_threads : 1, maxparts = MAXSLICES;
str msg = MAL_SUCCEED;
@@ -156,56 +156,44 @@ OPTmitosisImplementation(Client cntxt, M
cntxt->idle = 0; // this one is definitely not idle
MT_lock_unset(&mal_contextLock);
-/* This code was used to experiment with block sizes, mis-using the
memorylimit variable
- if (cntxt->memorylimit){
- // the new mitosis scheme uses a maximum chunck size in MB from
the client context
- m = (((size_t) cntxt->memorylimit) * 1048576) / (size_t)
row_size;
- pieces = (int) (rowcnt / m + (rowcnt - m * pieces > 0));
- }
- if (cntxt->memorylimit == 0 || pieces <= 1){
-*/
- if (pieces <= 1){
- /* improve memory usage estimation */
- if (nr_cols > 1 || nr_aggrs > 1 || nr_maps > 1)
- argsize = (nr_cols + nr_aggrs + nr_maps) * sizeof(lng);
- /* We haven't assigned the number of pieces.
- * Determine the memory available for this client
- */
+ /* improve memory usage estimation */
+ if (nr_cols > 1 || nr_aggrs > 1 || nr_maps > 1)
+ argsize = (nr_cols + nr_aggrs + nr_maps) * sizeof(lng);
+ /* We haven't assigned the number of pieces.
+ * Determine the memory available for this client
+ */
+
+ /* respect the memory limit size set for the user
+ * and determine the column part size
+ */
+ m = GDK_mem_maxsize / MCactiveClients(); /* use temporarily */
+ if (cntxt->memorylimit > 0 && (size_t) cntxt->memorylimit << 20 < m)
+ m = ((size_t) cntxt->memorylimit << 20) / argsize;
+ else if (cntxt->maxmem > 0 && cntxt->maxmem < (lng) m)
+ m = (size_t) (cntxt->maxmem / argsize);
+ else
+ m = m / argsize;
- /* respect the memory limit size set for the user
- * and determine the column part size
- */
- if( cntxt->memorylimit)
- m = (((size_t) cntxt->memorylimit) * 1048576) / argsize;
- else {
- memclaim= MCmemoryClaim();
- if(memclaim == GDK_mem_maxsize){
- m = GDK_mem_maxsize / (size_t)
MCactiveClients() / argsize;
- } else
- m = (GDK_mem_maxsize - memclaim) / argsize;
- }
-
- /* if data exceeds memory size,
- * i.e., (rowcnt*argsize > GDK_mem_maxsize),
- * i.e., (rowcnt > GDK_mem_maxsize/argsize = m) */
- if (rowcnt > m && m / threads > 0) {
- /* create |pieces| > |threads| partitions such that
- * |threads| partitions at a time fit in memory,
- * i.e., (threads*(rowcnt/pieces) <= m),
- * i.e., (rowcnt/pieces <= m/threads),
- * i.e., (pieces => rowcnt/(m/threads))
- * (assuming that (m > threads*MIN_PART_SIZE)) */
- /* the number of pieces affects SF-100, going beyond 8x
increases
- * the optimizer costs beyond the execution time
- */
- pieces = ((int) ceil((double)rowcnt / (m / threads)));
- if (pieces <= threads)
- pieces = threads;
- } else if (rowcnt > MIN_PART_SIZE) {
- /* exploit parallelism, but ensure minimal partition size to
- * limit overhead */
- pieces = MIN((int) ceil((double)rowcnt /
MIN_PART_SIZE), MAX_PARTS2THREADS_RATIO * threads);
- }
+ /* if data exceeds memory size,
+ * i.e., (rowcnt*argsize > GDK_mem_maxsize),
+ * i.e., (rowcnt > GDK_mem_maxsize/argsize = m) */
+ if (rowcnt > m && m / threads > 0) {
+ /* create |pieces| > |threads| partitions such that
+ * |threads| partitions at a time fit in memory,
+ * i.e., (threads*(rowcnt/pieces) <= m),
+ * i.e., (rowcnt/pieces <= m/threads),
+ * i.e., (pieces => rowcnt/(m/threads))
+ * (assuming that (m > threads*MIN_PART_SIZE)) */
+ /* the number of pieces affects SF-100, going beyond 8x
increases
+ * the optimizer costs beyond the execution time
+ */
+ pieces = ((int) ceil((double)rowcnt / (m / threads)));
+ if (pieces <= threads)
+ pieces = threads;
+ } else if (rowcnt > MIN_PART_SIZE) {
+ /* exploit parallelism, but ensure minimal partition size to
+ * limit overhead */
+ pieces = MIN((int) ceil((double)rowcnt / MIN_PART_SIZE),
MAX_PARTS2THREADS_RATIO * threads);
}
/* when testing, always aim for full parallelism, but avoid
diff --git a/sql/backends/monet5/UDF/capi/capi.c
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -473,7 +473,7 @@ static str CUDFeval(Client cntxt, MalBlk
lng initial_output_count = -1;
- struct sigaction sa, oldsa, oldsb;
+ struct sigaction sa = (struct sigaction) {.sa_flags = 0}, oldsa, oldsb;
sigset_t signal_set;
#ifdef NDEBUG
@@ -526,8 +526,6 @@ static str CUDFeval(Client cntxt, MalBlk
(void)sigaddset(&signal_set, SIGSEGV);
(void)sigaddset(&signal_set, SIGBUS);
(void)pthread_sigmask(SIG_UNBLOCK, &signal_set, NULL);
-
- sa = (struct sigaction) {.sa_flags = 0,};
}
sqlfun = *(sql_func **)getArgReference_ptr(stk, pci, pci->retc);
@@ -1405,7 +1403,7 @@ static str CUDFeval(Client cntxt, MalBlk
errno = 0;
goto wrapup;
}
- sa = (struct sigaction) {.sa_flags = 0,};
+ sa = (struct sigaction) {.sa_flags = 0};
}
if (msg) {
diff --git a/sql/backends/monet5/sql_scenario.c
b/sql/backends/monet5/sql_scenario.c
--- a/sql/backends/monet5/sql_scenario.c
+++ b/sql/backends/monet5/sql_scenario.c
@@ -270,11 +270,14 @@ SQLprepareClient(Client c, int login)
default:
break;
}
- lng maxmem;
- if (monet5_user_get_max_memory(m, m->user_id, &maxmem) == 0)
- c->qryctx.maxmem = (ATOMIC_BASE_TYPE) (maxmem > 0 ?
maxmem : 0);
- else
+ if (monet5_user_get_max_memory(m, m->user_id, &c->maxmem) == 0)
{
+ c->qryctx.maxmem = (ATOMIC_BASE_TYPE) (c->maxmem > 0 ?
c->maxmem : 0);
+ } else {
+ c->maxmem = 0;
c->qryctx.maxmem = 0;
+ }
+ if (c->memorylimit > 0 && c->qryctx.maxmem >
((ATOMIC_BASE_TYPE) c->memorylimit << 20))
+ c->qryctx.maxmem = (ATOMIC_BASE_TYPE) c->memorylimit <<
20;
}
if (c->handshake_options) {
diff --git a/sql/test/bincopy/Tests/All b/sql/test/bincopy/Tests/All
--- a/sql/test/bincopy/Tests/All
+++ b/sql/test/bincopy/Tests/All
@@ -52,7 +52,6 @@ bincopy_big_endians_on_server
bincopy_native_endians_on_client
bincopy_native_endians_on_server
-bincopy_invalid_json
bincopy_default_values
bincopy_decimal_range
bincopy_string_width
diff --git a/sql/test/bincopy/Tests/bincopy_nulls.py
b/sql/test/bincopy/Tests/bincopy_nulls.py
--- a/sql/test/bincopy/Tests/bincopy_nulls.py
+++ b/sql/test/bincopy/Tests/bincopy_nulls.py
@@ -24,16 +24,14 @@ class TestCase:
little_endian_bits: bytes
big_endian_bits: bytes
expected: List[Any]
- if_exists: bool
def __init__(self,
sqltype: str, byteswap_size: int, expected: List[Any],
little_endian_bits: bytes,
- big_endian_bits: Optional[bytes] = None, if_exists=False):
+ big_endian_bits: Optional[bytes] = None):
self.sqltype = sqltype
_______________________________________________
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org
