MonetDB: Dec2016 - Fix security vulnerability in FITS and NETCDF...
Changeset: 9415609bc718 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9415609bc718
Modified Files:
sql/backends/monet5/sql_execute.c
sql/backends/monet5/sql_scenario.h
sql/backends/monet5/vaults/fits/fits.c
sql/backends/monet5/vaults/netcdf/netcdf.c
Branch: Dec2016
Log Message:
Fix security vulnerability in FITS and NETCDF datavaults
This solves Bug 6258
diffs (truncated from 528 to 300 lines):
diff --git a/sql/backends/monet5/sql_execute.c
b/sql/backends/monet5/sql_execute.c
--- a/sql/backends/monet5/sql_execute.c
+++ b/sql/backends/monet5/sql_execute.c
@@ -341,6 +341,49 @@ SQLrun(Client c, backend *be, mvc *m){
return msg;
}
+/*
+ * Escape single quotes and backslashes. This is important to do before calling
+ * SQLstatementIntern, if we are pasting user provided strings into queries
+ * passed to the SQLstatementIntern. Otherwise we open ourselves to SQL
+ * injection attacks.
+ *
+ * It returns the input string with all the single quotes(') and the
backslashes
+ * (\) doubled, or NULL, if it could not allocate enough space.
+ *
+ * The caller is responsible to free the returned value.
+ */
+str
+SQLescapeString(str s)
+{
+ str ret = NULL;
+ char *p, *q;
+ int len = 0;
+
+ if(!s) {
+ return NULL;
+ }
+
+ /* At most we will need 2*strlen(s) + 1 characters */
+ len = strlen(s);
+ ret = (str)GDKmalloc(2*len + 1);
+ if (!ret) {
+ return NULL;
+ }
+
+ for (p = s, q = ret; *p != '\0'; p++, q++) {
+ *q = *p;
+ if (*p == '\'') {
+ *(++q) = '\'';
+ }
+ else if (*p == '\\') {
+ *(++q) = '\\';
+ }
+ }
+
+ *q = '\0';
+ return ret;
+}
+
str
SQLstatementIntern(Client c, str *expr, str nme, bit execute, bit output,
res_table **result)
{
diff --git a/sql/backends/monet5/sql_scenario.h
b/sql/backends/monet5/sql_scenario.h
--- a/sql/backends/monet5/sql_scenario.h
+++ b/sql/backends/monet5/sql_scenario.h
@@ -43,6 +43,7 @@ sql5_export str SQLstatementIntern(Clien
sql5_export str SQLcompile(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
sql5_export str SQLinclude(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
sql5_export str SQLCacheRemove(Client c, str nme);
+sql5_export str SQLescapeString(str s);
sql5_export MT_Lock sql_contextLock;
#endif /* _SQL_SCENARIO_H_ */
diff --git a/sql/backends/monet5/vaults/fits/fits.c
b/sql/backends/monet5/vaults/fits/fits.c
--- a/sql/backends/monet5/vaults/fits/fits.c
+++ b/sql/backends/monet5/vaults/fits/fits.c
@@ -31,8 +31,6 @@
#define FITS_INS_COL "INSERT INTO sys.fits_columns(id, name, type, units,
number, table_id) \
VALUES(%d,'%s','%s','%s',%d,%d);"
-#define FILE_INS "INSERT INTO sys.fits_files(id, name) VALUES (%d, '%s');"
-#define DEL_TABLE "DELETE FROM sys.fitsfiles;"
#define ATTACHDIR "call sys.fitsattach('%s');"
static void
@@ -570,14 +568,25 @@ str FITSdir(Client cntxt, MalBlkPtr mb,
s = stmt;
while ((ep = readdir(dp)) != NULL && !msg) {
- snprintf(fname, BUFSIZ, "%s%s", dir, ep->d_name);
+ char *filename = SQLescapeString(ep->d_name);
+ if (!filename) {
+ msg = createException(MAL, "fits.listdir",
MAL_MALLOC_FAIL);
+ break;
+ }
+
+ snprintf(fname, BUFSIZ, "%s%s", dir, filename);
status = 0;
fits_open_file(&fptr, fname, READONLY, &status);
if (status == 0) {
snprintf(stmt, BUFSIZ, ATTACHDIR, fname);
+#ifndef NDEBUG
+ fprintf(stderr, "Executing:\n%s\n", s);
+#endif
msg = SQLstatementIntern(cntxt, &s,
"fits.listofdir", TRUE, FALSE, NULL);
fits_close_file(fptr, &status);
}
+
+ GDKfree(filename);
}
(void)closedir(dp);
} else
@@ -591,6 +600,7 @@ str FITSdirpat(Client cntxt, MalBlkPtr m
str msg = MAL_SUCCEED;
str dir = *getArgReference_str(stk, pci, 1);
str pat = *getArgReference_str(stk, pci, 2);
+ char *filename = NULL;
fitsfile *fptr;
char *s;
int status = 0;
@@ -599,6 +609,7 @@ str FITSdirpat(Client cntxt, MalBlkPtr m
size_t j = 0;
(void)mb;
+
globbuf.gl_offs = 0;
snprintf(fulldirectory, BUFSIZ, "%s%s", dir, pat);
glob(fulldirectory, GLOB_DOOFFS, NULL, &globbuf);
@@ -606,7 +617,7 @@ str FITSdirpat(Client cntxt, MalBlkPtr m
/* fprintf(stderr,"#fulldir: %s \nSize: %lu\n",fulldirectory,
globbuf.gl_pathc);*/
if (globbuf.gl_pathc == 0)
-
MonetDB: python3udf - Consistent indentation.
Changeset: 4c617c68d0dc for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=4c617c68d0dc
Modified Files:
sql/backends/monet5/UDF/pyapi/pyapi.c
sql/backends/monet5/UDF/pyapi/type_conversion.c
Branch: python3udf
Log Message:
Consistent indentation.
diffs (truncated from 362 to 300 lines):
diff --git a/sql/backends/monet5/UDF/pyapi/pyapi.c
b/sql/backends/monet5/UDF/pyapi/pyapi.c
--- a/sql/backends/monet5/UDF/pyapi/pyapi.c
+++ b/sql/backends/monet5/UDF/pyapi/pyapi.c
@@ -31,24 +31,24 @@ static PyObject *marshal_module = NULL;
PyObject *marshal_loads = NULL;
typedef struct _AggrParams{
-PyInput **pyinput_values;
-void split_bats;
-size_t **group_counts;
-str **args;
-PyObject **connection;
-PyObject **function;
-PyObject **column_types_dict;
-PyObject **result_objects;
-str *pycall;
-str msg;
-size_t base;
-size_t additional_columns;
-size_t named_columns;
-size_t columns;
-size_t group_count;
-size_t group_start;
-size_t group_end;
-MT_Id thread;
+ PyInput **pyinput_values;
+ void split_bats;
+ size_t **group_counts;
+ str **args;
+ PyObject **connection;
+ PyObject **function;
+ PyObject **column_types_dict;
+ PyObject **result_objects;
+ str *pycall;
+ str msg;
+ size_t base;
+ size_t additional_columns;
+ size_t named_columns;
+ size_t columns;
+ size_t group_count;
+ size_t group_start;
+ size_t group_end;
+ MT_Id thread;
} AggrParams;
static void ComputeParallelAggregation(AggrParams *p);
@@ -73,7 +73,7 @@ static MT_Lock queryLock;
static int pyapiInitialized = FALSE;
int PYFUNCNAME(PyAPIInitialized)(void) {
-return pyapiInitialized;
+ return pyapiInitialized;
}
#ifdef HAVE_FORK
@@ -93,22 +93,22 @@ PyAPIeval(Client cntxt, MalBlkPtr mb, Ma
str
PYFUNCNAME(PyAPIevalStd)(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci) {
-return PyAPIeval(cntxt, mb, stk, pci, 0, 0);
+ return PyAPIeval(cntxt, mb, stk, pci, 0, 0);
}
str
PYFUNCNAME(PyAPIevalStdMap)(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr pci) {
-return PyAPIeval(cntxt, mb, stk, pci, 0, 1);
+ return PyAPIeval(cntxt, mb, stk, pci, 0, 1);
}
str
PYFUNCNAME(PyAPIevalAggr)(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci) {
-return PyAPIeval(cntxt, mb, stk, pci, 1, 0);
+ return PyAPIeval(cntxt, mb, stk, pci, 1, 0);
}
str
PYFUNCNAME(PyAPIevalAggrMap)(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr pci) {
-return PyAPIeval(cntxt, mb, stk, pci, 1, 1);
+ return PyAPIeval(cntxt, mb, stk, pci, 1, 1);
}
#define NP_SPLIT_BAT(tpe)
\
@@ -144,8 +144,8 @@ PYFUNCNAME(PyAPIevalAggrMap)(Client cntx
//! [RETURN_VALUES] Step 4: It collects the return values and converts them
back into BATs
//! If 'mapped' is set to True, it will fork a separate process at
[FORK_PROCESS] that executes Step 1-3, the process will then write the return
values into memory mapped files and exit, then Step 4 is executed by the main
process
static str PyAPIeval(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci,
bit grouped, bit mapped) {
-sql_func * sqlfun = NULL;
-str exprStr;
+ sql_func * sqlfun = NULL;
+ str exprStr;
const int additional_columns = 3;
int i = 1, ai = 0;
@@ -1420,55 +1420,55 @@ wrapup:
str
PYFUNCNAME(PyAPIprelude)(void *ret) {
-(void) ret;
-MT_lock_init(&pyapiLock, "pyapi_lock");
-MT_lock_init(&queryLock, "query_lock");
-MT_lock_set(&pyapiLock);
-if (!pyapiInitialized) {
-str msg = MAL_SUCCEED;
-Py_Initialize();
-_import_array();
-msg = _connection_init();
-if (msg != MAL_SUCCEED) {
-MT_lock_unset(&pyapiLock);
-return msg;
-}
-msg = _conversion_init();
-if (msg != MAL_SUCCEED) {
-MT_lock_unset(&pyapiLock);
-return msg;
-}
-_pytypes_init();
-_loader_init();
-marshal_module = PyImport_Import(PyString_FromString("marshal"));
-if (marshal_module == NULL) {
-return createException(MAL, "pyapi.eval", "Failed to load Marshal
module.");
-}
-marshal_loads = PyObject_GetAttrString(marshal_module, "loads");
-if (marshal_loads == NULL) {
-return createException(MAL, "pyapi.eval", "Failed to load function
\"loads\" from Marshal module.");
-}
-if (PyRun_SimpleString("import numpy") != 0) {
-return PyError_CreateException("Failed to initialize embedded
python", NULL);
-}
-PyEval_SaveThread();
-if (msg != MAL_SUCCEED) {
-MT_lock_unset(&pyapiLock);
-return msg;
-}
-pyapiInitialized++;
-fprintf(stdout, "# MonetDB/Python%d module loaded\n",
+
MonetDB: default - Merge with python3udf.
Changeset: 25993fc264b3 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=25993fc264b3 Added Files: sql/backends/monet5/UDF/pyapi3/50_pyapi3.mal sql/backends/monet5/UDF/pyapi3/Makefile.ag sql/backends/monet5/UDF/pyapi3/Tests/All sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_00.SQL.sh sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_00.stable.err sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_00.stable.out sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_01.SQL.sh sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_01.stable.err sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_01.stable.out sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_02.SQL.sh sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_02.stable.err sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_02.stable.out sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_03.SQL.sh sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_03.stable.err sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_03.stable.out sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_04.SQL.sh sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_04.stable.err sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_04.stable.out sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_05.SQL.sh sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_05.stable.err sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_05.stable.out sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_06.SQL.sh sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_06.stable.err sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_06.stable.out sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_07.SQL.sh sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_07.stable.err sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_07.stable.out sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_09.SQL.sh sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_09.stable.err sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_09.stable.out sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_10.SQL.sh sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_10.stable.err sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_10.stable.out sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_11.SQL.sh sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_11.stable.err sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_11.stable.out sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_12.SQL.sh sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_12.stable.err sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_12.stable.out sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_13.SQL.sh sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_13.stable.err sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_13.stable.out sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_15.SQL.sh sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_16.SQL.sh sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_16.stable.err sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_16.stable.out sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_17.SQL.sh sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_17.stable.err sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_17.stable.out sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_18.SQL.sh sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_18.stable.err sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_18.stable.out sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_19.SQL.sh sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_19.stable.err sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_19.stable.out sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_20.SQL.sh sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_20.stable.err sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_20.stable.out sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_21.SQL.sh sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_21.stable.err sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_21.stable.out sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_23.SQL.sh sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_23.stable.err sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_23.stable.out sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_24.SQL.sh sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_24.stable.err sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_24.stable.out sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_25.SQL.sh sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_25.stable.err sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_25.stable.out sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_26.SQL.sh sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_26.stable.err sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_26.stable.out sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_27.SQL.sh sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_27.stable.err sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_27.stable.out
MonetDB: python3udf - Close branch after merge with default.
Changeset: bc098254c603 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=bc098254c603 Branch: python3udf Log Message: Close branch after merge with default. ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - Merge with Dec2016 branch.
Changeset: 09dad7e2d985 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=09dad7e2d985
Added Files:
sql/test/BugTracker-2015/Tests/crash_timestamp_convert.Bug-3816.stable.out.Darwin.src
sql/test/BugTracker-2015/Tests/crash_timestamp_convert.Bug-3816.stable.out.FreeBSD
sql/test/BugTracker-2017/Tests/fullouterjoinfilter.Bug-6256.sql
sql/test/BugTracker-2017/Tests/fullouterjoinfilter.Bug-6256.stable.err
sql/test/BugTracker-2017/Tests/fullouterjoinfilter.Bug-6256.stable.out
sql/test/BugTracker-2017/Tests/wrong_aggregation_count.Bug-6257.sql
sql/test/BugTracker-2017/Tests/wrong_aggregation_count.Bug-6257.stable.err
sql/test/BugTracker-2017/Tests/wrong_aggregation_count.Bug-6257.stable.out
Modified Files:
sql/backends/monet5/sql_execute.c
sql/backends/monet5/sql_scenario.h
sql/backends/monet5/vaults/fits/fits.c
sql/backends/monet5/vaults/netcdf/netcdf.c
sql/test/BugTracker-2015/Tests/schema-trigger.Bug-3710.sql
sql/test/BugTracker-2015/Tests/schema-trigger.Bug-3710.stable.err
sql/test/BugTracker-2015/Tests/schema-trigger.Bug-3710.stable.out
sql/test/BugTracker-2016/Tests/convert-function-test.Bug-3460.stable.out
sql/test/BugTracker-2016/Tests/convert-function-test.Bug-3460.stable.out.int128
sql/test/BugTracker-2017/Tests/All
sql/test/BugTracker-2017/Tests/crash_correlated_subqueries_in_select.Bug-6254.sql
sql/test/BugTracker-2017/Tests/crash_correlated_subqueries_in_select.Bug-6254.stable.out
Branch: default
Log Message:
Merge with Dec2016 branch.
diffs (truncated from 1195 to 300 lines):
diff --git a/sql/backends/monet5/sql_execute.c
b/sql/backends/monet5/sql_execute.c
--- a/sql/backends/monet5/sql_execute.c
+++ b/sql/backends/monet5/sql_execute.c
@@ -352,6 +352,49 @@ SQLrun(Client c, backend *be, mvc *m){
return msg;
}
+/*
+ * Escape single quotes and backslashes. This is important to do before calling
+ * SQLstatementIntern, if we are pasting user provided strings into queries
+ * passed to the SQLstatementIntern. Otherwise we open ourselves to SQL
+ * injection attacks.
+ *
+ * It returns the input string with all the single quotes(') and the
backslashes
+ * (\) doubled, or NULL, if it could not allocate enough space.
+ *
+ * The caller is responsible to free the returned value.
+ */
+str
+SQLescapeString(str s)
+{
+ str ret = NULL;
+ char *p, *q;
+ int len = 0;
+
+ if(!s) {
+ return NULL;
+ }
+
+ /* At most we will need 2*strlen(s) + 1 characters */
+ len = strlen(s);
+ ret = (str)GDKmalloc(2*len + 1);
+ if (!ret) {
+ return NULL;
+ }
+
+ for (p = s, q = ret; *p != '\0'; p++, q++) {
+ *q = *p;
+ if (*p == '\'') {
+ *(++q) = '\'';
+ }
+ else if (*p == '\\') {
+ *(++q) = '\\';
+ }
+ }
+
+ *q = '\0';
+ return ret;
+}
+
str
SQLstatementIntern(Client c, str *expr, str nme, bit execute, bit output,
res_table **result)
{
diff --git a/sql/backends/monet5/sql_scenario.h
b/sql/backends/monet5/sql_scenario.h
--- a/sql/backends/monet5/sql_scenario.h
+++ b/sql/backends/monet5/sql_scenario.h
@@ -43,6 +43,7 @@ sql5_export str SQLstatementIntern(Clien
sql5_export str SQLcompile(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
sql5_export str SQLinclude(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
sql5_export str SQLCacheRemove(Client c, str nme);
+sql5_export str SQLescapeString(str s);
sql5_export MT_Lock sql_contextLock;
#endif /* _SQL_SCENARIO_H_ */
diff --git a/sql/backends/monet5/vaults/fits/fits.c
b/sql/backends/monet5/vaults/fits/fits.c
--- a/sql/backends/monet5/vaults/fits/fits.c
+++ b/sql/backends/monet5/vaults/fits/fits.c
@@ -31,8 +31,6 @@
#define FITS_INS_COL "INSERT INTO sys.fits_columns(id, name, type, units,
number, table_id) \
VALUES(%d,'%s','%s','%s',%d,%d);"
-#define FILE_INS "INSERT INTO sys.fits_files(id, name) VALUES (%d, '%s');"
-#define DEL_TABLE "DELETE FROM sys.fitsfiles;"
#define ATTACHDIR "call sys.fitsattach('%s');"
static void
@@ -572,14 +570,25 @@ str FITSdir(Client cntxt, MalBlkPtr mb,
s = stmt;
while ((ep = readdir(dp)) != NULL && !msg) {
- snprintf(fname, BUFSIZ, "%s%s", dir, ep->d_name);
+ char *filename = SQLescapeString(ep->d_name);
+ if (!filename) {
+ msg = createException(MAL, "fits.listdir",
MAL_MALLOC_FAIL);
+ break;
+ }
+
+ snprintf(fname, BUFSIZ, "%s%s", dir, filename);
status = 0;
fits_open_file(&fptr, fname, READONLY, &status);
if (
MonetDB: Dec2016 - fixed problem with system vacuum, ie make sur...
Changeset: d567c47aeaff for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d567c47aeaff
Modified Files:
sql/storage/bat/bat_table.c
sql/storage/store.c
Branch: Dec2016
Log Message:
fixed problem with system vacuum, ie make sure we only vacuum on the
pesistent bits (not the inserts/updates)
diffs (115 lines):
diff --git a/sql/storage/bat/bat_table.c b/sql/storage/bat/bat_table.c
--- a/sql/storage/bat/bat_table.c
+++ b/sql/storage/bat/bat_table.c
@@ -478,11 +478,15 @@ table_vacuum(sql_trans *tr, sql_table *t
BAT **cols;
node *n;
+ if (!tids)
+ return SQL_ERR;
cols = NEW_ARRAY(BAT*, cs_size(&t->columns));
for (n = t->columns.set->h; n; n = n->next) {
sql_column *c = n->data;
BAT *v = store_funcs.bind_col(tr, c, RDONLY);
+ if (!v)
+ return SQL_ERR;
cols[c->colnr] = BATproject(tids, v);
BBPunfix(v->batCacheid);
}
diff --git a/sql/storage/store.c b/sql/storage/store.c
--- a/sql/storage/store.c
+++ b/sql/storage/store.c
@@ -1698,6 +1698,7 @@ store_flush_log(void)
static int
store_needs_vacuum( sql_trans *tr )
{
+ size_t max_dels = GDKdebug & FORCEMITOMASK ? 1 : 128;
sql_schema *s = find_sql_schema(tr, "sys");
node *n;
@@ -1708,9 +1709,11 @@ store_needs_vacuum( sql_trans *tr )
if (!t->system)
continue;
/* no inserts, updates and enough deletes ? */
- if (!store_funcs.count_col(tr, c, 0) &&
+ if (store_funcs.count_col(tr, c, 0) &&
+ (store_funcs.count_col(tr, c, 1) -
+ store_funcs.count_col(tr, c, 0)) == 0 &&
!store_funcs.count_upd(tr, t) &&
- store_funcs.count_del(tr, t) > 128)
+ store_funcs.count_del(tr, t) >= max_dels)
return 1;
}
return 0;
@@ -1720,6 +1723,7 @@ static void
store_vacuum( sql_trans *tr )
{
/* tables */
+ size_t max_dels = GDKdebug & FORCEMITOMASK ? 1 : 128;
sql_schema *s = find_sql_schema(tr, "sys");
node *n;
@@ -1729,26 +1733,28 @@ store_vacuum( sql_trans *tr )
if (!t->system)
continue;
- if (!store_funcs.count_col(tr, c, 0) &&
+ if (store_funcs.count_col(tr, c, 0) &&
+ (store_funcs.count_col(tr, c, 1) -
+ store_funcs.count_col(tr, c, 0)) == 0 &&
!store_funcs.count_upd(tr, t) &&
- store_funcs.count_del(tr, t) > 128) {
+ store_funcs.count_del(tr, t) >= max_dels)
table_funcs.table_vacuum(tr, t);
- }
}
}
void
store_manager(void)
{
- const int timeout = GDKdebug & FORCEMITOMASK ? 10 : 50;
+ const int sleeptime = GDKdebug & FORCEMITOMASK ? 10 : 50;
+ const int timeout = GDKdebug & FORCEMITOMASK ? 500 : 5;
while (!GDKexiting()) {
int res = LOG_OK;
int t;
lng shared_transactions_drift = -1;
- for (t = 3; t > 0 && !need_flush; t -= timeout) {
- MT_sleep_ms(timeout);
+ for (t = timeout; t > 0 && !need_flush; t -= sleeptime) {
+ MT_sleep_ms(sleeptime);
if (GDKexiting())
return;
}
@@ -1778,7 +1784,7 @@ store_manager(void)
MT_lock_unset(&bs_lock);
if (GDKexiting())
return;
- MT_sleep_ms(timeout);
+ MT_sleep_ms(sleeptime);
MT_lock_set(&bs_lock);
}
@@ -1830,14 +1836,15 @@ store_manager(void)
void
idle_manager(void)
{
- const int timeout = GDKdebug & FORCEMITOMASK ? 10 : 50;
+ const int sleeptime = GDKdebug & FORCEMITOMASK ? 10 : 50;
+ const int timeout = GDKdebug & FORCEMITOMASK ? 50 : 5000;
while (!GDKexiting()) {
sql_session *s;
int t;
- for (t = 5000; t > 0; t -= timeout) {
- MT_sleep_ms(timeout);
+ for (t = timeout; t > 0; t -= sleeptime) {
+ MT_sleep_ms(sleeptime);
if (GDKexiting())
return;
}
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Dec2016 - Correctly compare types in BATifthenelse.
Changeset: ddf1181e1171 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ddf1181e1171
Modified Files:
gdk/gdk_calc.c
Branch: Dec2016
Log Message:
Correctly compare types in BATifthenelse.
diffs (12 lines):
diff --git a/gdk/gdk_calc.c b/gdk/gdk_calc.c
--- a/gdk/gdk_calc.c
+++ b/gdk/gdk_calc.c
@@ -13234,7 +13234,7 @@ BATcalcifthenelse(BAT *b, BAT *b1, BAT *
return NULL;
if (checkbats(b, b2, "BATcalcifthenelse") != GDK_SUCCEED)
return NULL;
- if (b->ttype != TYPE_bit || b1->ttype != b2->ttype) {
+ if (b->ttype != TYPE_bit || ATOMtype(b1->ttype) != ATOMtype(b2->ttype))
{
GDKerror("BATcalcifthenelse: \"then\" and \"else\" BATs have
different types.\n");
return NULL;
}
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - merged with 2016
Changeset: dc71eeaede6f for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=dc71eeaede6f
Modified Files:
gdk/gdk_calc.c
sql/storage/bat/bat_table.c
sql/storage/store.c
Branch: default
Log Message:
merged with 2016
diffs (129 lines):
diff --git a/gdk/gdk_calc.c b/gdk/gdk_calc.c
--- a/gdk/gdk_calc.c
+++ b/gdk/gdk_calc.c
@@ -13234,7 +13234,7 @@ BATcalcifthenelse(BAT *b, BAT *b1, BAT *
return NULL;
if (checkbats(b, b2, "BATcalcifthenelse") != GDK_SUCCEED)
return NULL;
- if (b->ttype != TYPE_bit || b1->ttype != b2->ttype) {
+ if (b->ttype != TYPE_bit || ATOMtype(b1->ttype) != ATOMtype(b2->ttype))
{
GDKerror("BATcalcifthenelse: \"then\" and \"else\" BATs have
different types.\n");
return NULL;
}
diff --git a/sql/storage/bat/bat_table.c b/sql/storage/bat/bat_table.c
--- a/sql/storage/bat/bat_table.c
+++ b/sql/storage/bat/bat_table.c
@@ -478,12 +478,17 @@ table_vacuum(sql_trans *tr, sql_table *t
BAT *tids = delta_cands(tr, t);
BAT **cols;
node *n;
+
+ if (!tids)
+ return SQL_ERR;
// FIXME unchecked_malloc NEW_ARRAY can return NULL
cols = NEW_ARRAY(BAT*, cs_size(&t->columns));
for (n = t->columns.set->h; n; n = n->next) {
sql_column *c = n->data;
BAT *v = store_funcs.bind_col(tr, c, RDONLY);
+ if (!v)
+ return SQL_ERR;
cols[c->colnr] = BATproject(tids, v);
BBPunfix(v->batCacheid);
}
diff --git a/sql/storage/store.c b/sql/storage/store.c
--- a/sql/storage/store.c
+++ b/sql/storage/store.c
@@ -1687,6 +1687,7 @@ store_flush_log(void)
static int
store_needs_vacuum( sql_trans *tr )
{
+ size_t max_dels = GDKdebug & FORCEMITOMASK ? 1 : 128;
sql_schema *s = find_sql_schema(tr, "sys");
node *n;
@@ -1697,9 +1698,11 @@ store_needs_vacuum( sql_trans *tr )
if (!t->system)
continue;
/* no inserts, updates and enough deletes ? */
- if (!store_funcs.count_col(tr, c, 0) &&
+ if (store_funcs.count_col(tr, c, 0) &&
+ (store_funcs.count_col(tr, c, 1) -
+ store_funcs.count_col(tr, c, 0)) == 0 &&
!store_funcs.count_upd(tr, t) &&
- store_funcs.count_del(tr, t) > 128)
+ store_funcs.count_del(tr, t) >= max_dels)
return 1;
}
return 0;
@@ -1709,6 +1712,7 @@ static void
store_vacuum( sql_trans *tr )
{
/* tables */
+ size_t max_dels = GDKdebug & FORCEMITOMASK ? 1 : 128;
sql_schema *s = find_sql_schema(tr, "sys");
node *n;
@@ -1718,26 +1722,28 @@ store_vacuum( sql_trans *tr )
if (!t->system)
continue;
- if (!store_funcs.count_col(tr, c, 0) &&
+ if (store_funcs.count_col(tr, c, 0) &&
+ (store_funcs.count_col(tr, c, 1) -
+ store_funcs.count_col(tr, c, 0)) == 0 &&
!store_funcs.count_upd(tr, t) &&
- store_funcs.count_del(tr, t) > 128) {
+ store_funcs.count_del(tr, t) >= max_dels)
table_funcs.table_vacuum(tr, t);
- }
}
}
void
store_manager(void)
{
- const int timeout = GDKdebug & FORCEMITOMASK ? 10 : 50;
+ const int sleeptime = GDKdebug & FORCEMITOMASK ? 10 : 50;
+ const int timeout = GDKdebug & FORCEMITOMASK ? 500 : 5;
while (!GDKexiting()) {
int res = LOG_OK;
int t;
lng shared_transactions_drift = -1;
- for (t = 3; t > 0 && !need_flush; t -= timeout) {
- MT_sleep_ms(timeout);
+ for (t = timeout; t > 0 && !need_flush; t -= sleeptime) {
+ MT_sleep_ms(sleeptime);
if (GDKexiting())
return;
}
@@ -1767,7 +1773,7 @@ store_manager(void)
MT_lock_unset(&bs_lock);
if (GDKexiting())
return;
- MT_sleep_ms(timeout);
+ MT_sleep_ms(sleeptime);
MT_lock_set(&bs_lock);
}
@@ -1819,14 +1825,15 @@ store_manager(void)
void
idle_manager(void)
{
- const int timeout = GDKdebug & FORCEMITOMASK ? 10 : 50;
+ const int sleeptime = GDKdebug & FORCEMITOMASK ? 10 : 50;
+ const int timeout = GDKdebug & FORCEMITOMASK ? 50 : 5000;
while (!GDKexiting()) {
sql_session *s;
int t;
- for (t = 5000; t > 0; t -= timeout) {
- MT_sleep_ms(timeout);
+ for (t = timeout; t > 0; t -= sleepti
MonetDB: Dec2016 - add missing exists handling in logical value ...
Changeset: d3e19b7567ac for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d3e19b7567ac
Modified Files:
sql/server/rel_select.c
Branch: Dec2016
Log Message:
add missing exists handling in logical value expression handing.
diffs (29 lines):
diff --git a/sql/server/rel_select.c b/sql/server/rel_select.c
--- a/sql/server/rel_select.c
+++ b/sql/server/rel_select.c
@@ -2128,6 +2128,25 @@ rel_logical_value_exp(mvc *sql, sql_rel
}
return NULL;
}
+ case SQL_EXISTS:
+ case SQL_NOT_EXISTS:
+ {
+ symbol *lo = sc->data.sym;
+ sql_exp *le = rel_value_exp(sql, rel, lo, f, ek);
+ sql_subfunc *f = NULL;
+
+ if (!le)
+ return NULL;
+
+ if (sc->token != SQL_EXISTS)
+ f = sql_bind_func(sql->sa, sql->session->schema,
"sql_not_exists", exp_subtype(le), NULL, F_FUNC);
+ else
+ f = sql_bind_func(sql->sa, sql->session->schema,
"sql_exists", exp_subtype(le), NULL, F_FUNC);
+
+ if (!f)
+ return sql_error(sql, 02, "exist operator on type %s
missing", exp_subtype(le)->type->sqlname);
+ return exp_unop(sql->sa, le, f);
+ }
case SQL_LIKE:
case SQL_NOT_LIKE:
{
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Dec2016 - Use correct type.
Changeset: d44c6572b0e9 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d44c6572b0e9
Modified Files:
sql/backends/monet5/sql_execute.c
Branch: Dec2016
Log Message:
Use correct type.
diffs (12 lines):
diff --git a/sql/backends/monet5/sql_execute.c
b/sql/backends/monet5/sql_execute.c
--- a/sql/backends/monet5/sql_execute.c
+++ b/sql/backends/monet5/sql_execute.c
@@ -357,7 +357,7 @@ SQLescapeString(str s)
{
str ret = NULL;
char *p, *q;
- int len = 0;
+ size_t len = 0;
if(!s) {
return NULL;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - BUNappend checks.
Changeset: ddba345b8dc0 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ddba345b8dc0
Modified Files:
monetdb5/mal/mal_profiler.c
sql/backends/monet5/sql.c
Branch: default
Log Message:
BUNappend checks.
diffs (223 lines):
diff --git a/monetdb5/mal/mal_profiler.c b/monetdb5/mal/mal_profiler.c
--- a/monetdb5/mal/mal_profiler.c
+++ b/monetdb5/mal/mal_profiler.c
@@ -858,8 +858,13 @@ cachedProfilerEvent(MalBlkPtr mb, MalStk
errors += BUNappend(TRACE_id_majflt, &v4, FALSE) != GDK_SUCCEED;
errors += BUNappend(TRACE_id_nvcsw, &v5, FALSE) != GDK_SUCCEED;
errors += BUNappend(TRACE_id_stmt, c, FALSE) != GDK_SUCCEED;
- TRACE_event++;
- eventcounter++;
+ if (errors > 0) {
+ /* stop profiling if an error occurred */
+ sqlProfiling = FALSE;
+ } else {
+ TRACE_event++;
+ eventcounter++;
+ }
MT_lock_unset(&mal_profileLock);
GDKfree(stmt);
}
diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -570,8 +570,12 @@ sql_variables(Client cntxt, MalBlkPtr mb
vars = COLnew(0, TYPE_str, m->topvars, TRANSIENT);
if (vars == NULL)
throw(SQL, "sql.variables", MAL_MALLOC_FAIL);
- for (i = 0; i < m->topvars && !m->vars[i].frame; i++)
- BUNappend(vars, m->vars[i].name, FALSE);
+ for (i = 0; i < m->topvars && !m->vars[i].frame; i++) {
+ if (BUNappend(vars, m->vars[i].name, FALSE) != GDK_SUCCEED) {
+ BBPreclaim(vars);
+ throw(SQL, "sql.variables", MAL_MALLOC_FAIL);
+ }
+ }
*res = vars->batCacheid;
BBPkeepref(vars->batCacheid);
return MAL_SUCCEED;
@@ -686,7 +690,12 @@ mvc_bat_next_value(Client cntxt, MalBlkP
seqbulk_destroy(sb);
throw(SQL, "sql.next_value", "error");
}
- BUNappend(r, &l, FALSE);
+ if (BUNappend(r, &l, FALSE) != GDK_SUCCEED) {
+ BBPunfix(b->batCacheid);
+ BBPunfix(r->batCacheid);
+ seqbulk_destroy(sb);
+ throw(SQL, "sql.next_value", MAL_MALLOC_FAIL);
+ }
}
if (sb)
seqbulk_destroy(sb);
@@ -2689,27 +2698,36 @@ mvc_bin_import_table_wrap(Client cntxt,
throw(SQL, "sql", MAL_MALLOC_FAIL);
/* this code should be extended to deal with larger
text strings. */
f = fopen(*getArgReference_str(stk, pci, i), "r");
- if (f == NULL)
+ if (f == NULL) {
+ BBPreclaim(c);
throw(SQL, "sql", "Failed to re-open file %s",
fname);
+ }
buf = GDKmalloc(bufsiz);
if (!buf) {
fclose(f);
+ BBPreclaim(c);
throw(SQL, "sql", "Failed to create buffer");
}
while (fgets(buf, bufsiz, f) != NULL) {
char *t = strrchr(buf, '\n');
if (t)
*t = 0;
- BUNappend(c, buf, FALSE);
+ if (BUNappend(c, buf, FALSE) != GDK_SUCCEED) {
+ BBPreclaim(c);
+ fclose(f);
+ throw(SQL, "sql", MAL_MALLOC_FAIL);
+ }
}
fclose(f);
GDKfree(buf);
} else {
throw(SQL, "sql", "Failed to attach file %s", fname);
}
- if (init && cnt != BATcount(c))
+ if (init && cnt != BATcount(c)) {
+ BBPunfix(c->batCacheid);
throw(SQL, "sql", "binary files for table '%s' have
inconsistent counts", tname);
+ }
cnt = BATcount(c);
init = 1;
*getArgReference_bat(stk, pci, i - (2 + pci->retc)) =
c->batCacheid;
@@ -2728,8 +2746,13 @@ mvc_bin_import_table_wrap(Client cntxt,
const void* nil = ATOMnilptr(tpe);
// fill the new BAT with NULL values
c = COLnew(0, tpe, cnt, TRANSIENT);
+ if (c == NULL)
+ throw(SQL, "sql", MAL_MALLOC_FAIL);
for(loop = 0; loop < cnt; loop++) {
- BUNappend(c, nil, 0);
+
MonetDB: default - Merge with Dec2016 branch.
Changeset: d0a6fc57012f for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d0a6fc57012f
Modified Files:
sql/backends/monet5/sql_execute.c
sql/server/rel_select.c
Branch: default
Log Message:
Merge with Dec2016 branch.
diffs (41 lines):
diff --git a/sql/backends/monet5/sql_execute.c
b/sql/backends/monet5/sql_execute.c
--- a/sql/backends/monet5/sql_execute.c
+++ b/sql/backends/monet5/sql_execute.c
@@ -368,7 +368,7 @@ SQLescapeString(str s)
{
str ret = NULL;
char *p, *q;
- int len = 0;
+ size_t len = 0;
if(!s) {
return NULL;
diff --git a/sql/server/rel_select.c b/sql/server/rel_select.c
--- a/sql/server/rel_select.c
+++ b/sql/server/rel_select.c
@@ -2140,6 +2140,25 @@ rel_logical_value_exp(mvc *sql, sql_rel
}
return NULL;
}
+ case SQL_EXISTS:
+ case SQL_NOT_EXISTS:
+ {
+ symbol *lo = sc->data.sym;
+ sql_exp *le = rel_value_exp(sql, rel, lo, f, ek);
+ sql_subfunc *f = NULL;
+
+ if (!le)
+ return NULL;
+
+ if (sc->token != SQL_EXISTS)
+ f = sql_bind_func(sql->sa, sql->session->schema,
"sql_not_exists", exp_subtype(le), NULL, F_FUNC);
+ else
+ f = sql_bind_func(sql->sa, sql->session->schema,
"sql_exists", exp_subtype(le), NULL, F_FUNC);
+
+ if (!f)
+ return sql_error(sql, 02, "exist operator on type %s
missing", exp_subtype(le)->type->sqlname);
+ return exp_unop(sql->sa, le, f);
+ }
case SQL_LIKE:
case SQL_NOT_LIKE:
{
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
