Changeset: 3ede22bba0ea for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/3ede22bba0ea Modified Files: clients/Tests/exports.stable.out gdk/gdk_atoms.h gdk/gdk_string.c monetdb5/modules/mal/tablet.c sql/backends/monet5/sql_gencode.c sql/backends/monet5/sql_upgrades.c sql/common/sql_string.c sql/common/sql_string.h sql/scripts/76_dump.sql sql/server/sql_atom.c sql/server/sql_scan.c sql/test/BugTracker-2020/Tests/copy-decimal-with-space.Bug-6917.test sql/test/BugTracker-2020/Tests/copy-empty-blob.Bug-6948.test sql/test/SQLancer/Tests/sqlancer01.test sql/test/SQLancer/Tests/sqlancer02.test sql/test/SQLancer/Tests/sqlancer03.test sql/test/Tests/regexp.test sql/test/copy/Tests/crlf_normalization.SQL.py sql/test/emptydb-previous-upgrade-chain-hge/Tests/upgrade.stable.out.int128 sql/test/emptydb-previous-upgrade-chain-hge/Tests/upgrade.stable.out.ppc64.int128 sql/test/emptydb-previous-upgrade-chain/Tests/upgrade.stable.out sql/test/emptydb-previous-upgrade-chain/Tests/upgrade.stable.out.32bit sql/test/emptydb-previous-upgrade-chain/Tests/upgrade.stable.out.int128 sql/test/emptydb-previous-upgrade-chain/Tests/upgrade.stable.out.ppc64 sql/test/emptydb-previous-upgrade-chain/Tests/upgrade.stable.out.ppc64.int128 sql/test/emptydb-previous-upgrade-hge/Tests/upgrade.stable.out.int128 sql/test/emptydb-previous-upgrade/Tests/upgrade.stable.out sql/test/emptydb-previous-upgrade/Tests/upgrade.stable.out.32bit sql/test/emptydb-previous-upgrade/Tests/upgrade.stable.out.int128 sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.int128 sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.ppc64.int128 sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.32bit sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.int128 sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.ppc64 sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.ppc64.int128 sql/test/emptydb-upgrade-hge/Tests/upgrade.stable.out.int128 sql/test/emptydb-upgrade/Tests/upgrade.stable.out sql/test/emptydb-upgrade/Tests/upgrade.stable.out.32bit sql/test/emptydb-upgrade/Tests/upgrade.stable.out.int128 sql/test/emptydb/Tests/check.stable.out sql/test/emptydb/Tests/check.stable.out.32bit sql/test/emptydb/Tests/check.stable.out.int128 sql/test/sys-schema/Tests/webExamplesComparisonFunctionsOperators.test sql/test/sysmon/Tests/sys_queue_expand.SQL.py sql/test/testdb-previous-upgrade-chain-hge/Tests/upgrade.stable.out.int128 sql/test/testdb-previous-upgrade-chain/Tests/upgrade.stable.out sql/test/testdb-previous-upgrade-chain/Tests/upgrade.stable.out.32bit sql/test/testdb-previous-upgrade-chain/Tests/upgrade.stable.out.int128 sql/test/testdb-previous-upgrade-hge/Tests/upgrade.stable.out.int128 sql/test/testdb-previous-upgrade/Tests/upgrade.stable.out sql/test/testdb-previous-upgrade/Tests/upgrade.stable.out.32bit sql/test/testdb-previous-upgrade/Tests/upgrade.stable.out.int128 sql/test/testdb-upgrade-chain-hge/Tests/upgrade.stable.out.int128 sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out.32bit sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out.int128 sql/test/testdb-upgrade-hge/Tests/upgrade.stable.out.int128 sql/test/testdb-upgrade/Tests/upgrade.stable.out sql/test/testdb-upgrade/Tests/upgrade.stable.out.32bit sql/test/testdb-upgrade/Tests/upgrade.stable.out.int128 Branch: default Log Message:
Fixes for raw strings. Storing function create queries that contain strings with single quotes in them needed to be done in such a way that the function can also be used if the server is started with raw strings after the function was first created in a server that was started without raw strings. Queries that contain backslash escapes in unadorned strings (i.e. without e prefix) needed to be fixed. diffs (truncated from 8505 to 300 lines): diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out --- a/clients/Tests/exports.stable.out +++ b/clients/Tests/exports.stable.out @@ -321,7 +321,7 @@ void GDKreset(int status); void GDKsetdebug(int debug); gdk_return GDKsetenv(const char *name, const char *value); void GDKsetmallocsuccesscount(lng count); -ssize_t GDKstrFromStr(unsigned char *restrict dst, const unsigned char *restrict src, ssize_t len); +ssize_t GDKstrFromStr(unsigned char *restrict dst, const unsigned char *restrict src, ssize_t len, char quote); str GDKstrdup(const char *s) __attribute__((__malloc__)) __attribute__((__warn_unused_result__)); str GDKstrndup(const char *s, size_t n) __attribute__((__malloc__)) __attribute__((__warn_unused_result__)); gdk_return GDKtracer_fill_comp_info(BAT *id, BAT *component, BAT *log_level); diff --git a/gdk/gdk_atoms.h b/gdk/gdk_atoms.h --- a/gdk/gdk_atoms.h +++ b/gdk/gdk_atoms.h @@ -132,7 +132,7 @@ gdk_export ssize_t fltFromStr(const char gdk_export ssize_t fltToStr(str *dst, size_t *len, const flt *src, bool external); gdk_export ssize_t dblFromStr(const char *src, size_t *len, dbl **dst, bool external); gdk_export ssize_t dblToStr(str *dst, size_t *len, const dbl *src, bool external); -gdk_export ssize_t GDKstrFromStr(unsigned char *restrict dst, const unsigned char *restrict src, ssize_t len); +gdk_export ssize_t GDKstrFromStr(unsigned char *restrict dst, const unsigned char *restrict src, ssize_t len, char quote); gdk_export ssize_t strFromStr(const char *restrict src, size_t *restrict len, str *restrict dst, bool external); gdk_export size_t escapedStrlen(const char *restrict src, const char *sep1, const char *sep2, int quote); gdk_export size_t escapedStr(char *restrict dst, const char *restrict src, size_t dstlen, const char *sep1, const char *sep2, int quote); diff --git a/gdk/gdk_string.c b/gdk/gdk_string.c --- a/gdk/gdk_string.c +++ b/gdk/gdk_string.c @@ -333,7 +333,7 @@ strPut(BAT *b, var_t *dst, const void *V #endif ssize_t -GDKstrFromStr(unsigned char *restrict dst, const unsigned char *restrict src, ssize_t len) +GDKstrFromStr(unsigned char *restrict dst, const unsigned char *restrict src, ssize_t len, char quote) { unsigned char *p = dst; const unsigned char *cur = src, *end = src + len; @@ -470,7 +470,6 @@ GDKstrFromStr(unsigned char *restrict ds } else if ((c = *cur) == '\\') { escaped = true; continue; -#if 0 } else if (c == quote && cur[1] == quote) { assert(c != 0); if (unlikely(n > 0)) @@ -478,7 +477,6 @@ GDKstrFromStr(unsigned char *restrict ds *p++ = quote; cur++; continue; -#endif } if (n > 0) { @@ -598,7 +596,8 @@ strFromStr(const char *restrict src, siz return GDKstrFromStr((unsigned char *) *dst, (const unsigned char *) start, - (ssize_t) (cur - start)); + (ssize_t) (cur - start), + '\0'); } /* diff --git a/monetdb5/modules/mal/tablet.c b/monetdb5/modules/mal/tablet.c --- a/monetdb5/modules/mal/tablet.c +++ b/monetdb5/modules/mal/tablet.c @@ -821,7 +821,7 @@ SQLinsert_val(READERtask *task, int col, size_t slen = strlen(s) + 1; char *data = slen <= sizeof(buf) ? buf : GDKmalloc(strlen(s) + 1); if (data == NULL || - GDKstrFromStr((unsigned char *) data, (unsigned char *) s, strlen(s)) < 0) + GDKstrFromStr((unsigned char *) data, (unsigned char *) s, strlen(s), '\0') < 0) adt = NULL; else adt = fmt->frstr(fmt, fmt->adt, data); diff --git a/sql/backends/monet5/sql_gencode.c b/sql/backends/monet5/sql_gencode.c --- a/sql/backends/monet5/sql_gencode.c +++ b/sql/backends/monet5/sql_gencode.c @@ -769,8 +769,6 @@ backend_dumpstmt(backend *be, MalBlkPtr /* Always keep the SQL query around for monitoring */ if (query) { - char *escaped_q; - while (*query && isspace((unsigned char) *query)) query++; @@ -780,11 +778,7 @@ backend_dumpstmt(backend *be, MalBlkPtr return -1; } setVarType(mb, getArg(q, 0), TYPE_void); - if (!(escaped_q = sql_escape_str(m->ta, query))) { - sql_error(m, 10, SQLSTATE(HY013) MAL_MALLOC_FAIL); - return -1; - } - q = pushStr(mb, q, escaped_q); + q = pushStr(mb, q, query); if (q == NULL) { sql_error(m, 10, SQLSTATE(HY013) MAL_MALLOC_FAIL); return -1; diff --git a/sql/backends/monet5/sql_upgrades.c b/sql/backends/monet5/sql_upgrades.c --- a/sql/backends/monet5/sql_upgrades.c +++ b/sql/backends/monet5/sql_upgrades.c @@ -5195,6 +5195,232 @@ sql_update_default(Client c, mvc *sql, s } } sql->sa = old_sa; + + /* fixes for handling single quotes in strings so that we can run + * with raw_strings after having created a database without (and + * v.v.) */ + pos = snprintf(buf, bufsize, + "select id from sys.functions where name = 'dump_table_data' and schema_id = 2000 and func like '%% R'')%%';\n"); + if ((err = SQLstatementIntern(c, buf, "update", true, false, &output)) == NULL) { + if ((b = BBPquickdesc(output->cols[0].b)) && BATcount(b) == 0) { + sql_table *t; + if ((t = mvc_bind_table(sql, s, "describe_tables")) != NULL) + t->system = 0; + if ((t = mvc_bind_table(sql, s, "dump_create_users")) != NULL) + t->system = 0; + if ((t = mvc_bind_table(sql, s, "dump_partition_tables")) != NULL) + t->system = 0; + if ((t = mvc_bind_table(sql, s, "dump_comments")) != NULL) + t->system = 0; + if ((t = mvc_bind_table(sql, s, "dump_tables")) != NULL) + t->system = 0; + pos = 0; + pos += snprintf(buf + pos, bufsize - pos, + "drop function if exists sys.dump_database(boolean);\n" + "drop procedure if exists sys.dump_table_data();\n" + "drop procedure if exists sys.dump_table_data(string, string);\n" + "drop view if exists sys.dump_tables;\n" + "drop view if exists sys.dump_comments;\n" + "drop function if exists sys.prepare_esc(string, string);\n" + "drop view if exists sys.dump_partition_tables;\n" + "drop view if exists sys.dump_create_users;\n" + "drop view if exists sys.describe_tables;\n" + "drop function if exists sys.get_remote_table_expressions(string, string);\n" + "drop function if exists sys.sq(string);\n"); + pos += snprintf(buf + pos, bufsize - pos, + "CREATE FUNCTION sys.SQ (s STRING) RETURNS STRING BEGIN RETURN '''' || sys.replace(s,'''','''''') || ''''; END;\n" + "CREATE FUNCTION sys.get_remote_table_expressions(s STRING, t STRING) RETURNS STRING BEGIN\n" + " RETURN SELECT ' ON ' || sys.SQ(uri) || ' WITH USER ' || sys.SQ(username) || ' ENCRYPTED PASSWORD ' || sys.SQ(\"hash\") FROM sys.remote_table_credentials(s ||'.' || t);\n" + "END;\n" + "CREATE VIEW sys.describe_tables AS\n" + " SELECT\n" + " t.id o,\n" + " s.name sch,\n" + " t.name tab,\n" + " ts.table_type_name typ,\n" + " (SELECT\n" + " ' (' ||\n" + " GROUP_CONCAT(\n" + " sys.DQ(c.name) || ' ' ||\n" + " sys.describe_type(c.type, c.type_digits, c.type_scale) ||\n" + " ifthenelse(c.\"null\" = 'false', ' NOT NULL', '')\n" + " , ', ') || ')'\n" + " FROM sys._columns c\n" + " WHERE c.table_id = t.id) col,\n" + " CASE ts.table_type_name\n" + " WHEN 'REMOTE TABLE' THEN\n" + " sys.get_remote_table_expressions(s.name, t.name)\n" + " WHEN 'MERGE TABLE' THEN\n" + " sys.get_merge_table_partition_expressions(t.id)\n" + " WHEN 'VIEW' THEN\n" + " sys.schema_guard(s.name, t.name, t.query)\n" + " ELSE\n" + " ''\n" + " END opt\n" + " FROM sys.schemas s, sys.table_types ts, sys.tables t\n" + " WHERE ts.table_type_name IN ('TABLE', 'VIEW', 'MERGE TABLE', 'REMOTE TABLE', 'REPLICA TABLE', 'UNLOGGED TABLE')\n" + " AND t.system = FALSE\n" + " AND s.id = t.schema_id\n" + " AND ts.table_type_id = t.type\n" + " AND s.name <> 'tmp';\n" + "CREATE VIEW sys.dump_create_users AS\n" + " SELECT\n" + " 'CREATE USER ' || sys.dq(ui.name) || ' WITH ENCRYPTED PASSWORD ' ||\n" + " sys.sq(sys.password_hash(ui.name)) ||\n" + " ' NAME ' || sys.sq(ui.fullname) || ' SCHEMA sys' || ifthenelse(ui.schema_path = '\"sys\"', '', ' SCHEMA PATH ' || sys.sq(ui.schema_path)) || ';' stmt,\n" + " ui.name user_name\n" + " FROM sys.db_user_info ui, sys.schemas s\n" + " WHERE ui.default_schema = s.id\n" + " AND ui.name <> 'monetdb'\n" + " AND ui.name <> '.snapshot';\n" + "CREATE VIEW sys.dump_partition_tables AS\n" + " SELECT\n" + " 'ALTER TABLE ' || sys.FQN(m_sch, m_tbl) || ' ADD TABLE ' || sys.FQN(p_sch, p_tbl) ||\n" + " CASE\n" + " WHEN tpe = 'VALUES' THEN ' AS PARTITION IN (' || pvalues || ')'\n" + " WHEN tpe = 'RANGE' THEN ' AS PARTITION FROM ' || ifthenelse(minimum IS NOT NULL, sys.SQ(minimum), 'RANGE MINVALUE') || ' TO ' || ifthenelse(maximum IS NOT NULL, sys.SQ(maximum), 'RANGE MAXVALUE')\n" + " WHEN tpe = 'FOR NULLS' THEN ' AS PARTITION FOR NULL VALUES'\n" + " ELSE '' --'READ ONLY'\n" + " END ||\n" + " CASE WHEN tpe in ('VALUES', 'RANGE') AND with_nulls THEN ' WITH NULL VALUES' ELSE '' END ||\n" + " ';' stmt,\n" + " m_sch merge_schema_name,\n" + " m_tbl merge_table_name,\n" + " p_sch partition_schema_name,\n" + " p_tbl partition_table_name\n" + " FROM sys.describe_partition_tables;\n" + "CREATE VIEW sys.dump_tables AS\n" + " SELECT\n" + " t.o o,\n" + " CASE\n" + " WHEN t.typ <> 'VIEW' THEN\n" + " 'CREATE ' || t.typ || ' ' || sys.FQN(t.sch, t.tab) || t.col || t.opt || ';'\n" + " ELSE\n" + " t.opt\n" + " END stmt,\n" + " t.sch schema_name,\n" + " t.tab table_name\n" + " FROM sys.describe_tables t;\n" + "CREATE VIEW sys.dump_comments AS\n" + " SELECT 'COMMENT ON ' || c.tpe || ' ' || c.fqn || ' IS ' || sys.SQ(c.rem) || ';' stmt FROM sys.describe_comments c;\n" + "CREATE FUNCTION sys.prepare_esc(s STRING, t STRING) RETURNS STRING\n" + "BEGIN\n" + " RETURN\n" + " CASE\n" + " WHEN (t = 'varchar' OR t ='char' OR t = 'clob' OR t = 'json' OR t = 'geometry' OR t = 'url') THEN\n" + " 'CASE WHEN ' || sys.DQ(s) || ' IS NULL THEN ''null'' ELSE ' || 'sys.esc(' || sys.DQ(s) || ')' || ' END'\n" + " ELSE\n" + " 'CASE WHEN ' || sys.DQ(s) || ' IS NULL THEN ''null'' ELSE CAST(' || sys.DQ(s) || ' AS STRING) END'\n" + " END;\n" + "END;\n" + "CREATE PROCEDURE sys.dump_table_data(sch STRING, tbl STRING)\n" + "BEGIN\n" + " DECLARE tid INT;\n" + " SET tid = (SELECT MIN(t.id) FROM sys.tables t, sys.schemas s WHERE t.name = tbl AND t.schema_id = s.id AND s.name = sch);\n" + " IF tid IS NOT NULL THEN\n" + " DECLARE k INT;\n" + " DECLARE m INT;\n" + " SET k = (SELECT MIN(c.id) FROM sys.columns c WHERE c.table_id = tid);\n" + " SET m = (SELECT MAX(c.id) FROM sys.columns c WHERE c.table_id = tid);\n" + " IF k IS NOT NULL AND m IS NOT NULL THEN\n" + " DECLARE cname STRING;\n" + " DECLARE ctype STRING;\n" + " DECLARE _cnt INT;\n" + " SET cname = (SELECT c.name FROM sys.columns c WHERE c.id = k);\n" + " SET ctype = (SELECT c.type FROM sys.columns c WHERE c.id = k);\n" + " SET _cnt = (SELECT count FROM sys.storage(sch, tbl, cname));\n" + " IF _cnt > 0 THEN\n" + " DECLARE COPY_INTO_STMT STRING;\n" + " DECLARE SELECT_DATA_STMT STRING;\n" + " SET COPY_INTO_STMT = 'COPY ' || _cnt || ' RECORDS INTO ' || sys.FQN(sch, tbl) || '(' || sys.DQ(cname);\n" + " SET SELECT_DATA_STMT = 'SELECT (SELECT COUNT(*) FROM sys.dump_statements) + RANK() OVER(), ' || sys.prepare_esc(cname, ctype);\n" + " WHILE (k < m) DO\n" + " SET k = (SELECT MIN(c.id) FROM sys.columns c WHERE c.table_id = tid AND c.id > k);\n" + " SET cname = (SELECT c.name FROM sys.columns c WHERE c.id = k);\n" + " SET ctype = (SELECT c.type FROM sys.columns c WHERE c.id = k);\n" + " SET COPY_INTO_STMT = (COPY_INTO_STMT || ', ' || sys.DQ(cname));\n" + " SET SELECT_DATA_STMT = (SELECT_DATA_STMT || '|| ''|'' || ' || sys.prepare_esc(cname, ctype));\n" + " END WHILE;\n" + " SET COPY_INTO_STMT = (COPY_INTO_STMT || R') FROM STDIN USING DELIMITERS ''|'',E''\\n'',''\"'';');\n" + " SET SELECT_DATA_STMT = (SELECT_DATA_STMT || ' FROM ' || sys.FQN(sch, tbl));\n" + " INSERT INTO sys.dump_statements VALUES ((SELECT COUNT(*) FROM sys.dump_statements) + 1, COPY_INTO_STMT);\n" + " CALL sys.EVAL('INSERT INTO sys.dump_statements ' || SELECT_DATA_STMT || ';');\n" + " END IF;\n" + " END IF;\n" + " END IF;\n" + " END;\n" + "CREATE PROCEDURE sys.dump_table_data()\n" + "BEGIN\n" + " DECLARE i INT;\n" + " SET i = (SELECT MIN(t.id) FROM sys.tables t, sys.table_types ts WHERE t.type = ts.table_type_id AND ts.table_type_name = 'TABLE' AND NOT t.system);\n" + " IF i IS NOT NULL THEN\n" + " DECLARE M INT;\n" + " SET M = (SELECT MAX(t.id) FROM sys.tables t, sys.table_types ts WHERE t.type = ts.table_type_id AND ts.table_type_name = 'TABLE' AND NOT t.system);\n" + " DECLARE sch STRING;\n" + " DECLARE tbl STRING;\n" + " WHILE i IS NOT NULL AND i <= M DO\n" + " SET sch = (SELECT s.name FROM sys.tables t, sys.schemas s WHERE s.id = t.schema_id AND t.id = i);\n" + " SET tbl = (SELECT t.name FROM sys.tables t, sys.schemas s WHERE s.id = t.schema_id AND t.id = i);\n" + " CALL sys.dump_table_data(sch, tbl);\n" + " SET i = (SELECT MIN(t.id) FROM sys.tables t, sys.table_types ts WHERE t.type = ts.table_type_id AND ts.table_type_name = 'TABLE' AND NOT t.system AND t.id > i);\n" + " END WHILE;\n" + " END IF;\n" + "END;\n" + "CREATE FUNCTION sys.dump_database(describe BOOLEAN) RETURNS TABLE(o int, stmt STRING)\n" + "BEGIN\n" + " SET SCHEMA sys;\n" + " TRUNCATE sys.dump_statements;\n" + " INSERT INTO sys.dump_statements VALUES (1, 'START TRANSACTION;');\n" + " INSERT INTO sys.dump_statements VALUES (2, 'SET SCHEMA \"sys\";');\n" + " INSERT INTO sys.dump_statements SELECT (SELECT COUNT(*) FROM sys.dump_statements) + RANK() OVER(), stmt FROM sys.dump_create_roles;\n" + " INSERT INTO sys.dump_statements SELECT (SELECT COUNT(*) FROM sys.dump_statements) + RANK() OVER(), stmt FROM sys.dump_create_users;\n" + " INSERT INTO sys.dump_statements SELECT (SELECT COUNT(*) FROM sys.dump_statements) + RANK() OVER(), stmt FROM sys.dump_create_schemas;\n" + " INSERT INTO sys.dump_statements SELECT (SELECT COUNT(*) FROM sys.dump_statements) + RANK() OVER(), stmt FROM sys.dump_user_defined_types;\n" + " INSERT INTO sys.dump_statements SELECT (SELECT COUNT(*) FROM sys.dump_statements) + RANK() OVER(), stmt FROM sys.dump_add_schemas_to_users;\n" + " INSERT INTO sys.dump_statements SELECT (SELECT COUNT(*) FROM sys.dump_statements) + RANK() OVER(), stmt FROM sys.dump_grant_user_privileges;\n" + " INSERT INTO sys.dump_statements SELECT (SELECT COUNT(*) FROM sys.dump_statements) + RANK() OVER(), stmt FROM sys.dump_sequences;\n" + " --functions and table-likes can be interdependent. They should be inserted in the order of their catalogue id.\n" + " INSERT INTO sys.dump_statements SELECT (SELECT COUNT(*) FROM sys.dump_statements) + RANK() OVER(ORDER BY stmts.o), stmts.s\n" + " FROM (\n" + " SELECT f.o, f.stmt FROM sys.dump_functions f\n" + " UNION ALL\n" + " SELECT t.o, t.stmt FROM sys.dump_tables t\n" + " ) AS stmts(o, s);\n" + " -- dump table data before adding constraints and fixing sequences\n" + " IF NOT DESCRIBE THEN\n" + " CALL sys.dump_table_data();\n" _______________________________________________ checkin-list mailing list -- checkin-list@monetdb.org To unsubscribe send an email to checkin-list-le...@monetdb.org