Changeset: 841ab61d0703 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=841ab61d0703
Added Files:
sql/test/copy/Tests/incorrect_columns.sql
sql/test/copy/Tests/incorrect_columns.stable.err
sql/test/copy/Tests/incorrect_columns.stable.out
Modified Files:
clients/odbc/driver/SQLDriverConnect.c
gdk/gdk_logger.c
monetdb5/modules/kernel/algebra.c
monetdb5/modules/mal/tablet.c
sql/test/copy/Tests/All
sql/test/copy/Tests/key_copy.sql
sql/test/copy/Tests/key_copy.stable.err
Branch: default
Log Message:
merged with jul2015
diffs (truncated from 462 to 300 lines):
diff --git a/clients/odbc/driver/SQLDriverConnect.c
b/clients/odbc/driver/SQLDriverConnect.c
--- a/clients/odbc/driver/SQLDriverConnect.c
+++ b/clients/odbc/driver/SQLDriverConnect.c
@@ -217,16 +217,17 @@ ODBCConnectionString(SQLRETURN rc,
/* calculate how much space was needed */
if (StringLength2Ptr)
- *StringLength2Ptr = (int) (strlen(dsn ? dsn : "DEFAULT") + 5 +
- (uid ? strlen(uid) + 5 : 0) +
- (pwd ? strlen(pwd) + 5 : 0) +
- (host ? strlen(host) + 6 : 0) +
- (port ? port + 6 : 0) +
- (database ? strlen(database) + 10 :
0)
+ *StringLength2Ptr = (SQLSMALLINT)
+ (strlen(dsn ? dsn : "DEFAULT") + 5 +
+ (uid ? strlen(uid) + 5 : 0) +
+ (pwd ? strlen(pwd) + 5 : 0) +
+ (host ? strlen(host) + 6 : 0) +
+ (port ? port + 6 : 0) +
+ (database ? strlen(database) + 10 : 0)
#ifdef ODBCDEBUG
- + (ODBCdebug && getenv("ODBCDEBUG")
== NULL ? strlen(ODBCdebug) + 9 : 0)
+ + (ODBCdebug && getenv("ODBCDEBUG") == NULL ?
strlen(ODBCdebug) + 9 : 0)
#endif
- );
+ );
#ifdef ODBCDEBUG
ODBCLOG("ConnectionString: \"%.*s\" %d\n", buf ? buflen : 6, buf ?
(char *) buf : "(null)", buflen);
diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -1643,6 +1643,9 @@ logger_load(int debug, const char* fn, c
if (BBPrename(lg->dsnapshots->batCacheid, bak) < 0)
logger_fatal("Logger_new: BBPrename to %s
failed", bak, 0, 0);
logger_add_bat(lg, lg->dsnapshots, "dsnapshots");
+
+ if (bm_subcommit(lg->catalog_bid, lg->catalog_nme,
lg->catalog_bid, lg->catalog_nme, lg->dcatalog, NULL, lg->debug) != GDK_SUCCEED)
+ logger_fatal("Logger_new: commit failed", 0, 0,
0);
}
}
lg->freed = logbat_new(TYPE_int, 1, TRANSIENT);
diff --git a/monetdb5/modules/kernel/algebra.c
b/monetdb5/modules/kernel/algebra.c
--- a/monetdb5/modules/kernel/algebra.c
+++ b/monetdb5/modules/kernel/algebra.c
@@ -108,6 +108,8 @@ CMDgen_group(BAT **result, BAT *gids, BA
r -> hdense = TRUE;
r -> hsorted = TRUE;
r -> hrevsorted = FALSE;
+ r -> tkey = FALSE;
+ r -> tdense = FALSE;
r -> tsorted = BATtordered(gids);
r -> trevsorted = BATtrevordered(gids);
r -> T ->nonil = gids->T->nonil;
diff --git a/monetdb5/modules/mal/tablet.c b/monetdb5/modules/mal/tablet.c
--- a/monetdb5/modules/mal/tablet.c
+++ b/monetdb5/modules/mal/tablet.c
@@ -838,14 +838,14 @@ mycpstr(char *t, const char *s)
}
static str
-SQLload_error(READERtask *task, lng idx)
+SQLload_error(READERtask *task, lng idx, BUN attrs)
{
str line;
char *s;
size_t sz = 0;
- unsigned int i;
+ BUN i;
- for (i = 0; i < task->as->nr_attrs; i++) {
+ for (i = 0; i < attrs; i++) {
if (task->fields[i][idx])
sz += mystrlen(task->fields[i][idx]);
sz += task->seplen;
@@ -856,10 +856,10 @@ SQLload_error(READERtask *task, lng idx)
tablet_error(task, idx, int_nil, "SQLload malloc error",
"SQLload_error");
return 0;
}
- for (i = 0; i < task->as->nr_attrs; i++) {
+ for (i = 0; i < attrs; i++) {
if (task->fields[i][idx])
s = mycpstr(s, task->fields[i][idx]);
- if (i < task->as->nr_attrs - 1)
+ if (i < attrs - 1)
s = mycpstr(s, task->csep);
}
strcat(line, task->rsep);
@@ -893,7 +893,7 @@ SQLinsert_val(READERtask *task, int col,
if (adt == NULL) {
lng row = task->cnt + idx + 1;
snprintf(buf, BUFSIZ, "'%s' expected", fmt->type);
- err = SQLload_error(task, idx);
+ err = SQLload_error(task, idx, task->as->nr_attrs);
if (task->rowerror) {
MT_lock_set(&errorlock, "insert_val");
col++;
@@ -925,7 +925,7 @@ SQLinsert_val(READERtask *task, int col,
BUNappend(task->cntxt->error_row, &row, FALSE);
BUNappend(task->cntxt->error_fld, &col, FALSE);
BUNappend(task->cntxt->error_msg, "insert failed", FALSE);
- err = SQLload_error(task, idx);
+ err = SQLload_error(task, idx,task->as->nr_attrs);
BUNappend(task->cntxt->error_input, err, FALSE);
GDKfree(err);
task->rowerror[row - 1]++;
@@ -953,8 +953,10 @@ SQLworker_column(READERtask *task, int c
MT_lock_unset(&mal_copyLock, "tablet insert value");
for (i = 0; i < task->top[task->cur]; i++) {
- if (!fmt[col].skip && SQLinsert_val(task, col, i) < 0)
- return -1;
+ if (!fmt[col].skip && SQLinsert_val(task, col, i) < 0){
+ if(task->besteffort == 0)
+ return -1;
+ }
}
return 0;
@@ -984,10 +986,11 @@ SQLload_parse_line(READERtask *task, int
//mnstr_printf(GDKout, "#SQL break line id %d state %d\n%s", task->id,
idx, line);
#endif
assert(idx < task->top[task->cur]);
+ assert(line);
errmsg[0] = 0;
if (task->quote || task->seplen != 1) {
- for (i = 0; i < as->nr_attrs; i++) {
+ for (i = 0; *line && i < as->nr_attrs; i++) {
task->fields[i][idx] = line;
/* recognize fields starting with a quote, keep them */
if (*line == task->quote) {
@@ -1000,7 +1003,7 @@ SQLload_parse_line(READERtask *task, int
mnstr_printf(GDKout, "after #1 %s\n", s);
#endif
if (!line) {
- errline = SQLload_error(task,
task->top[task->cur]);
+ errline = SQLload_error(task, idx, i+1);
snprintf(errmsg, BUFSIZ, "Quote (%c)
missing", task->quote);
tablet_error(task, idx, (int) i,
errmsg, errline);
GDKfree(errline);
@@ -1023,8 +1026,8 @@ SQLload_parse_line(READERtask *task, int
/* not enough fields */
if (i < as->nr_attrs - 1) {
- errline = SQLload_error(task,
task->top[task->cur]);
- snprintf(errmsg, BUFSIZ, "Separator missing
'%s' ", fmt->sep);
+ errline = SQLload_error(task, idx, i+1);
+ snprintf(errmsg, BUFSIZ, "Column value "BUNFMT"
missing ", i+1);
tablet_error(task, idx, (int) i, errmsg,
errline);
GDKfree(errline);
error++;
@@ -1033,18 +1036,14 @@ SQLload_parse_line(READERtask *task, int
for (; i < as->nr_attrs; i++)
task->fields[i][idx] = NULL;
i--;
- }
+ }
endoffieldcheck:
;
/* check for user defined NULL string */
if (!fmt->skip && fmt->nullstr && task->fields[i][idx]
&& strncasecmp(task->fields[i][idx], fmt->nullstr, fmt->null_length + 1) == 0)
task->fields[i][idx] = 0;
}
-#ifdef _DEBUG_TABLET_
- if (error)
- mnstr_printf(GDKout, "#line break failed %d:%s\n", idx,
line);
-#endif
- return error ? -1 : 0;
+ goto endofline;
}
assert(!task->quote);
assert(task->seplen == 1);
@@ -1068,8 +1067,8 @@ SQLload_parse_line(READERtask *task, int
#endif
/* not enough fields */
if (i < as->nr_attrs - 1) {
- errline = SQLload_error(task, task->top[task->cur]);
- snprintf(errmsg, BUFSIZ, "Separator missing '%s' ",
fmt->sep);
+ errline = SQLload_error(task, idx,i+1);
+ snprintf(errmsg, BUFSIZ, "Column value "BUNFMT"
missing",i+1);
tablet_error(task, idx, (int) i, errmsg, errline);
GDKfree(errline);
error++;
@@ -1085,6 +1084,15 @@ SQLload_parse_line(READERtask *task, int
task->fields[i][idx] = 0;
}
}
+endofline:
+ /* check for too many values as well*/
+ if (*line && i == as->nr_attrs) {
+ errline = SQLload_error(task, idx, task->as->nr_attrs);
+ snprintf(errmsg, BUFSIZ, "Leftover data '%s'",line);
+ tablet_error(task, idx, (int) i, errmsg, errline);
+ GDKfree(errline);
+ error++;
+ }
#ifdef _DEBUG_TABLET_
if (error)
mnstr_printf(GDKout, "#line break failed %d:%s\n", idx, line);
@@ -1125,8 +1133,10 @@ SQLworker(void *arg)
for (j = piece * task->id; j < task->top[task->cur] &&
j < piece * (task->id +1); j++)
if (task->lines[task->cur][j]) {
if (SQLload_parse_line(task, j) < 0) {
- task->error++;
- break;
+ task->errorcnt++;
+ // early break unless best
effort
+ if(task->besteffort == 0)
+ break;
}
}
task->wtime = GDKusec() - t0;
@@ -1550,7 +1560,7 @@ SQLproducer(void *p)
BUN
SQLload_file(Client cntxt, Tablet *as, bstream *b, stream *out, char *csep,
char *rsep, char quote, lng skip, lng maxrow, int best)
{
- BUN cnt = 0, cntstart = 0;
+ BUN cnt = 0, cntstart = 0, leftover = 0;
int res = 0; /* < 0: error, > 0: success, == 0: continue
processing */
int j;
BUN i, attr;
@@ -1794,7 +1804,7 @@ SQLload_file(Client cntxt, Tablet *as, b
#define trimerrors(TYPE)
\
do {
\
TYPE *src, *dst;
\
- BUN leftover= BATcount(task->as->format[attr].c);
\
+ leftover= BATcount(task->as->format[attr].c);
\
limit = leftover - cntstart;
\
dst =src= (TYPE *)
BUNtloc(task->as->format[attr].ci,cntstart); \
for(j = 0; j < (int) limit; j++, src++){
\
@@ -1813,8 +1823,14 @@ SQLload_file(Client cntxt, Tablet *as, b
#endif
if (best && BATcount(as->format[0].c)) {
BUN limit;
+ int width;
+
for (attr = 0; attr < as->nr_attrs; attr++) {
- switch (ATOMsize(as->format[attr].c->ttype)) {
+ if( as->format[attr].c->ttype == TYPE_str)
+ width = as->format[attr].c->T->width;
+ else
+ width =
ATOMsize(as->format[attr].c->ttype);
+ switch (width){
case 1:
trimerrors(unsigned char);
break;
diff --git a/sql/test/copy/Tests/All b/sql/test/copy/Tests/All
--- a/sql/test/copy/Tests/All
+++ b/sql/test/copy/Tests/All
@@ -11,3 +11,4 @@ overflow_error
format_date
key_copy
nonutf8
+incorrect_columns
diff --git a/sql/test/copy/Tests/incorrect_columns.sql
b/sql/test/copy/Tests/incorrect_columns.sql
new file mode 100644
--- /dev/null
+++ b/sql/test/copy/Tests/incorrect_columns.sql
@@ -0,0 +1,17 @@
+-- test the situation where the number of column values are incorrect
+create table t_columns(i int, t string);
+
+copy 7 records into t_columns from stdin delimiters ',','\n' best effort;
+1,hello
+2
+no tag
+3,too much,xyz
+4,world
+5,wereld
+6,maan
+
+select * from t_columns;
+
+select * from sys.rejects();
+
+drop table t_columns;
diff --git a/sql/test/copy/Tests/incorrect_columns.stable.err
b/sql/test/copy/Tests/incorrect_columns.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/test/copy/Tests/incorrect_columns.stable.err
@@ -0,0 +1,37 @@
+stderr of test 'incorrect_columns` in directory 'sql/test/copy` itself:
+
+
+# 08:11:47 >
+# 08:11:47 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set"
"mapi_open=true" "--set" "mapi_port=37579" "--set"
"mapi_usock=/var/tmp/mtest-26664/.s.monetdb.37579" "--set" "monet_prompt="
"--forcemito" "--set" "mal_listing=2"
"--dbpath=/export/scratch1/mk/Jul2015//Linux/var/MonetDB/mTests_sql_test_copy"
"--set" "mal_listing=0" "--set" "embedded_r=yes"
+# 08:11:47 >
+
+# builtin opt gdk_dbpath =
/export/scratch1/mk/Jul2015//Linux/var/monetdb5/dbfarm/demo
+# builtin opt gdk_debug = 0
+# builtin opt gdk_vmtrim = no
+# builtin opt monet_prompt = >
+# builtin opt monet_daemon = no
+# builtin opt mapi_port = 50000
+# builtin opt mapi_open = false
+# builtin opt mapi_autosense = false
+# builtin opt sql_optimizer = default_pipe
+# builtin opt sql_debug = 0
+# cmdline opt gdk_nr_threads = 0
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
