Changeset: 826ac37140cc for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=826ac37140cc
Modified Files:
clients/Tests/exports.stable.out
geom/BugTracker/Tests/copy_into_crash.SF-1975402.stable.err
monetdb5/modules/mal/tablet.c
sql/test/BugTracker-2008/Tests/copy-format.SF-2468513.stable.err
sql/test/BugTracker-2008/Tests/varchar.SF-2471371.stable.err
sql/test/BugTracker-2009/Tests/copy_into_error_off_by_one.SF-2833696.stable.err
sql/test/BugTracker-2010/Tests/copy-into-too-long-string.Bug-2358.stable.err
sql/test/BugTracker-2010/Tests/incomplete-utf8-sequence.Bug-2575.stable.err
sql/test/BugTracker/Tests/copy_decimal.SF-1582957.stable.err
sql/test/BugTracker/Tests/copy_overflow_null_bug.SF-1842550.stable.err
sql/test/copy/Tests/int_parse.stable.err
sql/test/copy/Tests/load_stdin_incorrect_line_nr.stable.err
sql/test/copy/Tests/null_as_string_errors.stable.err
sql/test/testdb/Tests/testdb-load.stable.out
Branch: default
Log Message:
Merge with Jan2014 branch.
diffs (truncated from 407 to 300 lines):
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -1251,7 +1251,6 @@ str CMDstartProfiler(Client cntxt, MalBl
str CMDstopProfiler(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
str CMDstr2qgrams(int *ret, str *val);
str CMDstrlength(int *ret, str *v);
-str CMDtablet_input(int *ret, int *nameid, int *sepid, int *typeid, stream *s,
int *nr);
str CMDtomograph(int *ret);
str CMDvarABS(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
str CMDvarADD(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
diff --git a/geom/BugTracker/Tests/copy_into_crash.SF-1975402.stable.err
b/geom/BugTracker/Tests/copy_into_crash.SF-1975402.stable.err
--- a/geom/BugTracker/Tests/copy_into_crash.SF-1975402.stable.err
+++ b/geom/BugTracker/Tests/copy_into_crash.SF-1975402.stable.err
@@ -13,7 +13,6 @@ stderr of test 'copy_into_crash.SF-19754
MAPI = (monetdb) /var/tmp/mtest-17297/.s.monetdb.35706
QUERY = COPY 1 RECORDS INTO nodes from STDIN USING DELIMITERS ',', '\n';
45111956, 'POINT(52.0697 4.3723)'
-
ERROR = !value ' 'POINT(52.0697 4.3723)'' from line 1 field 2 not inserted,
expecting type point
!failed to import table
diff --git a/monetdb5/modules/mal/tablet.c b/monetdb5/modules/mal/tablet.c
--- a/monetdb5/modules/mal/tablet.c
+++ b/monetdb5/modules/mal/tablet.c
@@ -53,7 +53,7 @@
#include <string.h>
#include <ctype.h>
-tablet_export str CMDtablet_input(int *ret, int *nameid, int *sepid, int
*typeid, stream *s, int *nr);
+/* #define SQLLOADTHREAD */ /* define to get separate reader thread
*/
static MT_Lock errorlock MT_LOCK_INITIALIZER("errorlock");
@@ -682,8 +682,10 @@ typedef struct {
lng *time, wtime; /* time per col + time per
thread */
int rounds; /* how often did we
divide the work */
MT_Id tid;
+#ifdef SQLLOADTHREAD
MT_Sema producer; /* reader waits for call */
MT_Sema consumer; /* data available */
+#endif
int ateof; /* io control */
bstream *b;
stream *out;
@@ -1067,9 +1069,10 @@ SQLworkdivider(READERtask *task, READERt
GDKfree(loc);
}
+#ifdef SQLLOADTHREAD
/*
* Reading is handled by a separate task as a preparation for
- * mode parallelism
+ * more parallelism
*/
static void
SQLloader(void *p)
@@ -1090,6 +1093,7 @@ SQLloader(void *p)
MT_sema_up(&task->consumer, "SQLloader");
}
}
+#endif
#define MAXWORKERS 64
@@ -1128,7 +1132,6 @@ SQLload_file(Client cntxt, Tablet *as, b
assert(rsep);
assert(csep);
assert(maxrow < 0 || maxrow <= (lng) BUN_MAX);
- rseplen = strlen(rsep);
task->fields = (char ***) GDKzalloc(as->nr_attrs * sizeof(char **));
task->cols = (int *) GDKzalloc(as->nr_attrs * sizeof(int));
task->time = (lng *) GDKzalloc(as->nr_attrs * sizeof(lng));
@@ -1148,13 +1151,15 @@ SQLload_file(Client cntxt, Tablet *as, b
task->csep = csep;
task->seplen = strlen(csep);
task->rsep = rsep;
- task->rseplen = strlen(rsep);
+ task->rseplen = rseplen = strlen(rsep);
task->errbuf = cntxt->errbuf;
task->input = task->base + 1; /* wrap the buffer with null bytes */
task->base[b->size + 1] = 0;
+#ifdef SQLLOADTHREAD
MT_sema_init(&task->consumer, 0, "task->consumer");
MT_sema_init(&task->producer, 0, "task->producer");
+#endif
task->ateof = 0;
task->b = b;
task->out = out;
@@ -1165,9 +1170,7 @@ SQLload_file(Client cntxt, Tablet *as, b
mlock(task->time, as->nr_attrs * sizeof(lng));
mlock(task->base, b->size + 2);
#endif
- MT_lock_set(&errorlock, "SQLload_file");
as->error = NULL;
- MT_lock_unset(&errorlock, "SQLload_file");
/* there is no point in creating more threads than we have columns */
if (as->nr_attrs < (BUN) threads)
@@ -1179,10 +1182,8 @@ SQLload_file(Client cntxt, Tablet *as, b
for (i = 0; i < as->nr_attrs; i++) {
task->fields[i] = GDKzalloc(sizeof(char *) * task->limit);
if (task->fields[i] == 0) {
- MT_lock_set(&errorlock, "SQLload_file");
if (task->as->error == NULL)
as->error = M5OutOfMemory;
- MT_lock_unset(&errorlock, "SQLload_file");
goto bailout;
}
#ifdef MLOCK_TST
@@ -1214,14 +1215,16 @@ SQLload_file(Client cntxt, Tablet *as, b
MT_create_thread(&ptask[j].tid, SQLworker, (void *) &ptask[j],
MT_THR_JOINABLE);
}
+#ifdef SQLLOADTHREAD
MT_create_thread(&task->tid, SQLloader, (void *) task, MT_THR_JOINABLE);
+#endif
#ifdef _DEBUG_TABLET_
mnstr_printf(GDKout, "parallel bulk load " LLFMT " - " LLFMT "\n",
skip, maxrow);
#endif
+ if (maxrow < 0)
+ maxrow = (lng) BUN_MAX;
tio = GDKusec();
- MT_sema_up(&task->producer, "SQLload_file");
- MT_sema_down(&task->consumer, "SQLload_file");
tio = GDKusec() - tio;
t1 = GDKusec();
#ifdef MLOCK_TST
@@ -1285,7 +1288,7 @@ SQLload_file(Client cntxt, Tablet *as, b
* the middle of the record separator). If this is too
* costly, we have to rethink the matter. */
e = s;
- while (s < end && task->next < task->limit && (maxrow < 0 ||
cnt < (BUN) maxrow)) {
+ while (s < end && task->next < task->limit && cnt < (BUN)
maxrow) {
char q = 0;
/* tokenize the record completely the format of the
input
* should comply to the following grammar rule [
@@ -1390,7 +1393,10 @@ SQLload_file(Client cntxt, Tablet *as, b
}
}
/* start feeding new data */
- MT_sema_up(&task->producer, "SQLload_file");
+#ifdef SQLLOADTHREAD
+ if ((e == NULL || s >= end || e >= end) && cnt < (BUN) maxrow)
+ MT_sema_up(&task->producer, "SQLload_file");
+#endif
t1 = GDKusec() - t1;
total += t1;
iototal += tio;
@@ -1444,9 +1450,15 @@ SQLload_file(Client cntxt, Tablet *as, b
for (j = 0; j < threads; j++)
MT_sema_down(&ptask[j].reply, "SQLload_file");
}
+ if ((e == NULL || s >= end || e >= end) && cnt < (BUN) maxrow) {
+#ifdef SQLLOADTHREAD
+ MT_sema_down(&task->consumer, "SQLload_file");
+#else
+ task->ateof = tablet_read_more(task->b, task->out,
task->b->size - (task->b->len - task->b->pos)) == EOF;
+#endif
+ }
if (task->ateof)
break;
- MT_sema_down(&task->consumer, "SQLload_file");
}
if (task->b->pos < task->b->len && cnt < (BUN) maxrow && task->ateof) {
@@ -1463,7 +1475,7 @@ SQLload_file(Client cntxt, Tablet *as, b
}
if (GDKdebug & GRPalgorithms) {
- if (cnt < (BUN) maxrow && maxrow > 0)
+ if (cnt < (BUN) maxrow)
/* providing a precise count is not always easy, instead
* consider maxrow as an upper bound */
mnstr_printf(GDKout, "#SQLload_file: read error, tuples
missing (after loading " BUNFMT " records)\n", BATcount(as->format[0].c));
@@ -1478,7 +1490,9 @@ SQLload_file(Client cntxt, Tablet *as, b
}
task->ateof = 1;
+#ifdef SQLLOADTHREAD
MT_sema_up(&task->producer, "SQLload_file");
+#endif
for (j = 0; j < threads; j++) {
ptask[j].next = -1;
MT_sema_up(&ptask[j].sema, "SQLload_file");
@@ -1495,7 +1509,9 @@ SQLload_file(Client cntxt, Tablet *as, b
MT_sema_destroy(&ptask[j].sema);
MT_sema_destroy(&ptask[j].reply);
}
+#ifdef SQLLOADTHREAD
MT_join_thread(task->tid);
+#endif
#ifdef _DEBUG_TABLET_
mnstr_printf(GDKout, "Found " BUNFMT " tuples\n", cnt);
@@ -1506,8 +1522,10 @@ SQLload_file(Client cntxt, Tablet *as, b
GDKfree(task->cols);
GDKfree(task->time);
GDKfree(task->base);
+#ifdef SQLLOADTHREAD
MT_sema_destroy(&task->consumer);
MT_sema_destroy(&task->producer);
+#endif
GDKfree(task);
#ifdef MLOCK_TST
munlockall();
diff --git a/sql/test/BugTracker-2008/Tests/copy-format.SF-2468513.stable.err
b/sql/test/BugTracker-2008/Tests/copy-format.SF-2468513.stable.err
--- a/sql/test/BugTracker-2008/Tests/copy-format.SF-2468513.stable.err
+++ b/sql/test/BugTracker-2008/Tests/copy-format.SF-2468513.stable.err
@@ -91,35 +91,30 @@ MAPI = (monetdb) /var/tmp/mtest-23209/.
QUERY = COPY 1 RECORDS INTO my_copytest FROM stdin USING DELIMITERS '|','\n'
NULL as '';
123|1.01||a|b
-
ERROR = !value '1.01' from line 1 field 2 not inserted, expecting type int
!failed to import table
MAPI = (monetdb) /var/tmp/mtest-23209/.s.monetdb.33225
QUERY = COPY 1 RECORDS INTO my_copytest FROM stdin USING DELIMITERS '|','\n'
NULL as '';
553|.02||a|b
-
ERROR = !value '.02' from line 1 field 2 not inserted, expecting type int
!failed to import table
MAPI = (monetdb) /var/tmp/mtest-23209/.s.monetdb.33225
QUERY = COPY 1 RECORDS INTO my_copytest FROM stdin USING DELIMITERS '|','\n'
NULL as '';
223|2.03||a|b
-
ERROR = !value '2.03' from line 1 field 2 not inserted, expecting type int
!failed to import table
MAPI = (monetdb) /var/tmp/mtest-23209/.s.monetdb.33225
QUERY = COPY 1 RECORDS INTO my_copytest FROM stdin USING DELIMITERS '|','\n'
NULL as '';
223|2||aaa|b
-
ERROR = !value 'aaa' from line 1 field 4 not inserted, expecting type
varchar(1)
!failed to import table
MAPI = (monetdb) /var/tmp/mtest-23209/.s.monetdb.33225
QUERY = COPY 1 RECORDS INTO my_copytest FROM stdin USING DELIMITERS '|','\n'
NULL as '';
223|3||a|bbb
-
ERROR = !value 'bbb' from line 1 field 5 not inserted, expecting type
varchar(1)
!failed to import table
diff --git a/sql/test/BugTracker-2008/Tests/varchar.SF-2471371.stable.err
b/sql/test/BugTracker-2008/Tests/varchar.SF-2471371.stable.err
--- a/sql/test/BugTracker-2008/Tests/varchar.SF-2471371.stable.err
+++ b/sql/test/BugTracker-2008/Tests/varchar.SF-2471371.stable.err
@@ -90,13 +90,11 @@ stderr of test 'varchar.SF-2471371` in d
MAPI = (monetdb) /var/tmp/mtest-23209/.s.monetdb.33225
QUERY = COPY 1 RECORDS INTO myvar_test FROM stdin USING DELIMITERS '|','\n'
NULL as '';
a|b2b
-
ERROR = !value 'b2b' from line 1 field 2 not inserted, expecting type
varchar(2)
!failed to import table
MAPI = (monetdb) /var/tmp/mtest-23209/.s.monetdb.33225
QUERY = COPY 1 RECORDS INTO myvar_test FROM stdin USING DELIMITERS '|','\n'
NULL as '';
aa|bb
-
ERROR = !value 'aa' from line 1 field 1 not inserted, expecting type varchar(1)
!failed to import table
diff --git
a/sql/test/BugTracker-2009/Tests/copy_into_error_off_by_one.SF-2833696.stable.err
b/sql/test/BugTracker-2009/Tests/copy_into_error_off_by_one.SF-2833696.stable.err
---
a/sql/test/BugTracker-2009/Tests/copy_into_error_off_by_one.SF-2833696.stable.err
+++
b/sql/test/BugTracker-2009/Tests/copy_into_error_off_by_one.SF-2833696.stable.err
@@ -73,7 +73,6 @@ MAPI = (monetdb) /var/tmp/mtest-23209/.
QUERY = COPY 2 RECORDS INTO number FROM stdin USING DELIMITERS '\n';
1
bla
- drop table number;
ERROR = !value 'bla' from line 2 field 1 not inserted, expecting type int
!failed to import table
diff --git
a/sql/test/BugTracker-2010/Tests/copy-into-too-long-string.Bug-2358.stable.err
b/sql/test/BugTracker-2010/Tests/copy-into-too-long-string.Bug-2358.stable.err
---
a/sql/test/BugTracker-2010/Tests/copy-into-too-long-string.Bug-2358.stable.err
+++
b/sql/test/BugTracker-2010/Tests/copy-into-too-long-string.Bug-2358.stable.err
@@ -75,7 +75,6 @@ MAPI = (monetdb) /var/tmp/mtest-23209/.
QUERY = COPY 1 RECORDS INTO t FROM STDIN USING DELIMITERS '|','
','"' NULL AS '';
"abcd"
-
ERROR = !value 'abcd' from line 1 field 1 not inserted, expecting type
varchar(3)
!failed to import table
diff --git
a/sql/test/BugTracker-2010/Tests/incomplete-utf8-sequence.Bug-2575.stable.err
b/sql/test/BugTracker-2010/Tests/incomplete-utf8-sequence.Bug-2575.stable.err
---
a/sql/test/BugTracker-2010/Tests/incomplete-utf8-sequence.Bug-2575.stable.err
+++
b/sql/test/BugTracker-2010/Tests/incomplete-utf8-sequence.Bug-2575.stable.err
@@ -71,7 +71,6 @@ MAPI = (monetdb) /var/tmp/mtest-23209/.
QUERY = copy 2 records into tbl_bug2575 from stdin using delimiters
'\t','\n','';
10001160000 29 .v.
10001690001 0 coÃ
-
ERROR = !value 'coÃ' from line 2 field 3 not inserted, expecting type char(3)
!failed to import table
diff --git a/sql/test/BugTracker/Tests/copy_decimal.SF-1582957.stable.err
b/sql/test/BugTracker/Tests/copy_decimal.SF-1582957.stable.err
--- a/sql/test/BugTracker/Tests/copy_decimal.SF-1582957.stable.err
+++ b/sql/test/BugTracker/Tests/copy_decimal.SF-1582957.stable.err
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
