Changeset: d15efa442679 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/d15efa442679
Removed Files:
sql/backends/monet5/sql_copyinto.c
sql/backends/monet5/sql_copyinto.h
sql/backends/monet5/sql_copyinto_dec_tmpl.h
sql/backends/monet5/sql_copyinto_int_tmpl.h
sql/test/copy/unit/Tests/All
sql/test/copy/unit/Tests/str2buf2str.maltest
Modified Files:
clients/Tests/exports.stable.out
gdk/gdk_atoms.h
gdk/gdk_string.c
monetdb5/modules/mal/tablet.c
monetdb5/modules/mal/tablet.h
monetdb5/optimizer/opt_prelude.c
monetdb5/optimizer/opt_prelude.h
monetdb5/optimizer/opt_support.c
sql/backends/monet5/CMakeLists.txt
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql.h
sql/backends/monet5/sql_result.c
sql/backends/monet5/sql_result.h
sql/common/sql_types.c
sql/storage/bat/bat_storage.c
testing/mapicursor.py
Branch: copyparpipe
Log Message:
Roll back the directappend code, we're not going to use it
diffs (truncated from 6241 to 300 lines):
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -583,7 +583,6 @@ ssize_t shtFromStr(const char *src, size
ssize_t shtToStr(str *dst, size_t *len, const sht *src, bool external);
const sht sht_nil;
ssize_t strFromStr(const char *restrict src, size_t *restrict len, str
*restrict dst, bool external);
-size_t strPrintWidth(const char *restrict s);
const char str_nil[2];
int strerror_r(int errnum, char *buf, size_t buflen);
timestamp timestamp_add_month(timestamp t, int m) __attribute__((__const__));
@@ -748,6 +747,8 @@ str BKCshrinkBAT(bat *ret, const bat *bi
ssize_t BLOBtostr(str *tostr, size_t *l, const void *pin, bool external);
str CLTsessions(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
str CLTshutdown(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
+str COPYrejects(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
+str COPYrejects_clear(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
str GRPgroup1(bat *ngid, bat *next, bat *nhis, const bat *bid);
str GRPsubgroup5(bat *ngid, bat *next, bat *nhis, const bat *bid, const bat
*sid, const bat *gid, const bat *eid, const bat *hid);
int MAL_MAXCLIENTS;
@@ -794,8 +795,12 @@ str QLOGenableThreshold(void *ret, int *
int QLOGisset(void);
QueryQueue QRYqueue;
str RMTdisconnect(void *ret, str *conn);
-void TABLETdestroy_outputformat(OutputTable *as);
-int TABLEToutput_file(OutputTable *as, BAT *order, stream *s);
+BUN SQLload_file(Client cntxt, Tablet *as, bstream *b, stream *out, const char
*csep, const char *rsep, char quote, lng skip, lng maxrow, int best, bool
from_stdin, const char *tabnam, bool escape);
+str TABLETcollect(BAT **bats, Tablet *as);
+str TABLETcollect_parts(BAT **bats, Tablet *as, BUN offset);
+str TABLETcreate_bats(Tablet *as, BUN est);
+void TABLETdestroy_format(Tablet *as);
+int TABLEToutput_file(Tablet *as, BAT *order, stream *s);
int TRACEtable(Client cntxt, BAT **r);
str TRNglobal_abort(bit *ret);
str TRNglobal_commit(bit *ret);
@@ -842,7 +847,6 @@ const char *alter_viewRef;
const char *andRef;
const char *appendBulkRef;
const char *appendRef;
-const char *append_fromRef;
const char *appendidxRef;
const char *arrayRef;
const char *assertRef;
diff --git a/gdk/gdk_atoms.h b/gdk/gdk_atoms.h
--- a/gdk/gdk_atoms.h
+++ b/gdk/gdk_atoms.h
@@ -136,7 +136,6 @@ gdk_export ssize_t GDKstrFromStr(unsigne
gdk_export ssize_t strFromStr(const char *restrict src, size_t *restrict len,
str *restrict dst, bool external);
gdk_export size_t escapedStrlen(const char *restrict src, const char *sep1,
const char *sep2, int quote);
gdk_export size_t escapedStr(char *restrict dst, const char *restrict src,
size_t dstlen, const char *sep1, const char *sep2, int quote);
-gdk_export int strPrintWidth(const char *restrict s);
/*
* @- nil values
* All types have a single value designated as a NIL value. It
diff --git a/gdk/gdk_string.c b/gdk/gdk_string.c
--- a/gdk/gdk_string.c
+++ b/gdk/gdk_string.c
@@ -585,172 +585,6 @@ strFromStr(const char *restrict src, siz
(ssize_t) (cur - start));
}
-int
-strPrintWidth(const char *restrict s)
-{
- int len = 0;
- int c;
- int n;
-
- if (strNil(s))
- return int_nil;
- c = 0;
- n = 0;
- while (*s != 0) {
- if ((*s & 0x80) == 0) {
- assert(n == 0);
- len++;
- n = 0;
- } else if ((*s & 0xC0) == 0x80) {
- c = (c << 6) | (*s & 0x3F);
- if (--n == 0) {
- /* last byte of a multi-byte character */
- len++;
- /* this list was created by combining
- * the code points marked as
- * Emoji_Presentation in
- * /usr/share/unicode/emoji/emoji-data.txt
- * and code points marked either F or
- * W in EastAsianWidth.txt; this list
- * is up-to-date with Unicode 9.0 */
- if ((0x1100 <= c && c <= 0x115F) ||
- (0x231A <= c && c <= 0x231B) ||
- (0x2329 <= c && c <= 0x232A) ||
- (0x23E9 <= c && c <= 0x23EC) ||
- c == 0x23F0 ||
- c == 0x23F3 ||
- (0x25FD <= c && c <= 0x25FE) ||
- (0x2614 <= c && c <= 0x2615) ||
- (0x2648 <= c && c <= 0x2653) ||
- c == 0x267F ||
- c == 0x2693 ||
- c == 0x26A1 ||
- (0x26AA <= c && c <= 0x26AB) ||
- (0x26BD <= c && c <= 0x26BE) ||
- (0x26C4 <= c && c <= 0x26C5) ||
- c == 0x26CE ||
- c == 0x26D4 ||
- c == 0x26EA ||
- (0x26F2 <= c && c <= 0x26F3) ||
- c == 0x26F5 ||
- c == 0x26FA ||
- c == 0x26FD ||
- c == 0x2705 ||
- (0x270A <= c && c <= 0x270B) ||
- c == 0x2728 ||
- c == 0x274C ||
- c == 0x274E ||
- (0x2753 <= c && c <= 0x2755) ||
- c == 0x2757 ||
- (0x2795 <= c && c <= 0x2797) ||
- c == 0x27B0 ||
- c == 0x27BF ||
- (0x2B1B <= c && c <= 0x2B1C) ||
- c == 0x2B50 ||
- c == 0x2B55 ||
- (0x2E80 <= c && c <= 0x2E99) ||
- (0x2E9B <= c && c <= 0x2EF3) ||
- (0x2F00 <= c && c <= 0x2FD5) ||
- (0x2FF0 <= c && c <= 0x2FFB) ||
- (0x3000 <= c && c <= 0x303E) ||
- (0x3041 <= c && c <= 0x3096) ||
- (0x3099 <= c && c <= 0x30FF) ||
- (0x3105 <= c && c <= 0x312D) ||
- (0x3131 <= c && c <= 0x318E) ||
- (0x3190 <= c && c <= 0x31BA) ||
- (0x31C0 <= c && c <= 0x31E3) ||
- (0x31F0 <= c && c <= 0x321E) ||
- (0x3220 <= c && c <= 0x3247) ||
- (0x3250 <= c && c <= 0x32FE) ||
- (0x3300 <= c && c <= 0x4DBF) ||
- (0x4E00 <= c && c <= 0xA48C) ||
- (0xA490 <= c && c <= 0xA4C6) ||
- (0xA960 <= c && c <= 0xA97C) ||
- (0xAC00 <= c && c <= 0xD7A3) ||
- (0xF900 <= c && c <= 0xFAFF) ||
- (0xFE10 <= c && c <= 0xFE19) ||
- (0xFE30 <= c && c <= 0xFE52) ||
- (0xFE54 <= c && c <= 0xFE66) ||
- (0xFE68 <= c && c <= 0xFE6B) ||
- (0xFF01 <= c && c <= 0xFF60) ||
- (0xFFE0 <= c && c <= 0xFFE6) ||
- c == 0x16FE0 ||
- (0x17000 <= c && c <= 0x187EC) ||
- (0x18800 <= c && c <= 0x18AF2) ||
- (0x1B000 <= c && c <= 0x1B001) ||
- c == 0x1F004 ||
- c == 0x1F0CF ||
- c == 0x1F18E ||
- (0x1F191 <= c && c <= 0x1F19A) ||
- /* removed 0x1F1E6..0x1F1FF */
- (0x1F200 <= c && c <= 0x1F202) ||
- (0x1F210 <= c && c <= 0x1F23B) ||
- (0x1F240 <= c && c <= 0x1F248) ||
- (0x1F250 <= c && c <= 0x1F251) ||
- (0x1F300 <= c && c <= 0x1F320) ||
- (0x1F32D <= c && c <= 0x1F335) ||
- (0x1F337 <= c && c <= 0x1F37C) ||
- (0x1F37E <= c && c <= 0x1F393) ||
- (0x1F3A0 <= c && c <= 0x1F3CA) ||
- (0x1F3CF <= c && c <= 0x1F3D3) ||
- (0x1F3E0 <= c && c <= 0x1F3F0) ||
- c == 0x1F3F4 ||
- (0x1F3F8 <= c && c <= 0x1F43E) ||
- c == 0x1F440 ||
- (0x1F442 <= c && c <= 0x1F4FC) ||
- (0x1F4FF <= c && c <= 0x1F53D) ||
- (0x1F54B <= c && c <= 0x1F54E) ||
- (0x1F550 <= c && c <= 0x1F567) ||
- c == 0x1F57A ||
- (0x1F595 <= c && c <= 0x1F596) ||
- c == 0x1F5A4 ||
- (0x1F5FB <= c && c <= 0x1F64F) ||
- (0x1F680 <= c && c <= 0x1F6C5) ||
- c == 0x1F6CC ||
- (0x1F6D0 <= c && c <= 0x1F6D2) ||
- (0x1F6EB <= c && c <= 0x1F6EC) ||
- (0x1F6F4 <= c && c <= 0x1F6F6) ||
- (0x1F910 <= c && c <= 0x1F91E) ||
- (0x1F920 <= c && c <= 0x1F927) ||
- c == 0x1F930 ||
- (0x1F933 <= c && c <= 0x1F93E) ||
- (0x1F940 <= c && c <= 0x1F94B) ||
- (0x1F950 <= c && c <= 0x1F95E) ||
- (0x1F980 <= c && c <= 0x1F991) ||
- c == 0x1F9C0 ||
- (0x20000 <= c && c <= 0x2FFFD) ||
- (0x30000 <= c && c <= 0x3FFFD))
- len++;
- }
- } else if ((*s & 0xE0) == 0xC0) {
- assert(n == 0);
- n = 1;
- c = *s & 0x1F;
- } else if ((*s & 0xF0) == 0xE0) {
- assert(n == 0);
- n = 2;
- c = *s & 0x0F;
- } else if ((*s & 0xF8) == 0xF0) {
- assert(n == 0);
- n = 3;
- c = *s & 0x07;
- } else if ((*s & 0xFC) == 0xF8) {
- assert(n == 0);
- n = 4;
- c = *s & 0x03;
- } else {
- assert(0);
- n = 0;
- }
- s++;
- }
- return len;
-
-}
-
-
-
-
/*
* Convert a GDK string value to something printable.
*/
diff --git a/monetdb5/modules/mal/tablet.c b/monetdb5/modules/mal/tablet.c
--- a/monetdb5/modules/mal/tablet.c
+++ b/monetdb5/modules/mal/tablet.c
@@ -6,15 +6,92 @@
* Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
*/
+/*
+ * Niels Nes, Martin Kersten
+ *
+ * Parallel bulk load for SQL
+ * The COPY INTO command for SQL is heavily CPU bound, which means
+ * that ideally we would like to exploit the multi-cores to do that
+ * work in parallel.
+ * Complicating factors are the initial record offset, the
+ * possible variable length of the input, and the original sort order
+ * that should preferable be maintained.
+ *
+ * The code below consists of a file reader, which breaks up the
+ * file into chunks of distinct rows. Then multiple parallel threads
+ * grab them, and break them on the field boundaries.
+ * After all fields are identified this way, the columns are converted
+ * and stored in the BATs.
+ *
+ * The threads get a reference to a private copy of the READERtask.
+ * It includes a list of columns they should handle. This is a basis
+ * to distributed cheap and expensive columns over threads.
+ *
+ * The file reader overlaps IO with updates of the BAT.
+ * Also the buffer size of the block stream might be a little small for
+ * this task (1MB). It has been increased to 8MB, which indeed improved.
+ *
+ * The work divider allocates subtasks to threads based on the
+ * observed time spending so far.
+ */
+
#include "monetdb_config.h"
#include "tablet.h"
+#include "mapi_prompt.h"
#include <string.h>
+#include <ctype.h>
+
+#define MAXWORKERS 64
+#define MAXBUFFERS 2
+/* We restrict the row length to be 32MB for the time being */
+#define MAXROWSIZE(X) (X > 32*1024*1024 ? X : 32*1024*1024)
+
+static MT_Lock errorlock = MT_LOCK_INITIALIZER(errorlock);
+
+static BAT *
+void_bat_create(int adt, BUN nr)
+{
+ BAT *b = COLnew(0, adt, nr, TRANSIENT);
+
+ /* check for correct structures */
+ if (b == NULL)
+ return NULL;
+ if ((b = BATsetaccess(b, BAT_APPEND)) == NULL) {
+ return NULL;
+ }
+
+ /* disable all properties here */
+ b->tsorted = false;
+ b->trevsorted = false;
+ b->tnosorted = 0;
+ b->tnorevsorted = 0;
+ b->tseqbase = oid_nil;
+ b->tkey = false;
+ b->tnokey[0] = 0;
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
