Changeset: f77be188a4c3 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f77be188a4c3
Added Files:
sql/test/BugTracker-2017/Tests/python_loader_clobbers_default_with_null.Bug-6464.sql
sql/test/BugTracker-2017/Tests/python_loader_clobbers_default_with_null.Bug-6464.stable.err
sql/test/BugTracker-2017/Tests/python_loader_clobbers_default_with_null.Bug-6464.stable.out
sql/test/BugTracker-2017/Tests/skip_problem_best_effort.Bug-6442.sql
sql/test/BugTracker-2017/Tests/skip_problem_best_effort.Bug-6442.stable.err
sql/test/BugTracker-2017/Tests/skip_problem_best_effort.Bug-6442.stable.out
sql/test/mergetables/Tests/sqlsmith-exist-lateral.sql
sql/test/mergetables/Tests/sqlsmith-exist-lateral.stable.err
sql/test/mergetables/Tests/sqlsmith-exist-lateral.stable.out
Modified Files:
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
clients/mapiclient/dump.c
gdk/gdk_heap.c
gdk/gdk_private.h
gdk/gdk_storage.c
monetdb5/modules/mal/tablet.c
sql/backends/monet5/Tests/pyloader01.sql
sql/backends/monet5/Tests/pyloader01.stable.out
sql/backends/monet5/UDF/pyapi/emit.c
sql/backends/monet5/UDF/pyapi/pyloader.c
sql/backends/monet5/sql.mal
sql/backends/monet5/sql_scenario.c
sql/backends/monet5/sql_scenario.h
sql/common/sql_types.c
sql/include/sql_catalog.h
sql/scripts/99_system.sql
sql/server/rel_select.c
sql/server/rel_updates.c
sql/storage/sql_storage.h
sql/storage/store.c
sql/test/BugTracker-2017/Tests/All
sql/test/Dependencies/Tests/Dependencies.stable.out
sql/test/Dependencies/Tests/Dependencies.stable.out.int128
sql/test/leaks/Tests/check1.stable.out
sql/test/leaks/Tests/check1.stable.out.int128
sql/test/mergetables/Tests/All
Branch: default
Log Message:
Merge with Jul2017 branch.
diffs (truncated from 1739 to 300 lines):
diff --git a/clients/Tests/MAL-signatures.stable.out
b/clients/Tests/MAL-signatures.stable.out
--- a/clients/Tests/MAL-signatures.stable.out
+++ b/clients/Tests/MAL-signatures.stable.out
@@ -8447,6 +8447,8 @@ Ready.
[ "sql", "transaction_release", "pattern
sql.transaction_release(chain:int, name:str):void ",
"SQLtransaction_release;", "A transaction statement (type can be
commit,release,rollback or start)" ]
[ "sql", "transaction_rollback", "pattern
sql.transaction_rollback(chain:int, name:str):void ",
"SQLtransaction_rollback;", "A transaction statement (type can be
commit,release,rollback or start)" ]
[ "sql", "update", "pattern sql.update(mvc:int, sname:str,
tname:str, cname:str, rids:any, upd:any):int ", "mvc_update_wrap;", "Update
the values of the column tname.cname. Returns sequence number for order
dependence)" ]
+[ "sql", "update_schemas", "pattern sql.update_schemas():void ",
"SYSupdate_schemas;", "Procedure triggered on update of the sys.schemas
table" ]
+[ "sql", "update_tables", "pattern sql.update_tables():void ",
"SYSupdate_tables;", "Procedure triggered on update of the sys._tables
table" ]
[ "sql", "vacuum", "pattern sql.vacuum(sch:str, tbl:str):void ",
"SQLvacuum;", "Choose an approach to consolidate the deletions" ]
[ "sql", "zero_or_one", "command sql.zero_or_one(col:bat[:any_1]):any_1
", "zero_or_one;", "if col contains exactly one value return this. Incase
of more raise an exception else return nil" ]
[ "sqlblob", "#fromstr", "command sqlblob.#fromstr():void ",
"SQLBLOBfromstr;", "" ]
diff --git a/clients/Tests/MAL-signatures.stable.out.int128
b/clients/Tests/MAL-signatures.stable.out.int128
--- a/clients/Tests/MAL-signatures.stable.out.int128
+++ b/clients/Tests/MAL-signatures.stable.out.int128
@@ -10811,6 +10811,8 @@ Ready.
[ "sql", "transaction_release", "pattern
sql.transaction_release(chain:int, name:str):void ",
"SQLtransaction_release;", "A transaction statement (type can be
commit,release,rollback or start)" ]
[ "sql", "transaction_rollback", "pattern
sql.transaction_rollback(chain:int, name:str):void ",
"SQLtransaction_rollback;", "A transaction statement (type can be
commit,release,rollback or start)" ]
[ "sql", "update", "pattern sql.update(mvc:int, sname:str,
tname:str, cname:str, rids:any, upd:any):int ", "mvc_update_wrap;", "Update
the values of the column tname.cname. Returns sequence number for order
dependence)" ]
+[ "sql", "update_schemas", "pattern sql.update_schemas():void ",
"SYSupdate_schemas;", "Procedure triggered on update of the sys.schemas
table" ]
+[ "sql", "update_tables", "pattern sql.update_tables():void ",
"SYSupdate_tables;", "Procedure triggered on update of the sys._tables
table" ]
[ "sql", "vacuum", "pattern sql.vacuum(sch:str, tbl:str):void ",
"SQLvacuum;", "Choose an approach to consolidate the deletions" ]
[ "sql", "zero_or_one", "command sql.zero_or_one(col:bat[:any_1]):any_1
", "zero_or_one;", "if col contains exactly one value return this. Incase
of more raise an exception else return nil" ]
[ "sqlblob", "#fromstr", "command sqlblob.#fromstr():void ",
"SQLBLOBfromstr;", "" ]
diff --git a/clients/mapiclient/dump.c b/clients/mapiclient/dump.c
--- a/clients/mapiclient/dump.c
+++ b/clients/mapiclient/dump.c
@@ -1238,6 +1238,7 @@ dump_function(Mapi mid, stream *toConsol
/* all information is stored in the func column */
mnstr_printf(toConsole, "%s\n", ffunc);
mapi_close_handle(hdl);
+ free(query);
return 0;
}
mnstr_printf(toConsole, "CREATE ");
@@ -1612,7 +1613,7 @@ dump_database(Mapi mid, stream *toConsol
"sys.schemas s, "
"sys._tables t "
"WHERE s.id = t.schema_id AND "
- "t.id = tr.table_id"
+ "t.id = tr.table_id AND t.system = FALSE"
") "
"SELECT sname, name, query, type FROM vft ORDER BY id";
char *sname = NULL;
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -1003,7 +1003,7 @@ HEAP_malloc(Heap *heap, size_t nbytes)
*/
ttrail = 0;
trail = 0;
- for (block = hheader->head; block != 0; block = HEAP_index(heap, block,
CHUNK)->next) {
+ for (block = hheader->head; block != 0; block = blockp->next) {
blockp = HEAP_index(heap, block, CHUNK);
#ifdef TRACE
@@ -1171,3 +1171,118 @@ HEAP_free(Heap *heap, var_t mem)
hheader->head = block;
}
}
+
+void
+HEAP_recover(Heap *h, const var_t *offsets, BUN noffsets)
+{
+ HEADER *hheader;
+ CHUNK *blockp;
+ size_t dirty = 0;
+ var_t maxoff = 0;
+ BUN i;
+
+ if (!h->cleanhash)
+ return;
+ hheader = HEAP_index(h, 0, HEADER);
+ assert(h->free >= sizeof(HEADER));
+ assert(hheader->version == HEAPVERSION);
+ assert(h->size >= hheader->firstblock);
+ for (i = 0; i < noffsets; i++)
+ if (offsets[i] > maxoff)
+ maxoff = offsets[i];
+ assert(maxoff < h->free);
+ if (maxoff == 0) {
+ if (hheader->head != hheader->firstblock) {
+ hheader->head = hheader->firstblock;
+ dirty = sizeof(HEADER);
+ }
+ blockp = HEAP_index(h, hheader->firstblock, CHUNK);
+ if (blockp->next != 0 ||
+ blockp->size != h->size - hheader->head) {
+ blockp->size = (size_t) (h->size - hheader->head);
+ blockp->next = 0;
+ dirty = hheader->firstblock + sizeof(CHUNK);
+ }
+ } else {
+ size_t block = maxoff - hheader->alignment;
+ size_t end = block + *HEAP_index(h, block, size_t);
+ size_t trail;
+
+ assert(end <= h->free);
+ if (end + sizeof(CHUNK) <= h->free) {
+ blockp = HEAP_index(h, end, CHUNK);
+ if (hheader->head <= end &&
+ blockp->next == 0 &&
+ blockp->size == h->free - end)
+ return;
+ } else if (hheader->head == 0) {
+ /* no free space after last allocated block
+ * and no free list */
+ return;
+ }
+ block = hheader->head;
+ trail = 0;
+ while (block < maxoff && block != 0) {
+ blockp = HEAP_index(h, block, CHUNK);
+ trail = block;
+ block = blockp->next;
+ }
+ if (trail == 0) {
+ /* no free list */
+ if (end + sizeof(CHUNK) > h->free) {
+ /* no free space after last allocated
+ * block */
+ if (hheader->head != 0) {
+ hheader->head = 0;
+ dirty = sizeof(HEADER);
+ }
+ } else {
+ /* there is free space after last
+ * allocated block */
+ if (hheader->head != end) {
+ hheader->head = end;
+ dirty = sizeof(HEADER);
+ }
+ blockp = HEAP_index(h, end, CHUNK);
+ if (blockp->next != 0 ||
+ blockp->size != h->free - end) {
+ blockp->next = 0;
+ blockp->size = h->free - end;
+ dirty = end + sizeof(CHUNK);
+ }
+ }
+ } else {
+ /* there is a free list */
+ blockp = HEAP_index(h, trail, CHUNK);
+ if (end + sizeof(CHUNK) > h->free) {
+ /* no free space after last allocated
+ * block */
+ if (blockp->next != 0) {
+ blockp->next = 0;
+ dirty = trail + sizeof(CHUNK);
+ }
+ } else {
+ /* there is free space after last
+ * allocated block */
+ if (blockp->next != end) {
+ blockp->next = end;
+ dirty = trail + sizeof(CHUNK);
+ }
+ blockp = HEAP_index(h, end, CHUNK);
+ if (blockp->next != 0 ||
+ blockp->size != h->free - end) {
+ blockp->next = 0;
+ blockp->size = h->free - end;
+ dirty = end + sizeof(CHUNK);
+ }
+ }
+ }
+ }
+ h->cleanhash = 0;
+ if (dirty) {
+ if (h->storage == STORE_MMAP)
+ (void) MT_msync(h->base, dirty);
+ else
+ h->dirty = 1;
+ }
+}
diff --git a/gdk/gdk_private.h b/gdk/gdk_private.h
--- a/gdk/gdk_private.h
+++ b/gdk/gdk_private.h
@@ -172,6 +172,8 @@ void BBPdump(void); /* never called: fo
__hidden gdk_return HEAPload(Heap *h, const char *nme, const char *ext, int
trunc)
__attribute__((__warn_unused_result__))
__attribute__((__visibility__("hidden")));
+__hidden void HEAP_recover(Heap *, const var_t *, BUN)
+ __attribute__((__visibility__("hidden")));
__hidden gdk_return HEAPsave(Heap *h, const char *nme, const char *ext)
__attribute__((__warn_unused_result__))
__attribute__((__visibility__("hidden")));
diff --git a/gdk/gdk_storage.c b/gdk/gdk_storage.c
--- a/gdk/gdk_storage.c
+++ b/gdk/gdk_storage.c
@@ -824,6 +824,9 @@ BATload_intern(bat bid, int lock)
}
if (ATOMstorage(b->ttype) == TYPE_str) {
strCleanHash(b->tvheap, FALSE); /* ensure consistency */
+ } else {
+ HEAP_recover(b->tvheap, (const var_t *) Tloc(b, 0),
+ BATcount(b));
}
}
diff --git a/monetdb5/modules/mal/tablet.c b/monetdb5/modules/mal/tablet.c
--- a/monetdb5/modules/mal/tablet.c
+++ b/monetdb5/modules/mal/tablet.c
@@ -1834,6 +1834,8 @@ SQLload_file(Client cntxt, Tablet *as, b
int width;
for (attr = 0; attr < as->nr_attrs; attr++) {
+ if (as->format[attr].skip)
+ continue;
width = as->format[attr].c->twidth;
switch (width){
case 1:
diff --git a/sql/backends/monet5/Tests/pyloader01.sql
b/sql/backends/monet5/Tests/pyloader01.sql
--- a/sql/backends/monet5/Tests/pyloader01.sql
+++ b/sql/backends/monet5/Tests/pyloader01.sql
@@ -1,7 +1,7 @@
START TRANSACTION;
-CREATE TABLE mytable(a DOUBLE, d int, s STRING);
+CREATE TABLE mytable(a DOUBLE, d int, s STRING DEFAULT 'hello');
CREATE LOADER myfunc() LANGUAGE PYTHON {
_emit.emit({'a':42,'d':1})
@@ -23,10 +23,10 @@ SELECT name,func,mod,language,type,side_
-- there is a reason for this, functions with 0, 1, 2 and 3+ arguments are
handled differently.
+COPY LOADER INTO mytable FROM myfunc();
COPY LOADER INTO mytable FROM myfunc3(46, 'asdf', 3.2);
COPY LOADER INTO mytable FROM myfunc2(45, 'asdf');
COPY LOADER INTO mytable FROM myfunc1(44);
-COPY LOADER INTO mytable FROM myfunc();
SELECT * FROM mytable;
diff --git a/sql/backends/monet5/Tests/pyloader01.stable.out
b/sql/backends/monet5/Tests/pyloader01.stable.out
--- a/sql/backends/monet5/Tests/pyloader01.stable.out
+++ b/sql/backends/monet5/Tests/pyloader01.stable.out
@@ -52,11 +52,11 @@ Ready.
% sys.mytable, sys.mytable, sys.mytable # table_name
% a, d, s # name
% double, int, clob # type
-% 24, 1, 5 # length
+% 24, 1, 7 # length
+[ 42, 1, "'hello'" ]
[ 46, 4, "hello" ]
-[ 45, 3, NULL ]
-[ 44, 2, NULL ]
-[ 42, 1, NULL ]
+[ 45, 3, "'hello'" ]
+[ 44, 2, "'hello'" ]
#DROP TABLE mytable;
#DROP ALL LOADER myfunc;
#CREATE LOADER myfunc() LANGUAGE PYTHON {
diff --git a/sql/backends/monet5/UDF/pyapi/emit.c
b/sql/backends/monet5/UDF/pyapi/emit.c
--- a/sql/backends/monet5/UDF/pyapi/emit.c
+++ b/sql/backends/monet5/UDF/pyapi/emit.c
@@ -199,6 +199,7 @@ PyObject *PyEmit_Emit(PyEmitObject *self
self->cols[self->ncols].b = COLnew(0, bat_type,
0, TRANSIENT);
self->cols[self->ncols].name = GDKstrdup(val);
+ self->cols[self->ncols].def = NULL;
if (self->nvals > 0) {
// insert NULL values up until the
current entry
for (ai = 0; ai < self->nvals; ai++) {
@@ -209,9 +210,9 @@ PyObject *PyEmit_Emit(PyEmitObject *self
goto wrapup;
}
}
- self->cols[i].b->tnil = 1;
- self->cols[i].b->tnonil = 0;
- BATsetcount(self->cols[i].b,
self->nvals);
+ self->cols[self->ncols].b->tnil = 1;
+ self->cols[self->ncols].b->tnonil = 0;
+ BATsetcount(self->cols[self->ncols].b,
self->nvals);
}
self->ncols++;
}
@@ -351,14 +352,21 @@ PyObject *PyEmit_Emit(PyEmitObject *self
self->cols[i].b->tnonil = 1 -
self->cols[i].b->tnil;
}
} else {
+ void* nill_value = ATOMnil(self->cols[i].b->ttype);
+ void* default_value = self->cols[i].def ?
+
self->cols[i].def :
+ nill_value;
for (ai = 0; ai < (size_t)el_count; ai++) {
- if (BUNappend(self->cols[i].b,
ATOMnil(self->cols[i].b->ttype),
+ if (BUNappend(self->cols[i].b,
+ default_value,
0) != GDK_SUCCEED) {
goto wrapup;
}
}
- self->cols[i].b->tnil = 1;
- self->cols[i].b->tnonil = 0;
+ if
(BATatoms[self->cols[i].b->ttype].atomCmp(default_value, nill_value) == 0) {
+ self->cols[i].b->tnil = 1;
+ self->cols[i].b->tnonil = 0;
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
