Changeset: a41582531767 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a41582531767
Modified Files:
sql/backends/monet5/sql.mx
sql/server/sql_statement.c
sql/server/sql_statement.h
Branch: sciql
Log Message:
Merge with default branch.
diffs (truncated from 1599 to 300 lines):
diff --git a/NT/Makefile b/NT/Makefile
--- a/NT/Makefile
+++ b/NT/Makefile
@@ -22,9 +22,12 @@ srcdir = $(TOPDIR)\..
prefix = $(MAKEDIR)
+# ensure "all" is first target
+all: _all
+
!INCLUDE "$(TOPDIR)\..\NT\rules.msc"
-all: update_winconfig_conds_py "$(srcdir)\Makefile.msc" monetdb_config.h
unistd.h inttypes.h .monetdb
+_all: update_winconfig_conds_py "$(srcdir)\Makefile.msc" monetdb_config.h
unistd.h inttypes.h .monetdb
$(MAKE) /nologo /f "$(srcdir)\Makefile.msc" "prefix=$(prefix)"
"bits=$(bits)" all
install: targetdirs all
diff --git a/gdk/gdk_bbp.mx b/gdk/gdk_bbp.mx
--- a/gdk/gdk_bbp.mx
+++ b/gdk/gdk_bbp.mx
@@ -507,7 +507,18 @@ static int BBPdiskscan(const char *);
* conversion are fatal. This function is "safe" in the sense that
* when it is interrupted, recovery will just start over. No
* permanent changes are made to the database until all string heaps
- * have been converted, and then the changes are made atomically. */
+ * have been converted, and then the changes are made atomically.
+ *
+ * In addition to doing the conversion to all BATs with OID or STR
+ * columns (the latter since the format of the string heap is
+ * different for 64/32 and 64/64 configurations), we also look out for
+ * the *_catalog BATs that are used by gdk_logger. If we encounter
+ * such a BAT, we create a file based on the name of that BAT
+ * (i.e. using the first part of the name) to inform the logger that
+ * it too needs to convert 32 bit OIDs to 64 bits. If the process
+ * here gets interrupted, the logger is never called, and if we get
+ * then restarted, we just create the file again, so this process is
+ * safe. */
static void
fixoidheapcolumn(BAT *b, const char *nme, const char *filename,
@@ -650,6 +661,7 @@ fixoidheap(void)
long_str filename;
int ht, tt;
char hs, hvs = '\0', ts, tvs = '\0'; /* saved storage types */
+ size_t len;
fprintf(stderr,
"# upgrading database from 32 bit OIDs to 64 bit OIDs\n");
@@ -658,6 +670,20 @@ fixoidheap(void)
for (bid = 1; bid < BBPsize; bid++) {
if ((bs = BBP[bid].cache) == NULL)
continue; /* not a valid BAT */
+ if (BBP[bid].nme[0] &&
+ (len = strlen(BBP[bid].nme[0])) > 8 &&
+ strcmp(BBP[bid].nme[0] + len - 8, "_catalog") == 0) {
+ /* this is one of the files used by the
+ * logger. We need to communicate to the
+ * logger that it also needs to do a
+ * conversion. That is done by creating a
+ * file here based on the name of this BAT. */
+ snprintf(filename, sizeof(filename),
+ "%.*s_32-64-convert",
+ (int) (len - 8), BBP[bid].nme[0]);
+ fclose(fopen(filename, "w"));
+ }
+
/* OID and (non-void) varsized columns have to be rewritten */
if (bs->H.type != TYPE_oid &&
(bs->H.type == TYPE_void || !bs->H.varsized) &&
diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -299,6 +299,10 @@ log_read_updates(logger *lg, trans *tr,
void *(*rt) (ptr, stream *, size_t) = BATatoms[tt].atomRead;
void *tv = ATOMnil(tt);
+#if SIZEOF_OID == 8
+ if (tt == TYPE_oid && lg->read32bitoid)
+ rt = BATatoms[TYPE_int].atomRead;
+#endif
r = BATnew(ht, tt, l->nr);
if (hseq)
@@ -314,8 +318,18 @@ log_read_updates(logger *lg, trans *tr,
res = LOG_ERR;
break;
}
- if (l->flag == LOG_INSERT)
+ if (l->flag == LOG_INSERT) {
+#if SIZEOF_OID == 8
+ if (tt == TYPE_oid && lg->read32bitoid)
{
+ int vi = * (int *) t;
+ if (vi == int_nil)
+ * (oid *) t = oid_nil;
+ else
+ * (oid *) t = vi;
+ }
+#endif
BUNappend(r, t, TRUE);
+ }
if (t != tv)
GDKfree(t);
}
@@ -323,6 +337,11 @@ log_read_updates(logger *lg, trans *tr,
void *(*rh) (ptr, stream *, size_t) = ht == TYPE_void ?
BATatoms[TYPE_oid].atomRead : BATatoms[ht].atomRead;
void *hv = ATOMnil(ht);
+#if SIZEOF_OID == 8
+ if ((ht == TYPE_oid || ht == TYPE_void) &&
+ lg->read32bitoid)
+ rh = BATatoms[TYPE_int].atomRead;
+#endif
for (; l->nr > 0; l->nr--) {
void *h = rh(hv, lg->log, 1);
void *t = rt(tv, lg->log, 1);
@@ -331,6 +350,24 @@ log_read_updates(logger *lg, trans *tr,
res = LOG_ERR;
break;
}
+#if SIZEOF_OID == 8
+ if (lg->read32bitoid) {
+ if (ht == TYPE_void || ht == TYPE_oid) {
+ int vi = * (int *) h;
+ if (vi == int_nil)
+ * (oid *) h = oid_nil;
+ else
+ * (oid *) h = vi;
+ }
+ if (tt == TYPE_oid) {
+ int vi = * (int *) t;
+ if (vi == int_nil)
+ * (oid *) t = oid_nil;
+ else
+ * (oid *) t = vi;
+ }
+ }
+#endif
BUNins(r, h, t, TRUE);
if (h != hv)
GDKfree(h);
@@ -968,6 +1005,9 @@ logger_new(int debug, char *fn, char *lo
lg->id = 1;
lg->tid = 0;
+#if SIZEOF_OID == 8
+ lg->read32bitoid = 0;
+#endif
/* if the path is absolute, it means someone is still calling
* logger_create/logger_new "manually" */
@@ -1107,14 +1147,109 @@ logger_new(int debug, char *fn, char *lo
BBPrename(lg->freed->batCacheid, bak);
if (fp != NULL) {
+#if SIZEOF_OID == 8
+ char cvfile[BUFSIZ];
+#endif
+
if (check_version(lg, fp)) {
goto error;
}
+#if SIZEOF_OID == 8
+ /* When a file *_32-64-convert exists in the dbfarm,
+ * it was left there by the BBP initialization code
+ * when it did a conversion of 32-bit OIDs to 64 bits
+ * (see the comment above fixoidheapcolumn and
+ * fixoidheap in gdk_bbp). It the file exists, we
+ * first create a file called convert-32-64 in the log
+ * directory and we write the current log ID into that
+ * file. After this file is created, we delete the
+ * *_32-64-convert file in the dbfarm. We then know
+ * that while reading the logs, we have to read OID
+ * values as 32 bits (this is indicated by setting the
+ * read32bitoid flag). When we're done reading the
+ * logs, we remove the file (and reset the flag). If
+ * we get interrupted before we have written this
+ * file, the file in the dbfarm will still exist, so
+ * the next time we're started, BBPinit will not
+ * convert OIDs (that was done before we got
+ * interrupted), but we will still know to convert the
+ * OIDs ourselves. If we get interrupted after we
+ * have deleted the file from the dbfarm, we check
+ * whether the file convert-32-64 exists and if it
+ * contains the expected ID. If it does, we again
+ * know that we have to convert. If the ID is not
+ * what we expect, the conversion was apparently done
+ * already, and so we can delete the file. */
+
+ snprintf(cvfile, sizeof(cvfile),
+ "%s%c%s%c%s%c%s%cconvert-32-64",
+ GDKgetenv("gdk_dbfarm"), DIR_SEP, dbname,
+ DIR_SEP, logdir, DIR_SEP, fn, DIR_SEP);
+ snprintf(bak, sizeof(bak), "%s_32-64-convert", fn);
+ {
+ FILE *fp1;
+ long off;
+ int curid;
+
+ /* read the current log id without disturbing
+ * the file pointer */
+ off = ftell(fp);
+ if (fscanf(fp, "%d", &curid) != 1)
+ curid = -1; /* shouldn't happen? */
+ fseek(fp, off, SEEK_SET);
+
+ if ((fp1 = fopen(bak, "r")) != NULL) {
+ /* file indicating that we need to do
+ * a 32->64 bit OID conversion exists;
+ * record the fact in case we get
+ * interrupted, and set the flag so
+ * that we actually do what's asked */
+ fclose(fp1);
+ /* first create a versioned file using
+ * the current log id */
+ fp1 = fopen(cvfile, "w");
+ fprintf(fp1, "%d\n", curid);
+ fclose(fp1);
+ /* then remove the unversioned file
+ * that gdk_bbp created (in this
+ * order!) */
+ unlink(bak);
+ /* set the flag that we need to convert */
+ lg->read32bitoid = 1;
+ } else if ((fp1 = fopen(cvfile, "r")) != NULL) {
+ /* the versioned conversion file
+ * exists: check version */
+ int newid;
+
+ if (fscanf(fp1, "%d", &newid) == 1 &&
+ newid == curid) {
+ /* versions match, we need to
+ * convert */
+ lg->read32bitoid = 1;
+ }
+ fclose(fp1);
+ if (!lg->read32bitoid) {
+ /* no conversion, so we can
+ * remove the versioned
+ * file */
+ unlink(cvfile);
+ }
+ }
+ }
+#endif
lg->changes++;
logger_readlogs(lg, fp, filename);
fclose(fp);
fp = NULL;
+#if SIZEOF_OID == 8
+ if (lg->read32bitoid) {
+ /* we converted, remove versioned file and
+ * reset conversion flag */
+ unlink(cvfile);
+ lg->read32bitoid = 0;
+ }
+#endif
if (lg->postfuncp)
(*lg->postfuncp)(lg);
}
diff --git a/gdk/gdk_logger.h b/gdk/gdk_logger.h
--- a/gdk/gdk_logger.h
+++ b/gdk/gdk_logger.h
@@ -53,6 +53,11 @@ typedef struct logger {
int version;
lng id;
int tid;
+#if SIZEOF_OID == 8
+ /* on 64-bit architecture, read OIDs as 32 bits (for upgrading
+ * oid size) */
+ int read32bitoid;
+#endif
char *fn;
char *dir;
preversionfix_fptr prefuncp;
diff --git a/monetdb5/modules/mal/pcre.c b/monetdb5/modules/mal/pcre.c
--- a/monetdb5/modules/mal/pcre.c
+++ b/monetdb5/modules/mal/pcre.c
@@ -447,6 +447,7 @@ pcre_uselect(BAT **res, str pattern, BAT
r->tsorted = FALSE;
my_pcre_free(re);
+ my_pcre_free(pe);
if (!(r->batDirty&2)) r = BATsetaccess(r, BAT_READ);
*res = r;
return MAL_SUCCEED;
diff --git a/monetdb5/modules/mal/tablet.mx b/monetdb5/modules/mal/tablet.mx
--- a/monetdb5/modules/mal/tablet.mx
+++ b/monetdb5/modules/mal/tablet.mx
@@ -1142,7 +1142,7 @@ create_loadformat(Tablet * as, BAT *name
fmt[p].name = (char *) bun_tail(names, p);
fmt[p].sep = sep_dup((char *) bun_tail(seps, p));
fmt[p].seplen = (int) strlen(fmt[p].sep);
- fmt[p].type = (char *) bun_tail(types, p);
+ fmt[p].type = GDKstrdup((char *) bun_tail(types, p));
fmt[p].adt = ATOMindex(fmt[p].type);
if (fmt[p].adt <= 0) {
GDKerror("create_loadformat: %s has unknown type %s
(using str instead).\n", fmt[p].name, fmt[p].name);
@@ -1196,7 +1196,7 @@ create_dumpformat(Tablet * as, BAT *name
fmt[p].name = (char *) bun_tail(names, p);
_______________________________________________
Checkin-list mailing list
Checkin-list@monetdb.org
http://mail.monetdb.org/mailman/listinfo/checkin-list
