Changeset: 6be7362eb489 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6be7362eb489
Modified Files:
README
common/stream/stream.c
monetdb5/extras/jaql/json.c
testing/Mtest.py.in
Branch: default
Log Message:
Merge with Oct2012 branch.
diffs (truncated from 451 to 300 lines):
diff --git a/README b/README
--- a/README
+++ b/README
@@ -35,3 +35,4 @@ The Initial Developer of the Original Co
Portions created by CWI are Copyright (C) 1997-July 2008 CWI.
Copyright August 2008-2012 MonetDB B.V.
All Rights Reserved.
+
diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -799,14 +799,57 @@ stream *open_gzwastream(const char *file
/* streams working on a bzip2-compressed disk file */
#ifdef HAVE_LIBBZ2
+struct bz {
+ BZFILE *b;
+ FILE *f;
+};
+
+static void
+stream_bzclose(stream *s)
+{
+ int err = BZ_OK;
+
+ if (s->stream_data.p) {
+ if (s->access == ST_READ)
+ BZ2_bzReadClose(&err, ((struct bz *)
s->stream_data.p)->b);
+ else
+ BZ2_bzWriteClose(&err, ((struct bz *)
s->stream_data.p)->b, 0, NULL, NULL);
+ fclose(((struct bz *) s->stream_data.p)->f);
+ free(s->stream_data.p);
+ }
+ s->stream_data.p = NULL;
+}
+
static ssize_t
stream_bzread(stream *s, void *buf, size_t elmsize, size_t cnt)
{
int size = (int) (elmsize * cnt);
-
- size = BZ2_bzread((BZFILE *) s->stream_data.p, buf, size);
- if (size)
- return size / elmsize;
+ int err;
+ void *punused;
+ int nunused;
+ char unused[BZ_MAX_UNUSED];
+
+ if (s->stream_data.p) {
+ size = BZ2_bzRead(&err, ((struct bz *) s->stream_data.p)->b,
buf, size);
+ if (err == BZ_STREAM_END) {
+ /* end of stream, but not necessarily end of
+ * file: get unused bits, close stream, and
+ * open again with the saved unused bits */
+ BZ2_bzReadGetUnused(&err, ((struct bz *)
s->stream_data.p)->b, &punused, &nunused);
+ if (err == BZ_OK &&
+ (nunused > 0 ||
+ !feof(((struct bz *) s->stream_data.p)->f))) {
+ if (nunused > 0)
+ memcpy(unused, punused, nunused);
+ BZ2_bzReadClose(&err, ((struct bz *)
s->stream_data.p)->b);
+ ((struct bz *) s->stream_data.p)->b =
BZ2_bzReadOpen(&err, ((struct bz *) s->stream_data.p)->f, 0, 0, unused,
nunused);
+ } else {
+ stream_bzclose(s);
+ }
+ }
+ if (err == BZ_OK)
+ return size / elmsize;
+ }
return 0;
}
@@ -814,27 +857,13 @@ static ssize_t
stream_bzwrite(stream *s, const void *buf, size_t elmsize, size_t cnt)
{
int size = (int) (elmsize * cnt);
+ int err;
if (size) {
- size = BZ2_bzwrite((BZFILE *) s->stream_data.p, (void *) buf,
size);
- return size / elmsize;
+ BZ2_bzWrite(&err, ((struct bz *) s->stream_data.p)->b, (void *)
buf, size);
+ if (err == BZ_OK)
+ return cnt;
}
- return cnt;
-}
-
-static void
-stream_bzclose(stream *s)
-{
- if (s->stream_data.p)
- BZ2_bzclose((BZFILE *) s->stream_data.p);
- s->stream_data.p = NULL;
-}
-
-static int
-stream_bzflush(stream *s)
-{
- if (s->access == ST_WRITE)
- BZ2_bzflush((BZFILE *) s->stream_data.p);
return 0;
}
@@ -842,17 +871,35 @@ static stream *
open_bzstream(const char *filename, const char *flags)
{
stream *s;
- BZFILE *fp;
-
- if ((s = create_stream(filename)) == NULL)
+ int err;
+ struct bz *bzp;
+
+ if ((bzp = malloc(sizeof(struct bz))) == NULL)
return NULL;
- if ((fp = BZ2_bzopen(filename, flags)) == NULL)
+ if ((s = create_stream(filename)) == NULL) {
+ free(bzp);
+ return NULL;
+ }
+ if ((bzp->f = fopen(filename, flags)) == NULL)
+ s->errnr = MNSTR_OPEN_ERROR;
+ if (strchr(flags, 'r') != NULL) {
+ bzp->b = BZ2_bzReadOpen(&err, bzp->f, 0, 0, NULL, 0);
+ s->access = ST_READ;
+ if (err == BZ_STREAM_END) {
+ BZ2_bzReadClose(&err, bzp->b);
+ bzp->b = NULL;
+ }
+ } else {
+ bzp->b = BZ2_bzWriteOpen(&err, bzp->f, 9, 0, 30);
+ s->access = ST_WRITE;
+ }
+ if (err != BZ_OK)
s->errnr = MNSTR_OPEN_ERROR;
s->read = stream_bzread;
s->write = stream_bzwrite;
s->close = stream_bzclose;
- s->flush = stream_bzflush;
- s->stream_data.p = (void *) fp;
+ s->flush = NULL;
+ s->stream_data.p = (void *) bzp;
return s;
}
@@ -865,7 +912,7 @@ open_bzrstream(const char *filename)
return NULL;
s->type = ST_BIN;
if (s->errnr == MNSTR_NO__ERROR &&
- BZ2_bzread((BZFILE *) s->stream_data.p, (void *) &s->byteorder,
sizeof(s->byteorder)) < (int) sizeof(s->byteorder)) {
+ stream_bzread(s, (void *) &s->byteorder, sizeof(s->byteorder), 1)
!= 1) {
stream_bzclose(s);
s->errnr = MNSTR_OPEN_ERROR;
}
@@ -882,7 +929,7 @@ open_bzwstream_(const char *filename, co
s->access = ST_WRITE;
s->type = ST_BIN;
if (s->errnr == MNSTR_NO__ERROR)
- BZ2_bzwrite((BZFILE *) s->stream_data.p, (void *)
&s->byteorder, sizeof(s->byteorder));
+ stream_bzwrite(s, (void *) &s->byteorder, sizeof(s->byteorder),
1);
return s;
}
diff --git a/monetdb5/extras/jaql/json.c b/monetdb5/extras/jaql/json.c
--- a/monetdb5/extras/jaql/json.c
+++ b/monetdb5/extras/jaql/json.c
@@ -54,35 +54,53 @@ static void json_error(jsonbat *, char *
_In_z_ _Printf_format_string_ const char *, ...)
__attribute__((__format__(__printf__, 3, 4)));
static void
-json_error(jsonbat *jb, char *p, const char *format, ...)
+json_error(jsonbat *jb, char *n, const char *format, ...)
{
va_list ap;
char message[8096];
size_t len;
char around[32];
- size_t off = p - jb->streambuf;
+ char *p, *q, *start = around;
char hadend = 0;
va_start(ap, format);
len = vsnprintf(message, sizeof(message), format, ap);
va_end(ap);
- if (off < 13)
- off = 13;
- off -= 13;
- if (snprintf(around, sizeof(around), "%s", jb->streambuf + off)
- <= (int)(sizeof(around)))
+ p = n;
+ q = around + 13 + 1;
+ *q = '\0';
+ for (; p >= jb->streambuf && q > around; p--) {
+ if (isspace(*p)) {
+ if (*q != ' ')
+ *--q = ' ';
+ } else if (*p == '\0') {
+ /* artifact from parse_json_string */
+ *--q = '"';
+ } else {
+ *--q = *p;
+ }
+ }
+ start = q;
+ p = n + 1;
+ q = around + 13;
+ for (; *p != '\0' && q - around < (ssize_t)sizeof(around) - 1; p++) {
+ if (isspace(*p)) {
+ if (*q != ' ')
+ *++q = ' ';
+ } else if (*p == '\0') {
+ /* artifact from parse_json_string */
+ *++q = '"';
+ } else {
+ *++q = *p;
+ }
+ }
+ *++q = '\0';
+ if (q - around < (ssize_t)sizeof(around))
hadend = 1;
- /* wrap at newline */
- for (p = around; *p != '\0'; p++)
- if (*p == '\n' || *p == '\r')
- *p = ' ';
- /* trim */
- for (--p; p > around && isspace(*p); p--)
- *p = '\0';
- for (p = around; *p != '\0' && isspace(*p); p++);
+
snprintf(message + len, sizeof(message) - len, " at or around '%s%s%s'",
- off == 0 ? "" : "...", p, hadend == 0 ? "..." : "");
+ start == around ? "..." : "", start, hadend == 0 ?
"..." : "");
if (jb->error != NULL)
GDKfree(jb->error);
@@ -108,7 +126,7 @@ read_from_stream(jsonbat *jb, char **pos
}
shift = *pos - jb->streambuf;
- if (*pos == jb->streambuf + jb->streambuflen) {
+ if (*pos == jb->streambuf + jb->streambuflen - 1) {
size_t rshift = *recall - jb->streambuf;
char *newbuf = realloc(jb->streambuf, jb->streambuflen += 8096);
if (newbuf == NULL)
@@ -124,30 +142,34 @@ read_from_stream(jsonbat *jb, char **pos
sret = mnstr_read(jb->is, *pos, 1, jb->streambuflen - shift - 1);
if (sret <= 0)
return 0;
- jb->streambuf[sret] = '\0';
+ jb->streambuf[shift + sret] = '\0';
+ assert(**pos != '\0');
return sret;
}
static char *
-parse_json_string(jsonbat *jb, oid *v, char pair, char *p)
+parse_json_string(jsonbat *jb, oid *v, char pair, char *p, char **recall)
{
char escape = 0;
+ char *r = p;
char *n = p;
char *w = p;
- for (; ; p++) {
+ for (; ; p++, w++) {
if (*p == '\0' &&
(jb->is == NULL || read_from_stream(jb, &p, &n,
&w) == 0))
- break;
+ {
+ json_error(jb, p, "unexpected end of stream while
reading string");
+ return NULL;
+ }
switch (*p) {
case '\\':
if (escape) {
*w = '\\';
} else if (w != p) {
- *w = *p;
+ w--;
}
- w++;
escape = !escape;
break;
case 'b':
@@ -156,7 +178,6 @@ parse_json_string(jsonbat *jb, oid *v, c
} else if (w != p) {
*w = *p;
}
- w++;
escape = 0;
break;
case 'f':
@@ -165,7 +186,6 @@ parse_json_string(jsonbat *jb, oid *v, c
} else if (w != p) {
*w = *p;
}
_______________________________________________
Checkin-list mailing list
Checkin-list@monetdb.org
http://mail.monetdb.org/mailman/listinfo/checkin-list
