Changeset: 5e105605f159 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/5e105605f159
Modified Files:
common/stream/stream_internal.h
gdk/gdk_logger.c
Branch: default
Log Message:
Merge 'binresultset' into 'default'
diffs (truncated from 1037 to 300 lines):
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -1625,6 +1625,7 @@ char *buffer_get_buf(buffer *b);
void buffer_init(buffer *restrict b, char *restrict buf, size_t size);
stream *buffer_rastream(buffer *restrict b, const char *restrict name);
stream *buffer_wastream(buffer *restrict b, const char *restrict name);
+stream *byte_counting_stream(stream *wrapped, uint64_t *counter);
stream *bz2_stream(stream *inner, int preset);
stream *callback_stream(void *restrict priv, ssize_t (*read)(void *restrict
priv, void *restrict buf, size_t elmsize, size_t cnt), ssize_t (*write)(void
*restrict priv, const void *restrict buf, size_t elmsize, size_t cnt), void
(*close)(void *priv), void (*destroy)(void *priv), const char *restrict name);
void close_stream(stream *s);
diff --git a/clients/examples/C/bincopydata.c b/clients/examples/C/bincopydata.c
--- a/clients/examples/C/bincopydata.c
+++ b/clients/examples/C/bincopydata.c
@@ -212,6 +212,35 @@ gen_null_strings(FILE *f, bool byteswap,
}
static void
+gen_null_blobs(FILE *f, bool byteswap, long nrecs)
+{
+ uint8_t *buffer = malloc(nrecs);
+ for (long i = 0; i < nrecs; i++) {
+ buffer[i] = 0xD0 + 3 - (i % 3);
+ }
+
+ for (long i = 0; i < nrecs; i++) {
+ uint64_t header;
+ size_t n;
+ if (i % 3 == 2) {
+ // null
+ n = 0;
+ header = (uint64_t)-1;
+ } else {
+ n = (i % 1000);
+ header = n;
+ }
+ if (byteswap)
+ copy_binary_convert64(&header);
+ assert(sizeof(header) == 8);
+ fwrite(&header, sizeof(header), 1, f);
+ if (n > 0)
+ fwrite(buffer, 1, n, f);
+ }
+ free(buffer);
+}
+
+static void
gen_json(FILE *f, bool byteswap, long nrecs)
{
(void)byteswap;
@@ -247,6 +276,7 @@ static struct gen {
{ "broken_strings", gen_broken_strings },
{ "newline_strings", gen_newline_strings },
{ "null_strings", gen_null_strings },
+ { "null_blobs", gen_null_blobs },
//
{ "timestamps", gen_timestamps },
{ "timestamp_times", gen_timestamp_times },
diff --git a/clients/examples/C/bincopytemporaldata.c
b/clients/examples/C/bincopytemporaldata.c
--- a/clients/examples/C/bincopytemporaldata.c
+++ b/clients/examples/C/bincopytemporaldata.c
@@ -10,11 +10,32 @@
#include "bincopydata.h"
+static const copy_binary_timestamp binary_nil_timestamp = {
+ .time = {
+ .ms = 0xFFFFFFFF,
+ .seconds = 255,
+ .minutes = 255,
+ .hours = 255,
+ .padding = 255,
+ },
+ .date = {
+ .day = 255,
+ .month = 255,
+ .year =-1,
+ },
+};
+
static copy_binary_timestamp
random_timestamp(struct rng *rng)
{
+ copy_binary_timestamp ts;
+ if (rng_next(rng) % 10 == 9) {
+ ts = binary_nil_timestamp;
+ return ts;
+ }
+
// the % trick gives a little skew but we don't care
- copy_binary_timestamp ts = {
+ ts = (copy_binary_timestamp){
.time = {
.ms = rng_next(rng) % 1000000,
.seconds = rng_next(rng) % 60, // 61 ??
@@ -62,7 +83,7 @@ gen_timestamps(FILE *f, bool byteswap, l
}
}
-#define GEN_TIMESTAMP_FIELD(name, fld) \
+#define GEN_TIMESTAMP_FIELD(name, typ, fld, nilvalue) \
void name \
(FILE *f, bool byteswap, long nrecs) \
{ \
@@ -70,20 +91,22 @@ gen_timestamps(FILE *f, bool byteswap, l
\
for (long i = 0; i < nrecs; i++) { \
copy_binary_timestamp ts = random_timestamp(&rng); \
+ typ *p = &ts.fld; \
+ typ tmp = ts.date.day == 255 ? nilvalue : *p; \
if (byteswap) { \
copy_binary_convert_timestamp(&ts); \
} \
- fwrite(&ts.fld, sizeof(ts.fld), 1, f); \
+ fwrite(&tmp, sizeof(tmp), 1, f); \
} \
}
-GEN_TIMESTAMP_FIELD(gen_timestamp_times, time)
-GEN_TIMESTAMP_FIELD(gen_timestamp_dates, date)
+GEN_TIMESTAMP_FIELD(gen_timestamp_times, copy_binary_time, time,
binary_nil_timestamp.time)
+GEN_TIMESTAMP_FIELD(gen_timestamp_dates, copy_binary_date, date,
binary_nil_timestamp.date)
-GEN_TIMESTAMP_FIELD(gen_timestamp_ms, time.ms)
-GEN_TIMESTAMP_FIELD(gen_timestamp_seconds, time.seconds)
-GEN_TIMESTAMP_FIELD(gen_timestamp_minutes, time.minutes)
-GEN_TIMESTAMP_FIELD(gen_timestamp_hours, time.hours)
-GEN_TIMESTAMP_FIELD(gen_timestamp_days, date.day)
-GEN_TIMESTAMP_FIELD(gen_timestamp_months, date.month)
-GEN_TIMESTAMP_FIELD(gen_timestamp_years, date.year)
+GEN_TIMESTAMP_FIELD(gen_timestamp_ms, uint32_t, time.ms, 0x80)
+GEN_TIMESTAMP_FIELD(gen_timestamp_seconds, uint8_t, time.seconds, 0x80)
+GEN_TIMESTAMP_FIELD(gen_timestamp_minutes, uint8_t, time.minutes, 0x80)
+GEN_TIMESTAMP_FIELD(gen_timestamp_hours, uint8_t, time.hours, 0x80)
+GEN_TIMESTAMP_FIELD(gen_timestamp_days, uint8_t, date.day, 0x80)
+GEN_TIMESTAMP_FIELD(gen_timestamp_months, uint8_t, date.month, 0x80)
+GEN_TIMESTAMP_FIELD(gen_timestamp_years, int16_t, date.year, -1)
diff --git a/common/stream/mapi_stream.c b/common/stream/mapi_stream.c
--- a/common/stream/mapi_stream.c
+++ b/common/stream/mapi_stream.c
@@ -13,6 +13,31 @@
#include "stream_internal.h"
#include "mapi_prompt.h"
+static ssize_t
+byte_counting_write(stream *restrict s, const void *restrict buf, size_t
elmsize, size_t cnt)
+{
+ uint64_t *counter = (uint64_t*) s->stream_data.p;
+ ssize_t nwritten = s->inner->write(s->inner, buf, elmsize, cnt);
+ if (nwritten >= 0) {
+ *counter += elmsize * nwritten;
+ }
+ return nwritten;
+}
+
+
+stream *
+byte_counting_stream(stream *wrapped, uint64_t *counter)
+{
+ stream *s = create_wrapper_stream(NULL, wrapped);
+ if (!s)
+ return NULL;
+ s->stream_data.p = counter;
+ s->write = &byte_counting_write;
+ s->destroy = &destroy_stream;
+ return s;
+}
+
+
static void
discard(stream *s)
diff --git a/common/stream/stream.h b/common/stream/stream.h
--- a/common/stream/stream.h
+++ b/common/stream/stream.h
@@ -269,4 +269,8 @@ stream_export stream *create_text_stream
stream_export stream *mapi_request_upload(const char *filename, bool binary,
bstream *rs, stream *ws);
stream_export stream *mapi_request_download(const char *filename, bool binary,
bstream *rs, stream *ws);
+// write-only
+stream_export stream *byte_counting_stream(stream *wrapped, uint64_t *counter);
+
+
#endif /*_STREAM_H_*/
diff --git a/common/stream/stream_internal.h b/common/stream/stream_internal.h
--- a/common/stream/stream_internal.h
+++ b/common/stream/stream_internal.h
@@ -268,8 +268,8 @@ typedef struct bs bs;
struct bs {
unsigned nr; /* how far we got in buf */
unsigned itotal; /* amount available in current read block */
- size_t blks; /* read/writen blocks (possibly partial) */
- size_t bytes; /* read/writen bytes */
+ int64_t blks; /* read/writen blocks (possibly partial) */
+ int64_t bytes; /* read/writen bytes */
char buf[BLOCK]; /* the buffered data (minus the size of
* size-short */
};
diff --git a/documentation/source/binary-resultset.rst
b/documentation/source/binary-resultset.rst
new file mode 100644
--- /dev/null
+++ b/documentation/source/binary-resultset.rst
@@ -0,0 +1,64 @@
+.. This Source Code Form is subject to the terms of the Mozilla Public
+.. License, v. 2.0. If a copy of the MPL was not distributed with this
+.. file, You can obtain one at http://mozilla.org/MPL/2.0/.
+..
+.. Copyright 1997 - July 2008 CWI, August 2008 - 2023 MonetDB B.V.
+
+************************
+Binary Result set format
+************************
+
+Note: this explanation will eventually be folded into a more comprehensive
+description of the MAPI protocol on the MonetDB website.
+In the mean time, it lives in this directory.
+
+
+Overview
+========
+
+When MonetDB executes a query it immediately sends the first `N` rows of the
+result set to the client, where `N` is determined by the ``reply_size``
setting.
+If the client needs more it can use the ``Xexport <startrow> <count>`` command
+to request more rows.
+
+Recently we have added the ``Xexportbin <startrow> <count>`` command which
+behaves the same but uses a binary format that may be more efficient to parse.
+
+The server advertises its support for ``Xexportbin`` in the eighth field of its
+connect challenge. For example,
+
+
bL1sNfkaa:mserver:9:RIPEMD160,SHA512,SHA384,SHA256,SHA224,SHA1,COMPRESSION_SNAPPY,COMPRESSION_LZ4:LIT:SHA512:sql=6:**BINARY=1**:
+
+Currently it sends ``BINARY=1``. In the future it may send a higher number if
+variants are added.
+
+
+Binary result set layout
+========================
+
+In response to ``Xexportbin <startrow> <count>`` the server returns a big blob
+of bytes delimited by a flush. In other words, the end is marked by the final
+8KiB MAPI block having its end-flag enabled, just like a regular response. (We
+can make this explanation more clear when the text is embedded in an overall
+description of the MAPI protocol.)
+
+To interpret the bytes, first look at the first character. If it starts with
+an exclamation mark, the rest of the message is an error message.
+Otherwise, look at the final 8 bytes. These form a 64 bit
+server-endian integer. In the current version of the protocol the number will
+always be either positive or negative and never zero.
+
+If the number is negative, an error has occurred, and the negated number is the
+byte offset of the error message, counting from the start of the response. The
+end of the error message is marked by a nul byte. Note: the error message
starts
+with an exclamation mark, just as in the textual protocol.
+
+If the number is positive, it is the byte offset of the table of contents of
+the response. This is a sequence of 16-byte entries, one for each column
+of the result set. Each entry consists of the starting offset and the length
+in bytes of the data for that column, again expressed as 64 bits server-endian
+integers.
+
+The byte layout of each individual column is identical to what would have been
+produced by ``COPY select_query INTO BINARY 'file1', 'file2', ...``.
+
diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -1097,8 +1097,7 @@ log_close_input(logger *lg)
static inline void
log_close_output(logger *lg)
{
- if (lg->flushing_output_log)
- return;
+ assert (!lg->flushing_output_log);
if (!LOG_DISABLED(lg))
close_stream(lg->output_log);
diff --git a/monetdb5/modules/mal/mal_mapi.c b/monetdb5/modules/mal/mal_mapi.c
--- a/monetdb5/modules/mal/mal_mapi.c
+++ b/monetdb5/modules/mal/mal_mapi.c
@@ -187,7 +187,7 @@ doChallenge(void *data)
}
// Send the challenge over the block stream
- mnstr_printf(fdout, "%s:mserver:9:%s:%s:%s:sql=%d:",
+ mnstr_printf(fdout, "%s:mserver:9:%s:%s:%s:sql=%d:BINARY=1:",
challenge,
mcrypt_getHashAlgorithms(),
#ifdef WORDS_BIGENDIAN
diff --git a/sql/backends/monet5/sql_bincopy.c
b/sql/backends/monet5/sql_bincopy.c
--- a/sql/backends/monet5/sql_bincopy.c
+++ b/sql/backends/monet5/sql_bincopy.c
@@ -23,6 +23,12 @@
#include "copybinary_support.h"
+#define bailout(...) do { \
_______________________________________________
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org
