Changeset: ef14cbd647eb for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/ef14cbd647eb
Branch: nested
Log Message:
merge newjson
diffs (truncated from 397 to 300 lines):
diff --git a/sql/backends/monet5/CMakeLists.txt
b/sql/backends/monet5/CMakeLists.txt
--- a/sql/backends/monet5/CMakeLists.txt
+++ b/sql/backends/monet5/CMakeLists.txt
@@ -187,6 +187,7 @@ target_include_directories(sql
target_link_libraries(sql
PRIVATE
monetdb_config_header
+ json
sqlstorage
sqlcommon
batstore
@@ -226,6 +227,7 @@ install(TARGETS
DESTINATION ${CMAKE_INSTALL_LIBDIR})
add_subdirectory(generator)
+add_subdirectory(vaults/json)
if(WITH_UDF)
add_subdirectory(UDF)
endif()
diff --git a/sql/backends/monet5/vaults/json/CMakeLists.txt
b/sql/backends/monet5/vaults/json/CMakeLists.txt
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/vaults/json/CMakeLists.txt
@@ -0,0 +1,47 @@
+#[[
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Copyright 2024, 2025 MonetDB Foundation;
+# Copyright August 2008 - 2023 MonetDB B.V.;
+# Copyright 1997 - July 2008 CWI.
+#]]
+
+add_library(json OBJECT)
+#if(MONETDB_STATIC)
+# add_library(json STATIC)
+#else()
+# add_library(json SHARED)
+#endif()
+
+target_sources(json
+ PRIVATE
+ json.c)
+
+target_include_directories(json
+ PRIVATE
+ $<TARGET_PROPERTY:mal,INTERFACE_INCLUDE_DIRECTORIES>
+ $<TARGET_PROPERTY:malmodules,INTERFACE_INCLUDE_DIRECTORIES>
+ $<TARGET_PROPERTY:atoms,INTERFACE_INCLUDE_DIRECTORIES>
+ $<TARGET_PROPERTY:sql,INTERFACE_INCLUDE_DIRECTORIES>
+ $<TARGET_PROPERTY:sqlcommon,INTERFACE_INCLUDE_DIRECTORIES>
+ $<TARGET_PROPERTY:sqlserver,INTERFACE_INCLUDE_DIRECTORIES>
+ $<TARGET_PROPERTY:sqlstorage,INTERFACE_INCLUDE_DIRECTORIES>
+ $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+ $<INSTALL_INTERFACE:${INCLUDEDIR}/monetdb>)
+
+target_link_libraries(json
+ PRIVATE
+ monetdb_config_header
+ sqlinclude
+ monetdb5
+ bat
+ stream
+)
+
+set_target_properties(json
+ PROPERTIES
+ POSITION_INDEPENDENT_CODE ON)
diff --git a/sql/backends/monet5/vaults/json/json.c
b/sql/backends/monet5/vaults/json/json.c
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/vaults/json/json.c
@@ -0,0 +1,295 @@
+/*
+ * SPDX-License-Identifier: MPL-2.0
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 2024 MonetDB Foundation;
+ * Copyright August 2008 - 2023 MonetDB B.V.;
+ * Copyright 1997 - July 2008 CWI.
+ */
+
+#include "monetdb_config.h"
+#include "rel_file_loader.h"
+#include "rel_exp.h"
+
+#include "mal_instruction.h"
+#include "mal_interpreter.h"
+#include "mal_parser.h"
+#include "mal_builder.h"
+#include "mal_namespace.h"
+#include "mal_exception.h"
+#include "mal_linker.h"
+#include "mal_backend.h"
+#include "sql_types.h"
+#include "rel_bin.h"
+#include "json.h"
+#include "mutils.h"
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+
+#include <unistd.h>
+// #include <glob.h> // not available on Windows
+
+
+typedef struct JSONFileHandle {
+ allocator *sa;
+ char *filename;
+ int fd;
+ size_t size;
+} JSONFileHandle;
+
+
+static JSONFileHandle *
+json_open(const char *fname, allocator *sa)
+{
+ if (!sa)
+ return NULL;
+ int fd = MT_open(fname, O_RDONLY);
+ if (fd < 0){
+ // TODO add relevant trace component
+ TRC_ERROR(SQL_EXECUTION, "Error opening file %s", fname);
+ return NULL;
+ }
+ struct stat stb;
+ if (MT_stat(fname, &stb) != 0) {
+ TRC_ERROR(SQL_EXECUTION, "Error stat file %s", fname);
+ close(fd);
+ return NULL;
+ }
+ JSONFileHandle *res = sa_alloc(sa, sizeof(JSONFileHandle));
+ res->sa = sa;
+ res->filename = sa_strdup(sa, fname);
+ res->fd = fd;
+ res->size = stb.st_size;
+ return res;
+}
+
+static void
+json_close(JSONFileHandle *jfh)
+{
+ if (jfh && jfh->fd)
+ close(jfh->fd);
+}
+
+
+static char *
+read_json_file(JSONFileHandle *jfh)
+{
+ char *content = NULL;
+ if (jfh) {
+ unsigned int length = (unsigned int)jfh->size;
+ content = sa_zalloc(jfh->sa, length + 1);
+ if (content) {
+ ssize_t nbytes = read(jfh->fd, content, length);
+ if (nbytes < 0)
+ return NULL;
+ content[length + 1] = '\0';
+ }
+ }
+ return content;
+}
+
+
+static size_t
+append_terms(allocator *sa, JSON *jt, size_t offset, BAT *b, char **error)
+{
+ JSONterm *t = jt->elm + offset;
+ char *v = NULL;
+ JSONterm *prev = offset > 0 ? (jt->elm + (offset - 1)) : NULL;
+ JSONterm *next = offset < (size_t)jt->free ? jt->elm + (offset + 1):
NULL;
+ switch(t->kind) {
+ case JSON_ARRAY:
+ if ( (prev == NULL && next && next->kind > JSON_ARRAY)
+ || (prev && prev->kind == JSON_ARRAY) )
{
+ // array of basic types or array of arrays
+ v = sa_strndup(sa, t->value, t->valuelen);
+ size_t depth = 0;
+ do {
+ offset += 1;
+ next = offset < (size_t)jt->free ?
jt->elm + offset : NULL;
+ if (next && next->kind <=JSON_ARRAY)
+ depth ++;
+ if ((depth > 0 && next && (next->kind
== JSON_VALUE || next->kind == 0))
+ || (depth > 0 && next
== NULL))
+ depth --;
+ } while((next && next->kind != JSON_VALUE) ||
depth > 0);
+ } else {
+ offset += 1;
+ }
+ break;
+ case JSON_OBJECT:
+ v = sa_strndup(sa, t->value, t->valuelen);
+ size_t depth = 0;
+ do {
+ offset += 1;
+ next = offset < (size_t)jt->free ? jt->elm +
offset : NULL;
+ if (next && next->kind <=JSON_ARRAY)
+ depth ++;
+ if ((depth > 0 && next && (next->kind ==
JSON_VALUE || next->kind == 0))
+ || (depth > 0 && next == NULL))
+ depth --;
+ } while((next && next->kind != JSON_VALUE) || depth >
0);
+ break;
+ case JSON_ELEMENT:
+ case JSON_STRING:
+ case JSON_NUMBER:
+ // should not happen
+ assert(0);
+ break;
+ case JSON_VALUE:
+ offset +=1;
+ break;
+ default:
+ *error = createException(SQL, "json.append_terms",
"unknown json term");
+ break;
+ }
+ if (v) {
+ if (BUNappend(b, v, false) != GDK_SUCCEED) {
+ *error = createException(SQL, "json.append_terms",
"BUNappend failed!");
+ }
+ }
+ return offset;
+}
+
+
+static str
+json_relation(mvc *sql, sql_subfunc *f, char *filename, list *res_exps, char
*tname)
+{
+ (void) filename;
+ char *res = MAL_SUCCEED;
+ list *types = sa_list(sql->sa);
+ list *names = sa_list(sql->sa);
+ // use file name as columnn name ?
+ char *cname = sa_strdup(sql->sa, "json");
+ list_append(names, cname);
+ sql_schema *jsons = mvc_bind_schema(sql, "sys");
+ if (!jsons)
+ return NULL;
+ sql_subtype *st = SA_NEW(sql->sa, sql_subtype);
+ st->digits = st->scale = 0;
+ st->multiset = 0;
+ st->type = schema_bind_type(sql, jsons, "json");
+ list_append(types, st);
+ sql_exp *ne = exp_column(sql->sa, a_create(sql->sa, tname), cname, st,
CARD_MULTI, 1, 0, 0);
+ set_basecol(ne);
+ ne->alias.label = -(sql->nid++);
+ list_append(res_exps, ne);
+ f->tname = tname;
+ f->res = types;
+ f->coltypes = types;
+ f->colnames = names;
+ return res;
+}
+
+
+static void *
+json_load(void *BE, sql_subfunc *f, char *filename, sql_exp *topn)
+{
+ (void) topn; // TODO include topn
+ backend *be = BE;
+ allocator *sa = be->mvc->sa;
+ sql_subtype *tpe = f->res->h->data;
+ const char *tname = f->tname;
+ const char *cname = f->colnames->h->data;
+
+ stmt *s = stmt_none(be);
+ InstrPtr q = newStmt(be->mb, "json", "read_json");
+ q = pushStr(be->mb, q, filename);
+ pushInstruction(be->mb, q);
+ s->nr = getDestVar(q);
+ s->q = q;
+ s->nrcols = 1;
+ s->op4.typeval = *tpe;
+ // is alias essential here?
+ s = stmt_alias(be, s, 1, a_create(sa, tname), cname);
+ return s;
+}
+
+int TYPE_json;
+
+static str
+JSONprelude(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
+{
+ (void)cntxt; (void)mb; (void)stk; (void)pci;
+ TYPE_json = ATOMindex("json");
+
+ fl_register("json", &json_relation, &json_load);
+ return MAL_SUCCEED;
+}
+
+static str
_______________________________________________
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org
