Changeset: d89be070142b for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d89be070142b
Modified Files:
CMakeLists.txt
buildtools/scripts/mal2h.py
buildtools/scripts/sql2h.py
cmake/BuildMacros.cmake
embedded/CMakeLists.txt
monetdb5/mal/CMakeLists.txt
sql/backends/monet5/CMakeLists.txt
sql/backends/monet5/sql_execute.c
Branch: cmake-monetdblite
Log Message:
Several changes:
- Allow to build MonetDBLite as a static library to embedded it to higher level
programming languages.
- Rename MonetDBLite library as monetdblite instead of monetdb5 to avoid
conflicts with the existing monetdb5 library.
- Make MAL scripts command comments strip optional for mserver5 compilation.
- Simplify embedded scripts generation.
diffs (truncated from 441 to 300 lines):
diff --git a/CMakeLists.txt b/CMakeLists.txt
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -116,7 +116,7 @@ set(OPTIONS_LIST CINTEGRATION EMBEDDED F
foreach(EXTENSION IN LISTS OPTIONS_LIST)
if(ENABLE_${EXTENSION})
string(TOUPPER "${ENABLE_${EXTENSION}}" ENABLE_${EXTENSION})
- if(NOT "${EXTENSION}" STREQUAL "REGEX" AND NOT
"${ENABLE_${EXTENSION}}" MATCHES "^YES|NO|AUTO$")
+ if(NOT "${EXTENSION}" MATCHES "^REGEX|EMBEDDED$" AND NOT
"${ENABLE_${EXTENSION}}" MATCHES "^YES|NO|AUTO$")
message(FATAL_ERROR "ENABLE_${EXTENSION} value must be
either YES, NO or AUTO")
endif()
endif()
@@ -304,7 +304,7 @@ set(DIR_SEP_STR ${OS_DIRSEP})
set(PATH_SEP "'${OS_PATHSEP}'")
set(SO_EXT "${CMAKE_SHARED_LIBRARY_SUFFIX}")
-# We need python3 for the testweb, python udfs and c udfs compilation
+# We need python3 for the testweb, python udfs, c udfs compilation and
generate embedded MAL and SQL scripts
if(CMAKE_VERSION VERSION_LESS 3.12)
set(Python_ADDITIONAL_VERSIONS 3.7 3.6 3.5)
find_package(PythonInterp 3.4)
@@ -355,7 +355,7 @@ if(NOT HAVE_PYTHON3)
else()
set(ENABLE_SQL "NO" CACHE STRING "A valid Python 3 interpreter
is required for MonetDB/SQL" FORCE)
endif()
- if("${ENABLE_EMBEDDED}" STREQUAL "YES")
+ if(${ENABLE_EMBEDDED} MATCHES "^SHARED|STATIC$")
message(FATAL_ERROR "A valid Python 3 interpreter is required
for MonetDB embedded library")
else()
set(ENABLE_EMBEDDED "NO" CACHE STRING "A valid Python 3
interpreter is required for MonetDB embedded library" FORCE)
@@ -368,7 +368,7 @@ if(NOT BISON_FOUND)
if(${ENABLE_SQL} STREQUAL "YES")
message(FATAL_ERROR "MonetDB/SQL requires bison")
endif()
- if(${ENABLE_EMBEDDED} STREQUAL "YES")
+ if(${ENABLE_EMBEDDED} MATCHES "^SHARED|STATIC$")
message(FATAL_ERROR "MonetDB embedded library requires bison")
endif()
set(ENABLE_SQL "NO" CACHE STRING "Bison is required for MonetDB/SQL"
FORCE)
@@ -601,10 +601,12 @@ if("${ENABLE_SQL}" MATCHES "^YES|AUTO$")
else()
set(HAVE_SQL OFF CACHE INTERNAL "MonetDB/SQL is available" FORCE)
endif()
-if(${ENABLE_EMBEDDED} MATCHES "^YES|AUTO$")
+if(${ENABLE_EMBEDDED} MATCHES "^SHARED|STATIC$")
set(HAVE_EMBEDDED ON CACHE INTERNAL "MonetDB embedded library is
enabled" FORCE)
+elseif(${ENABLE_EMBEDDED} STREQUAL "NO")
+ set(HAVE_EMBEDDED OFF CACHE INTERNAL "MonetDB embedded library is
enabled" FORCE)
else()
- set(HAVE_EMBEDDED OFF CACHE INTERNAL "MonetDB embedded library is
enabled" FORCE)
+ message(FATAL_ERROR "ENABLE_EMBEDDED parameter must be either SHARED,
STATIC or NO")
endif()
if(${ENABLE_STATIC_ANALYSIS} MATCHES "^YES|AUTO$")
set(STATIC_CODE_ANALYSIS ON CACHE INTERNAL "Static code analysis is
available" FORCE)
diff --git a/buildtools/scripts/mal2h.py b/buildtools/scripts/mal2h.py
--- a/buildtools/scripts/mal2h.py
+++ b/buildtools/scripts/mal2h.py
@@ -8,19 +8,25 @@ from __future__ import print_function
import os, sys
-total_files = len(sys.argv)
-if total_files < 2:
+total_files = len(sys.argv) # first argument is reserved for compression
+if total_files < 3:
raise Exception("There is no output file")
-if total_files < 3:
+if total_files < 4:
raise Exception("There are no input files")
+removing_inline_comments = False
+if sys.argv[1] == '--remove-command-comments':
+ removing_inline_comments = True
+elif sys.argv[1] != '--no-remove-command-comments':
+ raise Exception("The first argument must be either
--remove-command-comments or --no-remove-command-comments")
+
# The output file will be the first argument
-output_file_base = os.path.basename(sys.argv[1])
+output_file_base = os.path.basename(sys.argv[2])
output_file_split = os.path.splitext(output_file_base)
if len(output_file_split) < 2 or output_file_split[1] != '.h':
raise Exception("Only .h files are supported for the output file")
-mal_h_output_file = open(sys.argv[1], 'w')
+mal_h_output_file = open(sys.argv[2], 'w')
# write the C array entry
insert1 = ''.join([
@@ -36,15 +42,9 @@ insert1 = ''.join([
'static char ', output_file_split[0], '[] = {'])
mal_h_output_file.write(insert1)
-file_stat = os.stat(sys.argv[1])
-if os.name == 'nt':
- CACHE_SIZE = 512
-else:
- CACHE_SIZE = file_stat.st_blksize # we will set the cache size to the
filesystem blocksize
-
-buffer = ['\0'] * CACHE_SIZE
-current_output_file_pointer = 0
-current_input_file_number = 2
+file_stat = os.stat(sys.argv[2])
+buffer = bytearray()
+current_input_file_number = 3
# Iterate over the input files
while current_input_file_number < total_files:
@@ -110,21 +110,13 @@ while current_input_file_number < total_
cur_state = 1
i += 1
continue
- elif c == 'c' and i + 8 < endloop and mal_content[i:i+7] == 'comment'
and mal_content[i+8] in (' ', '\t', '\n'):
+ elif removing_inline_comments and c == 'c' and i + 8 < endloop and
mal_content[i:i+7] == 'comment' \
+ and mal_content[i+8] in (' ', '\t', '\n'):
cur_state = 2
i += 6
continue
- if current_output_file_pointer == CACHE_SIZE:
- mal_h_output_file.write("".join(buffer))
- current_output_file_pointer = 0
- buffer[current_output_file_pointer] = str(ord(c))
- current_output_file_pointer += 1
- if current_output_file_pointer == CACHE_SIZE:
- mal_h_output_file.write("".join(buffer))
- current_output_file_pointer = 0
- buffer[current_output_file_pointer] = ','
- current_output_file_pointer += 1
+ buffer.append(ord(c))
if c in (' ', '\t', '\n'):
cur_state = 4
@@ -132,8 +124,9 @@ while current_input_file_number < total_
current_input_file_number += 1
-if current_output_file_pointer > 0:
- mal_h_output_file.write("".join(buffer[:current_output_file_pointer]))
+if len(buffer) > 0: # write only if something was found
+ result = ",".join(str(c) for c in buffer) + ','
+ mal_h_output_file.write(result)
# finish C array entry
mal_h_output_file.write("0};\n")
diff --git a/buildtools/scripts/sql2h.py b/buildtools/scripts/sql2h.py
--- a/buildtools/scripts/sql2h.py
+++ b/buildtools/scripts/sql2h.py
@@ -37,13 +37,7 @@ insert1 = ''.join([
sql_h_output_file.write(insert1)
file_stat = os.stat(sys.argv[1])
-if os.name == 'nt':
- CACHE_SIZE = 512
-else:
- CACHE_SIZE = file_stat.st_blksize # we will set the cache size to the
filesystem blocksize
-
-buffer = ['\0'] * CACHE_SIZE
-current_output_file_pointer = 0
+buffer = bytearray()
current_input_file_number = 2
# Iterate over the input files
@@ -114,16 +108,7 @@ while current_input_file_number < total_
i += 2
continue
- if current_output_file_pointer == CACHE_SIZE:
- sql_h_output_file.write("".join(buffer))
- current_output_file_pointer = 0
- buffer[current_output_file_pointer] = str(ord(c))
- current_output_file_pointer += 1
- if current_output_file_pointer == CACHE_SIZE:
- sql_h_output_file.write("".join(buffer))
- current_output_file_pointer = 0
- buffer[current_output_file_pointer] = ','
- current_output_file_pointer += 1
+ buffer.append(ord(c))
if c in (' ', '\t', '\n'):
cur_state = 4 # Trim always
@@ -131,8 +116,9 @@ while current_input_file_number < total_
current_input_file_number += 1
-if current_output_file_pointer > 0:
- sql_h_output_file.write("".join(buffer[:current_output_file_pointer]))
+if len(buffer) > 0: # write only if something was found
+ result = ",".join(str(c) for c in buffer) + ','
+ sql_h_output_file.write(result)
# finish C array entry
sql_h_output_file.write("0};\n")
diff --git a/cmake/BuildMacros.cmake b/cmake/BuildMacros.cmake
--- a/cmake/BuildMacros.cmake
+++ b/cmake/BuildMacros.cmake
@@ -9,10 +9,10 @@
# This file holds macros for compilation objects common to MonetDB and
MonetDBLite
# Create C array with MAL scripts content as well the module names bundled
-macro(BUILD_EMBEDDED_MAL_SCRIPTS BUNDLE_NAME VARIABLE_NAME SCRIPTS_LIST)
+macro(BUILD_EMBEDDED_MAL_SCRIPTS BUNDLE_NAME VARIABLE_NAME REMOVE_COMMENTS
SCRIPTS_LIST)
execute_process(COMMAND "${Python3_EXECUTABLE}"
"${CMAKE_SOURCE_DIR}/buildtools/scripts/mal2h.py"
- "${CMAKE_CURRENT_BINARY_DIR}/${BUNDLE_NAME}.h"
${SCRIPTS_LIST}
- RESULT_VARIABLE PY_SCRIPT_RC OUTPUT_QUIET)
+ "${REMOVE_COMMENTS}"
"${CMAKE_CURRENT_BINARY_DIR}/${BUNDLE_NAME}.h" ${SCRIPTS_LIST}
+ RESULT_VARIABLE PY_SCRIPT_RC
OUTPUT_QUIET)
if(NOT PY_SCRIPT_RC EQUAL 0)
message(FATAL_ERROR "Could not generate sql_mal_inline.h file")
endif()
@@ -31,8 +31,8 @@ endmacro()
# Create C array with SQL scripts content
macro(BUILD_EMBEDDED_SQL_SCRIPTS BUNDLE_NAME SCRIPTS_LIST)
execute_process(COMMAND "${Python3_EXECUTABLE}"
"${CMAKE_SOURCE_DIR}/buildtools/scripts/sql2h.py"
- "${CMAKE_CURRENT_BINARY_DIR}/${BUNDLE_NAME}.h"
${SCRIPTS_LIST}
- RESULT_VARIABLE PY_SCRIPT_RC OUTPUT_QUIET)
+
"${CMAKE_CURRENT_BINARY_DIR}/${BUNDLE_NAME}.h" ${SCRIPTS_LIST}
+ RESULT_VARIABLE PY_SCRIPT_RC
OUTPUT_QUIET)
if(NOT PY_SCRIPT_RC EQUAL 0)
message(FATAL_ERROR "Could not generate ${BUNDLE_NAME}.h file")
endif()
diff --git a/embedded/CMakeLists.txt b/embedded/CMakeLists.txt
--- a/embedded/CMakeLists.txt
+++ b/embedded/CMakeLists.txt
@@ -82,7 +82,7 @@ set(SQL_SCRIPTS_LIST
"${CMAKE_SOURCE_DIR}/sql/scripts/90_generator.sql"
"${CMAKE_SOURCE_DIR}/sql/scripts/99_system.sql")
-build_embedded_mal_scripts(mal_inline malModules "${MAL_SCRIPTS_LIST}")
+build_embedded_mal_scripts(mal_inline malModules --remove-command-comments
"${MAL_SCRIPTS_LIST}")
build_embedded_sql_scripts(createdb_inline1 "${SQL_SCRIPTS_LIST}")
# All symbols present in monetdblite_no_export and and not required by
monetdblite will be ditched on Windows
@@ -204,88 +204,92 @@ set_target_properties(monetdblite_no_exp
target_link_libraries(monetdblite_no_export PUBLIC ${MATH_LIBRARIES}
${THREAD_LIBRARIES} ${DL_LIBRARIES}
${KVM_LIBRARIES} ${PSAPI_LIBRARIES})
-add_library(monetdblite SHARED
- monetdb_embedded.c
- ../monetdb5/modules/kernel/aggr.c
- ../monetdb5/modules/atoms/streams.c
- ../monetdb5/modules/kernel/algebra.c
- ../monetdb5/modules/kernel/bat5.c
- ../monetdb5/modules/mal/batcalc.c
- ../monetdb5/modules/mal/batExtensions.c
- ../monetdb5/modules/kernel/batmmath.c
- ../monetdb5/modules/kernel/batstr.c
- ../monetdb5/modules/atoms/blob.c
- ../monetdb5/modules/mal/bbp.c
- ../monetdb5/modules/mal/calc.c
- ../monetdb5/modules/kernel/group.c
- ../monetdb5/modules/mal/iterator.c
- ../monetdb5/modules/mal/language.c
- ../monetdb5/modules/mal/manifold.c
- ../monetdb5/modules/mal/mat.c
- ../monetdb5/modules/mal/mkey.c
- ../monetdb5/modules/kernel/mmath.c
- ../monetdb5/modules/atoms/mtime.c
- ../monetdb5/modules/atoms/mtime_analytic.c
- ../monetdb5/modules/atoms/strptime.c
- ../monetdb5/modules/mal/orderidx.c
- ../monetdb5/modules/mal/pcre.c
- ../monetdb5/modules/mal/projectionpath.c
- ../monetdb5/modules/mal/sample.c
- ../monetdb5/modules/atoms/str.c
- ../monetdb5/modules/mal/tablet.c
- ../monetdb5/optimizer/opt_aliases.c
- ../monetdb5/optimizer/opt_candidates.c
- ../monetdb5/optimizer/opt_coercion.c
- ../monetdb5/optimizer/opt_commonTerms.c
- ../monetdb5/optimizer/opt_constants.c
- ../monetdb5/optimizer/opt_costModel.c
- ../monetdb5/optimizer/opt_dataflow.c
- ../monetdb5/optimizer/opt_deadcode.c
- ../monetdb5/optimizer/opt_emptybind.c
- ../monetdb5/optimizer/opt_evaluate.c
- ../monetdb5/optimizer/opt_garbageCollector.c
- ../monetdb5/optimizer/opt_generator.c
- ../monetdb5/optimizer/opt_inline.c
- ../monetdb5/optimizer/opt_macro.c
- ../monetdb5/optimizer/opt_matpack.c
- ../monetdb5/optimizer/opt_mergetable.c
- ../monetdb5/optimizer/opt_mitosis.c
- ../monetdb5/optimizer/opt_multiplex.c
- ../monetdb5/optimizer/opt_pipes.c
- ../monetdb5/optimizer/opt_prelude.c
- ../monetdb5/optimizer/opt_profiler.c
- ../monetdb5/optimizer/opt_projectionpath.c
- ../monetdb5/optimizer/opt_pushselect.c
- ../monetdb5/optimizer/opt_remap.c
- ../monetdb5/optimizer/opt_reorder.c
- ../monetdb5/optimizer/opt_support.c
- ../monetdb5/optimizer/opt_wrapper.c
- ../monetdb5/optimizer/optimizer.c
- ../sql/backends/monet5/sql.c
- ../sql/backends/monet5/sql_assert.c
- ../sql/backends/monet5/sql_bat2time.c
- ../sql/backends/monet5/sql_cast.c
- ../sql/backends/monet5/sql_cat.c
- ../sql/backends/monet5/sql_datetrunc.c
- ../sql/backends/monet5/sql_execute.c
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
