Changeset: 5eef0f4f54b1 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5eef0f4f54b1
Modified Files:
buildtools/scripts/mal2h.py
buildtools/scripts/sql2h.py
monetdb5/embedded/CMakeLists.txt
monetdb5/embedded/mal_embedded.c
monetdb5/embedded/mal_embedded.h
monetdb5/mal/mal_import.c
monetdb5/mal/mal_import.h
monetdb5/modules/mal/mdb.c
sql/backends/monet5/CMakeLists.txt
sql/backends/monet5/sql_scenario.c
sql/scripts/CMakeLists.txt
sql/scripts/sql_scripts.c
Branch: cmake-monetdblite
Log Message:
Generating binary arrays instead of C literal strings.
The previous approach was difficult to compile with MSVC because, it restricts
string literals to 655536 characters in length, which is not long enough for
some of the scripts.
diffs (truncated from 1454 to 300 lines):
diff --git a/buildtools/scripts/mal2h.py b/buildtools/scripts/mal2h.py
--- a/buildtools/scripts/mal2h.py
+++ b/buildtools/scripts/mal2h.py
@@ -6,217 +6,135 @@
from __future__ import print_function
-import os
-import argparse
-
-
-parser = argparse.ArgumentParser(description='Convert MonetDB MAL scripts to C
header files to be inlined')
-parser.add_argument('-f', '--file', metavar='F', dest='file', type=str,
help='The MAL file to convert')
-parser.add_argument('-o', '--output', metavar='O', dest='output', type=str,
help='The output header file to write to')
-parser.add_argument('-t', '--trim', dest='trim', action='store_true',
help='trim whitespaces')
-parser.add_argument('-r', '--rcomments', dest='rcom', action='store_true',
help='remove comments')
-
-# check arguments and veracity of the input file first
-args = parser.parse_args()
+import os, sys
-if not os.path.exists(args.file):
- raise Exception("File {0} doesn't exist".format(args.file))
-if not os.path.isfile(args.file):
- raise Exception("{0} is not a file".format(args.file))
+total_files = len(sys.argv)
+if total_files < 2:
+ raise Exception("There is no output file")
+if total_files < 3:
+ raise Exception("There are no input files")
-file_stat = os.stat(args.file)
-if file_stat.st_size > (1 << 29):
- raise Exception("File {0} is too large to process".format(args.file))
-
-# get the file name and extension
-input_file_base = os.path.basename(args.file)
-input_file_split = os.path.splitext(input_file_base)
-if len(input_file_split) < 2 or input_file_split[1] != '.mal':
- raise Exception("Only .mal files are supported for the input file")
-
-output_file_base = os.path.basename(args.output)
+# The output file will be the first argument
+output_file_base = os.path.basename(sys.argv[1])
output_file_split = os.path.splitext(output_file_base)
if len(output_file_split) < 2 or output_file_split[1] != '.h':
raise Exception("Only .h files are supported for the output file")
-mal_content_file = open(args.file, 'r')
-mal_content = mal_content_file.read()
-mal_content_file.close()
-
-filebasename = output_file_split[0]
-# output file will be written on the same directory
-mal_h_output_file = open(args.output, 'w')
+mal_h_output_file = open(sys.argv[1], 'w')
-# write the common header, plus the C array entry
-insert1 = (
-'/*\n'
-'* This Source Code Form is subject to the terms of the Mozilla Public\n'
-'* License, v. 2.0. If a copy of the MPL was not distributed with this\n'
-'* file, You can obtain one at http://mozilla.org/MPL/2.0/.\n'
-'*\n'
-'* Copyright 1997 - July 2008 CWI, August 2008 - 2019 MonetDB B.V.\n'
-'*/\n'
-'\n'
-'// This file was generated automatically with mal2h.py. Do not edit this file
directly.\n'
-'{{ "{0}", "').format(filebasename)
-
+# write the C array entry
+insert1 = ''.join([
+ '/*\n',
+ '* This Source Code Form is subject to the terms of the Mozilla Public\n',
+ '* License, v. 2.0. If a copy of the MPL was not distributed with this\n',
+ '* file, You can obtain one at http://mozilla.org/MPL/2.0/.\n',
+ '*\n',
+ '* Copyright 1997 - July 2008 CWI, August 2008 - 2019 MonetDB B.V.\n',
+ '*/\n',
+ '\n',
+ '// This file was generated automatically with mal2h.py. Do not edit this
file directly.\n',
+ 'char ', output_file_split[0], '[] = {'])
mal_h_output_file.write(insert1)
-# Let's remove comments from the mal script with a Markov chain :) Bugs might
still be there
-# STATES 0 - OK, 1 in # comment, 2 between comment keyword and comment block
(removing comments),
-# 3 inside address comment block (removing comments),
-# 4 between comment keyword and comment block (not removing comments),
-# 5 inside address comment block (not removing comments), 6 inside " string, 7
inside whitespaces
+file_stat = os.stat(sys.argv[1])
if os.name == 'nt':
CACHE_SIZE = 512
else:
CACHE_SIZE = file_stat.st_blksize # we will set the cache size to the
filesystem blocksize
buffer = ['\0'] * CACHE_SIZE
-cur_state = 0
-current_pointer = 0
-i = 0
-endloop = len(mal_content) - 1
+current_output_file_pointer = 0
+current_input_file_number = 2
+# Iterate over the input files
+while current_input_file_number < total_files:
+ next_file = sys.argv[current_input_file_number]
-def write_to_buffer(input_c):
- global current_pointer, mal_h_output_file, buffer
- if current_pointer == CACHE_SIZE:
- mal_h_output_file.write("".join(buffer))
- current_pointer = 0
- buffer[current_pointer] = input_c
- current_pointer += 1
+ if not os.path.exists(next_file):
+ raise Exception(''.join(["File ", next_file, " doesn't exist"]))
+ if not os.path.isfile(next_file):
+ raise Exception(''.join([next_file, " is not a file"]))
+ file_stat = os.stat(next_file)
+ if file_stat.st_size > (1 << 29):
+ raise Exception(''.join(["File ", next_file, " is too large to
process"]))
-while i < endloop:
- c = mal_content[i]
+ # get the file name and extension
+ input_file_base = os.path.basename(next_file)
+ input_file_split = os.path.splitext(input_file_base)
+ if len(input_file_split) < 2 or input_file_split[1] != '.mal':
+ raise Exception("Only .mal files are supported as input")
+
+ mal_content_file = open(next_file, 'r')
+ mal_content = mal_content_file.read()
+ mal_content_file.close()
- if cur_state == 1: # inside an hashtag comment
- if c == '\n':
- cur_state = 0
- i += 1
- continue
- elif cur_state == 2: # before entering a MAL comment, and removing it
- if c == '"':
- cur_state = 3
- i += 1
- continue
- elif cur_state == 3: # inside a MAL comment, and removing it
- if c == '\\':
- i += 2
- elif c == '"':
- cur_state = 0
+ i = 0
+ cur_state = 0
+ endloop = len(mal_content) - 1
+
+ # Let's remove comments from the mal script with a Markov chain :) Bugs
might still be there
+ # STATES 0 - OK, 1 in # comment, 2 between comment keyword and comment
block (removing comments),
+ # 3 inside address comment block (removing comments),
+ # 4 inside whitespaces
+ while i < endloop:
+ c = mal_content[i]
+
+ if cur_state == 1: # inside an hashtag comment
+ if c == '\n':
+ cur_state = 0
i += 1
- else:
- i += 1
- continue
- elif cur_state == 4: # before entering a MAL comment, but not removing it
- if c == '"':
- write_to_buffer('\\')
- write_to_buffer('"')
- cur_state = 5
- i += 1
- continue
- elif cur_state == 5: # inside a MAL comment, but not removing it
- if c == '\\' and i + 1 < endloop:
- write_to_buffer('\\')
- write_to_buffer('\\')
- write_to_buffer('\\')
- write_to_buffer('\\')
- write_to_buffer('\\')
- write_to_buffer(mal_content[i + 1])
- i += 2
- elif c == '\n':
- write_to_buffer('\\')
- write_to_buffer('n')
- i += 1
- elif c == '\t':
- write_to_buffer('\\')
- write_to_buffer('t')
- i += 1
- else:
+ continue
+ elif cur_state == 2: # before entering a MAL comment, and removing it
if c == '"':
- write_to_buffer('\\')
- write_to_buffer('\"')
- cur_state = 0
- else:
- write_to_buffer(c)
+ cur_state = 3
i += 1
- continue
- elif cur_state == 6: # inside a string
- if c == '\\' and i + 1 < endloop:
- write_to_buffer('\\')
- write_to_buffer('\\')
- write_to_buffer('\\')
- write_to_buffer(mal_content[i + 1])
- i += 2
- elif c == '\n':
- write_to_buffer('\\')
- write_to_buffer('n')
+ continue
+ elif cur_state == 3: # inside a MAL comment, and removing it
+ if c == '\\':
+ i += 2
+ elif c == '"':
+ cur_state = 0
+ i += 1
+ else:
+ i += 1
+ continue
+ elif cur_state == 4: # inside whitespaces
+ if c not in (' ', '\t', '\n'):
+ cur_state = 0
+ continue
i += 1
- elif c == '\t':
- write_to_buffer('\\')
- write_to_buffer('t')
+ continue
+
+ if c == '#':
+ cur_state = 1
i += 1
- else:
- if c == '"':
- cur_state = 0
- write_to_buffer('\\')
- write_to_buffer(c)
- i += 1
- continue
- elif cur_state == 7: # inside whitespaces
- if c not in (' ', '\t', '\n'):
- cur_state = 0
+ continue
+ elif c == 'c' and i + 7 < endloop and mal_content[i:i+7] == 'comment':
+ cur_state = 2
+ i += 6
continue
- i += 1
- continue
- if c == '#':
- cur_state = 1
+ if current_output_file_pointer == CACHE_SIZE:
+ mal_h_output_file.write("".join(buffer))
+ current_output_file_pointer = 0
+ buffer[current_output_file_pointer] = str(ord(c))
+ current_output_file_pointer += 1
+ if current_output_file_pointer == CACHE_SIZE:
+ mal_h_output_file.write("".join(buffer))
+ current_output_file_pointer = 0
+ buffer[current_output_file_pointer] = ','
+ current_output_file_pointer += 1
+
+ if c in (' ', '\t', '\n'):
+ cur_state = 4
i += 1
- continue
- elif c == 'c':
- if i + 8 < endloop and mal_content[i:i+8] == 'comment ':
- if args.rcom:
- cur_state = 2
- else:
- for mchar in 'comment ':
- write_to_buffer(mchar)
- cur_state = 4
- i += 7
- else:
- write_to_buffer('c')
- i += 1
- continue
- elif c == '"':
- write_to_buffer('\\')
- write_to_buffer(c)
- cur_state = 6
- i += 1
- continue
- elif c in (' ', '\t', '\n'):
- if c == '\n':
- write_to_buffer('\\')
- write_to_buffer('n')
- elif c == '\t':
- write_to_buffer('\\')
- write_to_buffer('t')
- else:
- write_to_buffer(c)
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
