MonetDB: group-commit - merged with default
Changeset: ecd8c8346361 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/ecd8c8346361
Modified Files:
gdk/gdk_logger.c
sql/storage/bat/bat_logger.c
Branch: group-commit
Log Message:
merged with default
diffs (truncated from 5977 to 300 lines):
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -756,3 +756,5 @@ f458e1c71c73d6bd9636369c1406eadb74f016bf
00463fdd0d51d7ce058549a82bc74efaea6035a2 Jul2021_15
00463fdd0d51d7ce058549a82bc74efaea6035a2 Jul2021_SP3_release
db3cec8ea853884e857fcfb413428116cb95e786 Jul2021_17
+8c015afafb5903ea59b0e2cffac1138a0d82e007 Jul2021_19
+8c015afafb5903ea59b0e2cffac1138a0d82e007 Jul2021_SP4_release
diff --git a/CMakeLists.txt b/CMakeLists.txt
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -97,7 +97,7 @@ if(NOT ${CMAKE_INSTALL_PREFIX} STREQUAL
# SET(CMAKE_SKIP_RPATH TRUE)
endif()
-# required for some instalation files
+# required for some installation files
set(PROGRAM_PERMISSIONS_DEFAULT
OWNER_WRITE
OWNER_READ
diff --git a/clients/Tests/MAL-signatures-hge.test
b/clients/Tests/MAL-signatures-hge.test
--- a/clients/Tests/MAL-signatures-hge.test
+++ b/clients/Tests/MAL-signatures-hge.test
@@ -45798,6 +45798,11 @@ similarity
command battxtsim.similarity(X_0:bat[:str], X_1:bat[:str]):bat[:dbl]
fstrcmp0_impl_bulk;
Normalized edit distance between two strings
+baturl
+extractURLHost
+command baturl.extractURLHost(X_0:bat[:str], X_1:bit):bat[:str]
+BATextractURLHost;
+Extract host from BAT of URLs
batuuid
isaUUID
command batuuid.isaUUID(X_0:bat[:str]):bat[:bit]
@@ -64589,6 +64594,11 @@ command txtsim.stringdiff(X_0:str, X_1:s
stringdiff_impl;
calculate the soundexed editdistance
url
+extractURLHost
+command url.extractURLHost(X_0:str, X_1:bit):str
+extractURLHost;
+Extract host from a URL relaxed version
+url
getAnchor
command url.getAnchor(X_0:url):str
URLgetAnchor;
@@ -64622,7 +64632,7 @@ url
getHost
command url.getHost(X_0:url):str
URLgetHost;
-Extract the server name from the URL
+Extract the server name from the URL strict version
url
getPort
command url.getPort(X_0:url):str
diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -4358,7 +4358,7 @@ read_into_cache(MapiHdl hdl, int lookahe
for (;;) {
line = read_line(mid);
if (line == NULL) {
- if (mnstr_eof(mid->from)) {
+ if (mid->from && mnstr_eof(mid->from)) {
mapi_log_record(mid, "unexpected end of file");
mapi_log_record(mid, __func__);
close_connection(mid);
diff --git a/cmake/monetdb-defines.cmake b/cmake/monetdb-defines.cmake
--- a/cmake/monetdb-defines.cmake
+++ b/cmake/monetdb-defines.cmake
@@ -300,16 +300,18 @@ macro(monetdb_configure_misc)
"PASSWORD_BACKEND invalid, choose one of MD5, SHA1, RIPEMD160, SHA224,
SHA256, SHA384, SHA512")
endif()
- # Used for installing testing python module (don't pass a location, else we
need to strip this again)
- execute_process(COMMAND "${Python3_EXECUTABLE}" "-c" "import sysconfig;
print(sysconfig.get_path('purelib', vars={'base': ''})[1:])"
-RESULT_VARIABLE PY3_LIBDIR_CODE
-OUTPUT_VARIABLE PYTHON3_SITEDIR
-OUTPUT_STRIP_TRAILING_WHITESPACE)
- if (PY3_LIBDIR_CODE)
-message(WARNING
- "Could not determine MonetDB Python3 site-packages instalation
directory")
+ if(NOT DEFINED PYTHON3_LIBDIR)
+# Used for installing testing python module (don't pass a location, else
we need to strip this again)
+execute_process(COMMAND "${Python3_EXECUTABLE}" "-c" "import sysconfig;
print(sysconfig.get_path('purelib', vars={'base': ''})[1:])"
+ RESULT_VARIABLE PY3_LIBDIR_CODE
+ OUTPUT_VARIABLE PYTHON3_SITEDIR
+ OUTPUT_STRIP_TRAILING_WHITESPACE)
+if (PY3_LIBDIR_CODE)
+ message(WARNING
+"Could not determine MonetDB Python3 site-packages installation
directory")
+endif()
+set(PYTHON3_LIBDIR "${PYTHON3_SITEDIR}")
endif()
- set(PYTHON3_LIBDIR "${PYTHON3_SITEDIR}")
set(PYTHON "${Python3_EXECUTABLE}")
if(MSVC)
diff --git a/cmake/monetdb-packages.cmake b/cmake/monetdb-packages.cmake
--- a/cmake/monetdb-packages.cmake
+++ b/cmake/monetdb-packages.cmake
@@ -23,7 +23,7 @@ include(monetdb-wix-packages)
if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
get_os_release_info(LINUX_DISTRO LINUX_DISTRO_VERSION)
- if (${LINUX_DISTRO} STREQUAL "debian")
+ if ("${LINUX_DISTRO}" STREQUAL "debian")
monetdb_debian_extra_files()
endif()
endif()
diff --git a/debian/rules b/debian/rules
--- a/debian/rules
+++ b/debian/rules
@@ -13,6 +13,7 @@ DH_VERBOSE=1
override_dh_auto_configure:
dh_auto_configure -- \
+ -DPYTHON3_LIBDIR=lib/python3/dist-packages
-DCMAKE_INSTALL_RUNSTATEDIR=/run \
-DRELEASE_VERSION=ON \
-DASSERT=OFF \
diff --git a/documentation/source/developers_ha
MonetDB: Jul2021 - Added changelog blurb.
Changeset: 751bc783443d for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/751bc783443d Modified Files: gdk/ChangeLog.Jul2021 Branch: Jul2021 Log Message: Added changelog blurb. diffs (12 lines): diff --git a/gdk/ChangeLog.Jul2021 b/gdk/ChangeLog.Jul2021 --- a/gdk/ChangeLog.Jul2021 +++ b/gdk/ChangeLog.Jul2021 @@ -1,3 +1,8 @@ # ChangeLog file for GDK # This file is updated with Maddlog +* Fri Mar 18 2022 Sjoerd Mullender +- Fixed a race condition which could cause a too large size being written + for a .theap file to the BBP.dir file after the correct size file had + been saved to disk. + ___ checkin-list mailing list -- checkin-list@monetdb.org To unsubscribe send an email to checkin-list-le...@monetdb.org
MonetDB: Jan2022 - Merge with Jul2021 branch.
Changeset: 1f5203e4a3ac for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/1f5203e4a3ac Modified Files: gdk/ChangeLog.Jan2022 Branch: Jan2022 Log Message: Merge with Jul2021 branch. diffs (15 lines): diff --git a/gdk/ChangeLog.Jan2022 b/gdk/ChangeLog.Jan2022 --- a/gdk/ChangeLog.Jan2022 +++ b/gdk/ChangeLog.Jan2022 @@ -1,6 +1,11 @@ # ChangeLog file for GDK # This file is updated with Maddlog +* Fri Mar 18 2022 Sjoerd Mullender +- Fixed a race condition which could cause a too large size being written + for a .theap file to the BBP.dir file after the correct size file had + been saved to disk. + * Wed Mar 9 2022 Sjoerd Mullender - Fixed a bug in the append code for msk (bit mask) bats. - Conversions from floating point types to integral types that involve ___ checkin-list mailing list -- checkin-list@monetdb.org To unsubscribe send an email to checkin-list-le...@monetdb.org
MonetDB: default - Approve 32-bit and power upgrades
Changeset: b014a51863a0 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/b014a51863a0
Modified Files:
clients/Tests/MAL-signatures.test
sql/test/emptydb-previous-upgrade-chain-hge/Tests/upgrade.stable.out.ppc64.int128
sql/test/emptydb-previous-upgrade-chain/Tests/upgrade.stable.out.32bit
sql/test/emptydb-previous-upgrade-chain/Tests/upgrade.stable.out.ppc64
sql/test/emptydb-previous-upgrade-chain/Tests/upgrade.stable.out.ppc64.int128
sql/test/emptydb-previous-upgrade/Tests/upgrade.stable.out.32bit
sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.ppc64.int128
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.32bit
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.ppc64
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.ppc64.int128
sql/test/emptydb-upgrade/Tests/upgrade.stable.out.32bit
sql/test/emptydb/Tests/check.stable.out.32bit
sql/test/testdb-previous-upgrade-chain/Tests/upgrade.stable.out.32bit
sql/test/testdb-previous-upgrade/Tests/upgrade.stable.out.32bit
sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out.32bit
sql/test/testdb-upgrade/Tests/upgrade.stable.out.32bit
Branch: default
Log Message:
Approve 32-bit and power upgrades
diffs (223 lines):
diff --git a/clients/Tests/MAL-signatures.test
b/clients/Tests/MAL-signatures.test
--- a/clients/Tests/MAL-signatures.test
+++ b/clients/Tests/MAL-signatures.test
@@ -33008,6 +33008,11 @@ similarity
command battxtsim.similarity(X_0:bat[:str], X_1:bat[:str]):bat[:dbl]
fstrcmp0_impl_bulk;
Normalized edit distance between two strings
+baturl
+extractURLHost
+command baturl.extractURLHost(X_0:bat[:str], X_1:bit):bat[:str]
+BATextractURLHost;
+Extract host from BAT of URLs
batuuid
isaUUID
command batuuid.isaUUID(X_0:bat[:str]):bat[:bit]
@@ -47999,6 +48004,11 @@ command txtsim.stringdiff(X_0:str, X_1:s
stringdiff_impl;
calculate the soundexed editdistance
url
+extractURLHost
+command url.extractURLHost(X_0:str, X_1:bit):str
+extractURLHost;
+Extract host from a URL relaxed version
+url
getAnchor
command url.getAnchor(X_0:url):str
URLgetAnchor;
@@ -48032,7 +48042,7 @@ url
getHost
command url.getHost(X_0:url):str
URLgetHost;
-Extract the server name from the URL
+Extract the server name from the URL strict version
url
getPort
command url.getPort(X_0:url):str
diff --git
a/sql/test/emptydb-previous-upgrade-chain-hge/Tests/upgrade.stable.out.ppc64.int128
b/sql/test/emptydb-previous-upgrade-chain-hge/Tests/upgrade.stable.out.ppc64.int128
---
a/sql/test/emptydb-previous-upgrade-chain-hge/Tests/upgrade.stable.out.ppc64.int128
+++
b/sql/test/emptydb-previous-upgrade-chain-hge/Tests/upgrade.stable.out.ppc64.int128
@@ -4234,4 +4234,7 @@ RETURN sys.dump_statements;
END;
update sys._tables set system = true where name in
('describe_partition_tables', 'dump_partition_tables', 'dump_sequences',
'dump_start_sequences') AND schema_id = 2000;
update sys.functions set system = true where system <> true and name in
('dump_database') and schema_id = 2000 and type = 5;
-
+CREATE function sys.url_extract_host(url string, no_www bool) RETURNS STRING
+EXTERNAL NAME url."extractURLHost";
+GRANT EXECUTE ON FUNCTION url_extract_host(string, bool) TO PUBLIC;
+update sys.functions set system = true where system <> true and name =
'url_extract_host' and schema_id = 2000 and type = 1;
diff --git
a/sql/test/emptydb-previous-upgrade-chain/Tests/upgrade.stable.out.32bit
b/sql/test/emptydb-previous-upgrade-chain/Tests/upgrade.stable.out.32bit
--- a/sql/test/emptydb-previous-upgrade-chain/Tests/upgrade.stable.out.32bit
+++ b/sql/test/emptydb-previous-upgrade-chain/Tests/upgrade.stable.out.32bit
@@ -3887,4 +3887,7 @@ RETURN sys.dump_statements;
END;
update sys._tables set system = true where name in
('describe_partition_tables', 'dump_partition_tables', 'dump_sequences',
'dump_start_sequences') AND schema_id = 2000;
update sys.functions set system = true where system <> true and name in
('dump_database') and schema_id = 2000 and type = 5;
-
+CREATE function sys.url_extract_host(url string, no_www bool) RETURNS STRING
+EXTERNAL NAME url."extractURLHost";
+GRANT EXECUTE ON FUNCTION url_extract_host(string, bool) TO PUBLIC;
+update sys.functions set system = true where system <> true and name =
'url_extract_host' and schema_id = 2000 and type = 1;
diff --git
a/sql/test/emptydb-previous-upgrade-chain/Tests/upgrade.stable.out.ppc64
b/sql/test/emptydb-previous-upgrade-chain/Tests/upgrade.stable.out.ppc64
--- a/sql/test/emptydb-previous-upgrade-chain/Tests/upgrade.stable.out.ppc64
+++ b/sql/test/emptydb-previous-upgrade-chain/Tests/upgrade.stable.out.ppc64
@@ -3887,4 +3887,7 @@ RETURN sys.dump_statements;
END;
update sys._tables set system = true where name in
('describe_partition_tables', 'dump_partition_tables', 'dump_sequences',
'dump_start_sequences') AND schema_id = 2000;
update s
MonetDB: sqloptimizer - Merged with default
Changeset: db8144a929c8 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/db8144a929c8
Branch: sqloptimizer
Log Message:
Merged with default
diffs (truncated from 2926 to 300 lines):
diff --git a/clients/Tests/MAL-signatures.test
b/clients/Tests/MAL-signatures.test
--- a/clients/Tests/MAL-signatures.test
+++ b/clients/Tests/MAL-signatures.test
@@ -33008,6 +33008,11 @@ similarity
command battxtsim.similarity(X_0:bat[:str], X_1:bat[:str]):bat[:dbl]
fstrcmp0_impl_bulk;
Normalized edit distance between two strings
+baturl
+extractURLHost
+command baturl.extractURLHost(X_0:bat[:str], X_1:bit):bat[:str]
+BATextractURLHost;
+Extract host from BAT of URLs
batuuid
isaUUID
command batuuid.isaUUID(X_0:bat[:str]):bat[:bit]
@@ -47999,6 +48004,11 @@ command txtsim.stringdiff(X_0:str, X_1:s
stringdiff_impl;
calculate the soundexed editdistance
url
+extractURLHost
+command url.extractURLHost(X_0:str, X_1:bit):str
+extractURLHost;
+Extract host from a URL relaxed version
+url
getAnchor
command url.getAnchor(X_0:url):str
URLgetAnchor;
@@ -48032,7 +48042,7 @@ url
getHost
command url.getHost(X_0:url):str
URLgetHost;
-Extract the server name from the URL
+Extract the server name from the URL strict version
url
getPort
command url.getPort(X_0:url):str
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -758,8 +758,9 @@ typedef struct {
#define GDKLIBRARY_MINMAX_POS 061042U /* first in Nov2019: no min/max
position; no BBPinfo value */
#define GDKLIBRARY_TAILN 061043U /* first in Jul2021: str offset heaps
names don't take width into account */
#define GDKLIBRARY_HASHASH 061044U /* first in Jul2021: hashash bit in
string heaps */
+#define GDKLIBRARY_HSIZE 061045U /* first in Jan2022: heap "size" values
*/
/* if the version number is updated, also fix snapshot_bats() in bat_logger.c
*/
-#define GDKLIBRARY 061045U /* first after Jul2021 */
+#define GDKLIBRARY 061046U /* first after Jan2022 */
typedef struct BAT {
/* static bat properties */
diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c
--- a/gdk/gdk_bat.c
+++ b/gdk/gdk_bat.c
@@ -180,28 +180,6 @@ BATsetdims(BAT *b, uint16_t width)
}
const char *
-gettailnamebi(const BATiter *bi)
-{
- if (bi->type == TYPE_str) {
- switch (bi->width) {
- case 1:
- return "tail1";
- case 2:
- return "tail2";
- case 4:
-#if SIZEOF_VAR_T == 8
- return "tail4";
- case 8:
-#endif
- break;
- default:
- MT_UNREACHABLE();
- }
- }
- return "tail";
-}
-
-const char *
gettailname(const BAT *b)
{
if (b->ttype == TYPE_str) {
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -443,6 +443,8 @@ heapinit(BAT *b, const char *buf,
(void) bbpversion; /* could be used to implement compatibility */
minpos = maxpos = (uint64_t) oid_nil; /* for GDKLIBRARY_MINMAX_POS case
*/
+ size = 0; /* for GDKLIBRARY_HSIZE case */
+ storage = STORE_INVALID; /* for GDKLIBRARY_HSIZE case */
if (bbpversion <= GDKLIBRARY_MINMAX_POS ?
sscanf(buf,
" %10s %" SCNu16 " %" SCNu16 " %" SCNu16 " %" SCNu64
@@ -453,6 +455,7 @@ heapinit(BAT *b, const char *buf,
&nokey1, &nosorted, &norevsorted, &base,
&free, &size, &storage,
&n) < 12 :
+ bbpversion <= GDKLIBRARY_HSIZE ?
sscanf(buf,
" %10s %" SCNu16 " %" SCNu16 " %" SCNu16 " %" SCNu64
" %" SCNu64 " %" SCNu64 " %" SCNu64 " %" SCNu64
@@ -461,7 +464,16 @@ heapinit(BAT *b, const char *buf,
type, &width, &var, &properties, &nokey0,
&nokey1, &nosorted, &norevsorted, &base,
&free, &size, &storage, &minpos, &maxpos,
- &n) < 14) {
+ &n) < 14 :
+ sscanf(buf,
+ " %10s %" SCNu16 " %" SCNu16 " %" SCNu16 " %" SCNu64
+ " %" SCNu64 " %" SCNu64 " %" SCNu64 " %" SCNu64
+ " %" SCNu64 " %" SCNu64 " %" SCNu64
+ "%n",
+ type, &width, &var, &properties, &nokey0,
+ &nokey1, &nosorted, &norevsorted, &base,
+ &free, &minpos, &maxpos,
+ &n) < 12) {
TRC_CRITICAL(GDK, "invalid format for BBP.dir on line %d",
lineno);
return -1;
}
@@ -515,7 +527,14 @@ heapinit(BAT *b, const char *buf,
/* (properties & 0x0200) is the old tdense flag */
b->tseqbase = (properties & 0x0200) == 0 || base >= (uint64_t) oid_nil
? oid_nil : (oid) base;
b->theap->free = (size_t) free;
- b->theap->size = (size_t) size;
+ /* set heap size to mat
MonetDB: Jul2021 - Deal with not-yet-known types during initiali...
Changeset: 708ee5158058 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/708ee5158058
Modified Files:
gdk/gdk_bbp.c
gdk/gdk_string.c
Branch: Jul2021
Log Message:
Deal with not-yet-known types during initialization.
diffs (31 lines):
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -549,7 +549,8 @@ vheapinit(BAT *b, const char *buf, int h
TRC_CRITICAL(GDK, "cannot allocate memory for heap.");
return -1;
}
- if (ATOMstorage(b->ttype) == TYPE_str &&
+ if (b->ttype >= 0 &&
+ ATOMstorage(b->ttype) == TYPE_str &&
free < GDK_STRHASHTABLE * sizeof(stridx_t) + BATTINY *
GDK_VARALIGN)
size = GDK_STRHASHTABLE * sizeof(stridx_t) + BATTINY *
GDK_VARALIGN;
else if (free < 512)
diff --git a/gdk/gdk_string.c b/gdk/gdk_string.c
--- a/gdk/gdk_string.c
+++ b/gdk/gdk_string.c
@@ -88,6 +88,14 @@ strCleanHash(Heap *h, bool rebuild)
(void) rebuild;
if (!h->cleanhash)
return;
+ if (h->size < GDK_STRHASHTABLE * sizeof(stridx_t) &&
+ HEAPextend(h, GDK_STRHASHTABLE * sizeof(stridx_t) + BATTINY *
GDK_VARALIGN, true) != GDK_SUCCEED) {
+ GDKclrerr();
+ if (h->size > 0)
+ memset(h->base, 0, h->size);
+ return;
+ }
+
/* rebuild hash table for double elimination
*
* If appending strings to the BAT was aborted, if the heap
___
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org
MonetDB: Jan2022 - Merge with Jul2021 branch.
Changeset: ddd7eca55077 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/ddd7eca55077
Modified Files:
gdk/gdk_bbp.c
gdk/gdk_string.c
Branch: Jan2022
Log Message:
Merge with Jul2021 branch.
diffs (31 lines):
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -561,7 +561,8 @@ vheapinit(BAT *b, const char *buf, bat b
TRC_CRITICAL(GDK, "cannot allocate memory for heap.");
return -1;
}
- if (ATOMstorage(b->ttype) == TYPE_str &&
+ if (b->ttype >= 0 &&
+ ATOMstorage(b->ttype) == TYPE_str &&
free < GDK_STRHASHTABLE * sizeof(stridx_t) + BATTINY *
GDK_VARALIGN)
size = GDK_STRHASHTABLE * sizeof(stridx_t) + BATTINY *
GDK_VARALIGN;
else if (free < 512)
diff --git a/gdk/gdk_string.c b/gdk/gdk_string.c
--- a/gdk/gdk_string.c
+++ b/gdk/gdk_string.c
@@ -86,6 +86,14 @@ strCleanHash(Heap *h, bool rebuild)
(void) rebuild;
if (!h->cleanhash)
return;
+ if (h->size < GDK_STRHASHTABLE * sizeof(stridx_t) &&
+ HEAPextend(h, GDK_STRHASHTABLE * sizeof(stridx_t) + BATTINY *
GDK_VARALIGN, true) != GDK_SUCCEED) {
+ GDKclrerr();
+ if (h->size > 0)
+ memset(h->base, 0, h->size);
+ return;
+ }
+
/* rebuild hash table for double elimination
*
* If appending strings to the BAT was aborted, if the heap
___
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org
MonetDB: default - Merge with Jan2022 branch.
Changeset: 124433b46e8d for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/124433b46e8d
Modified Files:
gdk/gdk_bbp.c
gdk/gdk_string.c
Branch: default
Log Message:
Merge with Jan2022 branch.
diffs (46 lines):
diff --git a/gdk/ChangeLog.Jan2022 b/gdk/ChangeLog.Jan2022
--- a/gdk/ChangeLog.Jan2022
+++ b/gdk/ChangeLog.Jan2022
@@ -1,6 +1,11 @@
# ChangeLog file for GDK
# This file is updated with Maddlog
+* Fri Mar 18 2022 Sjoerd Mullender
+- Fixed a race condition which could cause a too large size being written
+ for a .theap file to the BBP.dir file after the correct size file had
+ been saved to disk.
+
* Wed Mar 9 2022 Sjoerd Mullender
- Fixed a bug in the append code for msk (bit mask) bats.
- Conversions from floating point types to integral types that involve
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -582,7 +582,8 @@ vheapinit(BAT *b, const char *buf, bat b
TRC_CRITICAL(GDK, "cannot allocate memory for heap.");
return -1;
}
- if (ATOMstorage(b->ttype) == TYPE_str &&
+ if (b->ttype >= 0 &&
+ ATOMstorage(b->ttype) == TYPE_str &&
free < GDK_STRHASHTABLE * sizeof(stridx_t) + BATTINY *
GDK_VARALIGN)
size = GDK_STRHASHTABLE * sizeof(stridx_t) + BATTINY *
GDK_VARALIGN;
else if (free < 512)
diff --git a/gdk/gdk_string.c b/gdk/gdk_string.c
--- a/gdk/gdk_string.c
+++ b/gdk/gdk_string.c
@@ -86,6 +86,14 @@ strCleanHash(Heap *h, bool rebuild)
(void) rebuild;
if (!h->cleanhash)
return;
+ if (h->size < GDK_STRHASHTABLE * sizeof(stridx_t) &&
+ HEAPextend(h, GDK_STRHASHTABLE * sizeof(stridx_t) + BATTINY *
GDK_VARALIGN, true) != GDK_SUCCEED) {
+ GDKclrerr();
+ if (h->size > 0)
+ memset(h->base, 0, h->size);
+ return;
+ }
+
/* rebuild hash table for double elimination
*
* If appending strings to the BAT was aborted, if the heap
___
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org
MonetDB: Jan2022 - Removed superfluous operation.
Changeset: 90ce50cd82c0 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/90ce50cd82c0 Modified Files: gdk/gdk_bbp.c Branch: Jan2022 Log Message: Removed superfluous operation. diffs (12 lines): diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c --- a/gdk/gdk_bbp.c +++ b/gdk/gdk_bbp.c @@ -1849,7 +1849,7 @@ heap_entry(FILE *fp, BATiter *bi, BUN si b->tvarsized, (unsigned short) b->tsorted | ((unsigned short) b->trevsorted << 7) | - (((unsigned short) b->tkey & 0x01) << 8) | + ((unsigned short) b->tkey << 8) | ((unsigned short) BATtdense(b) << 9) | ((unsigned short) b->tnonil << 10) | ((unsigned short) b->tnil << 11), ___ checkin-list mailing list -- checkin-list@monetdb.org To unsubscribe send an email to checkin-list-le...@monetdb.org
MonetDB: default - Get rid of macro BUNlast, just use BATcount i...
Changeset: 7b7730305ebf for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/7b7730305ebf
Modified Files:
gdk/ChangeLog
gdk/gdk.h
gdk/gdk_aggr.c
gdk/gdk_align.c
gdk/gdk_bat.c
gdk/gdk_batop.c
gdk/gdk_delta.c
gdk/gdk_delta.h
gdk/gdk_group.c
gdk/gdk_join.c
gdk/gdk_logger.c
gdk/gdk_logger_old.c
gdk/gdk_select.c
gdk/gdk_unique.c
monetdb5/modules/atoms/batxml.c
monetdb5/modules/kernel/algebra.c
monetdb5/modules/kernel/bat5.c
monetdb5/modules/mal/batMask.c
monetdb5/modules/mal/iterator.c
monetdb5/modules/mal/manifold.c
monetdb5/modules/mal/pcre.c
monetdb5/modules/mal/wlc.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_subquery.c
sql/storage/bat/bat_logger.c
sql/storage/bat/bat_storage.c
Branch: default
Log Message:
Get rid of macro BUNlast, just use BATcount instead.
diffs (truncated from 747 to 300 lines):
diff --git a/gdk/ChangeLog b/gdk/ChangeLog
--- a/gdk/ChangeLog
+++ b/gdk/ChangeLog
@@ -1,6 +1,9 @@
# ChangeLog file for GDK
# This file is updated with Maddlog
+* Fri Mar 18 2022 Sjoerd Mullender
+- Get rid of macro BUNlast, just use BATcount instead.
+
* Mon Jan 31 2022 Sjoerd Mullender
- The BLOB type has been moved into the GDK layer.
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -1170,8 +1170,6 @@ typedef var_t stridx_t;
#define BUNtvar(bi,p) (assert((bi).type && (bi).b->tvarsized), (void *)
((bi).vh->base+BUNtvaroff(bi,p)))
#define BUNtail(bi,p)
((bi).type?(bi).b->tvarsized?BUNtvar(bi,p):(bi).type==TYPE_msk?BUNtmsk(bi,p):BUNtloc(bi,p):BUNtpos(bi,p))
-#define BUNlast(b) (assert((b)->batCount <= BUN_MAX), (b)->batCount)
-
#define BATcount(b)((b)->batCount)
#include "gdk_atoms.h"
@@ -2228,7 +2226,7 @@ gdk_export void VIEWbounds(BAT *b, BAT *
* is the iteration variable.
*/
#define BATloop(r, p, q) \
- for (q = BUNlast(r), p = 0; p < q; p++)
+ for (q = BATcount(r), p = 0; p < q; p++)
/*
* @+ Common BAT Operations
diff --git a/gdk/gdk_aggr.c b/gdk/gdk_aggr.c
--- a/gdk/gdk_aggr.c
+++ b/gdk/gdk_aggr.c
@@ -102,7 +102,7 @@ BATgroupaggrinit(BAT *b, BAT *g, BAT *e,
if (!is_oid_nil(min)) {
/* found a non-nil, max must be last
* value (and there is one!) */
- max = gids[BUNlast(g) - 1];
+ max = gids[BATcount(g) - 1];
}
} else {
/* we'll do a complete scan */
diff --git a/gdk/gdk_align.c b/gdk/gdk_align.c
--- a/gdk/gdk_align.c
+++ b/gdk/gdk_align.c
@@ -179,7 +179,7 @@ BATmaterialize(BAT *b)
if ((tail = GDKmalloc(sizeof(Heap))) == NULL)
return GDK_FAIL;
p = 0;
- q = BUNlast(b);
+ q = BATcount(b);
assert(cnt >= q - p);
TRC_DEBUG(ALGO, "BATmaterialize(" ALGOBATFMT ")\n", ALGOBATPAR(b));
diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c
--- a/gdk/gdk_bat.c
+++ b/gdk/gdk_bat.c
@@ -631,7 +631,7 @@ BATclear(BAT *b, bool force)
if (tatmdel) {
BATiter bi = bat_iterator_nolock(b);
- for (p = b->batInserted, q = BUNlast(b); p < q; p++)
+ for (p = b->batInserted, q = BATcount(b); p < q; p++)
(*tatmdel)(b->tvheap, (var_t*) BUNtloc(bi,p));
b->tvheap->dirty = true;
}
@@ -951,7 +951,7 @@ COLcopy(BAT *b, int tt, bool writable, r
bn->tunique_est = bi.unique_est;
} else if (ATOMstorage(tt) == ATOMstorage(b->ttype) &&
ATOMcompare(tt) == ATOMcompare(b->ttype)) {
- BUN h = BUNlast(b);
+ BUN h = BATcount(b);
bn->tsorted = b->tsorted;
bn->trevsorted = b->trevsorted;
if (b->tkey)
@@ -1018,7 +1018,7 @@ BUNappendmulti(BAT *b, const void *value
TRC_DEBUG(ALGO, ALGOBATFMT " appending " BUNFMT " values%s\n",
ALGOBATPAR(b), count, values ? "" : " (all nil)");
- p = BUNlast(b); /* insert at end */
+ p = BATcount(b);/* insert at end */
if (p == BUN_MAX || BATcount(b) + count >= BUN_MAX) {
GDKerror("bat too large\n");
return GDK_FAIL;
@@ -1326,7 +1326,7 @@ BUNdelete(BAT *b, oid o)
return GDK_FAIL;
HASHdelete(b, p, val);
ATOMdel(b->ttype, b->tvheap, (var_t *) BUNtloc(bi, p));
- if (p != BUNlast(b) - 1 &&
+ if (p != BATcount(b) - 1 &&
(b->ttype != TYPE_void || BATtdense(b))) {
/* replace to-be-delete BUN with last BUN; materialize
* void column before doing so */
@@ -1334
MonetDB: Jan2022 - Use BUN for output count, so overflows are av...
Changeset: bdfa64c97652 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/bdfa64c97652
Modified Files:
sql/backends/monet5/generator/generator.c
Branch: Jan2022
Log Message:
Use BUN for output count, so overflows are avoided. Hopefully fixes cnt <=
BUN_MAX assertion
diffs (39 lines):
diff --git a/sql/backends/monet5/generator/generator.c
b/sql/backends/monet5/generator/generator.c
--- a/sql/backends/monet5/generator/generator.c
+++ b/sql/backends/monet5/generator/generator.c
@@ -419,7 +419,7 @@ VLTgenerator_subselect(Client cntxt, Mal
}
if (cand)
BBPunfix(cand->batCacheid);
- BATsetcount(bn, (BUN) n);
+ BATsetcount(bn, n);
bn->tsorted = true;
bn->trevsorted = BATcount(bn) <= 1;
bn->tkey = true;
@@ -567,11 +567,11 @@ VLTgenerator_subselect(Client cntxt, Mal
str VLTgenerator_thetasubselect(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr pci)
{
- int c= 0, anti =0,tpe;
+ int anti =0,tpe;
bat cndid =0;
BAT *cand = 0, *bn = NULL;
struct canditer ci = (struct canditer) {.tpe = cand_dense};
- BUN cap,j;
+ BUN cap,j, c = 0;
oid o = 0;
InstrPtr p;
str oper;
@@ -741,10 +741,10 @@ str VLTgenerator_thetasubselect(Client c
str VLTgenerator_projection(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr pci)
{
- int c= 0, tpe;
+ int tpe;
bat *ret;
BAT *b, *bn = NULL;
- BUN cnt;
+ BUN cnt, c = 0;
oid *ol = NULL, o= 0;
InstrPtr p;
___
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org
MonetDB: default - Merged with Jan2022
Changeset: b729c688d7d7 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/b729c688d7d7
Modified Files:
gdk/gdk_bbp.c
Branch: default
Log Message:
Merged with Jan2022
diffs (51 lines):
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -1878,7 +1878,7 @@ heap_entry(FILE *fp, BATiter *bi, BUN si
b->tvarsized,
(unsigned short) b->tsorted |
((unsigned short) b->trevsorted << 7) |
- (((unsigned short) b->tkey & 0x01) << 8) |
+ ((unsigned short) b->tkey << 8) |
((unsigned short) BATtdense(b) << 9) |
((unsigned short) b->tnonil << 10) |
((unsigned short) b->tnil << 11),
diff --git a/sql/backends/monet5/generator/generator.c
b/sql/backends/monet5/generator/generator.c
--- a/sql/backends/monet5/generator/generator.c
+++ b/sql/backends/monet5/generator/generator.c
@@ -419,7 +419,7 @@ VLTgenerator_subselect(Client cntxt, Mal
}
if (cand)
BBPunfix(cand->batCacheid);
- BATsetcount(bn, (BUN) n);
+ BATsetcount(bn, n);
bn->tsorted = true;
bn->trevsorted = BATcount(bn) <= 1;
bn->tkey = true;
@@ -567,11 +567,11 @@ VLTgenerator_subselect(Client cntxt, Mal
str VLTgenerator_thetasubselect(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr pci)
{
- int c= 0, anti =0,tpe;
+ int anti =0,tpe;
bat cndid =0;
BAT *cand = 0, *bn = NULL;
struct canditer ci = (struct canditer) {.tpe = cand_dense};
- BUN cap,j;
+ BUN cap,j, c = 0;
oid o = 0;
InstrPtr p;
str oper;
@@ -741,10 +741,10 @@ str VLTgenerator_thetasubselect(Client c
str VLTgenerator_projection(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr pci)
{
- int c= 0, tpe;
+ int tpe;
bat *ret;
BAT *b, *bn = NULL;
- BUN cnt;
+ BUN cnt, c = 0;
oid *ol = NULL, o= 0;
InstrPtr p;
___
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org
MonetDB: default - Get rid of macro Tsize, use ->twidth instead.
Changeset: 9dfa3162fce2 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/9dfa3162fce2
Modified Files:
gdk/ChangeLog
gdk/gdk.h
gdk/gdk_bat.c
gdk/gdk_batop.c
gdk/gdk_delta.h
gdk/gdk_join.c
monetdb5/modules/mal/txtsim.c
Branch: default
Log Message:
Get rid of macro Tsize, use ->twidth instead.
diffs (132 lines):
diff --git a/gdk/ChangeLog b/gdk/ChangeLog
--- a/gdk/ChangeLog
+++ b/gdk/ChangeLog
@@ -2,6 +2,7 @@
# This file is updated with Maddlog
* Fri Mar 18 2022 Sjoerd Mullender
+- Get rid of macro Tsize, use ->twidth instead.
- Get rid of macro BUNlast, just use BATcount instead.
* Mon Jan 31 2022 Sjoerd Mullender
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -1148,8 +1148,6 @@ gdk_export BUN BUNfnd(BAT *b, const void
#define BATttype(b)(BATtdense(b) ? TYPE_oid : (b)->ttype)
-#define Tsize(b) ((b)->twidth)
-
#define tailsize(b,p) ((b)->ttype ? \
(ATOMstorage((b)->ttype) == TYPE_msk ? \
(((size_t) (p) + 31) / 32) * 4 : \
diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c
--- a/gdk/gdk_bat.c
+++ b/gdk/gdk_bat.c
@@ -1342,7 +1342,7 @@ BUNdelete(BAT *b, oid o)
} else {
val = Tloc(b, BATcount(b) - 1);
HASHdelete(b, BATcount(b) - 1, val);
- memcpy(Tloc(b, p), val, Tsize(b));
+ memcpy(Tloc(b, p), val, b->twidth);
HASHinsert(b, p, val);
MT_lock_set(&b->theaplock);
if (b->tminpos == BATcount(b) - 1)
diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c
--- a/gdk/gdk_batop.c
+++ b/gdk/gdk_batop.c
@@ -934,7 +934,7 @@ BATdel(BAT *b, BAT *d)
} else {
memmove(Tloc(b, o),
Tloc(b, o + c),
- Tsize(b) * (BATcount(b) - (o + c)));
+ b->twidth * (BATcount(b) - (o + c)));
}
b->theap->dirty = true;
// o += b->hseqbase; // if this were to be used again
@@ -999,7 +999,7 @@ BATdel(BAT *b, BAT *d)
}
pos += n;
} else {
- n *= Tsize(b);
+ n *= b->twidth;
memmove(p,
Tloc(b, o[-1] + 1 -
b->hseqbase),
n);
@@ -2460,7 +2460,7 @@ BATsort(BAT **sorted, BAT **order, BAT *
if (do_sort(Tloc(bn, r),
ords ? ords + r : NULL,
bn->tvheap ? bn->tvheap->base :
NULL,
- p - r, Tsize(bn), ords ?
sizeof(oid) : 0,
+ p - r, bn->twidth, ords ?
sizeof(oid) : 0,
bn->ttype, reverse, nilslast,
stable) != GDK_SUCCEED)
goto error;
r = p;
@@ -2471,7 +2471,7 @@ BATsort(BAT **sorted, BAT **order, BAT *
if (do_sort(Tloc(bn, r),
ords ? ords + r : NULL,
bn->tvheap ? bn->tvheap->base : NULL,
- p - r, Tsize(bn), ords ? sizeof(oid) : 0,
+ p - r, bn->twidth, ords ? sizeof(oid) : 0,
bn->ttype, reverse, nilslast, stable) !=
GDK_SUCCEED)
goto error;
/* if single group (r==0) the result is (rev)sorted,
@@ -2503,7 +2503,7 @@ BATsort(BAT **sorted, BAT **order, BAT *
do_sort(Tloc(bn, 0),
ords,
bn->tvheap ? bn->tvheap->base : NULL,
-BATcount(bn), Tsize(bn), ords ? sizeof(oid) : 0,
+BATcount(bn), bn->twidth, ords ? sizeof(oid) : 0,
bn->ttype, reverse, nilslast, stable) !=
GDK_SUCCEED)) {
if (m != NULL) {
HEAPfree(m, true);
diff --git a/gdk/gdk_delta.h b/gdk/gdk_delta.h
--- a/gdk/gdk_delta.h
+++ b/gdk/gdk_delta.h
@@ -21,7 +21,7 @@ DELTAinit(BAT *b)
BATsetcount(b, 0);
b->theap->free = 0;
b->batInserted = 0;
- b->tshift = ATOMelmshift(Tsize(b));
+ b->tshift = ATOMelmshift(b->twidth);
TRC_DEBUG(DELTA,
"%s free %zu ins " BUNFMT " base %p\n",
BBP_logical(b->batCacheid),
diff --git a/gdk/gdk_join.c b/gdk/gdk_join.c
--- a/gdk/gdk_join.c
+++ b/gdk/gdk_join.c
@@ -3856,7 +3856,7
MonetDB: default - Removed BATmsync. It wasn't actually used.
Changeset: 4462862344a4 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/4462862344a4
Modified Files:
clients/Tests/exports.stable.out
gdk/gdk.h
gdk/gdk_storage.c
monetdb5/modules/kernel/bat5.c
monetdb5/modules/mal/tablet.c
Branch: default
Log Message:
Removed BATmsync. It wasn't actually used.
diffs (222 lines):
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -166,7 +166,6 @@ BAT *BATmergecand(BAT *a, BAT *b);
void *BATmin(BAT *b, void *aggr);
void *BATmin_skipnil(BAT *b, void *aggr, bit skipnil);
gdk_return BATmode(BAT *b, bool transient);
-void BATmsync(BAT *b);
BAT *BATnegcands(BUN nr, BAT *odels);
BAT *BATnil_grp(BAT *l, BAT *g, BAT *e, BAT *s);
bool BATordered(BAT *b);
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -1306,7 +1306,6 @@ gdk_export gdk_return BATgroup(BAT **gro
gdk_export gdk_return BATsave(BAT *b)
__attribute__((__warn_unused_result__));
-gdk_export void BATmsync(BAT *b);
#define NOFARM (-1) /* indicate to GDKfilepath to create relative path */
diff --git a/gdk/gdk_storage.c b/gdk/gdk_storage.c
--- a/gdk/gdk_storage.c
+++ b/gdk/gdk_storage.c
@@ -666,103 +666,6 @@ DESCload(int i)
return b;
}
-/* spawning the background msync should be done carefully
- * because there is a (small) chance that the BAT has been
- * deleted by the time you issue the msync.
- * This leaves you with possibly deadbeef BAT descriptors.
- */
-
-/* #define DISABLE_MSYNC */
-#define MSYNC_BACKGROUND
-
-#ifndef DISABLE_MSYNC
-#ifndef MS_ASYNC
-struct msync {
- bat id;
- Heap *h;
-};
-
-static void
-BATmsyncImplementation(void *arg)
-{
- Heap *h = ((struct msync *) arg)->h;
-
- (void) MT_msync(h->base, h->size);
- BBPunfix(((struct msync *) arg)->id);
- GDKfree(arg);
-}
-#endif
-#endif
-
-void
-BATmsync(BAT *b)
-{
- /* we don't sync views or if we're told not to */
- if (isVIEW(b) || GDKinmemory(b->theap->farmid) || (GDKdebug &
NOSYNCMASK))
- return;
- /* we don't sync transients */
- if (b->theap->farmid != 0 ||
- (b->tvheap != NULL && b->tvheap->farmid != 0))
- return;
-#ifndef DISABLE_MSYNC
-#ifdef MS_ASYNC
- if (b->theap->storage == STORE_MMAP &&
- msync(b->theap->base, b->theap->free, MS_ASYNC) < 0)
- GDKsyserror("msync heap of bat %d failed\n", b->batCacheid);
- if (b->tvheap && b->tvheap->storage == STORE_MMAP &&
- msync(b->tvheap->base, b->tvheap->free, MS_ASYNC) < 0)
- GDKsyserror("msync vheap of bat %d failed\n", b->batCacheid);
-#else
- {
- struct msync *arg;
-
- assert(!b->batTransient);
- if (b->theap->storage == STORE_MMAP &&
- (arg = GDKmalloc(sizeof(*arg))) != NULL) {
- arg->id = b->batCacheid;
- arg->h = b->theap;
- BBPfix(b->batCacheid);
-#ifdef MSYNC_BACKGROUND
- char name[MT_NAME_LEN];
- MT_Id tid;
- snprintf(name, sizeof(name), "msync%d", b->batCacheid);
- if (MT_create_thread(&tid, BATmsyncImplementation, arg,
-MT_THR_DETACHED, name) < 0) {
- /* don't bother if we can't create a thread */
- BBPunfix(b->batCacheid);
- GDKfree(arg);
- }
-#else
- BATmsyncImplementation(arg);
-#endif
- }
-
- if (b->tvheap && b->tvheap->storage == STORE_MMAP &&
- (arg = GDKmalloc(sizeof(*arg))) != NULL) {
- arg->id = b->batCacheid;
- arg->h = b->tvheap;
- BBPfix(b->batCacheid);
-#ifdef MSYNC_BACKGROUND
- char name[MT_NAME_LEN];
- MT_Id tid;
- snprintf(name, sizeof(name), "msync%d", b->batCacheid);
- if (MT_create_thread(&tid, BATmsyncImplementation, arg,
-MT_THR_DETACHED, name) < 0) {
- /* don't bother if we can't create a thread */
- BBPunfix(b->batCacheid);
- GDKfree(arg);
- }
-#else
- BATmsyncImplementation(arg);
-#endif
- }
- }
-#endif
-#else
- (void) b;
-#endif /* DISABLE_MSYNC */
-}
-
gdk_return
BATsave_locked(BAT *b, BATiter *bi, BUN size)
{
diff --git a/monetdb5/modules/kernel/bat5.c b/monetdb5/modules/kernel/bat5.c
--- a/monetdb5/modules/kernel/bat5.c
+++ b/monetdb5/modules/kernel/bat5.c
@@ -78,8 +78,6 @@ BKCattach(bat *ret, const int *tt, const
MonetDB: default - Validate aggregates without parameters inside...
Changeset: 466f075c39c2 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/466f075c39c2
Modified Files:
sql/server/rel_select.c
sql/test/SQLancer/Tests/sqlancer23.test
Branch: default
Log Message:
Validate aggregates without parameters inside subqueries
diffs (143 lines):
diff --git a/sql/server/rel_select.c b/sql/server/rel_select.c
--- a/sql/server/rel_select.c
+++ b/sql/server/rel_select.c
@@ -3418,14 +3418,14 @@ static sql_exp *
mvc *sql = query->sql;
exp_kind ek = {type_value, card_column, FALSE};
sql_subfunc *a = NULL;
- int no_nil = 0, group = 0;
+ int no_nil = 0, group = 0, all_aggr = query_has_outer(query), i;
unsigned int all_freevar = 0;
sql_rel *groupby = rel ? *rel : NULL, *sel = NULL, *gr, *og = NULL,
*res = groupby;
sql_rel *subquery = NULL;
- list *exps = NULL;
+ list *exps = NULL, *ungrouped_cols = NULL;
bool is_grouping = !strcmp(aname, "grouping"), has_args = false, found
= false, used_rel = false;
- if (!query_has_outer(query)) {
+ if (!all_aggr) {
if (!groupby) {
char *uaname = SA_NEW_ARRAY(sql->ta, char,
strlen(aname) + 1);
return sql_error(sql, 02, SQLSTATE(42000) "%s: missing
group by", toUpperCopy(uaname, aname));
@@ -3458,9 +3458,7 @@ static sql_exp *
exps = sa_list(sql->sa);
if (args && args->data.sym) {
- int i, all_aggr = query_has_outer(query);
bool arguments_correlated = true, all_const = true;
- list *ungrouped_cols = NULL;
all_freevar = all_aggr?1:0;
for (i = 0; args && args->data.sym; args = args->next, i++) {
@@ -3496,54 +3494,54 @@ static sql_exp *
}
if (all_const)
all_freevar = 0;
- if (!all_freevar) {
- if (is_sql_groupby(f)) {
- char *uaname = SA_NEW_ARRAY(sql->ta, char,
strlen(aname) + 1);
- return sql_error(sql, 02, SQLSTATE(42000) "%s:
aggregate function '%s' not allowed in GROUP BY clause", toUpperCopy(uaname,
aname), aname);
- } else if (is_sql_aggr(f) && groupby->grouped) {
- char *uaname = SA_NEW_ARRAY(sql->ta, char,
strlen(aname) + 1);
- return sql_error(sql, 02, SQLSTATE(42000) "%s:
aggregate functions cannot be nested", toUpperCopy(uaname, aname));
- } else if (is_sql_values(f)) {
- char *uaname = SA_NEW_ARRAY(sql->ta, char,
strlen(aname) + 1);
- return sql_error(sql, 02, SQLSTATE(42000) "%s:
aggregate functions not allowed on an unique value", toUpperCopy(uaname,
aname));
- } else if (is_sql_join(f)) { /* the is_sql_join test
must come before is_sql_where, because the join conditions are handled with
sql_where */
- char *uaname = SA_NEW_ARRAY(sql->ta, char,
strlen(aname) + 1);
- return sql_error(sql, 02, SQLSTATE(42000) "%s:
aggregate functions not allowed in JOIN conditions", toUpperCopy(uaname,
aname));
- } else if (is_sql_where(f)) {
- char *uaname = SA_NEW_ARRAY(sql->ta, char,
strlen(aname) + 1);
- return sql_error(sql, 02, SQLSTATE(42000) "%s:
aggregate functions not allowed in WHERE clause", toUpperCopy(uaname, aname));
- } else if (is_sql_from(f)) {
- char *uaname = SA_NEW_ARRAY(sql->ta, char,
strlen(aname) + 1);
- return sql_error(sql, 02, SQLSTATE(42000) "%s:
aggregate functions not allowed in functions in FROM", toUpperCopy(uaname,
aname));
- } else if (!all_aggr && ungrouped_cols &&
!list_empty(ungrouped_cols)) {
- for (node *n = ungrouped_cols->h ; n ; n =
n->next) {
- sql_rel *outer;
- sql_exp *e = (sql_exp*) n->data;
-
- if ((outer = query_fetch_outer(query,
is_freevar(e)-1))) {
- int of =
query_fetch_outer_state(query, is_freevar(e)-1);
- if (outer->grouped) {
- bool err = false,
was_processed = false;
-
- if
(is_processed(outer)) {
- was_processed =
true;
-
reset_processed(outer);
- }
- if
(!is_group
MonetDB: Jul2021 - Disabled logging into merovingian.log of clie...
Changeset: f8cba6941d5e for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/f8cba6941d5e
Modified Files:
tools/merovingian/ChangeLog.Jul2021
tools/merovingian/daemon/client.c
tools/merovingian/daemon/proxy.c
Branch: Jul2021
Log Message:
Disabled logging into merovingian.log of client connection info messages to
reduce size of log file
diffs (43 lines):
diff --git a/tools/merovingian/ChangeLog.Jul2021
b/tools/merovingian/ChangeLog.Jul2021
--- a/tools/merovingian/ChangeLog.Jul2021
+++ b/tools/merovingian/ChangeLog.Jul2021
@@ -1,3 +1,10 @@
# ChangeLog file for sql/src/backends/monet5/merovingian
# This file is updated with mchangelog
+* Fri Mar 18 2022 Sjoerd Mullender
+- Disabled logging into merovingian.log of next info message types:
+ "proxying client : for database '' to " and
+ "target connection is on local UNIX domain socket, passing on filedescriptor
instead of proxying".
+ These messages were written to the log file at each connection. In most
+ cases this information is not used. The disabling reduces the log file size.
+
diff --git a/tools/merovingian/daemon/client.c
b/tools/merovingian/daemon/client.c
--- a/tools/merovingian/daemon/client.c
+++ b/tools/merovingian/daemon/client.c
@@ -371,9 +371,11 @@ handleClient(void *data)
close_stream(fout);
close_stream(fdin);
} else {
+ /* Jan2022: disabled logging of next info message to reduce
merovingian.log size:
Mfprintf(stdout, "proxying client %s for database '%s' to "
"%s?database=%s\n",
host, database, redirs[0].conns->val,
redirs[0].dbname);
+ */
/* merovingian is in control, only consider the first redirect
*/
mnstr_printf(fout, "^mapi:merovingian://proxy?database=%s\n",
redirs[0].dbname);
diff --git a/tools/merovingian/daemon/proxy.c b/tools/merovingian/daemon/proxy.c
--- a/tools/merovingian/daemon/proxy.c
+++ b/tools/merovingian/daemon/proxy.c
@@ -184,8 +184,10 @@ startProxy(int psock, stream *cfdin, str
msg.msg_controllen = cmsg->cmsg_len;
msg.msg_flags = 0;
+ /* Jan2022: disabled logging of next info message to reduce
merovingian.log size:
Mfprintf(stdout, "target connection is on local UNIX domain
socket, "
"passing on filedescriptor instead of
proxying\n");
+ */
if (sendmsg(ssock, &msg, 0) < 0) {
closesocket(ssock);
return(newErr("could not send initial byte: %s",
strerror(errno)));
___
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org
MonetDB: Jan2022 - A bunch of smallish fixes, most important: fi...
Changeset: 63300fb65e60 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/63300fb65e60
Modified Files:
gdk/gdk.h
gdk/gdk_aggr.c
gdk/gdk_analytic_bounds.c
gdk/gdk_bat.c
gdk/gdk_calc.c
gdk/gdk_storage.c
Branch: Jan2022
Log Message:
A bunch of smallish fixes, most important: fix some bat_iterator_end calls.
diffs (truncated from 974 to 300 lines):
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -914,10 +914,10 @@ gdk_export void HEAPincref(Heap *h);
* The BAT iterator provides a number of fields that can (and often
* should) be used to access information about the BAT. For string
* BATs, if a parallel threads adds values, the offset heap (theap) may
- * get replaced by a one that is wider. This involves changing the
- * twidth and tshift values in the BAT structure. These changed values
- * should not be used to access the data in the iterator. Instead, use
- * the width and shift values in the iterator itself.
+ * get replaced by one that is wider. This involves changing the twidth
+ * and tshift values in the BAT structure. These changed values should
+ * not be used to access the data in the iterator. Instead, use the
+ * width and shift values in the iterator itself.
*/
typedef struct BATiter {
BAT *b;
diff --git a/gdk/gdk_aggr.c b/gdk/gdk_aggr.c
--- a/gdk/gdk_aggr.c
+++ b/gdk/gdk_aggr.c
@@ -1910,25 +1910,25 @@ BATgroupavg(BAT **bnp, BAT **cntsp, BAT
#endif
rems = GDKzalloc(ngrp * sizeof(lng));
if (rems == NULL)
- goto bailout;
+ goto bailout1;
break;
default:
break;
}
if (cntsp) {
if ((cn = COLnew(min, TYPE_lng, ngrp, TRANSIENT)) == NULL)
- goto bailout;
+ goto bailout1;
cnts = (lng *) Tloc(cn, 0);
memset(cnts, 0, ngrp * sizeof(lng));
} else {
cnts = GDKzalloc(ngrp * sizeof(lng));
if (cnts == NULL)
- goto bailout;
+ goto bailout1;
}
bn = COLnew(min, TYPE_dbl, ngrp, TRANSIENT);
if (bn == NULL)
- goto bailout;
+ goto bailout1;
dbls = (dbl *) Tloc(bn, 0);
if (BATtdense(g))
@@ -2001,6 +2001,7 @@ BATgroupavg(BAT **bnp, BAT **cntsp, BAT
return GDK_SUCCEED;
bailout:
bat_iterator_end(&bi);
+ bailout1:
if (bn)
BBPunfix(bn->batCacheid);
GDKfree(rems);
@@ -3196,8 +3197,7 @@ BATgroupcount(BAT *b, BAT *g, BAT *e, BA
AGGR_COUNT(dbl);
break;
default:
- while (ncand > 0) {
- ncand--;
+ TIMEOUT_LOOP(ncand, timeoffset) {
i = canditer_next(&ci) - b->hseqbase;
if (gids == NULL ||
(gids[i] >= min && gids[i] <= max)) {
@@ -3210,6 +3210,8 @@ BATgroupcount(BAT *b, BAT *g, BAT *e, BA
}
}
}
+ TIMEOUT_CHECK(timeoffset,
+ GOTO_LABEL_TIMEOUT_HANDLER(bailout));
break;
}
bat_iterator_end(&bi);
@@ -4120,7 +4122,7 @@ doBATgroupquantile(BAT *b, BAT *g, BAT *
if (g) {
g = BATproject(s, g);
if (g == NULL)
- goto bunins_failed;
+ goto bailout;
}
}
@@ -4147,14 +4149,14 @@ doBATgroupquantile(BAT *b, BAT *g, BAT *
return bn;
}
if (BATsort(&t1, &t2, NULL, g, NULL, NULL, false, false, false)
!= GDK_SUCCEED)
- goto bunins_failed;
+ goto bailout;
if (g != origg)
BBPunfix(g->batCacheid);
g = t1;
if (BATsort(&t1, NULL, NULL, b, t2, g, false, false, false) !=
GDK_SUCCEED) {
BBPunfix(t2->batCacheid);
- goto bunins_failed;
+ goto bailout;
}
if (b != origb)
BBPunfix(b->batCacheid);
@@ -4166,7 +4168,7 @@ doBATgroupquantile(BAT *b, BAT *g, BAT *
else
bn = COLnew(min, tp, ngrp, TRANSIENT);
if (bn == NULL)
- goto bunins_failed;
+ goto bailout;
bi = bat_iterator(b);
@@ -4239,7 +4241,7 @@ doBATgroupquantile(BAT *b, BAT *g, BAT *
nils += ngrp - BATcount(bn);
while (BATcount(bn) < ngrp) {
MonetDB: Jan2022 - Merge with Jul2021 branch.
Changeset: 2051240a9d6e for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/2051240a9d6e Modified Files: tools/merovingian/daemon/client.c tools/merovingian/daemon/proxy.c Branch: Jan2022 Log Message: Merge with Jul2021 branch. ___ checkin-list mailing list -- checkin-list@monetdb.org To unsubscribe send an email to checkin-list-le...@monetdb.org
MonetDB: sqloptimizer - Merged with default
Changeset: d3d4866c694c for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/d3d4866c694c
Modified Files:
sql/backends/monet5/sql.c
Branch: sqloptimizer
Log Message:
Merged with default
diffs (truncated from 1299 to 300 lines):
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -166,7 +166,6 @@ BAT *BATmergecand(BAT *a, BAT *b);
void *BATmin(BAT *b, void *aggr);
void *BATmin_skipnil(BAT *b, void *aggr, bit skipnil);
gdk_return BATmode(BAT *b, bool transient);
-void BATmsync(BAT *b);
BAT *BATnegcands(BUN nr, BAT *odels);
BAT *BATnil_grp(BAT *l, BAT *g, BAT *e, BAT *s);
bool BATordered(BAT *b);
diff --git a/gdk/ChangeLog b/gdk/ChangeLog
--- a/gdk/ChangeLog
+++ b/gdk/ChangeLog
@@ -1,6 +1,10 @@
# ChangeLog file for GDK
# This file is updated with Maddlog
+* Fri Mar 18 2022 Sjoerd Mullender
+- Get rid of macro Tsize, use ->twidth instead.
+- Get rid of macro BUNlast, just use BATcount instead.
+
* Mon Jan 31 2022 Sjoerd Mullender
- The BLOB type has been moved into the GDK layer.
diff --git a/gdk/ChangeLog.Jan2022 b/gdk/ChangeLog.Jan2022
--- a/gdk/ChangeLog.Jan2022
+++ b/gdk/ChangeLog.Jan2022
@@ -1,6 +1,11 @@
# ChangeLog file for GDK
# This file is updated with Maddlog
+* Fri Mar 18 2022 Sjoerd Mullender
+- Fixed a race condition which could cause a too large size being written
+ for a .theap file to the BBP.dir file after the correct size file had
+ been saved to disk.
+
* Wed Mar 9 2022 Sjoerd Mullender
- Fixed a bug in the append code for msk (bit mask) bats.
- Conversions from floating point types to integral types that involve
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -1148,8 +1148,6 @@ gdk_export BUN BUNfnd(BAT *b, const void
#define BATttype(b)(BATtdense(b) ? TYPE_oid : (b)->ttype)
-#define Tsize(b) ((b)->twidth)
-
#define tailsize(b,p) ((b)->ttype ? \
(ATOMstorage((b)->ttype) == TYPE_msk ? \
(((size_t) (p) + 31) / 32) * 4 : \
@@ -1170,8 +1168,6 @@ typedef var_t stridx_t;
#define BUNtvar(bi,p) (assert((bi).type && (bi).b->tvarsized), (void *)
((bi).vh->base+BUNtvaroff(bi,p)))
#define BUNtail(bi,p)
((bi).type?(bi).b->tvarsized?BUNtvar(bi,p):(bi).type==TYPE_msk?BUNtmsk(bi,p):BUNtloc(bi,p):BUNtpos(bi,p))
-#define BUNlast(b) (assert((b)->batCount <= BUN_MAX), (b)->batCount)
-
#define BATcount(b)((b)->batCount)
#include "gdk_atoms.h"
@@ -1310,7 +1306,6 @@ gdk_export gdk_return BATgroup(BAT **gro
gdk_export gdk_return BATsave(BAT *b)
__attribute__((__warn_unused_result__));
-gdk_export void BATmsync(BAT *b);
#define NOFARM (-1) /* indicate to GDKfilepath to create relative path */
@@ -2228,7 +2223,7 @@ gdk_export void VIEWbounds(BAT *b, BAT *
* is the iteration variable.
*/
#define BATloop(r, p, q) \
- for (q = BUNlast(r), p = 0; p < q; p++)
+ for (q = BATcount(r), p = 0; p < q; p++)
/*
* @+ Common BAT Operations
diff --git a/gdk/gdk_aggr.c b/gdk/gdk_aggr.c
--- a/gdk/gdk_aggr.c
+++ b/gdk/gdk_aggr.c
@@ -102,7 +102,7 @@ BATgroupaggrinit(BAT *b, BAT *g, BAT *e,
if (!is_oid_nil(min)) {
/* found a non-nil, max must be last
* value (and there is one!) */
- max = gids[BUNlast(g) - 1];
+ max = gids[BATcount(g) - 1];
}
} else {
/* we'll do a complete scan */
diff --git a/gdk/gdk_align.c b/gdk/gdk_align.c
--- a/gdk/gdk_align.c
+++ b/gdk/gdk_align.c
@@ -179,7 +179,7 @@ BATmaterialize(BAT *b)
if ((tail = GDKmalloc(sizeof(Heap))) == NULL)
return GDK_FAIL;
p = 0;
- q = BUNlast(b);
+ q = BATcount(b);
assert(cnt >= q - p);
TRC_DEBUG(ALGO, "BATmaterialize(" ALGOBATFMT ")\n", ALGOBATPAR(b));
diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c
--- a/gdk/gdk_bat.c
+++ b/gdk/gdk_bat.c
@@ -631,7 +631,7 @@ BATclear(BAT *b, bool force)
if (tatmdel) {
BATiter bi = bat_iterator_nolock(b);
- for (p = b->batInserted, q = BUNlast(b); p < q; p++)
+ for (p = b->batInserted, q = BATcount(b); p < q; p++)
(*tatmdel)(b->tvheap, (var_t*) BUNtloc(bi,p));
b->tvheap->dirty = true;
}
@@ -951,7 +951,7 @@ COLcopy(BAT *b, int tt, bool writable, r
bn->tunique_est = bi.unique_est;
} else if (ATOMstorage(tt) == ATOMstorage(b->ttype) &&
ATOMcompare(tt) == ATOMcompare(b->ttype)) {
- BUN h = BUNlast(b);
+ BUN h = BATcount(b);
bn->tsorted =
