Changeset: a8377256d356 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a8377256d356
Modified Files:
cmake/monetdb-toolchain.cmake
common/utils/matomic.h
documentation/source/manual_pages/monetdb.rst
documentation/source/manual_pages/monetdbd.rst.in
gdk/gdk_join.c
monetdb5/modules/mal/batcalc.c
monetdb_config.h.in
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql_statement.c
sql/backends/monet5/sql_statement.h
sql/include/sql_catalog.h
sql/server/rel_dump.c
sql/server/rel_optimizer.c
sql/server/rel_rel.c
sql/server/rel_rel.h
sql/server/rel_unnest.c
sql/server/sql_mvc.c
sql/server/sql_mvc.h
sql/storage/bat/bat_storage.c
sql/storage/sql_storage.h
sql/storage/store.c
tools/monetdbe/monetdbe.c
tools/monetdbe/monetdbe.h
Branch: hammertime
Log Message:
Merge heads.
diffs (truncated from 1229 to 300 lines):
diff --git a/cmake/monetdb-toolchain.cmake b/cmake/monetdb-toolchain.cmake
--- a/cmake/monetdb-toolchain.cmake
+++ b/cmake/monetdb-toolchain.cmake
@@ -67,8 +67,9 @@ function(monetdb_default_toolchain)
elseif(${CMAKE_C_COMPILER_ID} STREQUAL "Intel")
if(WIN32)
set(COMPILER_OPTION "/")
-# MT_addCompilerFlag("${COMPILER_OPTION}DNO_ATOMIC_INSTRUCTIONS"
"${COMPILER_OPTION}DNO_ATOMIC_INSTRUCTIONS=1" "${CMAKE_C_FLAGS}" "all"
CMAKE_C_FLAGS)
MT_addCompilerFlag("${COMPILER_OPTION}W3" "${COMPILER_OPTION}W3"
"${CMAKE_C_FLAGS}" "all" CMAKE_C_FLAGS)
+ MT_addCompilerFlag("${COMPILER_OPTION}Qdiag-disable:11074"
"${COMPILER_OPTION}Qdiag-disable:11074" "${CMAKE_C_FLAGS}" "all" CMAKE_C_FLAGS)
+ MT_addCompilerFlag("${COMPILER_OPTION}Qdiag-disable:11075"
"${COMPILER_OPTION}Qdiag-disable:11075" "${CMAKE_C_FLAGS}" "all" CMAKE_C_FLAGS)
else()
set(COMPILER_OPTION "-")
MT_addCompilerFlag("${COMPILER_OPTION}Wall" "${COMPILER_OPTION}Wall"
"${CMAKE_C_FLAGS}" "all" CMAKE_C_FLAGS)
diff --git a/common/utils/matomic.h b/common/utils/matomic.h
--- a/common/utils/matomic.h
+++ b/common/utils/matomic.h
@@ -57,7 +57,9 @@
/* define this if you don't want to use atomic instructions */
/* #define NO_ATOMIC_INSTRUCTIONS */
-#if defined(HAVE_STDATOMIC_H) && !defined(__STDC_NO_ATOMICS__) &&
!defined(NO_ATOMIC_INSTRUCTIONS)
+/* ignore __STDC_NO_ATOMICS__ if compiling using Intel compiler on
+ * Windows since otherwise we can't compile this at all in C99 mode */
+#if defined(HAVE_STDATOMIC_H) && (!defined(__STDC_NO_ATOMICS__) ||
(defined(__INTEL_COMPILER) && defined(_WINDOWS))) &&
!defined(NO_ATOMIC_INSTRUCTIONS)
#include <stdatomic.h>
diff --git a/documentation/source/manual_pages/monetdb.rst
b/documentation/source/manual_pages/monetdb.rst
--- a/documentation/source/manual_pages/monetdb.rst
+++ b/documentation/source/manual_pages/monetdb.rst
@@ -1,7 +1,3 @@
-=======
-MONETDB
-=======
-
NAME
====
@@ -56,10 +52,10 @@ COMMANDS
The commands for the *monetdb* utility are **create**, **destroy**,
**lock**, **release**, **status**, **start**, **stop**, **kill**,
-**profilerstart**, **profilerstop**, **set**, **get**, **inherit**,
-**discover**, **help**, and **version**. The commands facilitate adding,
-removing, maintaining, starting and stopping a database inside the
-MonetDB Database Server.
+**profilerstart**, **profilerstop**, **snapshot**, **set**, **get**,
+**inherit**, **discover**, **help**, and **version**. The commands
+facilitate adding, removing, maintaining, starting and stopping a
+database inside the MonetDB Database Server.
For all commands, database arguments can be glob-like expressions. This
allows to do wildcard matches. For details on the syntax, see
@@ -167,20 +163,21 @@ allows to do wildcard matches. For detai
**kill [-a]**\ *database*\ **[**\ *database*\ **...]**
Starts, stops or kills the given database, or, when **-a** is supplied,
-all known databases. The **kill** command immediately sends a SIGKILL
-and should only be used as last resort for a database that doesn't
-respond any more. Killing a database may result in (partial) data loss.
-It is more common to use the **stop** command to stop a database. It
-will first attempt to stop the database, waiting for
-**mero_exittimeout** seconds and if that fails, kill the database. When
-using the **start** command, *monetdb*\ (1) will output diagnostic
-messages if the requested action failed. When encountering an error, one
-should always consult the logfile of *monetdbd*\ (1) for more details.
-For the **kill** command a diagnostic message indicating the database
-has crashed is always emitted, due to the nature of that command. Note
-that in combination with **-a** the return code of *monetdb*\ (1)
-indicates failure if one of the databases had a failure, even though the
-operation on other databases was successful.
+all known databases. The **kill** command immediately terminates the
+database by sending the SIGKILL signal. Any data that hasn't been
+committed will be lost. This command should only be used as last resort
+for a database that doesn't respond any more. It is more common to use
+the **stop** command to stop a database. This will first attempt to stop
+the database, waiting for **exittimeout** seconds and if that fails,
+kill the database. When using the **start** command, *monetdb*\ (1) will
+output diagnostic messages if the requested action failed. When
+encountering an error, one should always consult the logfile of
+*monetdbd*\ (1) for more details. For the **kill** command a diagnostic
+message indicating the database has crashed is always emitted, due to
+the nature of that command. Note that in combination with **-a** the
+return code of *monetdb*\ (1) indicates failure if one of the databases
+had a failure, even though the operation on other databases was
+successful.
**profilerstart**\ *database*\ **[**\ *database*\ **...]**
@@ -196,6 +193,53 @@ recording the pid of the profiler is wri
therefore each database needs to have a different **profilerlogpath**
value.
+**monetdb snapshot write**\ *dbname*
+ Takes a snapshot of the given database and writes it to stdout.
+
+**monetdb snapshot create [-t**\ *targetfile*\ **]**\ *dbname*\ **[**\
*dbname*\ **..]**
+ Takes a snapshot of the given databases. Here, *dbname* can be either
+ the name of a single database or a pattern such as *staging\**
+ indicating multiple databases to snapshot. Unless **-t** is given,
+ the snapshots are written to files named
+ *<snapshotdir>/<dbname>_<YYYY><MM><DD>T<HH><MM>UTC<snapshotcompression>*
+ where *snapshotdir* is a *monetdbd* setting that has to be configured
+ explicitly using **monetdbd set** and *snapshotcompression* is
+ another **monetdbd** setting which defaults to *.tar.lz4* or *.tar*.
+ If **-t** is given, only a single database can be snapshotted and the
+ snapshot is written to *targetfile*, a file on the server which must
+ be somewhere under *snapshotdir* but which does not have to follow
+ any particular naming convention.
+
+**monetdb snapshot list [**\ *dbname*\ **..]**
+ Lists the snapshots for the given databases, or all databases if none
+ is given, showing the snapshot id, the time the snapshot was taken
+ and the (compressed) size of the snapshot file. Only snapshots
+ following the naming convention described under **monetdb snapshot
+ create** are listed. The snapshot id is of the form
+ *dbname*\ **@**\ *tag* where the tags are numbers starting at 1 for
+ the most recent snapshot of a database, 2 for the next most recent,
+ etc. For clarity, the first snapshot for each database shows the full
+ snapshot id (*dbname*\ **@1) and** older snapshots for the same
+ database are listed just as @2, @3, etc.
+
+**monetdb snapshot restore [-f]**\ *snapshotid*\ **[**\ *dbname*\ **]**
+ Restores a database from the given snapshot, where *snapshotid* is
+ either a path on the server or *name*\ **@**\ *tag*\ **as listed by**
+ **monetdb snapshot** **list.** The optional *dbname* argument sets
+ the name of the newly created database. It can be omitted unless
+ *snapshotid* is a full path. When **-f** is given, no confirmation is
+ asked when overwriting an existing database.
+
+**monetdb snapshot destroy [-f]**\ *name*\ **@**\ *tag*\ **..**
+ Delete the listed snapshots from the *snapshotdir* directory. When
+ **-f** is given, no confirmation is asked.
+
+**monetdb snapshot destroy [-f] -r**\ *N*\ *dbname*\ **..**
+ Delete all but the *N* latest snapshots for the given databases.
+ Again, *dbname* can be a pattern such as *staging\** or even *\** to
+ work on all snapshotted databases. When **-f** is given, no
+ confirmation is asked.
+
**get <all \|**\ *property*\ **[,**\ *property*\ **[,..]]> [**\ *database*\
**...]**
Prints the requested properties, or all known properties, for the
given database. For each property its source and value are printed.
diff --git a/documentation/source/manual_pages/monetdbd.rst.in
b/documentation/source/manual_pages/monetdbd.rst.in
--- a/documentation/source/manual_pages/monetdbd.rst.in
+++ b/documentation/source/manual_pages/monetdbd.rst.in
@@ -144,6 +144,21 @@ using the **set** command. The following
specified at configure time, which is @PASSWORD_BACKEND@. Changing
this property takes effect immediately at runtime.
+**snapshotdir**
+ This property specifies where the *monetdb snapshot create* command
+ stores its snapshots. There is no default value. If unset, no
+ snapshots can be created.
+
+**snapshotcompression**
+ This property specifies whether database snapshots created by
+ *monetdb snapshot create* are compressed, and if so, using which
+ algorithm. The default value is *.tar.lz4* if LZ4 compression is
+ available, or *.tar* otherwise. Other supported values are, if
+ available, *.tar.gz*, *.tar.xz*, and *.tar.bz2*. These more efficient
+ but much slower compression algorithms are usually not recommended
+ for backups of large databases because it would take enormous amounts
+ of time.
+
**discovery**
Specifies whether neighbor discovery is to be enabled using UDP
broadcasts or not. The broadcasts are done on the same portnumber as
@@ -161,18 +176,18 @@ using the **set** command. The following
reachable databases.
**exittimeout**
- mservers that were started by the MonetDB Database Server are shut
- down when *monetdbd* is shut down. Setting the **exittimeout**
- property to a positive non-zero value will shut down each running
- mserver with the given time-out in seconds. If the time-out expires,
- the mserver process is killed using the SIGKILL signal. A time-out
+ if *monetdbd* stops a running mserver, for example in response to a
+ *monetdb stop* command or when *monetdbd* itself is shutting down, it
+ first sends the mserver a SIGTERM signal to trigger a graceful
+ shutdown. If that takes more than **exittimeout** seconds, SIGKILL is
+ sent as a last resort. This may cause (partial) data loss. A time-out
value of 0 means no mservers will be shut down, and hence they will
continue to run after *monetdbd* has shut down. Note that this
particular configuration is extremely inconvenient. The default
- time-out is 60 seconds. If your databases are rather large and find
- your databases consistently being killed by *monetdbd* upon shutdown,
- you may want to increase this time-out. Changing this property takes
- effect immediately at runtime.
+ time-out is 60 seconds. If your databases are rather large and you
+ find your databases consistently being killed by *monetdbd* upon
+ shutdown, you may want to increase this time-out. Changing this
+ property takes effect immediately at runtime.
**forward**
*monetdbd* has two ways in which it can "attach" a connecting client
diff --git a/gdk/gdk_join.c b/gdk/gdk_join.c
--- a/gdk/gdk_join.c
+++ b/gdk/gdk_join.c
@@ -3063,7 +3063,6 @@ bitmaskjoin(BAT *l, BAT *r,
const char *reason, lng t0)
{
BAT *r1;
- const oid *rp = BATtdense(r) ? NULL : Tloc(r, 0);
size_t nmsk = (lci->ncand + 31) / 32;
uint32_t *mask = GDKzalloc(nmsk * sizeof(uint32_t));
BUN cnt = 0;
@@ -3073,13 +3072,9 @@ bitmaskjoin(BAT *l, BAT *r,
for (BUN n = 0; n < rci->ncand; n++) {
oid o = canditer_next(rci) - r->hseqbase;
- if (rp) {
- o = rp[o];
- if (is_oid_nil(o))
- continue;
- } else {
- o = o - r->hseqbase + r->tseqbase;
- }
+ o = BUNtoid(r, o);
+ if (is_oid_nil(o))
+ continue;
o += l->hseqbase;
if (o < lci->seq + l->tseqbase)
continue;
@@ -3362,39 +3357,58 @@ leftjoin(BAT **r1p, BAT **r2p, BAT *l, B
BBPunfix(sr->batCacheid);
if (rc != GDK_SUCCEED)
return rc;
+ if (r2p == NULL) {
+ BBPunfix(r2->batCacheid);
+ r2 = NULL;
+ }
if (semi)
r1->tkey = true;
- BAT *ob;
- rc = BATsort(&tmp, r2p ? &ob : NULL, NULL,
- r1, NULL, NULL, false, false, false);
- BBPunfix(r1->batCacheid);
- if (rc != GDK_SUCCEED) {
- BBPunfix(r2->batCacheid);
- return rc;
+ if (!VIEWtparent(r1) &&
+ r1->ttype == TYPE_oid &&
+ BBP_refs(r1->batCacheid) == 1 &&
+ (r2 == NULL ||
+ (!VIEWtparent(r2) &&
+ BBP_refs(r2->batCacheid) == 1 &&
+ r2->ttype == TYPE_oid))) {
+ /* in-place sort if we can */
+ if (r2) {
+ GDKqsort(r1->theap.base, r2->theap.base,
+ NULL, r1->batCount, r1->twidth,
+ r2->twidth, TYPE_oid, false,
+ false);
+ r2->tsorted = false;
+ r2->trevsorted = false;
+ *r2p = r2;
+ } else {
+ GDKqsort(r1->theap.base, NULL, NULL,
+ r1->batCount, r1->twidth, 0,
+ TYPE_oid, false, false);
+ }
+ r1->tsorted = true;
+ r1->trevsorted = false;
+ *r1p = r1;
+ } else {
+ BAT *ob;
+ rc = BATsort(&tmp, r2p ? &ob : NULL, NULL,
+ r1, NULL, NULL, false, false,
false);
+ BBPunfix(r1->batCacheid);
+ if (rc != GDK_SUCCEED) {
+ if (r2)
+ BBPunfix(r2->batCacheid);
+ return rc;
+ }
+ *r1p = r1 = tmp;
+ if (r2p) {
+ tmp = BATproject(ob, r2);
+ BBPunfix(r2->batCacheid);
+ BBPunfix(ob->batCacheid);
+ if (tmp == NULL) {
+ BBPunfix(r1->batCacheid);
+ return GDK_FAIL;
+ }
+ *r2p = tmp;
+ }
}
- *r1p = r1 = tmp;
- if (r2p) {
- tmp = BATproject(ob, r2);
- BBPunfix(r2->batCacheid);
- BBPunfix(ob->batCacheid);
- if (tmp == NULL) {
- BBPunfix(r1->batCacheid);
- return GDK_FAIL;
- }
- *r2p = tmp;
- }
-#if 0
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
