Changeset: 310a5d3fa000 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=310a5d3fa000
Added Files:
sql/test/BugTracker-2009/Tests/AVG_of_SQRT.SF-2757642.sql
sql/test/BugTracker-2017/Tests/insert_into_multiple_subqueries.Bug-6448.sql
sql/test/BugTracker-2017/Tests/insert_into_multiple_subqueries.Bug-6448.stable.err
sql/test/BugTracker-2017/Tests/insert_into_multiple_subqueries.Bug-6448.stable.out
sql/test/BugTracker-2017/Tests/sqlitelogictest-cast-decimal.Bug-6445.stable.err
sql/test/BugTracker-2017/Tests/sqlitelogictest-cast-decimal.Bug-6445.stable.out
sql/test/BugTracker-2017/Tests/sqlsmith.Bug-6449.sql
sql/test/BugTracker-2017/Tests/sqlsmith.Bug-6449.stable.err
sql/test/BugTracker-2017/Tests/sqlsmith.Bug-6449.stable.out
sql/test/BugTracker-2017/Tests/table_returning_with.Bug-6444.sql
sql/test/BugTracker-2017/Tests/table_returning_with.Bug-6444.stable.err
sql/test/BugTracker-2017/Tests/table_returning_with.Bug-6444.stable.out
sql/test/Tests/fsum.sql
sql/test/Tests/fsum.stable.err
sql/test/Tests/fsum.stable.out
Removed Files:
sql/test/BugTracker-2009/Tests/AVG_of_SQRT.SF-2757642.SQL.py
Modified Files:
gdk/ChangeLog.Jul2017
gdk/gdk_aggr.c
gdk/gdk_utils.c
monetdb5/mal/mal_parser.c
sql/backends/monet5/sql_gencode.c
sql/server/rel_optimizer.c
sql/server/rel_select.c
sql/server/rel_sequence.c
sql/server/rel_updates.c
sql/server/sql_parser.h
sql/server/sql_parser.y
sql/server/sql_privileges.c
sql/server/sql_scan.c
sql/test/BugTracker-2009/Tests/AVG_of_SQRT.SF-2757642.stable.out
sql/test/BugTracker-2017/Tests/All
sql/test/BugTracker/Tests/drop_schema_crash.SF-1504794.stable.err
sql/test/BugTracker/Tests/set_a_new_user_password.SF-1844050.stable.err
sql/test/Dependencies/Tests/Dependencies.stable.err
sql/test/Tests/All
sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade-hge/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade/Tests/upgrade.stable.out
sql/test/emptydb-upgrade/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out
sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade-hge/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade/Tests/upgrade.stable.out
sql/test/testdb-upgrade/Tests/upgrade.stable.out.int128
tools/merovingian/daemon/merovingian.c
Branch: default
Log Message:
Merge with Jul2017
diffs (truncated from 5073 to 300 lines):
diff --git a/gdk/ChangeLog.Jul2017 b/gdk/ChangeLog.Jul2017
--- a/gdk/ChangeLog.Jul2017
+++ b/gdk/ChangeLog.Jul2017
@@ -1,3 +1,10 @@
# ChangeLog file for MonetDB
# This file is updated with Maddlog
+* Mon Oct 30 2017 Sjoerd Mullender <sjo...@acm.org>
+- Reimplemented summing of a column of floating point (flt and dbl)
+ values. The old code could give wildly inaccurate results when adding
+ up lots and lots of values due to lack of precision. Try SELECT sum(c)
+ FROM t; where t is 100,000,000 rows, c is of type REAL and all values
+ are equal to 1.1. (The old code returned 33554432 instead of 1.1e8.)
+
diff --git a/gdk/gdk_aggr.c b/gdk/gdk_aggr.c
--- a/gdk/gdk_aggr.c
+++ b/gdk/gdk_aggr.c
@@ -147,6 +147,292 @@ BATgroupaggrinit(BAT *b, BAT *g, BAT *e,
/* ---------------------------------------------------------------------- */
/* sum */
+#if defined(_MSC_VER) && _MSC_VER < 1800
+#define isnan(x) _isnan(x)
+#define isinf(x) (_fpclass(x) & (_FPCLASS_NINF | _FPCLASS_PINF))
+#endif
+
+static inline int
+samesign(double x, double y)
+{
+ return (x >= 0) == (y >= 0);
+}
+
+/* Add two values, producing the sum and the remainder due to limited
+ * range of floating point arithmetic. This function depends on the
+ * fact that the sum returns INFINITY in *hi of the correct sign
+ * (i.e. isinf() returns TRUE) in case of overflow. */
+static inline void
+twosum(double *hi, double *lo, double x, double y)
+{
+ volatile double yr;
+
+ assert(fabs(x) >= fabs(y));
+
+ *hi = x + y;
+ yr = *hi - x;
+ *lo = y - yr;
+}
+
+static inline void
+exchange(double *x, double *y)
+{
+ double t = *x;
+ *x = *y;
+ *y = t;
+}
+
+static BUN
+dofsum(const void *restrict values, oid seqb, BUN start, BUN end,
+ void *restrict results, BUN ngrp, int tp1, int tp2,
+ const oid *restrict cand, const oid *candend, const oid *restrict gids,
+ oid min, oid max, int skip_nils, int abort_on_error,
+ int nil_if_empty, const char *func)
+{
+ struct pergroup {
+ size_t npartials;
+ size_t maxpartials;
+ double *partials;
+ int valseen;
+ } *pergroup;
+ size_t listi;
+ size_t parti;
+ size_t i;
+ BUN grp;
+ double x, y;
+ double lo, hi;
+ double twopow = pow((double) FLT_RADIX, (double) (DBL_MAX_EXP - 1));
+ BUN nils = 0;
+
+ ALGODEBUG fprintf(stderr, "#%s: floating point summation\n", func);
+ /* we only deal with the two floating point types */
+ assert(tp1 == TYPE_flt || tp1 == TYPE_dbl);
+ assert(tp2 == TYPE_flt || tp2 == TYPE_dbl);
+ /* if input is dbl, then so it output */
+ assert(tp2 == TYPE_flt || tp1 == TYPE_dbl);
+ /* if no gids, then we have a single group */
+ assert(ngrp == 1 || gids != NULL);
+ if (gids == NULL || ngrp == 1) {
+ min = max = 0;
+ ngrp = 1;
+ gids = NULL;
+ }
+ pergroup = GDKmalloc(ngrp * sizeof(*pergroup));
+ if (pergroup == NULL)
+ return BUN_NONE;
+ for (grp = 0; grp < ngrp; grp++) {
+ pergroup[grp].npartials = 1;
+ pergroup[grp].valseen = 0;
+ pergroup[grp].maxpartials = 32;
+ pergroup[grp].partials = GDKmalloc(pergroup[grp].maxpartials *
sizeof(double));
+ if (pergroup[grp].partials == NULL) {
+ while (grp > 0)
+ GDKfree(pergroup[--grp].partials);
+ GDKfree(pergroup);
+ return BUN_NONE;
+ }
+ pergroup[grp].partials[0] = 0;
+ }
+ for (;;) {
+ if (cand) {
+ if (cand >= candend)
+ break;
+ listi = *cand++ - seqb;
+ } else {
+ if (start >= end)
+ break;
+ listi = start++;
+ }
+ grp = gids ? gids[listi] : 0;
+ if (grp < min || grp > max)
+ continue;
+ if (pergroup[grp].partials == NULL)
+ continue;
+ if (tp1 == TYPE_flt && ((const flt *) values)[listi] != flt_nil)
+ x = ((const flt *) values)[listi];
+ else if (tp1 == TYPE_dbl && ((const dbl *) values)[listi] !=
dbl_nil)
+ x = ((const dbl *) values)[listi];
+ else {
+ /* it's a nil */
+ if (!skip_nils) {
+ if (tp2 == TYPE_flt)
+ ((flt *) results)[grp] = flt_nil;
+ else
+ ((dbl *) results)[grp] = dbl_nil;
+ GDKfree(pergroup[grp].partials);
+ pergroup[grp].partials = NULL;
+ if (++nils == ngrp)
+ break;
+ }
+ continue;
+ }
+ pergroup[grp].valseen = 1;
+ i = 1;
+ for (parti = 1; parti < pergroup[grp].npartials; parti++) {
+ y = pergroup[grp].partials[parti];
+ if (fabs(x) < fabs(y))
+ exchange(&x, &y);
+ twosum(&hi, &lo, x, y);
+ if (isinf(hi)) {
+ int sign = hi > 0 ? 1 : -1;
+ x = x - twopow * sign - twopow * sign;
+ pergroup[grp].partials[0] += sign;
+ if (fabs(x) < fabs(y))
+ exchange(&x, &y);
+ twosum(&hi, &lo, x, y);
+ }
+ if (lo != 0)
+ pergroup[grp].partials[i++] = lo;
+ x = hi;
+ }
+ if (x != 0) {
+ if (i == pergroup[grp].maxpartials - 1) {
+ /* -1 to make sure we have one spare
+ * for the final step below */
+ double *temp;
+ pergroup[grp].maxpartials +=
pergroup[grp].maxpartials;
+ temp = GDKrealloc(pergroup[grp].partials,
pergroup[grp].maxpartials);
+ if (temp == NULL)
+ goto bailout;
+ pergroup[grp].partials = temp;
+ }
+ pergroup[grp].partials[i++] = x;
+ }
+ pergroup[grp].npartials = i;
+ }
+
+ for (grp = 0; grp < ngrp; grp++) {
+ if (pergroup[grp].partials == NULL)
+ continue;
+ if (!pergroup[grp].valseen) {
+ if (tp2 == TYPE_flt)
+ ((flt *) results)[grp] = nil_if_empty ? flt_nil
: 0;
+ else
+ ((dbl *) results)[grp] = nil_if_empty ? dbl_nil
: 0;
+ nils += nil_if_empty;
+ GDKfree(pergroup[grp].partials);
+ pergroup[grp].partials = NULL;
+ continue;
+ }
+ if (isinf(pergroup[grp].partials[0]) ||
+ isnan(pergroup[grp].partials[0])) {
+ /* isnan: cannot happen: infinities of both
+ * signs in summands */
+ if (abort_on_error) {
+ goto overflow;
+ }
+ if (tp2 == TYPE_flt)
+ ((flt *) results)[grp] = flt_nil;
+ else
+ ((dbl *) results)[grp] = dbl_nil;
+ nils++;
+ GDKfree(pergroup[grp].partials);
+ pergroup[grp].partials = NULL;
+ continue;
+ }
+
+ if (fabs(pergroup[grp].partials[0]) == 1.0 &&
+ pergroup[grp].npartials > 1 &&
+ !samesign(pergroup[grp].partials[0],
pergroup[grp].partials[pergroup[grp].npartials - 1])) {
+ twosum(&hi, &lo, pergroup[grp].partials[0] * twopow,
pergroup[grp].partials[pergroup[grp].npartials - 1] / 2);
+ if (isinf(2 * hi)) {
+ if (hi + 2 * lo - hi == 2 * lo &&
+ pergroup[grp].npartials > 2 &&
+ samesign(lo,
pergroup[grp].partials[pergroup[grp].npartials - 2])) {
+ GDKfree(pergroup[grp].partials);
+ pergroup[grp].partials = NULL;
+ x = 2 * (hi + 2 * lo);
+ if (tp2 == TYPE_flt) {
+ if (x > GDK_flt_max ||
+ x <= GDK_flt_min) {
+ if (abort_on_error)
+ goto overflow;
+ ((flt *) results)[grp]
= flt_nil;
+ nils++;
+ } else
+ ((flt *) results)[grp]
= (flt) x;
+ } else if (x == GDK_dbl_min) {
+ if (abort_on_error)
+ goto overflow;
+ ((dbl *) results)[grp] =
dbl_nil;
+ nils++;
+ } else
+ ((dbl *) results)[grp] = x;
+ continue;
+ }
+ } else {
+ if (lo) {
+ /* we made sure above that we
+ * have space for one more
+ * partial */
+
pergroup[grp].partials[pergroup[grp].npartials - 1] = 2 * lo;
+
pergroup[grp].partials[pergroup[grp].npartials++] = 2 * hi;
+ } else {
+
pergroup[grp].partials[pergroup[grp].npartials - 1] = 2 * hi;
+ }
+ pergroup[grp].partials[0] = 0;
+ }
+ }
+
+ if (pergroup[grp].npartials == 0) {
+ GDKfree(pergroup[grp].partials);
+ pergroup[grp].partials = NULL;
+ if (tp2 == TYPE_flt)
+ ((flt *) results)[grp] = 0;
+ else
+ ((dbl *) results)[grp] = 0;
+ continue;
+ }
+ if (pergroup[grp].partials[0] != 0)
+ goto overflow;
+
+ /* accumulate into x */
+ x = pergroup[grp].partials[--pergroup[grp].npartials];
+ while (pergroup[grp].npartials > 0) {
+ twosum(&x, &lo, x,
pergroup[grp].partials[--pergroup[grp].npartials]);
+ if (lo) {
+
pergroup[grp].partials[pergroup[grp].npartials++] = lo;
+ break;
+ }
+ }
+
+ if (pergroup[grp].npartials >= 2 &&
+ samesign(pergroup[grp].partials[pergroup[grp].npartials -
1], pergroup[grp].partials[pergroup[grp].npartials - 2]) &&
+ x + 2 * pergroup[grp].partials[pergroup[grp].npartials - 1]
- x == 2 * pergroup[grp].partials[pergroup[grp].npartials - 1]) {
+ x += 2 * pergroup[grp].partials[pergroup[grp].npartials
- 1];
+ pergroup[grp].partials[pergroup[grp].npartials - 1] =
-pergroup[grp].partials[pergroup[grp].npartials - 1];
+ }
+
+ GDKfree(pergroup[grp].partials);
+ pergroup[grp].partials = NULL;
+ if (tp2 == TYPE_flt) {
+ if (x > GDK_flt_max || x <= GDK_flt_min) {
+ if (abort_on_error)
+ goto overflow;
+ ((flt *) results)[grp] = flt_nil;
+ nils++;
+ } else
+ ((flt *) results)[grp] = (flt) x;
+ } else if (x == GDK_dbl_min) {
+ if (abort_on_error)
+ goto overflow;
+ ((dbl *) results)[grp] = dbl_nil;
+ nils++;
+ } else
+ ((dbl *) results)[grp] = x;
+ }
+ GDKfree(pergroup);
+ return nils;
+
+ overflow:
+ GDKerror("22003!overflow in calculation.\n");
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
