sc/source/core/data/table4.cxx | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
New commits:
commit 8f46501233c164ff91d77a7f5adf74ea16cd0165
Author: Mike Kaganski <mike.kagan...@collabora.com>
AuthorDate: Wed Dec 25 10:23:41 2019 +0300
Commit: Mike Kaganski <mike.kagan...@collabora.com>
CommitDate: Wed Jan 8 15:57:19 2020 +0100
tdf#129606: limit precision in ScTable::FillAnalyse
... to 16th significant digit of least precise argument. This follows
the practice to only consider 16 significant digits of user-provided
values.
Change-Id: Ic44fff82396f4f383c96343f9b2f7d35ccce9070
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/85795
Tested-by: Jenkins
Reviewed-by: Mike Kaganski <mike.kagan...@collabora.com>
diff --git a/sc/source/core/data/table4.cxx b/sc/source/core/data/table4.cxx
index c79e81af397c..5aab834a1fed 100644
--- a/sc/source/core/data/table4.cxx
+++ b/sc/source/core/data/table4.cxx
@@ -202,8 +202,11 @@ double approxDiff( double a, double b )
// We now have two subtractions with a similar but not equal error. Obtain
// the exponent of the error magnitude and round accordingly.
const double e = fabs(d - c);
- const double fExp = floor( log10( e));
- return rtl::math::round( c, -static_cast<int>(fExp)-1);
+ const int nExp = static_cast<int>(floor(log10(e))) + 1;
+ // tdf#129606: Limit precision to the 16th significant digit of the least
precise argument.
+ // Cf. mnMaxGeneralPrecision in sc/source/core/data/column3.cxx.
+ const int nExpArg = static_cast<int>(floor(log10(std::max(aa, ab)))) - 15;
+ return rtl::math::round(c, -std::max(nExp, nExpArg));
}
}
_______________________________________________
Libreoffice-commits mailing list
libreoffice-comm...@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/libreoffice-commits
