sal/qa/rtl/math/test-rtl-math.cxx | 1 + sal/rtl/math.cxx | 10 +++++++--- sc/qa/unit/data/functions/statistical/fods/countif2.fods | 11 ++++++++++- 3 files changed, 18 insertions(+), 4 deletions(-)
New commits: commit 11c99476fd5af7fafaabd783ddfc7e9841b43db7 Author: Mike Kaganski <[email protected]> AuthorDate: Sun Oct 5 15:38:59 2025 +0500 Commit: Xisco Fauli <[email protected]> CommitDate: Mon Oct 20 17:48:44 2025 +0200 tdf#168673: approx equality threshold no less than 1/2 15th significand 01:26,47 is 0.001000810185185185, which has 16 significant decimals. It is converted to string as "0.00100081018518519" (15 significands). The problem was, that when criterion "=0.00100081018518519" is used to check value 0.001000810185185185, the two numbers differ in five least significant bits, but rtl_math_approxEqual uses 0x1p-48 to ignore last four bits only. This is a common problem for numbers close to 1.0E(N). This change introduces a second threshold, in addition to "only take into account 48 signigicant bits of mantissa": it calculates the value of half of 15th decimal unit of the smaller absolute value, and uses the greater of the two thresholds for the approximate equality. Change-Id: I2f98ab23c94ca3a2949c3762fbfb6ea563eef94b Reviewed-on: https://gerrit.libreoffice.org/c/core/+/191883 Reviewed-by: Mike Kaganski <[email protected]> Tested-by: Jenkins (cherry picked from commit d32c664c3df302b7107a70858aa55b11601225cf) Reviewed-on: https://gerrit.libreoffice.org/c/core/+/192698 Reviewed-by: Xisco Fauli <[email protected]> diff --git a/sal/qa/rtl/math/test-rtl-math.cxx b/sal/qa/rtl/math/test-rtl-math.cxx index 53b6827c818b..d684c028c910 100644 --- a/sal/qa/rtl/math/test-rtl-math.cxx +++ b/sal/qa/rtl/math/test-rtl-math.cxx @@ -508,6 +508,7 @@ public: CPPUNIT_ASSERT_EQUAL( true, rtl::math::approxEqual( 7.4124095894894475e+158, 7.4124095894894514e+158)); CPPUNIT_ASSERT_EQUAL( true, rtl::math::approxEqual( 1.2905754687023132e+79, 1.2905754687023098e+79)); CPPUNIT_ASSERT_EQUAL( true, rtl::math::approxEqual( 3.5612905090455637e+38, 3.5612905090455599e+38)); + CPPUNIT_ASSERT_EQUAL( true, rtl::math::approxEqual( 0.001000810185185185, 0.00100081018518519)); // 0.3 - 0.2 - 0.1 == 0.0 CPPUNIT_ASSERT_EQUAL( 0.0, rtl::math::approxSub( rtl::math::approxSub( 0.3, 0.2), 0.1)); // ((2^53)-1) - ((2^53)-2) == 1.0 diff --git a/sal/rtl/math.cxx b/sal/rtl/math.cxx index 49b38d6e74c7..4ab1b018f2b4 100644 --- a/sal/rtl/math.cxx +++ b/sal/rtl/math.cxx @@ -628,7 +628,10 @@ double SAL_CALL rtl_math_approxValue(double fValue) noexcept bool SAL_CALL rtl_math_approxEqual(double a, double b) noexcept { + // threshold is last four bits of mantissa: static const double e48 = 0x1p-48; + // threshold is half of the 15th significant decimal (see doubleToString): + static const double half15thSignificand = 5E-15; if (a == b) return true; @@ -641,10 +644,11 @@ bool SAL_CALL rtl_math_approxEqual(double a, double b) noexcept return false; // Nan or Inf involved a = fabs(a); - if (d >= (a * e48)) - return false; b = fabs(b); - if (d >= (b * e48)) + const double min_ab = std::min(a, b); + const double threshold1 = min_ab * e48; + const double threshold2 = getN10Exp(floor(log10(min_ab))) * half15thSignificand; + if (d >= std::max(threshold1, threshold2)) return false; if (isRepresentableInteger(a) && isRepresentableInteger(b)) diff --git a/sc/qa/unit/data/functions/statistical/fods/countif2.fods b/sc/qa/unit/data/functions/statistical/fods/countif2.fods index 35a61b34c642..3cf40c966b53 100644 --- a/sc/qa/unit/data/functions/statistical/fods/countif2.fods +++ b/sc/qa/unit/data/functions/statistical/fods/countif2.fods @@ -3143,8 +3143,17 @@ <table:table-cell table:style-name="Default"/> <table:table-cell/> </table:table-row> + <table:table-row table:style-name="ro2"> + <table:table-cell table:formula="of:=COUNTIF([.J10:.J10];"="&[.J10])" office:value-type="float"/> + <table:table-cell office:value-type="float" office:value="1"/> + <table:table-cell table:formula="of:=[.A10]=[.B10]" office:value-type="boolean"/> + <table:table-cell table:formula="of:=FORMULA([.A10])" office:value-type="string"/> + <table:table-cell office:value-type="string" office:string-value="tdf#168673"/> + <table:table-cell table:number-columns-repeated="4"/> + <table:table-cell office:value-type="time" office:time-value="PT00H01M26.47S"/> + </table:table-row> <calcext:conditional-formats> - <calcext:conditional-format calcext:target-range-address="Sheet2.C2:Sheet2.C9"> + <calcext:conditional-format calcext:target-range-address="Sheet2.C2:Sheet2.C100"> <calcext:condition calcext:apply-style-name="Default" calcext:value="=""" calcext:base-cell-address="Sheet2.C2"/> <calcext:condition calcext:apply-style-name="Untitled1" calcext:value="=0" calcext:base-cell-address="Sheet2.C2"/> <calcext:condition calcext:apply-style-name="Untitled2" calcext:value="=1" calcext:base-cell-address="Sheet2.C2"/>
