sc/source/core/tool/interpr3.cxx | 24 +++++++++++-------------
1 file changed, 11 insertions(+), 13 deletions(-)
New commits:
commit a9a734a680dda8c177f92cf2f14061c9b785ec8d
Author: Werner Koerner <wk66...@gmail.com>
Date: Tue Dec 11 22:23:39 2012 +0100
calc: fix mathematical error in CritBinom
CRITBINOM(462,0.8,0.9), expected result 381, calculated value is 462.
Similar errors in BinomDist, B, CritBinom and NegBinomDist were
fixed by commit 5cf55f5b7800e443c4f087e72ae05abc8b7fef45.
Change-Id: I9b12a1c4410ec72258ae1fb68409ad00c922b94c
Reviewed-on: https://gerrit.libreoffice.org/1301
Reviewed-by: Eike Rathke <er...@redhat.com>
Tested-by: Eike Rathke <er...@redhat.com>
diff --git a/sc/source/core/tool/interpr3.cxx b/sc/source/core/tool/interpr3.cxx
index ada1cff..638806d 100644
--- a/sc/source/core/tool/interpr3.cxx
+++ b/sc/source/core/tool/interpr3.cxx
@@ -1348,26 +1348,25 @@ void ScInterpreter::ScCritBinom()
RTL_LOGFILE_CONTEXT_AUTHOR( aLogger, "sc", "er",
"ScInterpreter::ScCritBinom" );
if ( MustHaveParamCount( GetByte(), 3 ) )
{
- double alpha = GetDouble(); // alpha
- double p = GetDouble(); // p
+ double alpha = GetDouble();
+ double p = GetDouble();
double n = ::rtl::math::approxFloor(GetDouble());
if (n < 0.0 || alpha <= 0.0 || alpha >= 1.0 || p < 0.0 || p > 1.0)
PushIllegalArgument();
else
{
- double q = 1.0 - p;
+ double q = (0.5 - p) + 0.5; // get one bit more for p
near 1.0
double fFactor = pow(q,n);
- if (fFactor == 0.0)
+ if (fFactor <= ::std::numeric_limits<double>::min())
{
fFactor = pow(p, n);
- if (fFactor == 0.0)
+ if (fFactor <= ::std::numeric_limits<double>::min())
PushNoValue();
else
{
- double fSum = 1.0 - fFactor; sal_uLong max = (sal_uLong) n;
- sal_uLong i;
-
- for ( i = 0; i < max && fSum >= alpha; i++)
+ double fSum = 1.0 - fFactor;
+ sal_uInt32 max = static_cast<sal_uInt32> (n), i;
+ for (i = 0; i < max && fSum >= alpha; i++)
{
fFactor *= (n-i)/(i+1)*q/p;
fSum -= fFactor;
@@ -1377,10 +1376,9 @@ void ScInterpreter::ScCritBinom()
}
else
{
- double fSum = fFactor; sal_uLong max = (sal_uLong) n;
- sal_uLong i;
-
- for ( i = 0; i < max && fSum < alpha; i++)
+ double fSum = fFactor;
+ sal_uInt32 max = static_cast<sal_uInt32> (n), i;
+ for (i = 0; i < max && fSum < alpha; i++)
{
fFactor *= (n-i)/(i+1)*p/q;
fSum += fFactor;
_______________________________________________
Libreoffice-commits mailing list
libreoffice-comm...@lists.freedesktop.org
http://lists.freedesktop.org/mailman/listinfo/libreoffice-commits
