I have implemented a version of OneWayAnova which uses SummaryStatistics rather than requiring the array of doubles.
I know that there is a whole process for submitting code but I am under a deadline so sending this email with the code is all I am going to do. A couple of notes: 1. I would have maintained the same signature structure simply adding methods that supported Collection<SummaryStatistics> but that creates an error because it has the same Signature due to type erasure on Collection<double[]>. 1. I did not see why certain cases such as having only one array or number of elements < 2 should necessarily throw exceptions -- the math still works out and gives reasonable results so given point #1, I differentiated the function signatures by adding a boolean to optionally skip the checks and therefore not not throw DimensionMismatchException. Minor note: I understand that you cannot change it at this point but there is no reason for this class not to abstract with static methods since it has no state. Do with this what you will. Thanks for the wonderful tools you provide. Peter Andrews
/** * this is a copy of the Apache Commons Math OneWayAnova which accepts a * list of StatisticalSummary objects rather than requiring the actual raw * data */ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.commons.math3.stat.inference; import java.util.ArrayList; import java.util.Collection; import org.apache.commons.math3.distribution.FDistribution; import org.apache.commons.math3.exception.ConvergenceException; import org.apache.commons.math3.exception.DimensionMismatchException; import org.apache.commons.math3.exception.MaxCountExceededException; import org.apache.commons.math3.exception.NullArgumentException; import org.apache.commons.math3.exception.OutOfRangeException; import org.apache.commons.math3.exception.util.LocalizedFormats; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; /** * Implements one-way ANOVA (analysis of variance) statistics. * * <p> * Tests for differences between two or more categories of univariate data (for * example, the body mass index of accountants, lawyers, doctors and computer * programmers). When two categories are given, this is equivalent to the * {@link org.apache.commons.math3.stat.inference.TTest}. * </p> * <p> * Uses the {@link org.apache.commons.math3.distribution.FDistribution * commons-math F Distribution implementation} to estimate exact p-values. * </p> * <p> * This implementation is based on a description at * http://faculty.vassar.edu/lowry/ch13pt1.html * </p> * * <pre> * Abbreviations: bg = between groups, * wg = within groups, * ss = sum squared deviations * </pre> * * @since 1.2 * @version $Id: OneWayAnova.java 1244107 2012-02-14 16:17:55Z erans $ */ public class OneWayAnova { /** * Default constructor. */ public OneWayAnova() { } /** * Computes the ANOVA F-value for a collection of <code>double[]</code> * arrays. * * <p> * <strong>Preconditions</strong>: * <ul> * <li>The categoryData <code>Collection</code> must contain * <code>double[]</code> arrays.</li> * <li>There must be at least two <code>double[]</code> arrays in the * <code>categoryData</code> collection and each of these arrays must * contain at least two values.</li> * </ul> * </p> * <p> * This implementation computes the F statistic using the definitional * formula * * <pre> * F = msbg / mswg * </pre> * * where * * <pre> * msbg = between group mean square * mswg = within group mean square * </pre> * * are as defined <a href="http://faculty.vassar.edu/lowry/ch13pt1.html";> * here</a> * </p> * * @param categoryData * <code>Collection</code> of <code>double[]</code> arrays each * containing data for one category * @return Fvalue * @throws NullArgumentException * if <code>categoryData</code> is <code>null</code> * @throws DimensionMismatchException * if the length of the <code>categoryData</code> array is less * than 2 or a contained <code>double[]</code> array does not * have at least two values */ public double anovaFValue(final Collection<double[]> categoryData) throws NullArgumentException, DimensionMismatchException { final AnovaStats a = anovaStats(categoryData); return a.F; } /** * Computes the ANOVA F-value for a collection of <code>double[]</code> * arrays. * * <p> * <strong>Preconditions</strong>: * <ul> * <li>The categoryData <code>Collection</code> must contain * <code>double[]</code> arrays.</li> * <li>There must be at least two <code>double[]</code> arrays in the * <code>categoryData</code> collection and each of these arrays must * contain at least two values.</li> * </ul> * </p> * <p> * This implementation computes the F statistic using the definitional * formula * * <pre> * F = msbg / mswg * </pre> * * where * * <pre> * msbg = between group mean square * mswg = within group mean square * </pre> * * are as defined <a href="http://faculty.vassar.edu/lowry/ch13pt1.html";> * here</a> * </p> * * @param categoryData * <code>Collection</code> of <code>SummaryStatistics</code> * arrays each containing data for one category * @return Fvalue * @throws NullArgumentException * if <code>categoryData</code> is <code>null</code> * @throws DimensionMismatchException * if the length of the <code>categoryData</code> array is less * than 2 or a contained <code>double[]</code> array does not * have at least two values */ public double anovaFValue(final Collection<SummaryStatistics> categoryData, final boolean doNotCheckDimensionMismatchException) { final AnovaStats a = anovaStats(categoryData, doNotCheckDimensionMismatchException); return a.F; } /** * Computes the ANOVA P-value for a collection of <code>double[]</code> * arrays. * * <p> * <strong>Preconditions</strong>: * <ul> * <li>The categoryData <code>Collection</code> must contain * <code>double[]</code> arrays.</li> * <li>There must be at least two <code>double[]</code> arrays in the * <code>categoryData</code> collection and each of these arrays must * contain at least two values.</li> * </ul> * </p> * <p> * This implementation uses the * {@link org.apache.commons.math3.distribution.FDistribution commons-math F * Distribution implementation} to estimate the exact p-value, using the * formula * * <pre> * p = 1 - cumulativeProbability(F) * </pre> * * where <code>F</code> is the F value and * <code>cumulativeProbability</code> is the commons-math implementation of * the F distribution. * </p> * * @param categoryData * <code>Collection</code> of <code>double[]</code> arrays each * containing data for one category * @return Pvalue * @throws NullArgumentException * if <code>categoryData</code> is <code>null</code> * @throws DimensionMismatchException * if the length of the <code>categoryData</code> array is less * than 2 or a contained <code>double[]</code> array does not * have at least two values * @throws ConvergenceException * if the p-value can not be computed due to a convergence error * @throws MaxCountExceededException * if the maximum number of iterations is exceeded */ public double anovaPValue(final Collection<double[]> categoryData) throws NullArgumentException, DimensionMismatchException, ConvergenceException, MaxCountExceededException { final AnovaStats a = anovaStats(categoryData); final FDistribution fdist = new FDistribution(a.dfbg, a.dfwg); return 1.0 - fdist.cumulativeProbability(a.F); } /** * Computes the ANOVA P-value for a collection of <code>double[]</code> * arrays. * * <p> * <strong>Preconditions</strong>: * <ul> * <li>The categoryData <code>Collection</code> must contain * <code>double[]</code> arrays.</li> * <li>There must be at least two <code>double[]</code> arrays in the * <code>categoryData</code> collection and each of these arrays must * contain at least two values.</li> * </ul> * </p> * <p> * This implementation uses the * {@link org.apache.commons.math3.distribution.FDistribution commons-math F * Distribution implementation} to estimate the exact p-value, using the * formula * * <pre> * p = 1 - cumulativeProbability(F) * </pre> * * where <code>F</code> is the F value and * <code>cumulativeProbability</code> is the commons-math implementation of * the F distribution. * </p> * * @param categoryData * <code>Collection</code> of <code>double[]</code> arrays each * containing data for one category * @return Pvalue * @throws NullArgumentException * if <code>categoryData</code> is <code>null</code> * @throws DimensionMismatchException * if the length of the <code>categoryData</code> array is less * than 2 or a contained <code>SummaryStatistics</code> array * does not have at least two values * @throws ConvergenceException * if the p-value can not be computed due to a convergence error * @throws MaxCountExceededException * if the maximum number of iterations is exceeded */ public double anovaPValue(final Collection<SummaryStatistics> categoryData, final boolean doNotCheckDimensionMismatchException) throws NullArgumentException, DimensionMismatchException, ConvergenceException, MaxCountExceededException { final AnovaStats a = anovaStats(categoryData, doNotCheckDimensionMismatchException); final FDistribution fdist = new FDistribution(a.dfbg, a.dfwg); return 1.0 - fdist.cumulativeProbability(a.F); } /** * This method calls the method that actually does the calculations (except * P-value). * * @param categoryData * <code>Collection</code> of <code>double[]</code> arrays each * containing data for one category * @return computed AnovaStats * @throws NullArgumentException * if <code>categoryData</code> is <code>null</code> * @throws DimensionMismatchException * if the length of the <code>categoryData</code> array is less * than 2 or a contained <code>double[]</code> array does not * contain at least two values */ private AnovaStats anovaStats(final Collection<double[]> categoryData) throws NullArgumentException, DimensionMismatchException { if (categoryData == null) { throw new NullArgumentException(); } final Collection<SummaryStatistics> categoryDataSummaryStatistics = new ArrayList<SummaryStatistics>( categoryData.size()); // check if each category has enough data and all is double[] for (final double[] data : categoryData) { final SummaryStatistics dataSummaryStatistics = new SummaryStatistics(); categoryDataSummaryStatistics.add(dataSummaryStatistics); for (final double val : data) { dataSummaryStatistics.addValue(val); } } return anovaStats(categoryDataSummaryStatistics, false /* doNotCheckDimensionMismatchException */); } /** * This method actually does the calculations (except P-value). * * @param categoryData * <code>Collection</code> of <code>double[]</code> arrays each * containing data for one category * @return computed AnovaStats * @throws NullArgumentException * if <code>categoryData</code> is <code>null</code> * @throws DimensionMismatchException * if the length of the <code>categoryData</code> array is less * than 2 or a contained <code>double[]</code> array does not * contain at least two values */ private AnovaStats anovaStats( final Collection<SummaryStatistics> categoryData, final boolean doNotCheckDimensionMismatchException) throws NullArgumentException, DimensionMismatchException { if (categoryData == null) { throw new NullArgumentException(); } if (!doNotCheckDimensionMismatchException) { // check if we have enough categories if (categoryData.size() < 2) { throw new DimensionMismatchException( LocalizedFormats.TWO_OR_MORE_CATEGORIES_REQUIRED, categoryData.size(), 2); } // check if each category has enough data and all is double[] for (final SummaryStatistics array : categoryData) { if (array.getN() <= 1) { throw new DimensionMismatchException( LocalizedFormats.TWO_OR_MORE_VALUES_IN_CATEGORY_REQUIRED, (int) array.getN(), 2); } } } int dfwg = 0; double sswg = 0; double totsum = 0; double totsumsq = 0; int totnum = 0; for (final SummaryStatistics data : categoryData) { final double sum = data.getSum(); final double sumsq = data.getSumsq(); final int num = (int) data.getN(); totnum += num; totsum += sum; totsumsq += sumsq; dfwg += num - 1; final double ss = sumsq - ((sum * sum) / num); sswg += ss; } final double sst = totsumsq - ((totsum * totsum) / totnum); final double ssbg = sst - sswg; final int dfbg = categoryData.size() - 1; final double msbg = ssbg / dfbg; final double mswg = sswg / dfwg; final double F = msbg / mswg; return new AnovaStats(dfbg, dfwg, F); } /** * Performs an ANOVA test, evaluating the null hypothesis that there is no * difference among the means of the data categories. * * <p> * <strong>Preconditions</strong>: * <ul> * <li>The categoryData <code>Collection</code> must contain * <code>double[]</code> arrays.</li> * <li>There must be at least two <code>double[]</code> arrays in the * <code>categoryData</code> collection and each of these arrays must * contain at least two values.</li> * <li>alpha must be strictly greater than 0 and less than or equal to 0.5.</li> * </ul> * </p> * <p> * This implementation uses the * {@link org.apache.commons.math3.distribution.FDistribution commons-math F * Distribution implementation} to estimate the exact p-value, using the * formula * * <pre> * p = 1 - cumulativeProbability(F) * </pre> * * where <code>F</code> is the F value and * <code>cumulativeProbability</code> is the commons-math implementation of * the F distribution. * </p> * <p> * True is returned iff the estimated p-value is less than alpha. * </p> * * @param categoryData * <code>Collection</code> of <code>double[]</code> arrays each * containing data for one category * @param alpha * significance level of the test * @return true if the null hypothesis can be rejected with confidence 1 - * alpha * @throws NullArgumentException * if <code>categoryData</code> is <code>null</code> * @throws DimensionMismatchException * if the length of the <code>categoryData</code> array is less * than 2 or a contained <code>double[]</code> array does not * have at least two values * @throws OutOfRangeException * if <code>alpha</code> is not in the range (0, 0.5] * @throws ConvergenceException * if the p-value can not be computed due to a convergence error * @throws MaxCountExceededException * if the maximum number of iterations is exceeded */ public boolean anovaTest(final Collection<double[]> categoryData, final double alpha) throws NullArgumentException, DimensionMismatchException, OutOfRangeException, ConvergenceException, MaxCountExceededException { if ((alpha <= 0) || (alpha > 0.5)) { throw new OutOfRangeException( LocalizedFormats.OUT_OF_BOUND_SIGNIFICANCE_LEVEL, alpha, 0, 0.5); } return anovaPValue(categoryData) < alpha; } /** * Convenience class to pass dfbg,dfwg,F values around within AnovaImpl. No * get/set methods provided. */ private static class AnovaStats { /** Degrees of freedom in numerator (between groups). */ private final int dfbg; /** Degrees of freedom in denominator (within groups). */ private final int dfwg; /** Statistic. */ private final double F; /** * Constructor * * @param dfbg * degrees of freedom in numerator (between groups) * @param dfwg * degrees of freedom in denominator (within groups) * @param F * statistic */ private AnovaStats(final int dfbg, final int dfwg, final double F) { this.dfbg = dfbg; this.dfwg = dfwg; this.F = F; } } }
--------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@commons.apache.org For additional commands, e-mail: dev-h...@commons.apache.org
