Hi hackers,
While testing ANALYZE with large statistics targets, I found that
building ndistinct extended statistics leaks memory proportional to the
sample size for every combination of columns.
Example:
postgresql.conf settings
```
default_statistics_target = 1000
```
Commands
```
SET client_min_messages = log;
SET log_statement_stats = on;
DROP TABLE IF EXISTS ndist_leak;
CREATE TABLE ndist_leak AS
SELECT (random() * 3000)::int AS a,
(random() * 3000)::int AS b,
(random() * 50)::int AS c,
(random() * 20)::int AS d,
(random() * 10)::int AS e,
(random() * 5)::int AS f
FROM generate_series(1, 1000000);
CREATE STATISTICS ndist_leak_nd (ndistinct) ON a, b, c, d, e, f FROM
ndist_leak;
ANALYZE ndist_leak;
```
*Peak memory of the backend:*
Before patch - 971076 kB max resident size
After patch - 112780 kB max resident size
ndistinct_for_combination() allocates the items, values and isnull
arrays for the whole sample, plus the MultiSortSupport, and never frees
them. statext_ndistinct_build() calls it once per combination of
columns, so these allocations accumulate until the whole ndistinct
object is built.
The attached patch changes statext_ndistinct_build() to call
ndistinct_for_combination() in a separate memory context and reset it
after each combination, the same way as is already done for
dependency_degree() in statext_dependencies_build().
Similar memory issues in extended statistics were fixed in [0]. The
original report there actually used ndistinct objects, but the fixes
addressed memory accumulated across statistics objects and in
dependency_degree(); the per-combination allocations in
ndistinct_for_combination() were not discussed there.
Any suggestions?
[0]:
https://www.postgresql.org/message-id/20210915200928.GP831%40telsasoft.com
--
Best regards,
Ilia Evdokimov,
Tantor Labs LLC,
https://tantorlabs.com/
From 70dac855e18ce19aad4d0c9b98028da1a6d9e470 Mon Sep 17 00:00:00 2001
From: Evdokimov Ilia <[email protected]>
Date: Fri, 9 Oct 2026 00:02:22 +0500
Subject: [PATCH v1] Release memory allocated by ndistinct_for_combination
Building ndistinct extended statistics calls ndistinct_for_combination()
once for every combination of columns. Each call allocates arrays for
the whole sample (SortItems, Datums and null flags), plus the
MultiSortSupport, and never frees them, so the memory accumulates until
the whole statistics object is built. The sample has 300 * target rows,
so with large statistics targets this can add up to gigabytes;
Fixed by calling ndistinct_for_combination() in a dedicated memory
context and resetting it after each combination, as is already done
for dependency_degree() in statext_dependencies_build(). We only need
the estimated number of distinct values, so we don't need to copy
anything out of the context.
---
src/backend/statistics/mvdistinct.c | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 601f108a2b4..0add6455f6a 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -28,6 +28,7 @@
#include "catalog/pg_statistic_ext.h"
#include "catalog/pg_statistic_ext_data.h"
#include "statistics/extended_stats_internal.h"
+#include "utils/memutils.h"
#include "utils/syscache.h"
#include "utils/typcache.h"
#include "varatt.h"
@@ -89,6 +90,7 @@ statext_ndistinct_build(double totalrows, StatsBuildData *data)
uint32 itemcnt;
int numattrs = data->nattnums;
int numcombs = num_combinations(numattrs);
+ MemoryContext cxt;
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
@@ -96,6 +98,10 @@ statext_ndistinct_build(double totalrows, StatsBuildData *data)
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
+ cxt = AllocSetContextCreate(CurrentMemoryContext,
+ "ndistinct_for_combination cxt",
+ ALLOCSET_DEFAULT_SIZES);
+
itemcnt = 0;
for (k = 2; k <= numattrs; k++)
{
@@ -109,6 +115,7 @@ statext_ndistinct_build(double totalrows, StatsBuildData *data)
{
MVNDistinctItem *item = &result->items[itemcnt];
int j;
+ MemoryContext oldcxt;
item->attributes = palloc_array(AttrNumber, k);
item->nattributes = k;
@@ -121,8 +128,12 @@ statext_ndistinct_build(double totalrows, StatsBuildData *data)
Assert(AttributeNumberIsValid(item->attributes[j]));
}
+ /* release memory used by ndistinct calculation */
+ oldcxt = MemoryContextSwitchTo(cxt);
item->ndistinct =
ndistinct_for_combination(totalrows, data, k, combination);
+ MemoryContextSwitchTo(oldcxt);
+ MemoryContextReset(cxt);
itemcnt++;
Assert(itemcnt <= result->nitems);
@@ -131,6 +142,8 @@ statext_ndistinct_build(double totalrows, StatsBuildData *data)
generator_free(generator);
}
+ MemoryContextDelete(cxt);
+
/* must consume exactly the whole output array */
Assert(itemcnt == result->nitems);
--
2.53.0
