A memory chunk allocated through the existing palloc.h interfaces is limited
to MaxAllocSize (~1 GiB). This is best for most callers; SET_VARSIZE() need
not check its own 1 GiB limit, and algorithms that grow a buffer by doubling
need not check for overflow. However, a handful of callers are quite happy to
navigate those hazards in exchange for the ability to allocate a larger chunk.
This patch introduces MemoryContextAllocHuge() and repalloc_huge() that check
a higher MaxAllocHugeSize limit of SIZE_MAX/2. Chunks don't bother recording
whether they were allocated as huge; one can start with palloc() and then
repalloc_huge() to grow the value. To demonstrate, I put this to use in
tuplesort.c; the patch also updates tuplestore.c to keep them similar. Here's
the trace_sort from building the pgbench_accounts primary key at scale factor
7500, maintenance_work_mem = '56GB'; memtuples itself consumed 17.2 GiB:
LOG: internal sort ended, 48603324 KB used: CPU 75.65s/305.46u sec elapsed
391.21 sec
Compare:
LOG: external sort ended, 1832846 disk blocks used: CPU 77.45s/988.11u sec
elapsed 1146.05 sec
This was made easier by tuplesort growth algorithm improvements in commit
8ae35e91807508872cabd3b0e8db35fc78e194ac. The problem has come up before
(TODO item "Allow sorts to use more available memory"), and Tom floated the
idea[1] behind the approach I've used. The next limit faced by sorts is
INT_MAX concurrent tuples in memory, which limits helpful work_mem to about
150 GiB when sorting int4.
I have not added variants like palloc_huge() and palloc0_huge(), and I have
not added to the frontend palloc.h interface. There's no particular barrier
to doing any of that. I don't expect more than a dozen or so callers, so most
of the variations might go unused.
The comment at MaxAllocSize said that aset.c expects doubling the size of an
arbitrary allocation to never overflow, but I couldn't find the code in
question. AllocSetAlloc() does double sizes of blocks used to aggregate small
allocations, so maxBlockSize had better stay under SIZE_MAX/2. Nonetheless,
that expectation does apply to dozens of repalloc() users outside aset.c, and
I preserved it for repalloc_huge(). 64-bit builds will never notice, and I
won't cry for the resulting 2 GiB limit on 32-bit.
Thanks,
nm
[1] http://www.postgresql.org/message-id/19908.1297696...@sss.pgh.pa.us
--
Noah Misch
EnterpriseDB http://www.enterprisedb.com
*** a/src/backend/utils/mmgr/aset.c
--- b/src/backend/utils/mmgr/aset.c
***************
*** 557,562 **** AllocSetDelete(MemoryContext context)
--- 557,566 ----
* AllocSetAlloc
* Returns pointer to allocated memory of given size; memory is
added
* to the set.
+ *
+ * No request may exceed:
+ * MAXALIGN_DOWN(SIZE_MAX) - ALLOC_BLOCKHDRSZ - ALLOC_CHUNKHDRSZ
+ * All callers use a much-lower limit.
*/
static void *
AllocSetAlloc(MemoryContext context, Size size)
*** a/src/backend/utils/mmgr/mcxt.c
--- b/src/backend/utils/mmgr/mcxt.c
***************
*** 451,464 **** MemoryContextContains(MemoryContext context, void *pointer)
header = (StandardChunkHeader *)
((char *) pointer - STANDARDCHUNKHEADERSIZE);
! /*
! * If the context link doesn't match then we certainly have a non-member
! * chunk. Also check for a reasonable-looking size as extra guard
against
! * being fooled by bogus pointers.
! */
! if (header->context == context && AllocSizeIsValid(header->size))
! return true;
! return false;
}
/*--------------------
--- 451,457 ----
header = (StandardChunkHeader *)
((char *) pointer - STANDARDCHUNKHEADERSIZE);
! return header->context == context;
}
/*--------------------
***************
*** 735,740 **** repalloc(void *pointer, Size size)
--- 728,790 ----
}
/*
+ * MemoryContextAllocHuge
+ * Allocate (possibly-expansive) space within the specified
context.
+ *
+ * See considerations in comment at MaxAllocHugeSize.
+ */
+ void *
+ MemoryContextAllocHuge(MemoryContext context, Size size)
+ {
+ AssertArg(MemoryContextIsValid(context));
+
+ if (!AllocHugeSizeIsValid(size))
+ elog(ERROR, "invalid memory alloc request size %lu",
+ (unsigned long) size);
+
+ context->isReset = false;
+
+ return (*context->methods->alloc) (context, size);
+ }
+
+ /*
+ * repalloc_huge
+ * Adjust the size of a previously allocated chunk, permitting a
large
+ * value. The previous allocation need not have been "huge".
+ */
+ void *
+ repalloc_huge(void *pointer, Size size)
+ {
+ StandardChunkHeader *header;
+
+ /*
+ * Try to detect bogus pointers handed to us, poorly though we can.
+ * Presumably, a pointer that isn't MAXALIGNED isn't pointing at an
+ * allocated chunk.
+ */
+ Assert(pointer != NULL);
+ Assert(pointer == (void *) MAXALIGN(pointer));
+
+ /*
+ * OK, it's probably safe to look at the chunk header.
+ */
+ header = (StandardChunkHeader *)
+ ((char *) pointer - STANDARDCHUNKHEADERSIZE);
+
+ AssertArg(MemoryContextIsValid(header->context));
+
+ if (!AllocHugeSizeIsValid(size))
+ elog(ERROR, "invalid memory alloc request size %lu",
+ (unsigned long) size);
+
+ /* isReset must be false already */
+ Assert(!header->context->isReset);
+
+ return (*header->context->methods->realloc) (header->context,
+
pointer, size);
+ }
+
+ /*
* MemoryContextStrdup
* Like strdup(), but allocate from the specified context
*/
*** a/src/backend/utils/sort/tuplesort.c
--- b/src/backend/utils/sort/tuplesort.c
***************
*** 211,218 **** struct Tuplesortstate
* tuples to
return? */
bool boundUsed; /* true if we made use of a
bounded heap */
int bound; /* if bounded, the
maximum number of tuples */
! long availMem; /* remaining memory available,
in bytes */
! long allowedMem; /* total memory allowed, in
bytes */
int maxTapes; /* number of tapes
(Knuth's T) */
int tapeRange; /* maxTapes-1 (Knuth's
P) */
MemoryContext sortcontext; /* memory context holding all sort data
*/
--- 211,218 ----
* tuples to
return? */
bool boundUsed; /* true if we made use of a
bounded heap */
int bound; /* if bounded, the
maximum number of tuples */
! Size availMem; /* remaining memory available,
in bytes */
! Size allowedMem; /* total memory allowed, in
bytes */
int maxTapes; /* number of tapes
(Knuth's T) */
int tapeRange; /* maxTapes-1 (Knuth's
P) */
MemoryContext sortcontext; /* memory context holding all sort data
*/
***************
*** 308,314 **** struct Tuplesortstate
int *mergenext; /* first preread tuple for each
source */
int *mergelast; /* last preread tuple for each
source */
int *mergeavailslots; /* slots left for prereading
each tape */
! long *mergeavailmem; /* availMem for prereading each tape */
int mergefreelist; /* head of freelist of recycled
slots */
int mergefirstfree; /* first slot never used in
this merge */
--- 308,314 ----
int *mergenext; /* first preread tuple for each
source */
int *mergelast; /* last preread tuple for each
source */
int *mergeavailslots; /* slots left for prereading
each tape */
! Size *mergeavailmem; /* availMem for prereading each tape */
int mergefreelist; /* head of freelist of recycled
slots */
int mergefirstfree; /* first slot never used in
this merge */
***************
*** 961,985 **** tuplesort_end(Tuplesortstate *state)
}
/*
! * Grow the memtuples[] array, if possible within our memory constraint.
! * Return TRUE if we were able to enlarge the array, FALSE if not.
*
! * Normally, at each increment we double the size of the array. When we no
! * longer have enough memory to do that, we attempt one last, smaller increase
! * (and then clear the growmemtuples flag so we don't try any more). That
! * allows us to use allowedMem as fully as possible; sticking to the pure
! * doubling rule could result in almost half of allowedMem going unused.
! * Because availMem moves around with tuple addition/removal, we need some
! * rule to prevent making repeated small increases in memtupsize, which would
! * just be useless thrashing. The growmemtuples flag accomplishes that and
! * also prevents useless recalculations in this function.
*/
static bool
grow_memtuples(Tuplesortstate *state)
{
int newmemtupsize;
int memtupsize = state->memtupsize;
! long memNowUsed = state->allowedMem - state->availMem;
/* Forget it if we've already maxed out memtuples, per comment above */
if (!state->growmemtuples)
--- 961,986 ----
}
/*
! * Grow the memtuples[] array, if possible within our memory constraint. We
! * must not exceed INT_MAX tuples in memory or the caller-provided memory
! * limit. Return TRUE if we were able to enlarge the array, FALSE if not.
*
! * Normally, at each increment we double the size of the array. When doing
! * that would exceed a limit, we attempt one last, smaller increase (and then
! * clear the growmemtuples flag so we don't try any more). That allows us to
! * use memory as fully as permitted; sticking to the pure doubling rule could
! * result in almost half going unused. Because availMem moves around with
! * tuple addition/removal, we need some rule to prevent making repeated small
! * increases in memtupsize, which would just be useless thrashing. The
! * growmemtuples flag accomplishes that and also prevents useless
! * recalculations in this function.
*/
static bool
grow_memtuples(Tuplesortstate *state)
{
int newmemtupsize;
int memtupsize = state->memtupsize;
! Size memNowUsed = state->allowedMem - state->availMem;
/* Forget it if we've already maxed out memtuples, per comment above */
if (!state->growmemtuples)
***************
*** 989,1002 **** grow_memtuples(Tuplesortstate *state)
if (memNowUsed <= state->availMem)
{
/*
! * It is surely safe to double memtupsize if we've used no more
than
! * half of allowedMem.
! *
! * Note: it might seem that we need to worry about memtupsize *
2
! * overflowing an int, but the MaxAllocSize clamp applied below
! * ensures the existing memtupsize can't be large enough for
that.
*/
! newmemtupsize = memtupsize * 2;
}
else
{
--- 990,1005 ----
if (memNowUsed <= state->availMem)
{
/*
! * We've used no more than half of allowedMem; double our usage,
! * clamping at INT_MAX.
*/
! if (memtupsize < INT_MAX / 2)
! newmemtupsize = memtupsize * 2;
! else
! {
! newmemtupsize = INT_MAX;
! state->growmemtuples = false;
! }
}
else
{
***************
*** 1012,1018 **** grow_memtuples(Tuplesortstate *state)
* we've already seen, and thus we can extrapolate from the
space
* consumption so far to estimate an appropriate new size for
the
* memtuples array. The optimal value might be higher or lower
than
! * this estimate, but it's hard to know that in advance.
*
* This calculation is safe against enlarging the array so much
that
* LACKMEM becomes true, because the memory currently used
includes
--- 1015,1022 ----
* we've already seen, and thus we can extrapolate from the
space
* consumption so far to estimate an appropriate new size for
the
* memtuples array. The optimal value might be higher or lower
than
! * this estimate, but it's hard to know that in advance. We
again
! * clamp at INT_MAX tuples.
*
* This calculation is safe against enlarging the array so much
that
* LACKMEM becomes true, because the memory currently used
includes
***************
*** 1020,1035 **** grow_memtuples(Tuplesortstate *state)
* new array elements even if no other memory were currently
used.
*
* We do the arithmetic in float8, because otherwise the
product of
! * memtupsize and allowedMem could overflow. (A little algebra
shows
! * that grow_ratio must be less than 2 here, so we are not
risking
! * integer overflow this way.) Any inaccuracy in the result
should be
! * insignificant; but even if we computed a completely insane
result,
! * the checks below will prevent anything really bad from
happening.
*/
double grow_ratio;
grow_ratio = (double) state->allowedMem / (double) memNowUsed;
! newmemtupsize = (int) (memtupsize * grow_ratio);
/* We won't make any further enlargement attempts */
state->growmemtuples = false;
--- 1024,1041 ----
* new array elements even if no other memory were currently
used.
*
* We do the arithmetic in float8, because otherwise the
product of
! * memtupsize and allowedMem could overflow. Any inaccuracy in
the
! * result should be insignificant; but even if we computed a
! * completely insane result, the checks below will prevent
anything
! * really bad from happening.
*/
double grow_ratio;
grow_ratio = (double) state->allowedMem / (double) memNowUsed;
! if (memtupsize * grow_ratio < INT_MAX)
! newmemtupsize = (int) (memtupsize * grow_ratio);
! else
! newmemtupsize = INT_MAX;
/* We won't make any further enlargement attempts */
state->growmemtuples = false;
***************
*** 1040,1051 **** grow_memtuples(Tuplesortstate *state)
goto noalloc;
/*
! * On a 64-bit machine, allowedMem could be more than MaxAllocSize.
Clamp
! * to ensure our request won't be rejected by palloc.
*/
! if ((Size) newmemtupsize >= MaxAllocSize / sizeof(SortTuple))
{
! newmemtupsize = (int) (MaxAllocSize / sizeof(SortTuple));
state->growmemtuples = false; /* can't grow any more */
}
--- 1046,1058 ----
goto noalloc;
/*
! * On a 32-bit machine, allowedMem could exceed MaxAllocHugeSize. Clamp
! * to ensure our request won't be rejected. Note that we can easily
! * exhaust address space before facing this outcome.
*/
! if ((Size) newmemtupsize >= MaxAllocHugeSize / sizeof(SortTuple))
{
! newmemtupsize = (int) (MaxAllocHugeSize / sizeof(SortTuple));
state->growmemtuples = false; /* can't grow any more */
}
***************
*** 1060,1074 **** grow_memtuples(Tuplesortstate *state)
* palloc would be treating both old and new arrays as separate chunks.
* But we'll check LACKMEM explicitly below just in case.)
*/
! if (state->availMem < (long) ((newmemtupsize - memtupsize) *
sizeof(SortTuple)))
goto noalloc;
/* OK, do it */
FREEMEM(state, GetMemoryChunkSpace(state->memtuples));
state->memtupsize = newmemtupsize;
state->memtuples = (SortTuple *)
! repalloc(state->memtuples,
! state->memtupsize * sizeof(SortTuple));
USEMEM(state, GetMemoryChunkSpace(state->memtuples));
if (LACKMEM(state))
elog(ERROR, "unexpected out-of-memory situation during sort");
--- 1067,1081 ----
* palloc would be treating both old and new arrays as separate chunks.
* But we'll check LACKMEM explicitly below just in case.)
*/
! if (state->availMem < (Size) ((newmemtupsize - memtupsize) *
sizeof(SortTuple)))
goto noalloc;
/* OK, do it */
FREEMEM(state, GetMemoryChunkSpace(state->memtuples));
state->memtupsize = newmemtupsize;
state->memtuples = (SortTuple *)
! repalloc_huge(state->memtuples,
! state->memtupsize *
sizeof(SortTuple));
USEMEM(state, GetMemoryChunkSpace(state->memtuples));
if (LACKMEM(state))
elog(ERROR, "unexpected out-of-memory situation during sort");
***************
*** 1715,1721 **** tuplesort_getdatum(Tuplesortstate *state, bool forward,
* This is exported for use by the planner. allowedMem is in bytes.
*/
int
! tuplesort_merge_order(long allowedMem)
{
int mOrder;
--- 1722,1728 ----
* This is exported for use by the planner. allowedMem is in bytes.
*/
int
! tuplesort_merge_order(Size allowedMem)
{
int mOrder;
***************
*** 1749,1755 **** inittapes(Tuplesortstate *state)
int maxTapes,
ntuples,
j;
! long tapeSpace;
/* Compute number of tapes to use: merge order plus 1 */
maxTapes = tuplesort_merge_order(state->allowedMem) + 1;
--- 1756,1762 ----
int maxTapes,
ntuples,
j;
! Size tapeSpace;
/* Compute number of tapes to use: merge order plus 1 */
maxTapes = tuplesort_merge_order(state->allowedMem) + 1;
***************
*** 1798,1804 **** inittapes(Tuplesortstate *state)
state->mergenext = (int *) palloc0(maxTapes * sizeof(int));
state->mergelast = (int *) palloc0(maxTapes * sizeof(int));
state->mergeavailslots = (int *) palloc0(maxTapes * sizeof(int));
! state->mergeavailmem = (long *) palloc0(maxTapes * sizeof(long));
state->tp_fib = (int *) palloc0(maxTapes * sizeof(int));
state->tp_runs = (int *) palloc0(maxTapes * sizeof(int));
state->tp_dummy = (int *) palloc0(maxTapes * sizeof(int));
--- 1805,1811 ----
state->mergenext = (int *) palloc0(maxTapes * sizeof(int));
state->mergelast = (int *) palloc0(maxTapes * sizeof(int));
state->mergeavailslots = (int *) palloc0(maxTapes * sizeof(int));
! state->mergeavailmem = (Size *) palloc0(maxTapes * sizeof(Size));
state->tp_fib = (int *) palloc0(maxTapes * sizeof(int));
state->tp_runs = (int *) palloc0(maxTapes * sizeof(int));
state->tp_dummy = (int *) palloc0(maxTapes * sizeof(int));
***************
*** 2026,2032 **** mergeonerun(Tuplesortstate *state)
int srcTape;
int tupIndex;
SortTuple *tup;
! long priorAvail,
spaceFreed;
/*
--- 2033,2039 ----
int srcTape;
int tupIndex;
SortTuple *tup;
! Size priorAvail,
spaceFreed;
/*
***************
*** 2100,2106 **** beginmerge(Tuplesortstate *state)
int tapenum;
int srcTape;
int slotsPerTape;
! long spacePerTape;
/* Heap should be empty here */
Assert(state->memtupcount == 0);
--- 2107,2113 ----
int tapenum;
int srcTape;
int slotsPerTape;
! Size spacePerTape;
/* Heap should be empty here */
Assert(state->memtupcount == 0);
***************
*** 2221,2227 **** mergeprereadone(Tuplesortstate *state, int srcTape)
unsigned int tuplen;
SortTuple stup;
int tupIndex;
! long priorAvail,
spaceUsed;
if (!state->mergeactive[srcTape])
--- 2228,2234 ----
unsigned int tuplen;
SortTuple stup;
int tupIndex;
! Size priorAvail,
spaceUsed;
if (!state->mergeactive[srcTape])
*** a/src/backend/utils/sort/tuplestore.c
--- b/src/backend/utils/sort/tuplestore.c
***************
*** 104,111 **** struct Tuplestorestate
bool backward; /* store extra length words in
file? */
bool interXact; /* keep open through
transactions? */
bool truncated; /* tuplestore_trim has removed
tuples? */
! long availMem; /* remaining memory available,
in bytes */
! long allowedMem; /* total memory allowed, in
bytes */
BufFile *myfile; /* underlying file, or NULL if
none */
MemoryContext context; /* memory context for holding tuples */
ResourceOwner resowner; /* resowner for holding temp files */
--- 104,111 ----
bool backward; /* store extra length words in
file? */
bool interXact; /* keep open through
transactions? */
bool truncated; /* tuplestore_trim has removed
tuples? */
! Size availMem; /* remaining memory available,
in bytes */
! Size allowedMem; /* total memory allowed, in
bytes */
BufFile *myfile; /* underlying file, or NULL if
none */
MemoryContext context; /* memory context for holding tuples */
ResourceOwner resowner; /* resowner for holding temp files */
***************
*** 531,555 **** tuplestore_ateof(Tuplestorestate *state)
}
/*
! * Grow the memtuples[] array, if possible within our memory constraint.
! * Return TRUE if we were able to enlarge the array, FALSE if not.
*
! * Normally, at each increment we double the size of the array. When we no
! * longer have enough memory to do that, we attempt one last, smaller increase
! * (and then clear the growmemtuples flag so we don't try any more). That
! * allows us to use allowedMem as fully as possible; sticking to the pure
! * doubling rule could result in almost half of allowedMem going unused.
! * Because availMem moves around with tuple addition/removal, we need some
! * rule to prevent making repeated small increases in memtupsize, which would
! * just be useless thrashing. The growmemtuples flag accomplishes that and
! * also prevents useless recalculations in this function.
*/
static bool
grow_memtuples(Tuplestorestate *state)
{
int newmemtupsize;
int memtupsize = state->memtupsize;
! long memNowUsed = state->allowedMem - state->availMem;
/* Forget it if we've already maxed out memtuples, per comment above */
if (!state->growmemtuples)
--- 531,556 ----
}
/*
! * Grow the memtuples[] array, if possible within our memory constraint. We
! * must not exceed INT_MAX tuples in memory or the caller-provided memory
! * limit. Return TRUE if we were able to enlarge the array, FALSE if not.
*
! * Normally, at each increment we double the size of the array. When doing
! * that would exceed a limit, we attempt one last, smaller increase (and then
! * clear the growmemtuples flag so we don't try any more). That allows us to
! * use memory as fully as permitted; sticking to the pure doubling rule could
! * result in almost half going unused. Because availMem moves around with
! * tuple addition/removal, we need some rule to prevent making repeated small
! * increases in memtupsize, which would just be useless thrashing. The
! * growmemtuples flag accomplishes that and also prevents useless
! * recalculations in this function.
*/
static bool
grow_memtuples(Tuplestorestate *state)
{
int newmemtupsize;
int memtupsize = state->memtupsize;
! Size memNowUsed = state->allowedMem - state->availMem;
/* Forget it if we've already maxed out memtuples, per comment above */
if (!state->growmemtuples)
***************
*** 559,572 **** grow_memtuples(Tuplestorestate *state)
if (memNowUsed <= state->availMem)
{
/*
! * It is surely safe to double memtupsize if we've used no more
than
! * half of allowedMem.
! *
! * Note: it might seem that we need to worry about memtupsize *
2
! * overflowing an int, but the MaxAllocSize clamp applied below
! * ensures the existing memtupsize can't be large enough for
that.
*/
! newmemtupsize = memtupsize * 2;
}
else
{
--- 560,575 ----
if (memNowUsed <= state->availMem)
{
/*
! * We've used no more than half of allowedMem; double our usage,
! * clamping at INT_MAX.
*/
! if (memtupsize < INT_MAX / 2)
! newmemtupsize = memtupsize * 2;
! else
! {
! newmemtupsize = INT_MAX;
! state->growmemtuples = false;
! }
}
else
{
***************
*** 582,588 **** grow_memtuples(Tuplestorestate *state)
* we've already seen, and thus we can extrapolate from the
space
* consumption so far to estimate an appropriate new size for
the
* memtuples array. The optimal value might be higher or lower
than
! * this estimate, but it's hard to know that in advance.
*
* This calculation is safe against enlarging the array so much
that
* LACKMEM becomes true, because the memory currently used
includes
--- 585,592 ----
* we've already seen, and thus we can extrapolate from the
space
* consumption so far to estimate an appropriate new size for
the
* memtuples array. The optimal value might be higher or lower
than
! * this estimate, but it's hard to know that in advance. We
again
! * clamp at INT_MAX tuples.
*
* This calculation is safe against enlarging the array so much
that
* LACKMEM becomes true, because the memory currently used
includes
***************
*** 590,605 **** grow_memtuples(Tuplestorestate *state)
* new array elements even if no other memory were currently
used.
*
* We do the arithmetic in float8, because otherwise the
product of
! * memtupsize and allowedMem could overflow. (A little algebra
shows
! * that grow_ratio must be less than 2 here, so we are not
risking
! * integer overflow this way.) Any inaccuracy in the result
should be
! * insignificant; but even if we computed a completely insane
result,
! * the checks below will prevent anything really bad from
happening.
*/
double grow_ratio;
grow_ratio = (double) state->allowedMem / (double) memNowUsed;
! newmemtupsize = (int) (memtupsize * grow_ratio);
/* We won't make any further enlargement attempts */
state->growmemtuples = false;
--- 594,611 ----
* new array elements even if no other memory were currently
used.
*
* We do the arithmetic in float8, because otherwise the
product of
! * memtupsize and allowedMem could overflow. Any inaccuracy in
the
! * result should be insignificant; but even if we computed a
! * completely insane result, the checks below will prevent
anything
! * really bad from happening.
*/
double grow_ratio;
grow_ratio = (double) state->allowedMem / (double) memNowUsed;
! if (memtupsize * grow_ratio < INT_MAX)
! newmemtupsize = (int) (memtupsize * grow_ratio);
! else
! newmemtupsize = INT_MAX;
/* We won't make any further enlargement attempts */
state->growmemtuples = false;
***************
*** 610,621 **** grow_memtuples(Tuplestorestate *state)
goto noalloc;
/*
! * On a 64-bit machine, allowedMem could be more than MaxAllocSize.
Clamp
! * to ensure our request won't be rejected by palloc.
*/
! if ((Size) newmemtupsize >= MaxAllocSize / sizeof(void *))
{
! newmemtupsize = (int) (MaxAllocSize / sizeof(void *));
state->growmemtuples = false; /* can't grow any more */
}
--- 616,628 ----
goto noalloc;
/*
! * On a 32-bit machine, allowedMem could exceed MaxAllocHugeSize. Clamp
! * to ensure our request won't be rejected. Note that we can easily
! * exhaust address space before facing this outcome.
*/
! if ((Size) newmemtupsize >= MaxAllocHugeSize / sizeof(void *))
{
! newmemtupsize = (int) (MaxAllocHugeSize / sizeof(void *));
state->growmemtuples = false; /* can't grow any more */
}
***************
*** 630,644 **** grow_memtuples(Tuplestorestate *state)
* palloc would be treating both old and new arrays as separate chunks.
* But we'll check LACKMEM explicitly below just in case.)
*/
! if (state->availMem < (long) ((newmemtupsize - memtupsize) *
sizeof(void *)))
goto noalloc;
/* OK, do it */
FREEMEM(state, GetMemoryChunkSpace(state->memtuples));
state->memtupsize = newmemtupsize;
state->memtuples = (void **)
! repalloc(state->memtuples,
! state->memtupsize * sizeof(void *));
USEMEM(state, GetMemoryChunkSpace(state->memtuples));
if (LACKMEM(state))
elog(ERROR, "unexpected out-of-memory situation during sort");
--- 637,651 ----
* palloc would be treating both old and new arrays as separate chunks.
* But we'll check LACKMEM explicitly below just in case.)
*/
! if (state->availMem < (Size) ((newmemtupsize - memtupsize) *
sizeof(void *)))
goto noalloc;
/* OK, do it */
FREEMEM(state, GetMemoryChunkSpace(state->memtuples));
state->memtupsize = newmemtupsize;
state->memtuples = (void **)
! repalloc_huge(state->memtuples,
! state->memtupsize * sizeof(void *));
USEMEM(state, GetMemoryChunkSpace(state->memtuples));
if (LACKMEM(state))
elog(ERROR, "unexpected out-of-memory situation during sort");
*** a/src/include/utils/memutils.h
--- b/src/include/utils/memutils.h
***************
*** 21,46 ****
/*
! * MaxAllocSize
! * Quasi-arbitrary limit on size of allocations.
*
* Note:
! * There is no guarantee that allocations smaller than MaxAllocSize
! * will succeed. Allocation requests larger than MaxAllocSize will
! * be summarily denied.
*
! * XXX This is deliberately chosen to correspond to the limiting size
! * of varlena objects under TOAST. See VARSIZE_4B() and related macros
! * in postgres.h. Many datatypes assume that any allocatable size can
! * be represented in a varlena header.
! *
! * XXX Also, various places in aset.c assume they can compute twice an
! * allocation's size without overflow, so beware of raising this.
*/
#define MaxAllocSize ((Size) 0x3fffffff) /* 1 gigabyte - 1 */
#define AllocSizeIsValid(size) ((Size) (size) <= MaxAllocSize)
/*
* All chunks allocated by any memory context manager are required to be
* preceded by a StandardChunkHeader at a spacing of STANDARDCHUNKHEADERSIZE.
--- 21,49 ----
/*
! * MaxAllocSize, MaxAllocHugeSize
! * Quasi-arbitrary limits on size of allocations.
*
* Note:
! * There is no guarantee that smaller allocations will succeed, but
! * larger requests will be summarily denied.
*
! * palloc() enforces MaxAllocSize, chosen to correspond to the limiting size
! * of varlena objects under TOAST. See VARSIZE_4B() and related macros in
! * postgres.h. Many datatypes assume that any allocatable size can be
! * represented in a varlena header. Callers that never use the allocation as
! * a varlena can access the higher limit with MemoryContextAllocHuge(). Both
! * limits permit code to assume that it may compute (in size_t math) twice an
! * allocation's size without overflow.
*/
#define MaxAllocSize ((Size) 0x3fffffff) /* 1 gigabyte - 1 */
#define AllocSizeIsValid(size) ((Size) (size) <= MaxAllocSize)
+ #define MaxAllocHugeSize ((Size) -1 >> 1) /* SIZE_MAX / 2 */
+
+ #define AllocHugeSizeIsValid(size) ((Size) (size) <= MaxAllocHugeSize)
+
/*
* All chunks allocated by any memory context manager are required to be
* preceded by a StandardChunkHeader at a spacing of STANDARDCHUNKHEADERSIZE.
*** a/src/include/utils/palloc.h
--- b/src/include/utils/palloc.h
***************
*** 51,56 **** extern void *MemoryContextAlloc(MemoryContext context, Size
size);
--- 51,60 ----
extern void *MemoryContextAllocZero(MemoryContext context, Size size);
extern void *MemoryContextAllocZeroAligned(MemoryContext context, Size size);
+ /* Higher-limit allocators. */
+ extern void *MemoryContextAllocHuge(MemoryContext context, Size size);
+ extern void *repalloc_huge(void *pointer, Size size);
+
/*
* The result of palloc() is always word-aligned, so we can skip testing
* alignment of the pointer when deciding which MemSet variant to use.
*** a/src/include/utils/tuplesort.h
--- b/src/include/utils/tuplesort.h
***************
*** 106,112 **** extern void tuplesort_get_stats(Tuplesortstate *state,
const char **spaceType,
long *spaceUsed);
! extern int tuplesort_merge_order(long allowedMem);
/*
* These routines may only be called if randomAccess was specified 'true'.
--- 106,112 ----
const char **spaceType,
long *spaceUsed);
! extern int tuplesort_merge_order(Size allowedMem);
/*
* These routines may only be called if randomAccess was specified 'true'.
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
