I'd like to propose adding a new GUC to limit the amount of memory a backend
can allocate for its own use. The problem this addresses is that sometimes
one needs to set work_mem fairly high to get good query plans for large joins.
However, some complex queries will then use huge amounts of memory so that
one or a few of them will consume all the memory on the host and run it deep
into swap or trigger the oom killer or worse.
I've attached a patch based on 8.4.1. It works by keeping a track of the
total memory allocated via malloc to AllocBlocks (aset.c). If this is not
shot down/up too badly I will rebase it on CVS and submit it for the next
commit fest.
I would also like to propose a similar limit on temp space use. It is quite
easy for an unintended cartesion product to use hundreds of gigabytes of
scratch space and cause other processes to fail due to lack of disk space.
If this is not objectionable, I'll work on it too.
-dg
--
David Gould da...@sonic.net 510 536 1443 510 282 0869
If simplicity worked, the world would be overrun with insects.
*** ./src/include/utils/memutils.h.orig 2009-09-30 01:54:36.000000000 -0700
--- ./src/include/utils/memutils.h 2009-09-30 03:33:44.000000000 -0700
***************
*** 114,119 ****
--- 114,122 ----
*/
/* aset.c */
+
+ extern int max_allocated_mem;
+
extern MemoryContext AllocSetContextCreate(MemoryContext parent,
const char *name,
Size minContextSize,
*** ./src/backend/utils/mmgr/aset.c.orig 2009-09-29 16:14:23.000000000
-0700
--- ./src/backend/utils/mmgr/aset.c 2009-10-01 03:07:34.000000000 -0700
***************
*** 168,173 ****
--- 168,187 ----
} AllocBlockData;
/*
+ * AllocBlock accounting maintains total allocated memory to enforce the
memory use limit.
+ */
+ int max_allocated_mem = 0;
+ Size AllocBlockAccountingMemUsed = 0;
+
+ #define AllocBlockAccountingFree(block) \
+ (AllocBlockAccountingMemUsed -= block->endptr - (char
*) (block))
+ #define AllocBlockAccountingAlloc(block) \
+ (AllocBlockAccountingMemUsed += block->endptr - (char
*) (block))
+ #define AllocBlockAccountingOverLimit() \
+ (max_allocated_mem != 0 \
+ && AllocBlockAccountingMemUsed / 1024 >
max_allocated_mem)
+
+ /*
* AllocChunk
* The prefix of each piece of memory in an AllocBlock
*
***************
*** 393,398 ****
--- 407,423 ----
context->blocks = block;
/* Mark block as not to be released at reset time */
context->keeper = block;
+
+ AllocBlockAccountingAlloc(block);
+ if (AllocBlockAccountingOverLimit())
+ {
+ MemoryContextStats(TopMemoryContext);
+ ereport(ERROR,
+ (errcode(ERRCODE_OUT_OF_MEMORY),
+ errmsg("memory limit exceeded"),
+ errdetail("Failed while creating
memory context \"%s\".",
+ name)));
+ }
}
context->isReset = true;
***************
*** 476,481 ****
--- 501,507 ----
else
{
/* Normal case, release the block */
+ AllocBlockAccountingFree(block);
#ifdef CLOBBER_FREED_MEMORY
/* Wipe freed memory for debugging purposes */
memset(block, 0x7F, block->freeptr - ((char *) block));
***************
*** 521,526 ****
--- 547,553 ----
{
AllocBlock next = block->next;
+ AllocBlockAccountingFree(block);
#ifdef CLOBBER_FREED_MEMORY
/* Wipe freed memory for debugging purposes */
memset(block, 0x7F, block->freeptr - ((char *) block));
***************
*** 597,602 ****
--- 624,640 ----
set->blocks = block;
}
+ AllocBlockAccountingAlloc(block);
+ if (AllocBlockAccountingOverLimit())
+ {
+ MemoryContextStats(TopMemoryContext);
+ ereport(ERROR,
+ (errcode(ERRCODE_OUT_OF_MEMORY),
+ errmsg("memory limit exceeded"),
+ errdetail("Failed on request of size
%lu.",
+ (unsigned long)
size)));
+ }
+
set->isReset = false;
AllocAllocInfo(set, chunk);
***************
*** 767,772 ****
--- 805,821 ----
block->next = set->blocks;
set->blocks = block;
+
+ AllocBlockAccountingAlloc(block);
+ if (AllocBlockAccountingOverLimit())
+ {
+ MemoryContextStats(TopMemoryContext);
+ ereport(ERROR,
+ (errcode(ERRCODE_OUT_OF_MEMORY),
+ errmsg("memory limit exceeded"),
+ errdetail("Failed on request of size
%lu.",
+ (unsigned long)
size)));
+ }
}
/*
***************
*** 843,848 ****
--- 892,898 ----
set->blocks = block->next;
else
prevblock->next = block->next;
+ AllocBlockAccountingFree(block);
#ifdef CLOBBER_FREED_MEMORY
/* Wipe freed memory for debugging purposes */
memset(block, 0x7F, block->freeptr - ((char *) block));
*** ./src/backend/utils/misc/guc.c.orig 2009-09-30 01:39:18.000000000 -0700
--- ./src/backend/utils/misc/guc.c 2009-10-01 03:03:13.000000000 -0700
***************
*** 167,172 ****
--- 167,173 ----
static bool assign_maxconnections(int newval, bool doit, GucSource source);
static bool assign_autovacuum_max_workers(int newval, bool doit, GucSource
source);
static bool assign_effective_io_concurrency(int newval, bool doit, GucSource
source);
+ static bool assign_max_allocated_mem(int newval, bool doit, GucSource source);
static const char *assign_pgstat_temp_directory(const char *newval, bool
doit, GucSource source);
static char *config_enum_get_options(struct config_enum * record,
***************
*** 1415,1420 ****
--- 1416,1431 ----
},
{
+ {"max_allocated_mem", PGC_USERSET, RESOURCES_MEM,
+ gettext_noop("Sets the maximum memory that can be
allocated by a session."),
+ gettext_noop("Exceeding this limit will cause the
current operation to fail."),
+ GUC_UNIT_KB
+ },
+ &max_allocated_mem,
+ 0, 0, MAX_KILOBYTES, assign_max_allocated_mem, NULL
+ },
+
+ {
{"max_stack_depth", PGC_SUSET, RESOURCES_MEM,
gettext_noop("Sets the maximum stack depth, in
kilobytes."),
NULL,
***************
*** 7673,7676 ****
--- 7684,7696 ----
return newval;
}
+ static bool
+ assign_max_allocated_mem(int newval, bool doit, GucSource source)
+ {
+ /* minimum enforceable limit is 16MB */
+ if (newval != 0 && newval < 16384)
+ return false;
+ return true;
+ }
+
#include "guc-file.c"
*** ./doc/src/sgml/config.sgml.orig 2009-10-01 02:15:56.000000000 -0700
--- ./doc/src/sgml/config.sgml 2009-10-01 03:02:19.000000000 -0700
***************
*** 859,864 ****
--- 859,887 ----
</listitem>
</varlistentry>
+ <varlistentry id="guc-max-allocated-mem" xreflabel="max_allocated_mem">
+ <term><varname>max_allocated_mem</varname> (<type>integer</type>)</term>
+ <indexterm>
+ <primary><varname>max_allocated_mem</> configuration
parameter</primary>
+ </indexterm>
+ <listitem>
+ <para>
+ Limits the maximum amount of memory that an individual backend process
+ can allocate for its own use. This prevents a single session running
a large
+ or badly behaved query from consuming excessive system memory. The
default
+ setting is zero (<literal>0</>) which disables this limit. This is
+ appropriate for most situations.
+ </para>
+ The minimum non-zero setting is 16 megabytes (<literal>16MB</>). It
should
+ always be set larger than <varname>maintenance_work_mem</varname>, and
+ usually several times larger than <varname>work_mem</varname>. Useful
+ settings will depend on the nature of the workload. A starting point
might
+ be one fourth the total memory on the host machine. Setting this too
small
+ may prevent large or complex queries from running.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry id="guc-max-stack-depth" xreflabel="max_stack_depth">
<term><varname>max_stack_depth</varname> (<type>integer</type>)</term>
<indexterm>
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
