Consider this:
create table arrays as select array[random(), random(), random(),
random(), random(), random()] as a from generate_series(1, 1000000);
create or replace function plpython_outputfunc() returns void as $$
c = plpy.cursor('select a from arrays')
for row in c:
pass
$$ language plpythonu;
When running the function, every datum will get transformed into a
Python dict, which includes calling the type's output function,
resulting in a memory allocation. The memory is allocated in the SPI
context, so it accumulates until the function is finished.
This is annoying for functions that plough through large tables, doing
some calculation. Attached is a patch that does the conversion of
PostgreSQL Datums into Python dict objects in a scratch memory context
that gets reset every time.
Cheers,
Jan
diff --git a/src/pl/plpython/plpy_main.c b/src/pl/plpython/plpy_main.c
index ae9d87e..79d7784 100644
*** a/src/pl/plpython/plpy_main.c
--- b/src/pl/plpython/plpy_main.c
***************
*** 12,17 ****
--- 12,18 ----
#include "executor/spi.h"
#include "miscadmin.h"
#include "utils/guc.h"
+ #include "utils/memutils.h"
#include "utils/syscache.h"
#include "plpython.h"
*************** plpython_inline_handler(PG_FUNCTION_ARGS
*** 268,273 ****
--- 269,279 ----
MemSet(&proc, 0, sizeof(PLyProcedure));
proc.pyname = PLy_strdup("__plpython_inline_block");
+ proc.tmp_ctx = AllocSetContextCreate(TopMemoryContext,
+ "PL/Python temporary ctx",
+ ALLOCSET_DEFAULT_MINSIZE,
+ ALLOCSET_DEFAULT_INITSIZE,
+ ALLOCSET_DEFAULT_MAXSIZE);
proc.result.out.d.typoid = VOIDOID;
PG_TRY();
diff --git a/src/pl/plpython/plpy_procedure.c b/src/pl/plpython/plpy_procedure.c
index 229966a..f539cec 100644
*** a/src/pl/plpython/plpy_procedure.c
--- b/src/pl/plpython/plpy_procedure.c
***************
*** 12,17 ****
--- 12,18 ----
#include "catalog/pg_type.h"
#include "utils/builtins.h"
#include "utils/hsearch.h"
+ #include "utils/memutils.h"
#include "utils/syscache.h"
#include "plpython.h"
*************** PLy_procedure_create(HeapTuple procTup,
*** 169,174 ****
--- 170,180 ----
proc->setof = NULL;
proc->src = NULL;
proc->argnames = NULL;
+ proc->tmp_ctx = AllocSetContextCreate(TopMemoryContext,
+ "PL/Python temporary ctx",
+ ALLOCSET_DEFAULT_MINSIZE,
+ ALLOCSET_DEFAULT_INITSIZE,
+ ALLOCSET_DEFAULT_MAXSIZE);
PG_TRY();
{
*************** PLy_procedure_delete(PLyProcedure *proc)
*** 411,416 ****
--- 417,424 ----
PLy_free(proc->src);
if (proc->argnames)
PLy_free(proc->argnames);
+ if (proc->tmp_ctx)
+ MemoryContextDelete(proc->tmp_ctx);
}
/*
diff --git a/src/pl/plpython/plpy_procedure.h b/src/pl/plpython/plpy_procedure.h
index e986c7e..5ee73fa 100644
*** a/src/pl/plpython/plpy_procedure.h
--- b/src/pl/plpython/plpy_procedure.h
*************** typedef struct PLyProcedure
*** 30,35 ****
--- 30,36 ----
PyObject *code; /* compiled procedure code */
PyObject *statics; /* data saved across calls, local scope */
PyObject *globals; /* data saved across calls, global scope */
+ MemoryContext tmp_ctx;
} PLyProcedure;
/* the procedure cache entry */
diff --git a/src/pl/plpython/plpy_typeio.c b/src/pl/plpython/plpy_typeio.c
index d5cac9f..6f0eb46 100644
*** a/src/pl/plpython/plpy_typeio.c
--- b/src/pl/plpython/plpy_typeio.c
***************
*** 23,28 ****
--- 23,29 ----
#include "plpy_typeio.h"
#include "plpy_elog.h"
+ #include "plpy_procedure.h"
/* I/O function caching */
*************** PLy_output_record_funcs(PLyTypeInfo *arg
*** 258,268 ****
Assert(arg->is_rowtype == 1);
}
PyObject *
PLyDict_FromTuple(PLyTypeInfo *info, HeapTuple tuple, TupleDesc desc)
{
! PyObject *volatile dict;
! int i;
if (info->is_rowtype != 1)
elog(ERROR, "PLyTypeInfo structure describes a datum");
--- 259,274 ----
Assert(arg->is_rowtype == 1);
}
+ /*
+ * Transform a tuple into a Python dict object. The transformation can result
+ * in memory allocation, so it's done in a scratch memory context.
+ */
PyObject *
PLyDict_FromTuple(PLyTypeInfo *info, HeapTuple tuple, TupleDesc desc)
{
! PyObject *volatile dict;
! MemoryContext oldcontext;
! int i;
if (info->is_rowtype != 1)
elog(ERROR, "PLyTypeInfo structure describes a datum");
*************** PLyDict_FromTuple(PLyTypeInfo *info, Hea
*** 271,276 ****
--- 277,284 ----
if (dict == NULL)
PLy_elog(ERROR, "could not create new dictionary");
+ oldcontext = MemoryContextSwitchTo(PLy_curr_procedure->tmp_ctx);
+
PG_TRY();
{
for (i = 0; i < info->in.r.natts; i++)
*************** PLyDict_FromTuple(PLyTypeInfo *info, Hea
*** 298,308 ****
--- 306,322 ----
}
PG_CATCH();
{
+ MemoryContextSwitchTo(oldcontext);
+ MemoryContextReset(PLy_curr_procedure->tmp_ctx);
+
Py_DECREF(dict);
PG_RE_THROW();
}
PG_END_TRY();
+ MemoryContextSwitchTo(oldcontext);
+ MemoryContextReset(PLy_curr_procedure->tmp_ctx);
+
return dict;
}
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
