Hello!
If I create a big bytea value and try to select it from a table, I get
an error, something like: "ERROR: invalid memory alloc request size
...".
So basically we can insert data into a table but then we can't even work
with it. Sounds like a bug. Attaching a patch that fixes it (applies to
2a41507dab0f293ff241fe8ae326065998668af8).
And as it seems like quite a serious issue, would it be possible to
backport a fix for it to earlier versions?
HOW TO RECREATE:
1) generate some random data (in this case, 600 MB):
dd if=/dev/urandom of=rand.dat bs=1M count=600
2) postgres=# select lo_import('/PATH/TO/rand.dat');
lo_import
-----------
16397 [USE THIS ID FOR THE NEXT STEP]
(1 row)
3) postgres=# create table big_data as select (string_agg(data,'')) as
data from pg_largeobject where loid =16397;
SELECT 1
4) postgres=# select * from big_data;
ERROR: invalid memory alloc request size 1468006403
--
Anna Akenteva
Postgres Professional: http://www.postgrespro.com
The Russian Postgres Company
diff --git a/src/backend/access/transam/parallel.c b/src/backend/access/transam/parallel.c
index 9d4efc0..85296e9 100644
--- a/src/backend/access/transam/parallel.c
+++ b/src/backend/access/transam/parallel.c
@@ -1143,7 +1143,7 @@ HandleParallelMessage(ParallelContext *pcxt, int i, StringInfo msg)
default:
{
- elog(ERROR, "unrecognized message type received from parallel worker: %c (message length %d bytes)",
+ elog(ERROR, "unrecognized message type received from parallel worker: %c (message length %zu bytes)",
msgtype, msg->len);
}
}
diff --git a/src/backend/lib/stringinfo.c b/src/backend/lib/stringinfo.c
index 798a823..d99b8bb 100644
--- a/src/backend/lib/stringinfo.c
+++ b/src/backend/lib/stringinfo.c
@@ -45,7 +45,7 @@ makeStringInfo(void)
void
initStringInfo(StringInfo str)
{
- int size = 1024; /* initial default buffer size */
+ size_t size = 1024; /* initial default buffer size */
str->data = (char *) palloc(size);
str->maxlen = size;
@@ -80,7 +80,7 @@ appendStringInfo(StringInfo str, const char *fmt,...)
for (;;)
{
va_list args;
- int needed;
+ size_t needed;
/* Try to format the data. */
va_start(args, fmt);
@@ -110,10 +110,10 @@ appendStringInfo(StringInfo str, const char *fmt,...)
* to redo va_start before you can rescan the argument list, and we can't do
* that from here.
*/
-int
+size_t
appendStringInfoVA(StringInfo str, const char *fmt, va_list args)
{
- int avail;
+ size_t avail;
size_t nprinted;
Assert(str != NULL);
@@ -127,24 +127,20 @@ appendStringInfoVA(StringInfo str, const char *fmt, va_list args)
if (avail < 16)
return 32;
- nprinted = pvsnprintf(str->data + str->len, (size_t) avail, fmt, args);
+ nprinted = pvsnprintf(str->data + str->len, avail, fmt, args);
- if (nprinted < (size_t) avail)
+ if (nprinted < avail)
{
/* Success. Note nprinted does not include trailing null. */
- str->len += (int) nprinted;
+ str->len += nprinted;
return 0;
}
/* Restore the trailing null so that str is unmodified. */
str->data[str->len] = '\0';
- /*
- * Return pvsnprintf's estimate of the space needed. (Although this is
- * given as a size_t, we know it will fit in int because it's not more
- * than MaxAllocSize.)
- */
- return (int) nprinted;
+ /* Return pvsnprintf's estimate of the space needed. */
+ return nprinted;
}
/*
@@ -189,7 +185,7 @@ appendStringInfoSpaces(StringInfo str, int count)
if (count > 0)
{
/* Make more room if needed */
- enlargeStringInfo(str, count);
+ enlargeStringInfo(str, (size_t) count);
/* OK, append the spaces */
while (--count >= 0)
@@ -205,7 +201,7 @@ appendStringInfoSpaces(StringInfo str, int count)
* if necessary. Ensures that a trailing null byte is present.
*/
void
-appendBinaryStringInfo(StringInfo str, const char *data, int datalen)
+appendBinaryStringInfo(StringInfo str, const char *data, size_t datalen)
{
Assert(str != NULL);
@@ -231,7 +227,7 @@ appendBinaryStringInfo(StringInfo str, const char *data, int datalen)
* if necessary. Does not ensure a trailing null-byte exists.
*/
void
-appendBinaryStringInfoNT(StringInfo str, const char *data, int datalen)
+appendBinaryStringInfoNT(StringInfo str, const char *data, size_t datalen)
{
Assert(str != NULL);
@@ -261,21 +257,19 @@ appendBinaryStringInfoNT(StringInfo str, const char *data, int datalen)
* This is the desired and indeed critical behavior!
*/
void
-enlargeStringInfo(StringInfo str, int needed)
+enlargeStringInfo(StringInfo str, size_t needed)
{
- int newlen;
+ size_t newlen;
/*
* Guard against out-of-range "needed" values. Without this, we can get
* an overflow or infinite loop in the following.
*/
- if (needed < 0) /* should not happen */
- elog(ERROR, "invalid string enlargement request size: %d", needed);
- if (((Size) needed) >= (MaxAllocSize - (Size) str->len))
+ if (needed >= (MaxAllocHugeSize - str->len))
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("out of memory"),
- errdetail("Cannot enlarge string buffer containing %d bytes by %d more bytes.",
+ errdetail("Cannot enlarge string buffer containing %zu bytes by %zu more bytes.",
str->len, needed)));
needed += str->len + 1; /* total space required now */
@@ -295,14 +289,14 @@ enlargeStringInfo(StringInfo str, int needed)
newlen = 2 * newlen;
/*
- * Clamp to MaxAllocSize in case we went past it. Note we are assuming
- * here that MaxAllocSize <= INT_MAX/2, else the above loop could
+ * Clamp to MaxAllocHugeSize in case we went past it. Note we know
+ * here that MaxAllocHugeSize <= SIZE_MAX/2, else the above loop could
* overflow. We will still have newlen >= needed.
*/
- if (newlen > (int) MaxAllocSize)
- newlen = (int) MaxAllocSize;
+ if (newlen > MaxAllocHugeSize)
+ newlen = MaxAllocHugeSize;
- str->data = (char *) repalloc(str->data, newlen);
+ str->data = (char *) repalloc_huge(str->data, newlen);
str->maxlen = newlen;
}
diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c
index 304cb26..fb55c94 100644
--- a/src/backend/utils/adt/varlena.c
+++ b/src/backend/utils/adt/varlena.c
@@ -358,7 +358,7 @@ byteaout(PG_FUNCTION_ARGS)
if (bytea_output == BYTEA_OUTPUT_HEX)
{
/* Print hex format */
- rp = result = palloc(VARSIZE_ANY_EXHDR(vlena) * 2 + 2 + 1);
+ rp = result = palloc_extended(VARSIZE_ANY_EXHDR(vlena) * 2 + 2 + 1, MCXT_ALLOC_HUGE);
*rp++ = '\\';
*rp++ = 'x';
rp += hex_encode(VARDATA_ANY(vlena), VARSIZE_ANY_EXHDR(vlena), rp);
@@ -367,7 +367,7 @@ byteaout(PG_FUNCTION_ARGS)
{
/* Print traditional escaped format */
char *vp;
- int len;
+ size_t len;
int i;
len = 1; /* empty string has 1 char */
@@ -381,7 +381,7 @@ byteaout(PG_FUNCTION_ARGS)
else
len++;
}
- rp = result = (char *) palloc(len);
+ rp = result = (char *) palloc_extended(len, MCXT_ALLOC_HUGE);
vp = VARDATA_ANY(vlena);
for (i = VARSIZE_ANY_EXHDR(vlena); i != 0; i--, vp++)
{
diff --git a/src/include/lib/stringinfo.h b/src/include/lib/stringinfo.h
index 8551237..7acea28 100644
--- a/src/include/lib/stringinfo.h
+++ b/src/include/lib/stringinfo.h
@@ -35,9 +35,9 @@
typedef struct StringInfoData
{
char *data;
- int len;
- int maxlen;
- int cursor;
+ size_t len;
+ size_t maxlen;
+ size_t cursor;
} StringInfoData;
typedef StringInfoData *StringInfo;
@@ -103,7 +103,7 @@ extern void appendStringInfo(StringInfo str, const char *fmt,...) pg_attribute_p
* pass the return value to enlargeStringInfo() before trying again; see
* appendStringInfo for standard usage pattern.
*/
-extern int appendStringInfoVA(StringInfo str, const char *fmt, va_list args) pg_attribute_printf(2, 0);
+extern size_t appendStringInfoVA(StringInfo str, const char *fmt, va_list args) pg_attribute_printf(2, 0);
/*------------------------
* appendStringInfoString
@@ -141,7 +141,7 @@ extern void appendStringInfoSpaces(StringInfo str, int count);
* if necessary.
*/
extern void appendBinaryStringInfo(StringInfo str,
- const char *data, int datalen);
+ const char *data, size_t datalen);
/*------------------------
* appendBinaryStringInfoNT
@@ -149,12 +149,12 @@ extern void appendBinaryStringInfo(StringInfo str,
* if necessary. Does not ensure a trailing null-byte exists.
*/
extern void appendBinaryStringInfoNT(StringInfo str,
- const char *data, int datalen);
+ const char *data, size_t datalen);
/*------------------------
* enlargeStringInfo
* Make sure a StringInfo's buffer can hold at least 'needed' more bytes.
*/
-extern void enlargeStringInfo(StringInfo str, int needed);
+extern void enlargeStringInfo(StringInfo str, size_t needed);
#endif /* STRINGINFO_H */
