Hi all, Toast compression is supported for LZ4, and thanks to the refactoring work done with compression methods assigned to an attribute, adding support for more methods is straight-forward, as long as we don't support more than 4 methods as the compression ID is stored within the first 2 bits of the raw length.
Do we have an argument against supporting zstd for this stuff? Zstandard compresses a bit more than LZ4 at the cost of some extra CPU, outclassing easily pglz, but those facts are known, and zstd has benefits over LZ4 when one is ready to pay more CPU for the extra compression. It took me a couple of hours to get that done. I have not added any tests for pg_dump or cross-checks with the default compressions as this is basically what compression.sql already does, so this patch includes a minimum to look after the compression, decompression and slice decompression. Another thing is that the errors generated by SET default_toast_compression make the output generated build-dependent, and that becomes annoying once there is more than one compression option. The attached removes those cases for simplicity, and perhaps we'd better remove from compression.sql all the LZ4-only tests. ZSTD_decompress() does not allow the use of a destination buffer lower than the full decompressed size, but similarly to base backups, streams seem to handle the case of slices fine. Thoughts? -- Michael
From e2adb3b37ed28ec42bf9d0af7fb9283a8cd162d7 Mon Sep 17 00:00:00 2001
From: Michael Paquier <mich...@paquier.xyz>
Date: Tue, 17 May 2022 13:09:23 +0900
Subject: [PATCH] Add zstd support for toast compression
---
src/include/access/toast_compression.h | 10 +-
src/include/access/toast_internals.h | 3 +-
src/include/postgres.h | 3 +-
src/backend/access/common/detoast.c | 12 +-
src/backend/access/common/toast_compression.c | 163 +++++++++++++++++-
src/backend/access/common/toast_internals.c | 4 +
src/backend/utils/adt/varlena.c | 3 +
src/backend/utils/misc/guc.c | 3 +
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/bin/psql/describe.c | 5 +-
src/test/regress/expected/compression.out | 9 -
src/test/regress/expected/compression_1.out | 11 --
.../regress/expected/compression_zstd.out | 79 +++++++++
.../regress/expected/compression_zstd_1.out | 60 +++++++
src/test/regress/parallel_schedule | 2 +-
src/test/regress/sql/compression.sql | 6 -
src/test/regress/sql/compression_zstd.sql | 35 ++++
doc/src/sgml/config.sgml | 6 +-
doc/src/sgml/ref/alter_table.sgml | 8 +-
doc/src/sgml/ref/create_table.sgml | 9 +-
20 files changed, 380 insertions(+), 53 deletions(-)
create mode 100644 src/test/regress/expected/compression_zstd.out
create mode 100644 src/test/regress/expected/compression_zstd_1.out
create mode 100644 src/test/regress/sql/compression_zstd.sql
diff --git a/src/include/access/toast_compression.h b/src/include/access/toast_compression.h
index deb8f99da5..cc90ba235d 100644
--- a/src/include/access/toast_compression.h
+++ b/src/include/access/toast_compression.h
@@ -38,7 +38,8 @@ typedef enum ToastCompressionId
{
TOAST_PGLZ_COMPRESSION_ID = 0,
TOAST_LZ4_COMPRESSION_ID = 1,
- TOAST_INVALID_COMPRESSION_ID = 2
+ TOAST_ZSTD_COMPRESSION_ID = 2,
+ TOAST_INVALID_COMPRESSION_ID = 3
} ToastCompressionId;
/*
@@ -48,6 +49,7 @@ typedef enum ToastCompressionId
*/
#define TOAST_PGLZ_COMPRESSION 'p'
#define TOAST_LZ4_COMPRESSION 'l'
+#define TOAST_ZSTD_COMPRESSION 'z'
#define InvalidCompressionMethod '\0'
#define CompressionMethodIsValid(cm) ((cm) != InvalidCompressionMethod)
@@ -65,6 +67,12 @@ extern struct varlena *lz4_decompress_datum(const struct varlena *value);
extern struct varlena *lz4_decompress_datum_slice(const struct varlena *value,
int32 slicelength);
+/* Zstandard compression/decompression routines */
+extern struct varlena *zstd_compress_datum(const struct varlena *value);
+extern struct varlena *zstd_decompress_datum(const struct varlena *value);
+extern struct varlena *zstd_decompress_datum_slice(const struct varlena *value,
+ int32 slicelength);
+
/* other stuff */
extern ToastCompressionId toast_get_compression_id(struct varlena *attr);
extern char CompressionNameToMethod(const char *compression);
diff --git a/src/include/access/toast_internals.h b/src/include/access/toast_internals.h
index 85e7dc0fc5..811dd16fb4 100644
--- a/src/include/access/toast_internals.h
+++ b/src/include/access/toast_internals.h
@@ -40,7 +40,8 @@ typedef struct toast_compress_header
do { \
Assert((len) > 0 && (len) <= VARLENA_EXTSIZE_MASK); \
Assert((cm_method) == TOAST_PGLZ_COMPRESSION_ID || \
- (cm_method) == TOAST_LZ4_COMPRESSION_ID); \
+ (cm_method) == TOAST_LZ4_COMPRESSION_ID || \
+ (cm_method) == TOAST_ZSTD_COMPRESSION_ID); \
((toast_compress_header *) (ptr))->tcinfo = \
(len) | ((uint32) (cm_method) << VARLENA_EXTSIZE_BITS); \
} while (0)
diff --git a/src/include/postgres.h b/src/include/postgres.h
index 31358110dc..d172bda341 100644
--- a/src/include/postgres.h
+++ b/src/include/postgres.h
@@ -376,7 +376,8 @@ typedef struct
#define VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, len, cm) \
do { \
Assert((cm) == TOAST_PGLZ_COMPRESSION_ID || \
- (cm) == TOAST_LZ4_COMPRESSION_ID); \
+ (cm) == TOAST_LZ4_COMPRESSION_ID || \
+ (cm) == TOAST_ZSTD_COMPRESSION_ID); \
((toast_pointer).va_extinfo = \
(len) | ((uint32) (cm) << VARLENA_EXTSIZE_BITS)); \
} while (0)
diff --git a/src/backend/access/common/detoast.c b/src/backend/access/common/detoast.c
index 92c9c658d3..086d0da707 100644
--- a/src/backend/access/common/detoast.c
+++ b/src/backend/access/common/detoast.c
@@ -246,10 +246,10 @@ detoast_attr_slice(struct varlena *attr,
* Determine maximum amount of compressed data needed for a prefix
* of a given length (after decompression).
*
- * At least for now, if it's LZ4 data, we'll have to fetch the
- * whole thing, because there doesn't seem to be an API call to
- * determine how much compressed data we need to be sure of being
- * able to decompress the required slice.
+ * At least for now, if it's LZ4 or Zstandard data, we'll have to
+ * fetch the whole thing, because there doesn't seem to be an API
+ * call to determine how much compressed data we need to be sure
+ * of being able to decompress the required slice.
*/
if (VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) ==
TOAST_PGLZ_COMPRESSION_ID)
@@ -485,6 +485,8 @@ toast_decompress_datum(struct varlena *attr)
return pglz_decompress_datum(attr);
case TOAST_LZ4_COMPRESSION_ID:
return lz4_decompress_datum(attr);
+ case TOAST_ZSTD_COMPRESSION_ID:
+ return zstd_decompress_datum(attr);
default:
elog(ERROR, "invalid compression method id %d", cmid);
return NULL; /* keep compiler quiet */
@@ -528,6 +530,8 @@ toast_decompress_datum_slice(struct varlena *attr, int32 slicelength)
return pglz_decompress_datum_slice(attr, slicelength);
case TOAST_LZ4_COMPRESSION_ID:
return lz4_decompress_datum_slice(attr, slicelength);
+ case TOAST_ZSTD_COMPRESSION_ID:
+ return zstd_decompress_datum_slice(attr, slicelength);
default:
elog(ERROR, "invalid compression method id %d", cmid);
return NULL; /* keep compiler quiet */
diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c
index f90f9f11e3..70b1e5240f 100644
--- a/src/backend/access/common/toast_compression.c
+++ b/src/backend/access/common/toast_compression.c
@@ -16,6 +16,9 @@
#ifdef USE_LZ4
#include <lz4.h>
#endif
+#ifdef USE_ZSTD
+#include <zstd.h>
+#endif
#include "access/detoast.h"
#include "access/toast_compression.h"
@@ -26,11 +29,11 @@
/* GUC */
int default_toast_compression = TOAST_PGLZ_COMPRESSION;
-#define NO_LZ4_SUPPORT() \
+#define NO_METHOD_SUPPORT(method) \
ereport(ERROR, \
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED), \
- errmsg("compression method lz4 not supported"), \
- errdetail("This functionality requires the server to be built with lz4 support.")))
+ errmsg("compression method %s not supported", method), \
+ errdetail("This functionality requires the server to be built with %s support.", method)))
/*
* Compress a varlena using PGLZ.
@@ -140,7 +143,7 @@ struct varlena *
lz4_compress_datum(const struct varlena *value)
{
#ifndef USE_LZ4
- NO_LZ4_SUPPORT();
+ NO_METHOD_SUPPORT("lz4");
return NULL; /* keep compiler quiet */
#else
int32 valsize;
@@ -183,7 +186,7 @@ struct varlena *
lz4_decompress_datum(const struct varlena *value)
{
#ifndef USE_LZ4
- NO_LZ4_SUPPORT();
+ NO_METHOD_SUPPORT("lz4");
return NULL; /* keep compiler quiet */
#else
int32 rawsize;
@@ -216,7 +219,7 @@ struct varlena *
lz4_decompress_datum_slice(const struct varlena *value, int32 slicelength)
{
#ifndef USE_LZ4
- NO_LZ4_SUPPORT();
+ NO_METHOD_SUPPORT("lz4");
return NULL; /* keep compiler quiet */
#else
int32 rawsize;
@@ -246,6 +249,143 @@ lz4_decompress_datum_slice(const struct varlena *value, int32 slicelength)
#endif
}
+/*
+ * Compress a varlena using Zstandard.
+ *
+ * Returns the compressed varlena, or NULL if compression fails.
+ */
+struct varlena *
+zstd_compress_datum(const struct varlena *value)
+{
+#ifndef USE_ZSTD
+ NO_METHOD_SUPPORT("zstd");
+ return NULL; /* keep compiler quiet */
+#else
+ int32 valsize;
+ int32 len;
+ int32 max_size;
+ struct varlena *tmp = NULL;
+
+ valsize = VARSIZE_ANY_EXHDR(value);
+
+ /*
+ * Figure out the maximum possible size of the ZSTD output, add the bytes
+ * that will be needed for varlena overhead, and allocate that amount.
+ */
+ max_size = ZSTD_compressBound(valsize);
+ tmp = (struct varlena *) palloc(max_size + VARHDRSZ_COMPRESSED);
+
+ len = ZSTD_compress((char *) tmp + VARHDRSZ_COMPRESSED,
+ max_size, VARDATA_ANY(value), valsize,
+ ZSTD_CLEVEL_DEFAULT);
+ if (ZSTD_isError(len))
+ elog(ERROR, "zstd compression failed: %s",
+ ZSTD_getErrorName(len));
+
+ /* data is incompressible so just free the memory and return NULL */
+ if (len > valsize)
+ {
+ pfree(tmp);
+ return NULL;
+ }
+
+ SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_COMPRESSED);
+
+ return tmp;
+#endif
+}
+
+/*
+ * Decompress a varlena that was compressed using Zstandard.
+ */
+struct varlena *
+zstd_decompress_datum(const struct varlena *value)
+{
+#ifndef USE_ZSTD
+ NO_METHOD_SUPPORT("zstd");
+ return NULL; /* keep compiler quiet */
+#else
+ int32 rawsize;
+ struct varlena *result;
+
+ /* allocate memory for the uncompressed data */
+ result = (struct varlena *) palloc(VARDATA_COMPRESSED_GET_EXTSIZE(value) + VARHDRSZ);
+
+ /* decompress the data */
+ rawsize = ZSTD_decompress(VARDATA(result),
+ VARDATA_COMPRESSED_GET_EXTSIZE(value),
+ (char *) value + VARHDRSZ_COMPRESSED,
+ VARSIZE(value) - VARHDRSZ_COMPRESSED);
+ if (ZSTD_isError(rawsize))
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg_internal("compressed zstd data is corrupt: %s",
+ ZSTD_getErrorName(rawsize))));
+
+ SET_VARSIZE(result, rawsize + VARHDRSZ);
+
+ return result;
+#endif
+}
+
+/*
+ * Decompress part of a varlena that was compressed using Zstandard.
+ *
+ * ZSTD_decompress() is not able to decompress a partial portion, but streams
+ * can do that.
+ */
+struct varlena *
+zstd_decompress_datum_slice(const struct varlena *value, int32 slicelength)
+{
+#ifndef USE_ZSTD
+ NO_METHOD_SUPPORT("zstd");
+ return NULL; /* keep compiler quiet */
+#else
+
+ struct varlena *result;
+
+ ZSTD_inBuffer inBuf;
+ ZSTD_outBuffer outBuf;
+ ZSTD_DCtx *dctx = ZSTD_createDCtx();
+
+ if (dctx == NULL)
+ elog(ERROR, "could not create zstd decompression context");
+
+ inBuf.src = (char *) value + VARHDRSZ_COMPRESSED;
+ inBuf.size = VARSIZE(value) - VARHDRSZ_COMPRESSED;
+ inBuf.pos = 0;
+
+ result = (struct varlena *) palloc(slicelength + VARHDRSZ);
+
+ outBuf.dst = (char *) result + VARHDRSZ;
+ outBuf.size = slicelength;
+ outBuf.pos = 0;
+
+ while (inBuf.pos < inBuf.size &&
+ outBuf.pos < outBuf.size)
+ {
+ size_t ret;
+
+ ret = ZSTD_decompressStream(dctx, &outBuf, &inBuf);
+
+ if (ZSTD_isError(ret))
+ {
+ ZSTD_freeDCtx(dctx);
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg_internal("compressed zstd data is corrupt: %s",
+ ZSTD_getErrorName(ret))));
+ }
+ }
+
+ Assert(outBuf.size == slicelength && outBuf.pos == slicelength);
+ SET_VARSIZE(result, outBuf.pos + VARHDRSZ);
+ ZSTD_freeDCtx(dctx);
+
+ return result;
+#endif
+}
+
/*
* Extract compression ID from a varlena.
*
@@ -290,10 +430,17 @@ CompressionNameToMethod(const char *compression)
else if (strcmp(compression, "lz4") == 0)
{
#ifndef USE_LZ4
- NO_LZ4_SUPPORT();
+ NO_METHOD_SUPPORT("lz4");
#endif
return TOAST_LZ4_COMPRESSION;
}
+ else if (strcmp(compression, "zstd") == 0)
+ {
+#ifndef USE_ZSTD
+ NO_METHOD_SUPPORT("zstd");
+#endif
+ return TOAST_ZSTD_COMPRESSION;
+ }
return InvalidCompressionMethod;
}
@@ -310,6 +457,8 @@ GetCompressionMethodName(char method)
return "pglz";
case TOAST_LZ4_COMPRESSION:
return "lz4";
+ case TOAST_ZSTD_COMPRESSION:
+ return "zstd";
default:
elog(ERROR, "invalid compression method %c", method);
return NULL; /* keep compiler quiet */
diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c
index 576e585a89..1242c77c57 100644
--- a/src/backend/access/common/toast_internals.c
+++ b/src/backend/access/common/toast_internals.c
@@ -72,6 +72,10 @@ toast_compress_datum(Datum value, char cmethod)
tmp = lz4_compress_datum((const struct varlena *) value);
cmid = TOAST_LZ4_COMPRESSION_ID;
break;
+ case TOAST_ZSTD_COMPRESSION:
+ tmp = zstd_compress_datum((const struct varlena *) value);
+ cmid = TOAST_ZSTD_COMPRESSION_ID;
+ break;
default:
elog(ERROR, "invalid compression method %c", cmethod);
}
diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c
index 919138eaf3..c60b297712 100644
--- a/src/backend/utils/adt/varlena.c
+++ b/src/backend/utils/adt/varlena.c
@@ -5324,6 +5324,9 @@ pg_column_compression(PG_FUNCTION_ARGS)
case TOAST_LZ4_COMPRESSION_ID:
result = "lz4";
break;
+ case TOAST_ZSTD_COMPRESSION_ID:
+ result = "zstd";
+ break;
default:
elog(ERROR, "invalid compression method id %d", cmid);
}
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 8e9b71375c..8f203c89b9 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -573,6 +573,9 @@ static struct config_enum_entry default_toast_compression_options[] = {
{"pglz", TOAST_PGLZ_COMPRESSION, false},
#ifdef USE_LZ4
{"lz4", TOAST_LZ4_COMPRESSION, false},
+#endif
+#ifdef USE_ZSTD
+ {"zstd", TOAST_ZSTD_COMPRESSION, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index a5a6d14cd4..133457d185 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -684,7 +684,7 @@
#row_security = on
#default_table_access_method = 'heap'
#default_tablespace = '' # a tablespace name, '' uses the default
-#default_toast_compression = 'pglz' # 'pglz' or 'lz4'
+#default_toast_compression = 'pglz' # 'pglz', 'lz4' or 'zstd'
#temp_tablespaces = '' # a list of tablespace names, '' uses
# only default tablespace
#check_function_bodies = on
diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c
index 1a5d924a23..9a40d43665 100644
--- a/src/bin/psql/describe.c
+++ b/src/bin/psql/describe.c
@@ -2088,8 +2088,9 @@ describeOneTableDetails(const char *schemaname,
/* these strings are literal in our syntax, so not translated. */
printTableAddCell(&cont, (compression[0] == 'p' ? "pglz" :
(compression[0] == 'l' ? "lz4" :
- (compression[0] == '\0' ? "" :
- "???"))),
+ (compression[0] == 'z' ? "zstd" :
+ (compression[0] == '\0' ? "" :
+ "???")))),
false, false);
}
diff --git a/src/test/regress/expected/compression.out b/src/test/regress/expected/compression.out
index 4c997e2602..4b3d8b6a89 100644
--- a/src/test/regress/expected/compression.out
+++ b/src/test/regress/expected/compression.out
@@ -232,15 +232,6 @@ CREATE TABLE cminh(f1 TEXT COMPRESSION lz4) INHERITS(cmdata);
NOTICE: merging column "f1" with inherited definition
ERROR: column "f1" has a compression method conflict
DETAIL: pglz versus lz4
--- test default_toast_compression GUC
-SET default_toast_compression = '';
-ERROR: invalid value for parameter "default_toast_compression": ""
-HINT: Available values: pglz, lz4.
-SET default_toast_compression = 'I do not exist compression';
-ERROR: invalid value for parameter "default_toast_compression": "I do not exist compression"
-HINT: Available values: pglz, lz4.
-SET default_toast_compression = 'lz4';
-SET default_toast_compression = 'pglz';
-- test alter compression method
ALTER TABLE cmdata ALTER COLUMN f1 SET COMPRESSION lz4;
INSERT INTO cmdata VALUES (repeat('123456789', 4004));
diff --git a/src/test/regress/expected/compression_1.out b/src/test/regress/expected/compression_1.out
index c0a47646eb..75a168867f 100644
--- a/src/test/regress/expected/compression_1.out
+++ b/src/test/regress/expected/compression_1.out
@@ -223,17 +223,6 @@ CREATE TABLE cminh(f1 TEXT COMPRESSION lz4) INHERITS(cmdata);
NOTICE: merging column "f1" with inherited definition
ERROR: column "f1" has a compression method conflict
DETAIL: pglz versus lz4
--- test default_toast_compression GUC
-SET default_toast_compression = '';
-ERROR: invalid value for parameter "default_toast_compression": ""
-HINT: Available values: pglz.
-SET default_toast_compression = 'I do not exist compression';
-ERROR: invalid value for parameter "default_toast_compression": "I do not exist compression"
-HINT: Available values: pglz.
-SET default_toast_compression = 'lz4';
-ERROR: invalid value for parameter "default_toast_compression": "lz4"
-HINT: Available values: pglz.
-SET default_toast_compression = 'pglz';
-- test alter compression method
ALTER TABLE cmdata ALTER COLUMN f1 SET COMPRESSION lz4;
ERROR: compression method lz4 not supported
diff --git a/src/test/regress/expected/compression_zstd.out b/src/test/regress/expected/compression_zstd.out
new file mode 100644
index 0000000000..a219642b6a
--- /dev/null
+++ b/src/test/regress/expected/compression_zstd.out
@@ -0,0 +1,79 @@
+\set HIDE_TOAST_COMPRESSION false
+-- ensure we get stable results regardless of installation's default
+SET default_toast_compression = 'pglz';
+-- test creating table with compression method
+CREATE TABLE cmdata_zstd(f1 TEXT COMPRESSION zstd);
+INSERT INTO cmdata_zstd VALUES(repeat('1234567890', 1004));
+\d+ cmdata_zstd
+ Table "public.cmdata_zstd"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | text | | | | extended | zstd | |
+
+-- verify stored compression method in the data
+SELECT pg_column_compression(f1) FROM cmdata_zstd;
+ pg_column_compression
+-----------------------
+ zstd
+(1 row)
+
+-- decompress data slice
+SELECT SUBSTR(f1, 2000, 50) FROM cmdata_zstd;
+ substr
+----------------------------------------------------
+ 01234567890123456789012345678901234567890123456789
+(1 row)
+
+-- test LIKE INCLUDING COMPRESSION
+CREATE TABLE cmdata_zstd_2 (LIKE cmdata_zstd INCLUDING COMPRESSION);
+\d+ cmdata_zstd_2
+ Table "public.cmdata_zstd_2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | text | | | | extended | zstd | |
+
+DROP TABLE cmdata_zstd_2;
+-- test externally stored compressed data
+CREATE OR REPLACE FUNCTION large_val() RETURNS TEXT LANGUAGE SQL AS
+'select array_agg(md5(g::text))::text from generate_series(1, 256) g';
+INSERT INTO cmdata_zstd SELECT large_val() || repeat('a', 4000);
+SELECT pg_column_compression(f1) FROM cmdata_zstd;
+ pg_column_compression
+-----------------------
+ zstd
+ zstd
+(2 rows)
+
+SELECT SUBSTR(f1, 200, 5) FROM cmdata_zstd;
+ substr
+--------
+ 01234
+ 8f14e
+(2 rows)
+
+-- test compression with materialized view
+CREATE MATERIALIZED VIEW compressmv_zstd(x) AS SELECT * FROM cmdata_zstd;
+\d+ compressmv_zstd
+ Materialized view "public.compressmv_zstd"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ x | text | | | | extended | | |
+View definition:
+ SELECT cmdata_zstd.f1 AS x
+ FROM cmdata_zstd;
+
+SELECT pg_column_compression(f1) FROM cmdata_zstd;
+ pg_column_compression
+-----------------------
+ zstd
+ zstd
+(2 rows)
+
+SELECT pg_column_compression(x) FROM compressmv_zstd;
+ pg_column_compression
+-----------------------
+ zstd
+ zstd
+(2 rows)
+
+\set HIDE_TOAST_COMPRESSION true
diff --git a/src/test/regress/expected/compression_zstd_1.out b/src/test/regress/expected/compression_zstd_1.out
new file mode 100644
index 0000000000..af2074a3c5
--- /dev/null
+++ b/src/test/regress/expected/compression_zstd_1.out
@@ -0,0 +1,60 @@
+\set HIDE_TOAST_COMPRESSION false
+-- ensure we get stable results regardless of installation's default
+SET default_toast_compression = 'pglz';
+-- test creating table with compression method
+CREATE TABLE cmdata_zstd(f1 TEXT COMPRESSION zstd);
+ERROR: compression method zstd not supported
+DETAIL: This functionality requires the server to be built with zstd support.
+INSERT INTO cmdata_zstd VALUES(repeat('1234567890', 1004));
+ERROR: relation "cmdata_zstd" does not exist
+LINE 1: INSERT INTO cmdata_zstd VALUES(repeat('1234567890', 1004));
+ ^
+\d+ cmdata_zstd
+-- verify stored compression method in the data
+SELECT pg_column_compression(f1) FROM cmdata_zstd;
+ERROR: relation "cmdata_zstd" does not exist
+LINE 1: SELECT pg_column_compression(f1) FROM cmdata_zstd;
+ ^
+-- decompress data slice
+SELECT SUBSTR(f1, 2000, 50) FROM cmdata_zstd;
+ERROR: relation "cmdata_zstd" does not exist
+LINE 1: SELECT SUBSTR(f1, 2000, 50) FROM cmdata_zstd;
+ ^
+-- test LIKE INCLUDING COMPRESSION
+CREATE TABLE cmdata_zstd_2 (LIKE cmdata_zstd INCLUDING COMPRESSION);
+ERROR: relation "cmdata_zstd" does not exist
+LINE 1: CREATE TABLE cmdata_zstd_2 (LIKE cmdata_zstd INCLUDING COMPR...
+ ^
+\d+ cmdata_zstd_2
+DROP TABLE cmdata_zstd_2;
+ERROR: table "cmdata_zstd_2" does not exist
+-- test externally stored compressed data
+CREATE OR REPLACE FUNCTION large_val() RETURNS TEXT LANGUAGE SQL AS
+'select array_agg(md5(g::text))::text from generate_series(1, 256) g';
+INSERT INTO cmdata_zstd SELECT large_val() || repeat('a', 4000);
+ERROR: relation "cmdata_zstd" does not exist
+LINE 1: INSERT INTO cmdata_zstd SELECT large_val() || repeat('a', 40...
+ ^
+SELECT pg_column_compression(f1) FROM cmdata_zstd;
+ERROR: relation "cmdata_zstd" does not exist
+LINE 1: SELECT pg_column_compression(f1) FROM cmdata_zstd;
+ ^
+SELECT SUBSTR(f1, 200, 5) FROM cmdata_zstd;
+ERROR: relation "cmdata_zstd" does not exist
+LINE 1: SELECT SUBSTR(f1, 200, 5) FROM cmdata_zstd;
+ ^
+-- test compression with materialized view
+CREATE MATERIALIZED VIEW compressmv_zstd(x) AS SELECT * FROM cmdata_zstd;
+ERROR: relation "cmdata_zstd" does not exist
+LINE 1: ...RIALIZED VIEW compressmv_zstd(x) AS SELECT * FROM cmdata_zst...
+ ^
+\d+ compressmv_zstd
+SELECT pg_column_compression(f1) FROM cmdata_zstd;
+ERROR: relation "cmdata_zstd" does not exist
+LINE 1: SELECT pg_column_compression(f1) FROM cmdata_zstd;
+ ^
+SELECT pg_column_compression(x) FROM compressmv_zstd;
+ERROR: relation "compressmv_zstd" does not exist
+LINE 1: SELECT pg_column_compression(x) FROM compressmv_zstd;
+ ^
+\set HIDE_TOAST_COMPRESSION true
diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule
index 103e11483d..1625af0869 100644
--- a/src/test/regress/parallel_schedule
+++ b/src/test/regress/parallel_schedule
@@ -127,7 +127,7 @@ test: plancache limit plpgsql copy2 temp domain rangefuncs prepare conversion tr
# The stats test resets stats, so nothing else needing stats access can be in
# this group.
# ----------
-test: partition_join partition_prune reloptions hash_part indexing partition_aggregate partition_info tuplesort explain compression memoize stats
+test: partition_join partition_prune reloptions hash_part indexing partition_aggregate partition_info tuplesort explain compression compression_zstd memoize stats
# event_trigger cannot run concurrently with any test that runs DDL
# oidjoins is read-only, though, and should run late for best coverage
diff --git a/src/test/regress/sql/compression.sql b/src/test/regress/sql/compression.sql
index 86332dcc51..ceccd90e7f 100644
--- a/src/test/regress/sql/compression.sql
+++ b/src/test/regress/sql/compression.sql
@@ -97,12 +97,6 @@ SELECT pg_column_compression(f1) FROM cmpart2;
CREATE TABLE cminh() INHERITS(cmdata, cmdata1);
CREATE TABLE cminh(f1 TEXT COMPRESSION lz4) INHERITS(cmdata);
--- test default_toast_compression GUC
-SET default_toast_compression = '';
-SET default_toast_compression = 'I do not exist compression';
-SET default_toast_compression = 'lz4';
-SET default_toast_compression = 'pglz';
-
-- test alter compression method
ALTER TABLE cmdata ALTER COLUMN f1 SET COMPRESSION lz4;
INSERT INTO cmdata VALUES (repeat('123456789', 4004));
diff --git a/src/test/regress/sql/compression_zstd.sql b/src/test/regress/sql/compression_zstd.sql
new file mode 100644
index 0000000000..b853741b40
--- /dev/null
+++ b/src/test/regress/sql/compression_zstd.sql
@@ -0,0 +1,35 @@
+\set HIDE_TOAST_COMPRESSION false
+
+-- ensure we get stable results regardless of installation's default
+SET default_toast_compression = 'pglz';
+
+-- test creating table with compression method
+CREATE TABLE cmdata_zstd(f1 TEXT COMPRESSION zstd);
+INSERT INTO cmdata_zstd VALUES(repeat('1234567890', 1004));
+\d+ cmdata_zstd
+
+-- verify stored compression method in the data
+SELECT pg_column_compression(f1) FROM cmdata_zstd;
+
+-- decompress data slice
+SELECT SUBSTR(f1, 2000, 50) FROM cmdata_zstd;
+
+-- test LIKE INCLUDING COMPRESSION
+CREATE TABLE cmdata_zstd_2 (LIKE cmdata_zstd INCLUDING COMPRESSION);
+\d+ cmdata_zstd_2
+DROP TABLE cmdata_zstd_2;
+
+-- test externally stored compressed data
+CREATE OR REPLACE FUNCTION large_val() RETURNS TEXT LANGUAGE SQL AS
+'select array_agg(md5(g::text))::text from generate_series(1, 256) g';
+INSERT INTO cmdata_zstd SELECT large_val() || repeat('a', 4000);
+SELECT pg_column_compression(f1) FROM cmdata_zstd;
+SELECT SUBSTR(f1, 200, 5) FROM cmdata_zstd;
+
+-- test compression with materialized view
+CREATE MATERIALIZED VIEW compressmv_zstd(x) AS SELECT * FROM cmdata_zstd;
+\d+ compressmv_zstd
+SELECT pg_column_compression(f1) FROM cmdata_zstd;
+SELECT pg_column_compression(x) FROM compressmv_zstd;
+
+\set HIDE_TOAST_COMPRESSION true
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 03986946a8..de80a30c16 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -8702,9 +8702,11 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv;
the <literal>COMPRESSION</literal> column option in
<command>CREATE TABLE</command> or
<command>ALTER TABLE</command>.)
- The supported compression methods are <literal>pglz</literal> and
+ The supported compression methods are <literal>pglz</literal>,
(if <productname>PostgreSQL</productname> was compiled with
- <option>--with-lz4</option>) <literal>lz4</literal>.
+ <option>--with-lz4</option>) <literal>lz4</literal> and
+ (if <productname>PostgreSQL</productname> was compiled with
+ <option>--with-zstd</option> <literal>zstd</literal>.
The default is <literal>pglz</literal>.
</para>
</listitem>
diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml
index a3c62bf056..15f0ab48dd 100644
--- a/doc/src/sgml/ref/alter_table.sgml
+++ b/doc/src/sgml/ref/alter_table.sgml
@@ -414,9 +414,11 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM
its existing compression method, rather than being recompressed with the
compression method of the target column.
The supported compression
- methods are <literal>pglz</literal> and <literal>lz4</literal>.
- (<literal>lz4</literal> is available only if <option>--with-lz4</option>
- was used when building <productname>PostgreSQL</productname>.) In
+ methods are <literal>pglz</literal>, <literal>lz4</literal> (if
+ <option>--with-lz4</option> was used when building
+ <productname>PostgreSQL</productname>) and <literal>zstd</literal> (if
+ <option>--with-zstd</option> was used when building
+ <productname>PostgreSQL</productname>). In
addition, <replaceable class="parameter">compression_method</replaceable>
can be <literal>default</literal>, which selects the default behavior of
consulting the <xref linkend="guc-default-toast-compression"/> setting
diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml
index 6c9918b0a1..7d78ac0425 100644
--- a/doc/src/sgml/ref/create_table.sgml
+++ b/doc/src/sgml/ref/create_table.sgml
@@ -309,10 +309,11 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM
column storage modes.) Setting this property for a partitioned table
has no direct effect, because such tables have no storage of their own,
but the configured value will be inherited by newly-created partitions.
- The supported compression methods are <literal>pglz</literal> and
- <literal>lz4</literal>. (<literal>lz4</literal> is available only if
- <option>--with-lz4</option> was used when building
- <productname>PostgreSQL</productname>.) In addition,
+ The supported compression methods are <literal>pglz</literal>,
+ <literal>lz4</literal> (if <option>--with-lz4</option> was used when
+ building <productname>PostgreSQL</productname>) and
+ <literal>zstd</literal> (if <option>--with-zstd</option> was used when
+ building <productname>PostgreSQL</productname>). In addition,
<replaceable class="parameter">compression_method</replaceable>
can be <literal>default</literal> to explicitly specify the default
behavior, which is to consult the
--
2.36.0
signature.asc
Description: PGP signature
