From e2adb3b37ed28ec42bf9d0af7fb9283a8cd162d7 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Tue, 17 May 2022 13:09:23 +0900 Subject: [PATCH] Add zstd support for toast compression --- src/include/access/toast_compression.h | 10 +- src/include/access/toast_internals.h | 3 +- src/include/postgres.h | 3 +- src/backend/access/common/detoast.c | 12 +- src/backend/access/common/toast_compression.c | 163 +++++++++++++++++- src/backend/access/common/toast_internals.c | 4 + src/backend/utils/adt/varlena.c | 3 + src/backend/utils/misc/guc.c | 3 + src/backend/utils/misc/postgresql.conf.sample | 2 +- src/bin/psql/describe.c | 5 +- src/test/regress/expected/compression.out | 9 - src/test/regress/expected/compression_1.out | 11 -- .../regress/expected/compression_zstd.out | 79 +++++++++ .../regress/expected/compression_zstd_1.out | 60 +++++++ src/test/regress/parallel_schedule | 2 +- src/test/regress/sql/compression.sql | 6 - src/test/regress/sql/compression_zstd.sql | 35 ++++ doc/src/sgml/config.sgml | 6 +- doc/src/sgml/ref/alter_table.sgml | 8 +- doc/src/sgml/ref/create_table.sgml | 9 +- 20 files changed, 380 insertions(+), 53 deletions(-) create mode 100644 src/test/regress/expected/compression_zstd.out create mode 100644 src/test/regress/expected/compression_zstd_1.out create mode 100644 src/test/regress/sql/compression_zstd.sql diff --git a/src/include/access/toast_compression.h b/src/include/access/toast_compression.h index deb8f99da5..cc90ba235d 100644 --- a/src/include/access/toast_compression.h +++ b/src/include/access/toast_compression.h @@ -38,7 +38,8 @@ typedef enum ToastCompressionId { TOAST_PGLZ_COMPRESSION_ID = 0, TOAST_LZ4_COMPRESSION_ID = 1, - TOAST_INVALID_COMPRESSION_ID = 2 + TOAST_ZSTD_COMPRESSION_ID = 2, + TOAST_INVALID_COMPRESSION_ID = 3 } ToastCompressionId; /* @@ -48,6 +49,7 @@ typedef enum ToastCompressionId */ #define TOAST_PGLZ_COMPRESSION 'p' #define TOAST_LZ4_COMPRESSION 'l' +#define TOAST_ZSTD_COMPRESSION 'z' #define InvalidCompressionMethod '\0' #define CompressionMethodIsValid(cm) ((cm) != InvalidCompressionMethod) @@ -65,6 +67,12 @@ extern struct varlena *lz4_decompress_datum(const struct varlena *value); extern struct varlena *lz4_decompress_datum_slice(const struct varlena *value, int32 slicelength); +/* Zstandard compression/decompression routines */ +extern struct varlena *zstd_compress_datum(const struct varlena *value); +extern struct varlena *zstd_decompress_datum(const struct varlena *value); +extern struct varlena *zstd_decompress_datum_slice(const struct varlena *value, + int32 slicelength); + /* other stuff */ extern ToastCompressionId toast_get_compression_id(struct varlena *attr); extern char CompressionNameToMethod(const char *compression); diff --git a/src/include/access/toast_internals.h b/src/include/access/toast_internals.h index 85e7dc0fc5..811dd16fb4 100644 --- a/src/include/access/toast_internals.h +++ b/src/include/access/toast_internals.h @@ -40,7 +40,8 @@ typedef struct toast_compress_header do { \ Assert((len) > 0 && (len) <= VARLENA_EXTSIZE_MASK); \ Assert((cm_method) == TOAST_PGLZ_COMPRESSION_ID || \ - (cm_method) == TOAST_LZ4_COMPRESSION_ID); \ + (cm_method) == TOAST_LZ4_COMPRESSION_ID || \ + (cm_method) == TOAST_ZSTD_COMPRESSION_ID); \ ((toast_compress_header *) (ptr))->tcinfo = \ (len) | ((uint32) (cm_method) << VARLENA_EXTSIZE_BITS); \ } while (0) diff --git a/src/include/postgres.h b/src/include/postgres.h index 31358110dc..d172bda341 100644 --- a/src/include/postgres.h +++ b/src/include/postgres.h @@ -376,7 +376,8 @@ typedef struct #define VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, len, cm) \ do { \ Assert((cm) == TOAST_PGLZ_COMPRESSION_ID || \ - (cm) == TOAST_LZ4_COMPRESSION_ID); \ + (cm) == TOAST_LZ4_COMPRESSION_ID || \ + (cm) == TOAST_ZSTD_COMPRESSION_ID); \ ((toast_pointer).va_extinfo = \ (len) | ((uint32) (cm) << VARLENA_EXTSIZE_BITS)); \ } while (0) diff --git a/src/backend/access/common/detoast.c b/src/backend/access/common/detoast.c index 92c9c658d3..086d0da707 100644 --- a/src/backend/access/common/detoast.c +++ b/src/backend/access/common/detoast.c @@ -246,10 +246,10 @@ detoast_attr_slice(struct varlena *attr, * Determine maximum amount of compressed data needed for a prefix * of a given length (after decompression). * - * At least for now, if it's LZ4 data, we'll have to fetch the - * whole thing, because there doesn't seem to be an API call to - * determine how much compressed data we need to be sure of being - * able to decompress the required slice. + * At least for now, if it's LZ4 or Zstandard data, we'll have to + * fetch the whole thing, because there doesn't seem to be an API + * call to determine how much compressed data we need to be sure + * of being able to decompress the required slice. */ if (VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) == TOAST_PGLZ_COMPRESSION_ID) @@ -485,6 +485,8 @@ toast_decompress_datum(struct varlena *attr) return pglz_decompress_datum(attr); case TOAST_LZ4_COMPRESSION_ID: return lz4_decompress_datum(attr); + case TOAST_ZSTD_COMPRESSION_ID: + return zstd_decompress_datum(attr); default: elog(ERROR, "invalid compression method id %d", cmid); return NULL; /* keep compiler quiet */ @@ -528,6 +530,8 @@ toast_decompress_datum_slice(struct varlena *attr, int32 slicelength) return pglz_decompress_datum_slice(attr, slicelength); case TOAST_LZ4_COMPRESSION_ID: return lz4_decompress_datum_slice(attr, slicelength); + case TOAST_ZSTD_COMPRESSION_ID: + return zstd_decompress_datum_slice(attr, slicelength); default: elog(ERROR, "invalid compression method id %d", cmid); return NULL; /* keep compiler quiet */ diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c index f90f9f11e3..70b1e5240f 100644 --- a/src/backend/access/common/toast_compression.c +++ b/src/backend/access/common/toast_compression.c @@ -16,6 +16,9 @@ #ifdef USE_LZ4 #include #endif +#ifdef USE_ZSTD +#include +#endif #include "access/detoast.h" #include "access/toast_compression.h" @@ -26,11 +29,11 @@ /* GUC */ int default_toast_compression = TOAST_PGLZ_COMPRESSION; -#define NO_LZ4_SUPPORT() \ +#define NO_METHOD_SUPPORT(method) \ ereport(ERROR, \ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), \ - errmsg("compression method lz4 not supported"), \ - errdetail("This functionality requires the server to be built with lz4 support."))) + errmsg("compression method %s not supported", method), \ + errdetail("This functionality requires the server to be built with %s support.", method))) /* * Compress a varlena using PGLZ. @@ -140,7 +143,7 @@ struct varlena * lz4_compress_datum(const struct varlena *value) { #ifndef USE_LZ4 - NO_LZ4_SUPPORT(); + NO_METHOD_SUPPORT("lz4"); return NULL; /* keep compiler quiet */ #else int32 valsize; @@ -183,7 +186,7 @@ struct varlena * lz4_decompress_datum(const struct varlena *value) { #ifndef USE_LZ4 - NO_LZ4_SUPPORT(); + NO_METHOD_SUPPORT("lz4"); return NULL; /* keep compiler quiet */ #else int32 rawsize; @@ -216,7 +219,7 @@ struct varlena * lz4_decompress_datum_slice(const struct varlena *value, int32 slicelength) { #ifndef USE_LZ4 - NO_LZ4_SUPPORT(); + NO_METHOD_SUPPORT("lz4"); return NULL; /* keep compiler quiet */ #else int32 rawsize; @@ -246,6 +249,143 @@ lz4_decompress_datum_slice(const struct varlena *value, int32 slicelength) #endif } +/* + * Compress a varlena using Zstandard. + * + * Returns the compressed varlena, or NULL if compression fails. + */ +struct varlena * +zstd_compress_datum(const struct varlena *value) +{ +#ifndef USE_ZSTD + NO_METHOD_SUPPORT("zstd"); + return NULL; /* keep compiler quiet */ +#else + int32 valsize; + int32 len; + int32 max_size; + struct varlena *tmp = NULL; + + valsize = VARSIZE_ANY_EXHDR(value); + + /* + * Figure out the maximum possible size of the ZSTD output, add the bytes + * that will be needed for varlena overhead, and allocate that amount. + */ + max_size = ZSTD_compressBound(valsize); + tmp = (struct varlena *) palloc(max_size + VARHDRSZ_COMPRESSED); + + len = ZSTD_compress((char *) tmp + VARHDRSZ_COMPRESSED, + max_size, VARDATA_ANY(value), valsize, + ZSTD_CLEVEL_DEFAULT); + if (ZSTD_isError(len)) + elog(ERROR, "zstd compression failed: %s", + ZSTD_getErrorName(len)); + + /* data is incompressible so just free the memory and return NULL */ + if (len > valsize) + { + pfree(tmp); + return NULL; + } + + SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_COMPRESSED); + + return tmp; +#endif +} + +/* + * Decompress a varlena that was compressed using Zstandard. + */ +struct varlena * +zstd_decompress_datum(const struct varlena *value) +{ +#ifndef USE_ZSTD + NO_METHOD_SUPPORT("zstd"); + return NULL; /* keep compiler quiet */ +#else + int32 rawsize; + struct varlena *result; + + /* allocate memory for the uncompressed data */ + result = (struct varlena *) palloc(VARDATA_COMPRESSED_GET_EXTSIZE(value) + VARHDRSZ); + + /* decompress the data */ + rawsize = ZSTD_decompress(VARDATA(result), + VARDATA_COMPRESSED_GET_EXTSIZE(value), + (char *) value + VARHDRSZ_COMPRESSED, + VARSIZE(value) - VARHDRSZ_COMPRESSED); + if (ZSTD_isError(rawsize)) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg_internal("compressed zstd data is corrupt: %s", + ZSTD_getErrorName(rawsize)))); + + SET_VARSIZE(result, rawsize + VARHDRSZ); + + return result; +#endif +} + +/* + * Decompress part of a varlena that was compressed using Zstandard. + * + * ZSTD_decompress() is not able to decompress a partial portion, but streams + * can do that. + */ +struct varlena * +zstd_decompress_datum_slice(const struct varlena *value, int32 slicelength) +{ +#ifndef USE_ZSTD + NO_METHOD_SUPPORT("zstd"); + return NULL; /* keep compiler quiet */ +#else + + struct varlena *result; + + ZSTD_inBuffer inBuf; + ZSTD_outBuffer outBuf; + ZSTD_DCtx *dctx = ZSTD_createDCtx(); + + if (dctx == NULL) + elog(ERROR, "could not create zstd decompression context"); + + inBuf.src = (char *) value + VARHDRSZ_COMPRESSED; + inBuf.size = VARSIZE(value) - VARHDRSZ_COMPRESSED; + inBuf.pos = 0; + + result = (struct varlena *) palloc(slicelength + VARHDRSZ); + + outBuf.dst = (char *) result + VARHDRSZ; + outBuf.size = slicelength; + outBuf.pos = 0; + + while (inBuf.pos < inBuf.size && + outBuf.pos < outBuf.size) + { + size_t ret; + + ret = ZSTD_decompressStream(dctx, &outBuf, &inBuf); + + if (ZSTD_isError(ret)) + { + ZSTD_freeDCtx(dctx); + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg_internal("compressed zstd data is corrupt: %s", + ZSTD_getErrorName(ret)))); + } + } + + Assert(outBuf.size == slicelength && outBuf.pos == slicelength); + SET_VARSIZE(result, outBuf.pos + VARHDRSZ); + ZSTD_freeDCtx(dctx); + + return result; +#endif +} + /* * Extract compression ID from a varlena. * @@ -290,10 +430,17 @@ CompressionNameToMethod(const char *compression) else if (strcmp(compression, "lz4") == 0) { #ifndef USE_LZ4 - NO_LZ4_SUPPORT(); + NO_METHOD_SUPPORT("lz4"); #endif return TOAST_LZ4_COMPRESSION; } + else if (strcmp(compression, "zstd") == 0) + { +#ifndef USE_ZSTD + NO_METHOD_SUPPORT("zstd"); +#endif + return TOAST_ZSTD_COMPRESSION; + } return InvalidCompressionMethod; } @@ -310,6 +457,8 @@ GetCompressionMethodName(char method) return "pglz"; case TOAST_LZ4_COMPRESSION: return "lz4"; + case TOAST_ZSTD_COMPRESSION: + return "zstd"; default: elog(ERROR, "invalid compression method %c", method); return NULL; /* keep compiler quiet */ diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c index 576e585a89..1242c77c57 100644 --- a/src/backend/access/common/toast_internals.c +++ b/src/backend/access/common/toast_internals.c @@ -72,6 +72,10 @@ toast_compress_datum(Datum value, char cmethod) tmp = lz4_compress_datum((const struct varlena *) value); cmid = TOAST_LZ4_COMPRESSION_ID; break; + case TOAST_ZSTD_COMPRESSION: + tmp = zstd_compress_datum((const struct varlena *) value); + cmid = TOAST_ZSTD_COMPRESSION_ID; + break; default: elog(ERROR, "invalid compression method %c", cmethod); } diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index 919138eaf3..c60b297712 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -5324,6 +5324,9 @@ pg_column_compression(PG_FUNCTION_ARGS) case TOAST_LZ4_COMPRESSION_ID: result = "lz4"; break; + case TOAST_ZSTD_COMPRESSION_ID: + result = "zstd"; + break; default: elog(ERROR, "invalid compression method id %d", cmid); } diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 8e9b71375c..8f203c89b9 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -573,6 +573,9 @@ static struct config_enum_entry default_toast_compression_options[] = { {"pglz", TOAST_PGLZ_COMPRESSION, false}, #ifdef USE_LZ4 {"lz4", TOAST_LZ4_COMPRESSION, false}, +#endif +#ifdef USE_ZSTD + {"zstd", TOAST_ZSTD_COMPRESSION, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index a5a6d14cd4..133457d185 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -684,7 +684,7 @@ #row_security = on #default_table_access_method = 'heap' #default_tablespace = '' # a tablespace name, '' uses the default -#default_toast_compression = 'pglz' # 'pglz' or 'lz4' +#default_toast_compression = 'pglz' # 'pglz', 'lz4' or 'zstd' #temp_tablespaces = '' # a list of tablespace names, '' uses # only default tablespace #check_function_bodies = on diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 1a5d924a23..9a40d43665 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -2088,8 +2088,9 @@ describeOneTableDetails(const char *schemaname, /* these strings are literal in our syntax, so not translated. */ printTableAddCell(&cont, (compression[0] == 'p' ? "pglz" : (compression[0] == 'l' ? "lz4" : - (compression[0] == '\0' ? "" : - "???"))), + (compression[0] == 'z' ? "zstd" : + (compression[0] == '\0' ? "" : + "???")))), false, false); } diff --git a/src/test/regress/expected/compression.out b/src/test/regress/expected/compression.out index 4c997e2602..4b3d8b6a89 100644 --- a/src/test/regress/expected/compression.out +++ b/src/test/regress/expected/compression.out @@ -232,15 +232,6 @@ CREATE TABLE cminh(f1 TEXT COMPRESSION lz4) INHERITS(cmdata); NOTICE: merging column "f1" with inherited definition ERROR: column "f1" has a compression method conflict DETAIL: pglz versus lz4 --- test default_toast_compression GUC -SET default_toast_compression = ''; -ERROR: invalid value for parameter "default_toast_compression": "" -HINT: Available values: pglz, lz4. -SET default_toast_compression = 'I do not exist compression'; -ERROR: invalid value for parameter "default_toast_compression": "I do not exist compression" -HINT: Available values: pglz, lz4. -SET default_toast_compression = 'lz4'; -SET default_toast_compression = 'pglz'; -- test alter compression method ALTER TABLE cmdata ALTER COLUMN f1 SET COMPRESSION lz4; INSERT INTO cmdata VALUES (repeat('123456789', 4004)); diff --git a/src/test/regress/expected/compression_1.out b/src/test/regress/expected/compression_1.out index c0a47646eb..75a168867f 100644 --- a/src/test/regress/expected/compression_1.out +++ b/src/test/regress/expected/compression_1.out @@ -223,17 +223,6 @@ CREATE TABLE cminh(f1 TEXT COMPRESSION lz4) INHERITS(cmdata); NOTICE: merging column "f1" with inherited definition ERROR: column "f1" has a compression method conflict DETAIL: pglz versus lz4 --- test default_toast_compression GUC -SET default_toast_compression = ''; -ERROR: invalid value for parameter "default_toast_compression": "" -HINT: Available values: pglz. -SET default_toast_compression = 'I do not exist compression'; -ERROR: invalid value for parameter "default_toast_compression": "I do not exist compression" -HINT: Available values: pglz. -SET default_toast_compression = 'lz4'; -ERROR: invalid value for parameter "default_toast_compression": "lz4" -HINT: Available values: pglz. -SET default_toast_compression = 'pglz'; -- test alter compression method ALTER TABLE cmdata ALTER COLUMN f1 SET COMPRESSION lz4; ERROR: compression method lz4 not supported diff --git a/src/test/regress/expected/compression_zstd.out b/src/test/regress/expected/compression_zstd.out new file mode 100644 index 0000000000..a219642b6a --- /dev/null +++ b/src/test/regress/expected/compression_zstd.out @@ -0,0 +1,79 @@ +\set HIDE_TOAST_COMPRESSION false +-- ensure we get stable results regardless of installation's default +SET default_toast_compression = 'pglz'; +-- test creating table with compression method +CREATE TABLE cmdata_zstd(f1 TEXT COMPRESSION zstd); +INSERT INTO cmdata_zstd VALUES(repeat('1234567890', 1004)); +\d+ cmdata_zstd + Table "public.cmdata_zstd" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | zstd | | + +-- verify stored compression method in the data +SELECT pg_column_compression(f1) FROM cmdata_zstd; + pg_column_compression +----------------------- + zstd +(1 row) + +-- decompress data slice +SELECT SUBSTR(f1, 2000, 50) FROM cmdata_zstd; + substr +---------------------------------------------------- + 01234567890123456789012345678901234567890123456789 +(1 row) + +-- test LIKE INCLUDING COMPRESSION +CREATE TABLE cmdata_zstd_2 (LIKE cmdata_zstd INCLUDING COMPRESSION); +\d+ cmdata_zstd_2 + Table "public.cmdata_zstd_2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | zstd | | + +DROP TABLE cmdata_zstd_2; +-- test externally stored compressed data +CREATE OR REPLACE FUNCTION large_val() RETURNS TEXT LANGUAGE SQL AS +'select array_agg(md5(g::text))::text from generate_series(1, 256) g'; +INSERT INTO cmdata_zstd SELECT large_val() || repeat('a', 4000); +SELECT pg_column_compression(f1) FROM cmdata_zstd; + pg_column_compression +----------------------- + zstd + zstd +(2 rows) + +SELECT SUBSTR(f1, 200, 5) FROM cmdata_zstd; + substr +-------- + 01234 + 8f14e +(2 rows) + +-- test compression with materialized view +CREATE MATERIALIZED VIEW compressmv_zstd(x) AS SELECT * FROM cmdata_zstd; +\d+ compressmv_zstd + Materialized view "public.compressmv_zstd" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + x | text | | | | extended | | | +View definition: + SELECT cmdata_zstd.f1 AS x + FROM cmdata_zstd; + +SELECT pg_column_compression(f1) FROM cmdata_zstd; + pg_column_compression +----------------------- + zstd + zstd +(2 rows) + +SELECT pg_column_compression(x) FROM compressmv_zstd; + pg_column_compression +----------------------- + zstd + zstd +(2 rows) + +\set HIDE_TOAST_COMPRESSION true diff --git a/src/test/regress/expected/compression_zstd_1.out b/src/test/regress/expected/compression_zstd_1.out new file mode 100644 index 0000000000..af2074a3c5 --- /dev/null +++ b/src/test/regress/expected/compression_zstd_1.out @@ -0,0 +1,60 @@ +\set HIDE_TOAST_COMPRESSION false +-- ensure we get stable results regardless of installation's default +SET default_toast_compression = 'pglz'; +-- test creating table with compression method +CREATE TABLE cmdata_zstd(f1 TEXT COMPRESSION zstd); +ERROR: compression method zstd not supported +DETAIL: This functionality requires the server to be built with zstd support. +INSERT INTO cmdata_zstd VALUES(repeat('1234567890', 1004)); +ERROR: relation "cmdata_zstd" does not exist +LINE 1: INSERT INTO cmdata_zstd VALUES(repeat('1234567890', 1004)); + ^ +\d+ cmdata_zstd +-- verify stored compression method in the data +SELECT pg_column_compression(f1) FROM cmdata_zstd; +ERROR: relation "cmdata_zstd" does not exist +LINE 1: SELECT pg_column_compression(f1) FROM cmdata_zstd; + ^ +-- decompress data slice +SELECT SUBSTR(f1, 2000, 50) FROM cmdata_zstd; +ERROR: relation "cmdata_zstd" does not exist +LINE 1: SELECT SUBSTR(f1, 2000, 50) FROM cmdata_zstd; + ^ +-- test LIKE INCLUDING COMPRESSION +CREATE TABLE cmdata_zstd_2 (LIKE cmdata_zstd INCLUDING COMPRESSION); +ERROR: relation "cmdata_zstd" does not exist +LINE 1: CREATE TABLE cmdata_zstd_2 (LIKE cmdata_zstd INCLUDING COMPR... + ^ +\d+ cmdata_zstd_2 +DROP TABLE cmdata_zstd_2; +ERROR: table "cmdata_zstd_2" does not exist +-- test externally stored compressed data +CREATE OR REPLACE FUNCTION large_val() RETURNS TEXT LANGUAGE SQL AS +'select array_agg(md5(g::text))::text from generate_series(1, 256) g'; +INSERT INTO cmdata_zstd SELECT large_val() || repeat('a', 4000); +ERROR: relation "cmdata_zstd" does not exist +LINE 1: INSERT INTO cmdata_zstd SELECT large_val() || repeat('a', 40... + ^ +SELECT pg_column_compression(f1) FROM cmdata_zstd; +ERROR: relation "cmdata_zstd" does not exist +LINE 1: SELECT pg_column_compression(f1) FROM cmdata_zstd; + ^ +SELECT SUBSTR(f1, 200, 5) FROM cmdata_zstd; +ERROR: relation "cmdata_zstd" does not exist +LINE 1: SELECT SUBSTR(f1, 200, 5) FROM cmdata_zstd; + ^ +-- test compression with materialized view +CREATE MATERIALIZED VIEW compressmv_zstd(x) AS SELECT * FROM cmdata_zstd; +ERROR: relation "cmdata_zstd" does not exist +LINE 1: ...RIALIZED VIEW compressmv_zstd(x) AS SELECT * FROM cmdata_zst... + ^ +\d+ compressmv_zstd +SELECT pg_column_compression(f1) FROM cmdata_zstd; +ERROR: relation "cmdata_zstd" does not exist +LINE 1: SELECT pg_column_compression(f1) FROM cmdata_zstd; + ^ +SELECT pg_column_compression(x) FROM compressmv_zstd; +ERROR: relation "compressmv_zstd" does not exist +LINE 1: SELECT pg_column_compression(x) FROM compressmv_zstd; + ^ +\set HIDE_TOAST_COMPRESSION true diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 103e11483d..1625af0869 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -127,7 +127,7 @@ test: plancache limit plpgsql copy2 temp domain rangefuncs prepare conversion tr # The stats test resets stats, so nothing else needing stats access can be in # this group. # ---------- -test: partition_join partition_prune reloptions hash_part indexing partition_aggregate partition_info tuplesort explain compression memoize stats +test: partition_join partition_prune reloptions hash_part indexing partition_aggregate partition_info tuplesort explain compression compression_zstd memoize stats # event_trigger cannot run concurrently with any test that runs DDL # oidjoins is read-only, though, and should run late for best coverage diff --git a/src/test/regress/sql/compression.sql b/src/test/regress/sql/compression.sql index 86332dcc51..ceccd90e7f 100644 --- a/src/test/regress/sql/compression.sql +++ b/src/test/regress/sql/compression.sql @@ -97,12 +97,6 @@ SELECT pg_column_compression(f1) FROM cmpart2; CREATE TABLE cminh() INHERITS(cmdata, cmdata1); CREATE TABLE cminh(f1 TEXT COMPRESSION lz4) INHERITS(cmdata); --- test default_toast_compression GUC -SET default_toast_compression = ''; -SET default_toast_compression = 'I do not exist compression'; -SET default_toast_compression = 'lz4'; -SET default_toast_compression = 'pglz'; - -- test alter compression method ALTER TABLE cmdata ALTER COLUMN f1 SET COMPRESSION lz4; INSERT INTO cmdata VALUES (repeat('123456789', 4004)); diff --git a/src/test/regress/sql/compression_zstd.sql b/src/test/regress/sql/compression_zstd.sql new file mode 100644 index 0000000000..b853741b40 --- /dev/null +++ b/src/test/regress/sql/compression_zstd.sql @@ -0,0 +1,35 @@ +\set HIDE_TOAST_COMPRESSION false + +-- ensure we get stable results regardless of installation's default +SET default_toast_compression = 'pglz'; + +-- test creating table with compression method +CREATE TABLE cmdata_zstd(f1 TEXT COMPRESSION zstd); +INSERT INTO cmdata_zstd VALUES(repeat('1234567890', 1004)); +\d+ cmdata_zstd + +-- verify stored compression method in the data +SELECT pg_column_compression(f1) FROM cmdata_zstd; + +-- decompress data slice +SELECT SUBSTR(f1, 2000, 50) FROM cmdata_zstd; + +-- test LIKE INCLUDING COMPRESSION +CREATE TABLE cmdata_zstd_2 (LIKE cmdata_zstd INCLUDING COMPRESSION); +\d+ cmdata_zstd_2 +DROP TABLE cmdata_zstd_2; + +-- test externally stored compressed data +CREATE OR REPLACE FUNCTION large_val() RETURNS TEXT LANGUAGE SQL AS +'select array_agg(md5(g::text))::text from generate_series(1, 256) g'; +INSERT INTO cmdata_zstd SELECT large_val() || repeat('a', 4000); +SELECT pg_column_compression(f1) FROM cmdata_zstd; +SELECT SUBSTR(f1, 200, 5) FROM cmdata_zstd; + +-- test compression with materialized view +CREATE MATERIALIZED VIEW compressmv_zstd(x) AS SELECT * FROM cmdata_zstd; +\d+ compressmv_zstd +SELECT pg_column_compression(f1) FROM cmdata_zstd; +SELECT pg_column_compression(x) FROM compressmv_zstd; + +\set HIDE_TOAST_COMPRESSION true diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 03986946a8..de80a30c16 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -8702,9 +8702,11 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; the COMPRESSION column option in CREATE TABLE or ALTER TABLE.) - The supported compression methods are pglz and + The supported compression methods are pglz, (if PostgreSQL was compiled with - ) lz4. + ) lz4 and + (if PostgreSQL was compiled with + zstd. The default is pglz. diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml index a3c62bf056..15f0ab48dd 100644 --- a/doc/src/sgml/ref/alter_table.sgml +++ b/doc/src/sgml/ref/alter_table.sgml @@ -414,9 +414,11 @@ WITH ( MODULUS numeric_literal, REM its existing compression method, rather than being recompressed with the compression method of the target column. The supported compression - methods are pglz and lz4. - (lz4 is available only if - was used when building PostgreSQL.) In + methods are pglz, lz4 (if + was used when building + PostgreSQL) and zstd (if + was used when building + PostgreSQL). In addition, compression_method can be default, which selects the default behavior of consulting the setting diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index 6c9918b0a1..7d78ac0425 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -309,10 +309,11 @@ WITH ( MODULUS numeric_literal, REM column storage modes.) Setting this property for a partitioned table has no direct effect, because such tables have no storage of their own, but the configured value will be inherited by newly-created partitions. - The supported compression methods are pglz and - lz4. (lz4 is available only if - was used when building - PostgreSQL.) In addition, + The supported compression methods are pglz, + lz4 (if was used when + building PostgreSQL) and + zstd (if was used when + building PostgreSQL). In addition, compression_method can be default to explicitly specify the default behavior, which is to consult the -- 2.36.0