From c28b0071e5be4edf165289d1c4fb20b83c78800b Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Veldanda Date: Tue, 15 Sep 2026 23:02:35 -0700 Subject: [PATCH v2 3/3] Add zstd as a TOAST compression method Allow columns to be compressed with Zstandard, via COMPRESSION zstd in CREATE TABLE and ALTER TABLE, and via default_toast_compression = 'zstd'. It is available when the server is built with --with-zstd (or -Dzstd for meson); the library was already linked into the backend for WAL compression, so no build system changes are needed. zstd is the first method to use the long form of the compressed header and of the TOAST pointer introduced by the previous commit, and is assigned ID 2. The compressor lays out the datum with VARHDRSZ_COMPRESSED_LONG bytes of header, and the decompressors expect that. Compression uses ZSTD_compress() at ZSTD_CLEVEL_DEFAULT with no dictionary. Slice decompression uses the streaming API, since the one-shot API can only decode a whole frame; as with lz4, the whole compressed value has to be fetched from the TOAST table first, because there is no way to bound how much compressed input a given prefix needs. Teach the usual places about the new method: pg_column_compression(), the default_toast_compression GUC, amcheck, pg_dump (which would otherwise silently drop the column's compression setting), psql's \d+ and tab completion, and the documentation. The regression test follows compression_lz4 and skips itself when the server lacks zstd support. Beyond the cases covered for lz4, it checks: slices at the start, end and past the end of a value, and longer than the value; values that stay uncompressed because they are below the TOAST threshold or because zstd cannot shrink them; bytea with zero bytes, jsonb and arrays; inline and external values with both oid and oid8 TOAST value IDs, including a value spanning hundreds of TOAST chunks; the EXTERNAL, MAIN and PLAIN storage modes; copying compressed values between tables and methods in both directions; VACUUM FULL, CLUSTER and a rewriting ALTER COLUMN TYPE; that updating or deleting a row removes its old TOAST value; indexes on compressed keys; and the GUC, including SET COMPRESSION default. A pg_dump test checks that the compression setting survives a dump. --- contrib/amcheck/verify_heapam.c | 1 + doc/src/sgml/catalogs.sgml | 6 +- doc/src/sgml/config.sgml | 6 +- doc/src/sgml/installation.sgml | 1 + doc/src/sgml/ref/alter_table.sgml | 9 +- doc/src/sgml/ref/create_table.sgml | 8 +- src/backend/access/common/detoast.c | 12 +- src/backend/access/common/toast_compression.c | 179 ++++ src/backend/access/common/toast_internals.c | 7 +- src/backend/utils/adt/varlena.c | 3 + src/backend/utils/misc/guc_tables.c | 3 + src/backend/utils/misc/postgresql.conf.sample | 2 +- src/bin/pg_dump/pg_dump.c | 3 + src/bin/pg_dump/t/006_pg_dump_compress.pl | 18 + src/bin/psql/describe.c | 5 +- src/bin/psql/tab-complete.in.c | 2 +- src/include/access/toast_compression.h | 8 + src/include/access/toast_internals.h | 3 +- .../regress/expected/compression_zstd.out | 839 ++++++++++++++++++ .../regress/expected/compression_zstd_1.out | 7 + src/test/regress/parallel_schedule | 2 +- src/test/regress/sql/compression_zstd.sql | 501 +++++++++++ 22 files changed, 1604 insertions(+), 21 deletions(-) create mode 100644 src/test/regress/expected/compression_zstd.out create mode 100644 src/test/regress/expected/compression_zstd_1.out create mode 100644 src/test/regress/sql/compression_zstd.sql diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c index 7c6a2ff635b..1cf0a0a1522 100644 --- a/contrib/amcheck/verify_heapam.c +++ b/contrib/amcheck/verify_heapam.c @@ -1811,6 +1811,7 @@ check_tuple_attribute(HeapCheckContext *ctx) /* List of all valid compression method IDs */ case TOAST_PGLZ_COMPRESSION_ID: case TOAST_LZ4_COMPRESSION_ID: + case TOAST_ZSTD_COMPRESSION_ID: valid = (toast_compression_id_needs_cmid_byte(cmid) == VARTAG_IS_ONDISK_LONG(va_tag_value)); break; diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 78cce7e370d..9e44bbf71d5 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -1250,9 +1250,11 @@ The current compression method of the column. Typically this is '\0' to specify use of the current default setting (see ). Otherwise, - 'p' selects pglz compression, while + 'p' selects pglz compression, 'l' selects LZ4 - compression. However, this field is ignored + compression, and 'z' selects + Zstandard compression. + However, this field is ignored whenever attstorage does not allow compression. diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 0165eb9ec02..54cb1274af8 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -10499,9 +10499,11 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; the COMPRESSION column option in CREATE TABLE or ALTER TABLE.) - The supported compression methods are pglz and + The supported compression methods are pglz, (if PostgreSQL was compiled with - ) lz4. + ) lz4, and + (if PostgreSQL was compiled with + ) zstd. The default is lz4 (if available); otherwise, pglz. diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index e4159522e18..607faca1113 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -335,6 +335,7 @@ You need Zstandard, if you want to support compression of data with that method; see + and . The minimum required version is 1.4.0. diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml index 0f9d698d170..004bfe58812 100644 --- a/doc/src/sgml/ref/alter_table.sgml +++ b/doc/src/sgml/ref/alter_table.sgml @@ -464,9 +464,12 @@ WITH ( MODULUS numeric_literal, REM its existing compression method, rather than being recompressed with the compression method of the target column. The supported compression - methods are pglz and lz4. - (lz4 is available only if - was used when building PostgreSQL.) In + methods are pglz, lz4, and + zstd. (lz4 and + zstd are available only if + or , + respectively, was used when building + PostgreSQL.) In addition, compression_method can be default, which selects the default behavior of consulting the setting diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index fef24d8f3a2..7340c61c1b1 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -363,9 +363,11 @@ WITH ( MODULUS numeric_literal, REM column storage modes.) Setting this property for a partitioned table has no direct effect, because such tables have no storage of their own, but the configured value will be inherited by newly-created partitions. - The supported compression methods are pglz and - lz4. (lz4 is available only if - was used when building + The supported compression methods are pglz, + lz4, and zstd. + (lz4 and zstd are available only + if or , + respectively, was used when building PostgreSQL.) In addition, compression_method can be default to explicitly specify the default diff --git a/src/backend/access/common/detoast.c b/src/backend/access/common/detoast.c index 25140372238..44ecdc851d3 100644 --- a/src/backend/access/common/detoast.c +++ b/src/backend/access/common/detoast.c @@ -252,10 +252,10 @@ detoast_attr_slice(varlena *attr, * Determine maximum amount of compressed data needed for a prefix * of a given length (after decompression). * - * At least for now, if it's LZ4 data, we'll have to fetch the - * whole thing, because there doesn't seem to be an API call to - * determine how much compressed data we need to be sure of being - * able to decompress the required slice. + * At least for now, if it's LZ4 or zstd data, we'll have to fetch + * the whole thing, because there doesn't seem to be an API call + * to determine how much compressed data we need to be sure of + * being able to decompress the required slice. */ if (compress_method == TOAST_PGLZ_COMPRESSION_ID) max_size = pglz_maximum_compressed_size(slicelimit, max_size); @@ -496,6 +496,8 @@ toast_decompress_datum(varlena *attr) return pglz_decompress_datum(attr); case TOAST_LZ4_COMPRESSION_ID: return lz4_decompress_datum(attr); + case TOAST_ZSTD_COMPRESSION_ID: + return zstd_decompress_datum(attr); default: elog(ERROR, "invalid compression method id %d", cmid); return NULL; /* keep compiler quiet */ @@ -539,6 +541,8 @@ toast_decompress_datum_slice(varlena *attr, int32 slicelength) return pglz_decompress_datum_slice(attr, slicelength); case TOAST_LZ4_COMPRESSION_ID: return lz4_decompress_datum_slice(attr, slicelength); + case TOAST_ZSTD_COMPRESSION_ID: + return zstd_decompress_datum_slice(attr, slicelength); default: elog(ERROR, "invalid compression method id %d", cmid); return NULL; /* keep compiler quiet */ diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c index 9bd4404fcf7..849ec174539 100644 --- a/src/backend/access/common/toast_compression.c +++ b/src/backend/access/common/toast_compression.c @@ -17,6 +17,10 @@ #include #endif +#ifdef USE_ZSTD +#include +#endif + #include "access/detoast.h" #include "access/toast_compression.h" #include "common/pg_lzcompress.h" @@ -253,6 +257,172 @@ lz4_decompress_datum_slice(const varlena *value, int32 slicelength) #endif } +/* + * Compress a varlena using Zstandard. + * + * Returns the compressed varlena, or NULL if compression fails. + * + * Unlike pglz and lz4, zstd's method ID does not fit in the two method bits + * of the compressed header, so the datum is laid out with the long form of + * the header (VARHDRSZ_COMPRESSED_LONG), whose extra byte will receive the ID. + */ +varlena * +zstd_compress_datum(const varlena *value) +{ +#ifndef USE_ZSTD + NO_COMPRESSION_SUPPORT("zstd"); + return NULL; /* keep compiler quiet */ +#else + int32 valsize; + size_t len; + size_t max_size; + varlena *tmp = NULL; + + valsize = VARSIZE_ANY_EXHDR(value); + + /* + * Figure out the maximum possible size of the zstd output, add the bytes + * that will be needed for varlena overhead, and allocate that amount. + */ + max_size = ZSTD_compressBound(valsize); + tmp = (varlena *) palloc(max_size + VARHDRSZ_COMPRESSED_LONG); + + len = ZSTD_compress((char *) tmp + VARHDRSZ_COMPRESSED_LONG, + max_size, + VARDATA_ANY(value), + valsize, + ZSTD_CLEVEL_DEFAULT); + if (ZSTD_isError(len)) + elog(ERROR, "zstd compression failed: %s", ZSTD_getErrorName(len)); + + /* data is incompressible so just free the memory and return NULL */ + if (len > (size_t) valsize) + { + pfree(tmp); + return NULL; + } + + SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_COMPRESSED_LONG); + + return tmp; +#endif +} + +/* + * Decompress a varlena that was compressed using Zstandard. + */ +varlena * +zstd_decompress_datum(const varlena *value) +{ +#ifndef USE_ZSTD + NO_COMPRESSION_SUPPORT("zstd"); + return NULL; /* keep compiler quiet */ +#else + size_t rawsize; + varlena *result; + + Assert(VARDATA_COMPRESSED_GET_COMPRESS_METHOD(value) == TOAST_ZSTD_COMPRESSION_ID); + + /* allocate memory for the uncompressed data */ + result = (varlena *) palloc(VARDATA_COMPRESSED_GET_EXTSIZE(value) + VARHDRSZ); + + /* decompress the data */ + rawsize = ZSTD_decompress(VARDATA(result), + VARDATA_COMPRESSED_GET_EXTSIZE(value), + (const char *) value + VARHDRSZ_COMPRESSED_LONG, + VARSIZE(value) - VARHDRSZ_COMPRESSED_LONG); + if (ZSTD_isError(rawsize)) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg_internal("compressed zstd data is corrupt"))); + + SET_VARSIZE(result, rawsize + VARHDRSZ); + + return result; +#endif +} + +/* + * Decompress part of a varlena that was compressed using Zstandard. + */ +varlena * +zstd_decompress_datum_slice(const varlena *value, int32 slicelength) +{ +#ifndef USE_ZSTD + NO_COMPRESSION_SUPPORT("zstd"); + return NULL; /* keep compiler quiet */ +#else + ZSTD_DCtx *dctx; + ZSTD_inBuffer inbuf; + ZSTD_outBuffer outbuf; + varlena *result; + bool failed = false; + + Assert(VARDATA_COMPRESSED_GET_COMPRESS_METHOD(value) == TOAST_ZSTD_COMPRESSION_ID); + + /* + * The one-shot API can only decompress a whole frame, so use the + * streaming API instead: it stops as soon as the output buffer is full. + * Note that this means we never reach the end of the frame, so its + * checksum (if any) is not verified; that matches the behavior of the + * other methods here. + */ + + /* + * Allocate memory for the uncompressed data first, so that a failure here + * cannot leak the decompression context, which is not palloc'd. + */ + result = (varlena *) palloc(slicelength + VARHDRSZ); + + dctx = ZSTD_createDCtx(); + if (dctx == NULL) + elog(ERROR, "could not create zstd decompression context"); + + inbuf.src = (const char *) value + VARHDRSZ_COMPRESSED_LONG; + inbuf.size = VARSIZE(value) - VARHDRSZ_COMPRESSED_LONG; + inbuf.pos = 0; + outbuf.dst = VARDATA(result); + outbuf.size = slicelength; + outbuf.pos = 0; + + while (outbuf.pos < outbuf.size) + { + size_t ret = ZSTD_decompressStream(dctx, &outbuf, &inbuf); + + if (ZSTD_isError(ret)) + { + failed = true; + break; + } + + /* + * A zero return means the frame is fully decoded, and a positive one + * means more input is wanted. Either way, if we could not fill the + * output buffer the data must be truncated or corrupt, since the + * caller has already established that the value decompresses to more + * than slicelength bytes. + */ + if (ret == 0 || inbuf.pos == inbuf.size) + { + failed = (outbuf.pos < outbuf.size); + break; + } + } + + /* release the context before any possible error is thrown */ + ZSTD_freeDCtx(dctx); + + if (failed) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg_internal("compressed zstd data is corrupt"))); + + SET_VARSIZE(result, outbuf.pos + VARHDRSZ); + + return result; +#endif +} + /* * Extract compression ID from a varlena. * @@ -301,6 +471,13 @@ CompressionNameToMethod(const char *compression) #endif return TOAST_LZ4_COMPRESSION; } + else if (strcmp(compression, "zstd") == 0) + { +#ifndef USE_ZSTD + NO_COMPRESSION_SUPPORT("zstd"); +#endif + return TOAST_ZSTD_COMPRESSION; + } return InvalidCompressionMethod; } @@ -317,6 +494,8 @@ GetCompressionMethodName(char method) return "pglz"; case TOAST_LZ4_COMPRESSION: return "lz4"; + case TOAST_ZSTD_COMPRESSION: + return "zstd"; default: elog(ERROR, "invalid compression method %c", method); return NULL; /* keep compiler quiet */ diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c index 5e187e99b4a..2e56f080c1f 100644 --- a/src/backend/access/common/toast_internals.c +++ b/src/backend/access/common/toast_internals.c @@ -73,6 +73,10 @@ toast_compress_datum(Datum value, char cmethod) tmp = lz4_compress_datum((const varlena *) DatumGetPointer(value)); cmid = TOAST_LZ4_COMPRESSION_ID; break; + case TOAST_ZSTD_COMPRESSION: + tmp = zstd_compress_datum((const varlena *) DatumGetPointer(value)); + cmid = TOAST_ZSTD_COMPRESSION_ID; + break; default: elog(ERROR, "invalid compression method %c", cmethod); } @@ -239,7 +243,8 @@ toast_save_datum(Relation rel, Datum value, /* set external size and compression method */ Assert(cmid == TOAST_PGLZ_COMPRESSION_ID || - cmid == TOAST_LZ4_COMPRESSION_ID); + cmid == TOAST_LZ4_COMPRESSION_ID || + cmid == TOAST_ZSTD_COMPRESSION_ID); if (toast_compression_id_needs_cmid_byte(cmid)) { pointer_long = true; diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index 01cc61c5778..3579248443e 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -4245,6 +4245,9 @@ pg_column_compression(PG_FUNCTION_ARGS) case TOAST_LZ4_COMPRESSION_ID: result = "lz4"; break; + case TOAST_ZSTD_COMPRESSION_ID: + result = "zstd"; + break; default: elog(ERROR, "invalid compression method id %d", cmid); } diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 342aaeef59a..6f0ec8757f4 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -473,6 +473,9 @@ static const struct config_enum_entry default_toast_compression_options[] = { {"pglz", TOAST_PGLZ_COMPRESSION, false}, #ifdef USE_LZ4 {"lz4", TOAST_LZ4_COMPRESSION, false}, +#endif +#ifdef USE_ZSTD + {"zstd", TOAST_ZSTD_COMPRESSION, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index e759f06b50f..f912cd91ea5 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -806,7 +806,7 @@ #row_security = on #default_table_access_method = 'heap' #default_tablespace = '' # a tablespace name, '' uses the default -#default_toast_compression = pglz # pglz or lz4 +#default_toast_compression = pglz # pglz, lz4, or zstd #temp_tablespaces = '' # a list of tablespace names, '' uses # only default tablespace #check_function_bodies = on diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 388c3b9c346..746b6f1d07f 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -17823,6 +17823,9 @@ dumpTableSchema(Archive *fout, const TableInfo *tbinfo) case 'l': cmname = "lz4"; break; + case 'z': + cmname = "zstd"; + break; default: cmname = NULL; break; diff --git a/src/bin/pg_dump/t/006_pg_dump_compress.pl b/src/bin/pg_dump/t/006_pg_dump_compress.pl index d4ce6b18077..320fc3cfd15 100644 --- a/src/bin/pg_dump/t/006_pg_dump_compress.pl +++ b/src/bin/pg_dump/t/006_pg_dump_compress.pl @@ -385,6 +385,24 @@ my %tests = ( like => {%full_runs}, }, + 'CREATE TABLE test_compression_zstd' => { + create_order => 4, + create_sql => 'CREATE TABLE test_compression_zstd ( + col1 int, + col2 text COMPRESSION zstd + );', + regexp => qr/^ + \QCREATE TABLE public.test_compression_zstd (\E\n + \s+\Qcol1 integer,\E\n + \s+\Qcol2 text\E\n + \);\n + .* + \QALTER TABLE ONLY public.test_compression_zstd ALTER COLUMN col2 SET COMPRESSION zstd;\E\n + /xms, + compile_option => 'zstd', + like => {%full_runs}, + }, + # Create a large object so we can test compression of blobs.toc 'LO create (using lo_from_bytea)' => { create_order => 50, diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index dc9b3841592..51940726069 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -2094,8 +2094,9 @@ describeOneTableDetails(const char *schemaname, /* these strings are literal in our syntax, so not translated. */ printTableAddCell(&cont, (compression[0] == 'p' ? "pglz" : (compression[0] == 'l' ? "lz4" : - (compression[0] == '\0' ? "" : - "???"))), + (compression[0] == 'z' ? "zstd" : + (compression[0] == '\0' ? "" : + "???")))), false, false); } diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c index b3bfe050b18..982a85bc558 100644 --- a/src/bin/psql/tab-complete.in.c +++ b/src/bin/psql/tab-complete.in.c @@ -2966,7 +2966,7 @@ match_previous_words(int pattern_id, /* ALTER TABLE ALTER [COLUMN] SET COMPRESSION */ else if (Matches("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET", "COMPRESSION") || Matches("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET", "COMPRESSION")) - COMPLETE_WITH("DEFAULT", "PGLZ", "LZ4"); + COMPLETE_WITH("DEFAULT", "PGLZ", "LZ4", "ZSTD"); /* ALTER TABLE ALTER [COLUMN] SET EXPRESSION */ else if (Matches("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET", "EXPRESSION") || Matches("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET", "EXPRESSION")) diff --git a/src/include/access/toast_compression.h b/src/include/access/toast_compression.h index e848d72805b..53d4f72257c 100644 --- a/src/include/access/toast_compression.h +++ b/src/include/access/toast_compression.h @@ -50,6 +50,7 @@ typedef enum ToastCompressionId { TOAST_PGLZ_COMPRESSION_ID = 0, TOAST_LZ4_COMPRESSION_ID = 1, + TOAST_ZSTD_COMPRESSION_ID = 2, TOAST_INVALID_COMPRESSION_ID = 3, } ToastCompressionId; @@ -79,6 +80,7 @@ toast_compression_id_needs_cmid_byte(ToastCompressionId cmid) */ #define TOAST_PGLZ_COMPRESSION 'p' #define TOAST_LZ4_COMPRESSION 'l' +#define TOAST_ZSTD_COMPRESSION 'z' #define InvalidCompressionMethod '\0' #define CompressionMethodIsValid(cm) ((cm) != InvalidCompressionMethod) @@ -105,6 +107,12 @@ extern varlena *lz4_decompress_datum(const varlena *value); extern varlena *lz4_decompress_datum_slice(const varlena *value, int32 slicelength); +/* zstd compression/decompression routines */ +extern varlena *zstd_compress_datum(const varlena *value); +extern varlena *zstd_decompress_datum(const varlena *value); +extern varlena *zstd_decompress_datum_slice(const varlena *value, + int32 slicelength); + /* other stuff */ extern ToastCompressionId toast_get_compression_id(varlena *attr); extern char CompressionNameToMethod(const char *compression); diff --git a/src/include/access/toast_internals.h b/src/include/access/toast_internals.h index 9f3dfa326cd..85366a23f5c 100644 --- a/src/include/access/toast_internals.h +++ b/src/include/access/toast_internals.h @@ -36,7 +36,8 @@ toast_compress_set_size_and_method(varlena *ptr, uint32 rawsize, Assert(rawsize > 0 && rawsize <= VARLENA_EXTSIZE_MASK); Assert(cmid == TOAST_PGLZ_COMPRESSION_ID || - cmid == TOAST_LZ4_COMPRESSION_ID); + cmid == TOAST_LZ4_COMPRESSION_ID || + cmid == TOAST_ZSTD_COMPRESSION_ID); if (toast_compression_id_needs_cmid_byte(cmid)) { diff --git a/src/test/regress/expected/compression_zstd.out b/src/test/regress/expected/compression_zstd.out new file mode 100644 index 00000000000..1575a2714d1 --- /dev/null +++ b/src/test/regress/expected/compression_zstd.out @@ -0,0 +1,839 @@ +-- Tests for TOAST compression with zstd +SELECT NOT(enumvals @> '{zstd}') AS skip_test FROM pg_settings WHERE + name = 'default_toast_compression' \gset +\if :skip_test + \echo '*** skipping TOAST tests with zstd (not supported) ***' + \quit +\endif +CREATE SCHEMA zstd; +SET search_path TO zstd, public; +\set HIDE_TOAST_COMPRESSION false +-- Ensure we get stable results regardless of the installation's default. +-- We rely on this GUC value for a few tests. +SET default_toast_compression = 'pglz'; +-- Helpers. Hash output is poorly compressible, so values built from it +-- stay large after compression and are stored out of line; a bytea of raw +-- hash bytes is not compressible at all. None of the checks below print +-- compressed sizes, which depend on the zstd version. +CREATE FUNCTION large_val_zstd() RETURNS TEXT LANGUAGE SQL AS +'select array_agg(fipshash(g::text))::text from generate_series(1, 256) g'; +CREATE FUNCTION incompressible_val() RETURNS bytea LANGUAGE SQL AS +$$ select decode(string_agg(fipshash(g::text), '' order by g), 'hex') + from generate_series(1, 200) g $$; +---------------------------------------------------------------------- +-- DDL +---------------------------------------------------------------------- +-- test creating table with compression method +CREATE TABLE cmdata_pglz(f1 text COMPRESSION pglz); +CREATE INDEX idx ON cmdata_pglz(f1); +INSERT INTO cmdata_pglz VALUES(repeat('1234567890', 1000)); +\d+ cmdata_pglz + Table "zstd.cmdata_pglz" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | pglz | | +Indexes: + "idx" btree (f1) + +CREATE TABLE cmdata_zstd(f1 TEXT COMPRESSION zstd); +INSERT INTO cmdata_zstd VALUES(repeat('1234567890', 1004)); +\d+ cmdata_zstd + Table "zstd.cmdata_zstd" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | zstd | | + +-- the method name is an identifier, so case does not matter +CREATE TABLE cmupper(f1 text COMPRESSION ZSTD); +\d+ cmupper + Table "zstd.cmupper" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | zstd | | + +DROP TABLE cmupper; +-- a domain over a varlena type can be compressed too +CREATE DOMAIN zstdtext AS text; +CREATE TABLE cmdomain(f1 zstdtext COMPRESSION zstd); +INSERT INTO cmdomain VALUES (repeat('1234567890', 1004)); +\d+ cmdomain + Table "zstd.cmdomain" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+----------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | zstdtext | | | | extended | zstd | | + +SELECT pg_column_compression(f1), length(f1) FROM cmdomain; + pg_column_compression | length +-----------------------+-------- + zstd | 10040 +(1 row) + +DROP TABLE cmdomain; +DROP DOMAIN zstdtext; +-- errors: incompressible data type, unknown method +CREATE TABLE cmbad(f1 int COMPRESSION zstd); +ERROR: column data type integer does not support compression +CREATE TABLE cmbad(f1 text); +ALTER TABLE cmbad ALTER COLUMN f1 SET COMPRESSION zstd_nodict; +ERROR: invalid compression method "zstd_nodict" +DROP TABLE cmbad; +---------------------------------------------------------------------- +-- Compressed-in-line values +---------------------------------------------------------------------- +-- verify stored compression method in the data, and that compression +-- did save space +SELECT pg_column_compression(f1), + pg_column_size(f1) < length(f1) AS smaller, + pg_column_toast_chunk_id(f1) IS NULL AS is_inline + FROM cmdata_zstd; + pg_column_compression | smaller | is_inline +-----------------------+---------+----------- + zstd | t | t +(1 row) + +-- decompress data slice +SELECT SUBSTR(f1, 200, 5) FROM cmdata_pglz; + substr +-------- + 01234 +(1 row) + +SELECT SUBSTR(f1, 2000, 50) FROM cmdata_zstd; + substr +---------------------------------------------------- + 01234567890123456789012345678901234567890123456789 +(1 row) + +-- check that slice decompression agrees with full decompression, at the +-- start, in the middle and at the very end of the value +SELECT SUBSTR(f1, 1, 100) = SUBSTR(repeat('1234567890', 1004), 1, 100) AS ok_head, + SUBSTR(f1, 5000, 100) = SUBSTR(repeat('1234567890', 1004), 5000, 100) AS ok_mid, + SUBSTR(f1, 10036, 5) = SUBSTR(repeat('1234567890', 1004), 10036, 5) AS ok_tail, + length(f1) AS len + FROM cmdata_zstd; + ok_head | ok_mid | ok_tail | len +---------+--------+---------+------- + t | t | t | 10040 +(1 row) + +-- slices of one byte, past the end, and longer than the whole value (the +-- latter falls back to a full decompression) +SELECT left(f1, 1) AS first_char, + right(f1, 1) AS last_char, + SUBSTR(f1, 10040, 1) AS at_end, + SUBSTR(f1, 10041, 10) = '' AS past_end, + length(SUBSTR(f1, 1, 1000000)) AS full_len + FROM cmdata_zstd; + first_char | last_char | at_end | past_end | full_len +------------+-----------+--------+----------+---------- + 1 | 0 | 0 | t | 10040 +(1 row) + +---------------------------------------------------------------------- +-- Values that do not get compressed +---------------------------------------------------------------------- +-- values below the TOAST threshold are stored as they are; NULL and the +-- empty string are fine too +CREATE TABLE cmshort(f1 text COMPRESSION zstd); +INSERT INTO cmshort VALUES (repeat('x', 100)), (''), (NULL); +SELECT pg_column_compression(f1), length(f1) FROM cmshort ORDER BY length(f1); + pg_column_compression | length +-----------------------+-------- + | 0 + | 100 + | +(3 rows) + +DROP TABLE cmshort; +-- data that zstd cannot shrink is stored uncompressed, out of line if +-- needed, with no compression method recorded +CREATE TABLE cmincompressible(f1 bytea COMPRESSION zstd); +INSERT INTO cmincompressible SELECT incompressible_val(); +SELECT pg_column_compression(f1) IS NULL AS uncompressed, + pg_column_toast_chunk_id(f1) IS NOT NULL AS is_external, + pg_column_size(f1) = octet_length(f1) AS same_size, + length(f1) AS len, + f1 = incompressible_val() AS ok_full, + substr(f1, 3000, 40) = substr(incompressible_val(), 3000, 40) AS ok_slice + FROM cmincompressible; + uncompressed | is_external | same_size | len | ok_full | ok_slice +--------------+-------------+-----------+------+---------+---------- + t | t | t | 3200 | t | t +(1 row) + +DROP TABLE cmincompressible; +---------------------------------------------------------------------- +-- Other varlena types +---------------------------------------------------------------------- +-- binary data with zero and high bytes +CREATE TABLE cmbytea(f1 bytea COMPRESSION zstd); +INSERT INTO cmbytea VALUES (decode(repeat('00ff10', 2000), 'hex')); +SELECT pg_column_compression(f1), length(f1), + f1 = decode(repeat('00ff10', 2000), 'hex') AS ok_full, + get_byte(f1, 0) AS byte0, get_byte(f1, 1) AS byte1, + substr(f1, 3001, 3) = decode('00ff10', 'hex') AS ok_slice + FROM cmbytea; + pg_column_compression | length | ok_full | byte0 | byte1 | ok_slice +-----------------------+--------+---------+-------+-------+---------- + zstd | 6000 | t | 0 | 255 | t +(1 row) + +DROP TABLE cmbytea; +-- jsonb, compressed inline and out of line +CREATE TABLE cmjsonb(j jsonb COMPRESSION zstd); +INSERT INTO cmjsonb SELECT jsonb_build_object('k', repeat('v', 5000), 'n', 1); +INSERT INTO cmjsonb SELECT jsonb_object_agg(g::text, fipshash(g::text)) + FROM generate_series(1, 300) g; +SELECT pg_column_compression(j), + pg_column_toast_chunk_id(j) IS NOT NULL AS is_external, + length(j ->> 'k') AS klen, + j ->> 'n' AS n, + j ->> '150' = fipshash('150') AS ok_150 + FROM cmjsonb ORDER BY is_external; + pg_column_compression | is_external | klen | n | ok_150 +-----------------------+-------------+------+---+-------- + zstd | f | 5000 | 1 | + zstd | t | | | t +(2 rows) + +DROP TABLE cmjsonb; +-- arrays +CREATE TABLE cmarray(f1 text[] COMPRESSION zstd); +INSERT INTO cmarray SELECT array_agg(repeat('x', 100) || g ORDER BY g) + FROM generate_series(1, 200) g; +SELECT pg_column_compression(f1), array_length(f1, 1) AS n, + f1[1] = repeat('x', 100) || '1' AS ok_first, + f1[200] = repeat('x', 100) || '200' AS ok_last + FROM cmarray; + pg_column_compression | n | ok_first | ok_last +-----------------------+-----+----------+--------- + zstd | 200 | t | t +(1 row) + +DROP TABLE cmarray; +---------------------------------------------------------------------- +-- Moving data between tables and methods +---------------------------------------------------------------------- +-- copy with table creation +SELECT * INTO cmmove1 FROM cmdata_zstd; +\d+ cmmove1 + Table "zstd.cmmove1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | | | + +SELECT pg_column_compression(f1) FROM cmmove1; + pg_column_compression +----------------------- + zstd +(1 row) + +-- test LIKE INCLUDING COMPRESSION. The GUC default_toast_compression +-- has no effect, the compression method is taken from the table being copied. +CREATE TABLE cmdata2 (LIKE cmdata_zstd INCLUDING COMPRESSION); +\d+ cmdata2 + Table "zstd.cmdata2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | zstd | | + +DROP TABLE cmdata2; +-- copy to existing table: a compressed datum keeps its own method, in both +-- directions, so a table can hold values compressed with different methods +CREATE TABLE cmmove3(f1 text COMPRESSION pglz); +INSERT INTO cmmove3 SELECT * FROM cmdata_pglz; +INSERT INTO cmmove3 SELECT * FROM cmdata_zstd; +SELECT pg_column_compression(f1) FROM cmmove3 ORDER BY 1; + pg_column_compression +----------------------- + pglz + zstd +(2 rows) + +INSERT INTO cmdata_zstd SELECT * FROM cmdata_pglz; +SELECT pg_column_compression(f1), count(*) FROM cmdata_zstd GROUP BY 1 ORDER BY 1; + pg_column_compression | count +-----------------------+------- + pglz | 1 + zstd | 1 +(2 rows) + +DELETE FROM cmdata_zstd WHERE pg_column_compression(f1) = 'pglz'; +-- update using datum from different table with zstd data. +CREATE TABLE cmmove2(f1 text COMPRESSION pglz); +INSERT INTO cmmove2 VALUES (repeat('1234567890', 1004)); +SELECT pg_column_compression(f1) FROM cmmove2; + pg_column_compression +----------------------- + pglz +(1 row) + +UPDATE cmmove2 SET f1 = cmdata_zstd.f1 FROM cmdata_zstd; +SELECT pg_column_compression(f1) FROM cmmove2; + pg_column_compression +----------------------- + zstd +(1 row) + +---------------------------------------------------------------------- +-- Externally stored compressed values +---------------------------------------------------------------------- +CREATE TABLE cmdata2 (f1 text COMPRESSION zstd); +INSERT INTO cmdata2 SELECT large_val_zstd() || repeat('a', 4000); +SELECT pg_column_compression(f1), pg_column_size(f1) < length(f1) AS smaller + FROM cmdata2; + pg_column_compression | smaller +-----------------------+--------- + zstd | t +(1 row) + +SELECT SUBSTR(f1, 200, 5) FROM cmdata2; + substr +-------- + 79026 +(1 row) + +SELECT SUBSTR(f1, 200, 5) = SUBSTR(large_val_zstd() || repeat('a', 4000), 200, 5) AS ok_slice + FROM cmdata2; + ok_slice +---------- + t +(1 row) + +DROP TABLE cmdata2; +-- test that both TOAST value ID widths carry the compression method through +-- an external TOAST pointer. The data here is poorly compressible on +-- purpose, so that the value is certain to be stored out of line. +CREATE TABLE cmdata_oid(f1 text COMPRESSION zstd) + WITH (toast_value_type = 'oid'); +CREATE TABLE cmdata_oid8(f1 text COMPRESSION zstd) + WITH (toast_value_type = 'oid8'); +INSERT INTO cmdata_oid SELECT large_val_zstd(); +INSERT INTO cmdata_oid8 SELECT large_val_zstd(); +-- confirm the values really are out of line, so that the checks below +-- exercise the external TOAST pointer rather than the inline header +SELECT pg_column_toast_chunk_id(f1) IS NOT NULL AS is_external, + pg_column_compression(f1) FROM cmdata_oid; + is_external | pg_column_compression +-------------+----------------------- + t | zstd +(1 row) + +SELECT pg_column_toast_chunk_id(f1) IS NOT NULL AS is_external, + pg_column_compression(f1) FROM cmdata_oid8; + is_external | pg_column_compression +-------------+----------------------- + t | zstd +(1 row) + +SELECT length(f1) = length(large_val_zstd()) AS ok_len, + f1 = large_val_zstd() AS ok_full, + SUBSTR(f1, 3000, 40) = SUBSTR(large_val_zstd(), 3000, 40) AS ok_slice + FROM cmdata_oid; + ok_len | ok_full | ok_slice +--------+---------+---------- + t | t | t +(1 row) + +SELECT length(f1) = length(large_val_zstd()) AS ok_len, + f1 = large_val_zstd() AS ok_full, + SUBSTR(f1, 3000, 40) = SUBSTR(large_val_zstd(), 3000, 40) AS ok_slice + FROM cmdata_oid8; + ok_len | ok_full | ok_slice +--------+---------+---------- + t | t | t +(1 row) + +-- a value spanning many TOAST chunks, with slices from the front, from +-- near the end (which has to decompress almost everything), and longer +-- than the value +CREATE TABLE cmbig(f1 text COMPRESSION zstd); +INSERT INTO cmbig SELECT string_agg(fipshash(g::text) || repeat('-', 40), '' ORDER BY g) + FROM generate_series(1, 10000) g; +SELECT pg_column_compression(f1), + pg_column_toast_chunk_id(f1) IS NOT NULL AS is_external, + pg_column_size(f1) < length(f1) AS smaller, + length(f1) AS len, + left(f1, 32) = fipshash('1') AS ok_head, + SUBSTR(f1, 360001, 32) = fipshash('5001') AS ok_mid, + SUBSTR(f1, 719929, 32) = fipshash('10000') AS ok_tail, + length(SUBSTR(f1, 1, 2000000)) AS full_len + FROM cmbig; + pg_column_compression | is_external | smaller | len | ok_head | ok_mid | ok_tail | full_len +-----------------------+-------------+---------+--------+---------+--------+---------+---------- + zstd | t | t | 720000 | t | t | t | 720000 +(1 row) + +DROP TABLE cmbig; +-- a smaller toast_tuple_target makes even a small compressed value go out +-- of line +CREATE TABLE cmtarget(f1 text COMPRESSION zstd) WITH (toast_tuple_target = 128); +INSERT INTO cmtarget SELECT substr(large_val_zstd(), 1, 3000); +SELECT pg_column_compression(f1), + pg_column_toast_chunk_id(f1) IS NOT NULL AS is_external, + f1 = substr(large_val_zstd(), 1, 3000) AS ok_full + FROM cmtarget; + pg_column_compression | is_external | ok_full +-----------------------+-------------+--------- + zstd | t | t +(1 row) + +DROP TABLE cmtarget; +-- an external zstd value copied into a table with a different chunk_id type +-- and compression setting keeps its compression, and is re-externalized +-- with a fresh TOAST pointer; try both chunk_id types as destination +CREATE TABLE cmmove_ext (f1 text COMPRESSION pglz) WITH (toast_value_type = 'oid'); +CREATE TABLE cmmove_ext8 (f1 text COMPRESSION pglz) WITH (toast_value_type = 'oid8'); +INSERT INTO cmmove_ext SELECT f1 FROM cmdata_oid8; +INSERT INTO cmmove_ext8 SELECT f1 FROM cmdata_oid; +SELECT pg_column_toast_chunk_id(f1) IS NOT NULL AS is_external, + pg_column_compression(f1), + f1 = large_val_zstd() AS ok_full + FROM cmmove_ext; + is_external | pg_column_compression | ok_full +-------------+-----------------------+--------- + t | zstd | t +(1 row) + +SELECT pg_column_toast_chunk_id(f1) IS NOT NULL AS is_external, + pg_column_compression(f1), + f1 = large_val_zstd() AS ok_full + FROM cmmove_ext8; + is_external | pg_column_compression | ok_full +-------------+-----------------------+--------- + t | zstd | t +(1 row) + +DROP TABLE cmmove_ext, cmmove_ext8; +---------------------------------------------------------------------- +-- Storage modes +---------------------------------------------------------------------- +-- EXTERNAL: stored out of line but never compressed, slices cross chunk +-- boundaries directly +CREATE TABLE cmstorage(f1 text COMPRESSION zstd); +ALTER TABLE cmstorage ALTER COLUMN f1 SET STORAGE EXTERNAL; +INSERT INTO cmstorage SELECT large_val_zstd(); +SELECT pg_column_compression(f1) IS NULL AS uncompressed, + pg_column_toast_chunk_id(f1) IS NOT NULL AS is_external, + pg_column_size(f1) = octet_length(f1) AS same_size, + SUBSTR(f1, 1990, 20) = SUBSTR(large_val_zstd(), 1990, 20) AS ok_slice + FROM cmstorage; + uncompressed | is_external | same_size | ok_slice +--------------+-------------+-----------+---------- + t | t | t | t +(1 row) + +-- MAIN: compressed, and kept in the tuple when that makes it fit +TRUNCATE cmstorage; +ALTER TABLE cmstorage ALTER COLUMN f1 SET STORAGE MAIN; +INSERT INTO cmstorage VALUES (repeat('1234567890', 1004)); +SELECT pg_column_compression(f1), + pg_column_toast_chunk_id(f1) IS NULL AS is_inline, + length(f1) AS len + FROM cmstorage; + pg_column_compression | is_inline | len +-----------------------+-----------+------- + zstd | t | 10040 +(1 row) + +DROP TABLE cmstorage; +-- PLAIN: new values are never compressed. An external compressed datum +-- coming from another table is fetched and decompressed before being +-- stored, while a datum that is already compressed in line is stored as it +-- is, like for the other methods. +CREATE TABLE cmsmall(f1 text COMPRESSION zstd) WITH (toast_tuple_target = 128); +INSERT INTO cmsmall VALUES (repeat('1234567890', 300)); +INSERT INTO cmsmall SELECT substr(large_val_zstd(), 1, 3000); +SELECT pg_column_compression(f1), + pg_column_toast_chunk_id(f1) IS NOT NULL AS is_external + FROM cmsmall ORDER BY is_external; + pg_column_compression | is_external +-----------------------+------------- + zstd | f + zstd | t +(2 rows) + +CREATE TABLE cmplain(f1 text COMPRESSION zstd); +ALTER TABLE cmplain ALTER COLUMN f1 SET STORAGE PLAIN; +INSERT INTO cmplain VALUES (repeat('1234567890', 300)); +INSERT INTO cmplain SELECT f1 FROM cmsmall; +SELECT pg_column_compression(f1), length(f1), + f1 IN (repeat('1234567890', 300), substr(large_val_zstd(), 1, 3000)) AS ok_full + FROM cmplain ORDER BY pg_column_compression(f1) NULLS FIRST, length(f1); + pg_column_compression | length | ok_full +-----------------------+--------+--------- + | 3000 | t + | 3000 | t + zstd | 3000 | t +(3 rows) + +DROP TABLE cmplain, cmsmall; +---------------------------------------------------------------------- +-- Table rewrites and updates of external values +---------------------------------------------------------------------- +CREATE TABLE cmrewrite(id int, f1 text COMPRESSION zstd) + WITH (toast_value_type = 'oid8'); +INSERT INTO cmrewrite VALUES (1, large_val_zstd()), (2, repeat('1234567890', 1004)); +CREATE INDEX cmrewrite_id ON cmrewrite(id); +-- VACUUM FULL and CLUSTER preserve the values and do not recompress +VACUUM FULL cmrewrite; +SELECT id, pg_column_compression(f1), + pg_column_toast_chunk_id(f1) IS NOT NULL AS is_external, + f1 = CASE id WHEN 1 THEN large_val_zstd() ELSE repeat('1234567890', 1004) END AS ok_full + FROM cmrewrite ORDER BY id; + id | pg_column_compression | is_external | ok_full +----+-----------------------+-------------+--------- + 1 | zstd | t | t + 2 | zstd | f | t +(2 rows) + +CLUSTER cmrewrite USING cmrewrite_id; +SELECT id, pg_column_compression(f1), + pg_column_toast_chunk_id(f1) IS NOT NULL AS is_external, + f1 = CASE id WHEN 1 THEN large_val_zstd() ELSE repeat('1234567890', 1004) END AS ok_full + FROM cmrewrite ORDER BY id; + id | pg_column_compression | is_external | ok_full +----+-----------------------+-------------+--------- + 1 | zstd | t | t + 2 | zstd | f | t +(2 rows) + +-- a column type change resets the column's compression method to the +-- default (as it does its storage), so the rewritten values follow the +-- default_toast_compression GUC; values that were external come back in +-- one piece +ALTER TABLE cmrewrite ALTER COLUMN f1 TYPE varchar(30000); +\d+ cmrewrite + Table "zstd.cmrewrite" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+--------------------------+-----------+----------+---------+----------+-------------+--------------+------------- + id | integer | | | | plain | | | + f1 | character varying(30000) | | | | extended | | | +Indexes: + "cmrewrite_id" btree (id) CLUSTER +Options: toast_value_type=oid8 + +SELECT id, pg_column_compression(f1), + f1 = CASE id WHEN 1 THEN large_val_zstd() ELSE repeat('1234567890', 1004) END AS ok_full + FROM cmrewrite ORDER BY id; + id | pg_column_compression | ok_full +----+-----------------------+--------- + 1 | | t + 2 | pglz | t +(2 rows) + +SET default_toast_compression = 'zstd'; +ALTER TABLE cmrewrite ALTER COLUMN f1 TYPE varchar(20000); +SELECT id, pg_column_compression(f1), + f1 = CASE id WHEN 1 THEN large_val_zstd() ELSE repeat('1234567890', 1004) END AS ok_full + FROM cmrewrite ORDER BY id; + id | pg_column_compression | ok_full +----+-----------------------+--------- + 1 | zstd | t + 2 | zstd | t +(2 rows) + +SET default_toast_compression = 'pglz'; +DROP TABLE cmrewrite; +-- an UPDATE that leaves the external value alone must keep the same TOAST +-- value rather than re-toasting it +ALTER TABLE cmdata_oid ADD COLUMN n int DEFAULT 0; +SELECT pg_column_toast_chunk_id(f1) AS chunk_before FROM cmdata_oid \gset +UPDATE cmdata_oid SET n = 1; +SELECT pg_column_toast_chunk_id(f1) = :'chunk_before' AS same_toast_value, + f1 = large_val_zstd() AS ok_full + FROM cmdata_oid; + same_toast_value | ok_full +------------------+--------- + t | t +(1 row) + +-- whereas an UPDATE of the value itself replaces the TOAST value; the old +-- chunks are gone from the TOAST table as soon as the update commits +SELECT reltoastrelid::regclass AS toastrel FROM pg_class + WHERE oid = 'cmdata_oid'::regclass \gset +UPDATE cmdata_oid SET f1 = f1 || 'x'; +SELECT pg_column_toast_chunk_id(f1) <> :'chunk_before' AS new_toast_value, + pg_column_compression(f1), + f1 = large_val_zstd() || 'x' AS ok_full + FROM cmdata_oid; + new_toast_value | pg_column_compression | ok_full +-----------------+-----------------------+--------- + t | zstd | t +(1 row) + +SELECT count(DISTINCT chunk_id) AS live_toast_values FROM :toastrel; + live_toast_values +------------------- + 1 +(1 row) + +-- deleting the row deletes its TOAST value, for both chunk_id types +DELETE FROM cmdata_oid; +SELECT count(*) AS chunks_left FROM :toastrel; + chunks_left +------------- + 0 +(1 row) + +SELECT reltoastrelid::regclass AS toastrel FROM pg_class + WHERE oid = 'cmdata_oid8'::regclass \gset +DELETE FROM cmdata_oid8; +SELECT count(*) AS chunks_left FROM :toastrel; + chunks_left +------------- + 0 +(1 row) + +DROP TABLE cmdata_oid, cmdata_oid8; +---------------------------------------------------------------------- +-- Indexes +---------------------------------------------------------------------- +-- a compressible key large enough to need compression to fit in an index +-- entry; the index must be usable to find the row +CREATE TABLE cmindex(f1 text COMPRESSION zstd); +CREATE INDEX cmindex_f1 ON cmindex(f1); +INSERT INTO cmindex VALUES (repeat('abcdefghij', 300)), (repeat('klmnopqrst', 300)); +SET enable_seqscan = off; +SET enable_bitmapscan = off; +SELECT length(f1), left(f1, 10) FROM cmindex WHERE f1 = repeat('klmnopqrst', 300); + length | left +--------+------------ + 3000 | klmnopqrst +(1 row) + +RESET enable_seqscan; +RESET enable_bitmapscan; +DROP TABLE cmindex; +-- test expression index +CREATE TABLE cmdata2 (f1 TEXT COMPRESSION pglz, f2 TEXT COMPRESSION zstd); +CREATE UNIQUE INDEX idx1 ON cmdata2 ((f1 || f2)); +INSERT INTO cmdata2 VALUES((SELECT array_agg(fipshash(g::TEXT))::TEXT FROM +generate_series(1, 50) g), VERSION()); +DROP TABLE cmdata2; +---------------------------------------------------------------------- +-- Materialized views, partitions, inheritance, temporary tables +---------------------------------------------------------------------- +-- test compression with materialized view +CREATE MATERIALIZED VIEW compressmv(x) AS SELECT * FROM cmdata_zstd; +\d+ compressmv + Materialized view "zstd.compressmv" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + x | text | | | | extended | | | +View definition: + SELECT f1 AS x + FROM cmdata_zstd; + +SELECT pg_column_compression(f1) FROM cmdata_zstd; + pg_column_compression +----------------------- + zstd +(1 row) + +SELECT pg_column_compression(x) FROM compressmv; + pg_column_compression +----------------------- + zstd +(1 row) + +-- test compression with partition +CREATE TABLE cmpart(f1 text COMPRESSION zstd) PARTITION BY HASH(f1); +CREATE TABLE cmpart1 PARTITION OF cmpart FOR VALUES WITH (MODULUS 2, REMAINDER 0); +CREATE TABLE cmpart2(f1 text COMPRESSION pglz); +ALTER TABLE cmpart ATTACH PARTITION cmpart2 FOR VALUES WITH (MODULUS 2, REMAINDER 1); +INSERT INTO cmpart VALUES (repeat('123456789', 1004)); +INSERT INTO cmpart VALUES (repeat('123456789', 4004)); +SELECT pg_column_compression(f1) FROM cmpart1; + pg_column_compression +----------------------- + zstd +(1 row) + +SELECT pg_column_compression(f1) FROM cmpart2; + pg_column_compression +----------------------- + pglz +(1 row) + +-- test compression with inheritance +CREATE TABLE cminh() INHERITS(cmdata_pglz, cmdata_zstd); -- error +NOTICE: merging multiple inherited definitions of column "f1" +ERROR: column "f1" has a compression method conflict +DETAIL: pglz versus zstd +CREATE TABLE cminh(f1 TEXT COMPRESSION zstd) INHERITS(cmdata_pglz); -- error +NOTICE: merging column "f1" with inherited definition +ERROR: column "f1" has a compression method conflict +DETAIL: pglz versus zstd +CREATE TABLE cmdata3(f1 text); +CREATE TABLE cminh() INHERITS (cmdata_pglz, cmdata3); +NOTICE: merging multiple inherited definitions of column "f1" +-- temporary table +CREATE TEMP TABLE cmtemp(f1 text COMPRESSION zstd); +INSERT INTO cmtemp SELECT large_val_zstd(); +SELECT pg_column_compression(f1), + pg_column_toast_chunk_id(f1) IS NOT NULL AS is_external, + f1 = large_val_zstd() AS ok_full + FROM cmtemp; + pg_column_compression | is_external | ok_full +-----------------------+-------------+--------- + zstd | t | t +(1 row) + +DROP TABLE cmtemp; +---------------------------------------------------------------------- +-- default_toast_compression GUC +---------------------------------------------------------------------- +-- test default_toast_compression GUC; the value is case-insensitive +SET default_toast_compression = 'ZSTD'; +SHOW default_toast_compression; + default_toast_compression +--------------------------- + zstd +(1 row) + +SET default_toast_compression = 'zstd'; +-- the GUC drives the method for columns with no explicit COMPRESSION, so +-- the column keeps a default attcompression but the data comes out zstd +CREATE TABLE cmdata_default(f1 text); +INSERT INTO cmdata_default VALUES(repeat('1234567890', 1004)); +\d+ cmdata_default + Table "zstd.cmdata_default" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | | | + +SELECT pg_column_compression(f1) FROM cmdata_default; + pg_column_compression +----------------------- + zstd +(1 row) + +SELECT f1 = repeat('1234567890', 1004) AS ok_full, + SUBSTR(f1, 4000, 20) = SUBSTR(repeat('1234567890', 1004), 4000, 20) AS ok_slice + FROM cmdata_default; + ok_full | ok_slice +---------+---------- + t | t +(1 row) + +-- SET COMPRESSION default makes a column follow the GUC again +ALTER TABLE cmdata_default ALTER COLUMN f1 SET COMPRESSION pglz; +INSERT INTO cmdata_default VALUES(repeat('1234567890', 1004)); +ALTER TABLE cmdata_default ALTER COLUMN f1 SET COMPRESSION default; +\d+ cmdata_default + Table "zstd.cmdata_default" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | | | + +INSERT INTO cmdata_default VALUES(repeat('1234567890', 1004)); +SELECT pg_column_compression(f1), count(*) FROM cmdata_default GROUP BY 1 ORDER BY 1; + pg_column_compression | count +-----------------------+------- + pglz | 1 + zstd | 2 +(2 rows) + +DROP TABLE cmdata_default; +-- test alter compression method +ALTER TABLE cmdata_pglz ALTER COLUMN f1 SET COMPRESSION zstd; +INSERT INTO cmdata_pglz VALUES (repeat('123456789', 4004)); +\d+ cmdata_pglz + Table "zstd.cmdata_pglz" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | zstd | | +Indexes: + "idx" btree (f1) +Child tables: + cminh + +SELECT pg_column_compression(f1) FROM cmdata_pglz; + pg_column_compression +----------------------- + pglz + zstd +(2 rows) + +ALTER TABLE cmdata_pglz ALTER COLUMN f1 SET COMPRESSION pglz; +-- test alter compression method for materialized views +ALTER MATERIALIZED VIEW compressmv ALTER COLUMN x SET COMPRESSION zstd; +\d+ compressmv + Materialized view "zstd.compressmv" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + x | text | | | | extended | zstd | | +View definition: + SELECT f1 AS x + FROM cmdata_zstd; + +-- test alter compression method for partitioned tables +ALTER TABLE cmpart1 ALTER COLUMN f1 SET COMPRESSION pglz; +ALTER TABLE cmpart2 ALTER COLUMN f1 SET COMPRESSION zstd; +-- new data should be compressed with the current compression method +INSERT INTO cmpart VALUES (repeat('123456789', 1004)); +INSERT INTO cmpart VALUES (repeat('123456789', 4004)); +SELECT pg_column_compression(f1) FROM cmpart1; + pg_column_compression +----------------------- + zstd + pglz +(2 rows) + +SELECT pg_column_compression(f1) FROM cmpart2; + pg_column_compression +----------------------- + pglz + zstd +(2 rows) + +-- VACUUM FULL does not recompress +SELECT pg_column_compression(f1) FROM cmdata_zstd; + pg_column_compression +----------------------- + zstd +(1 row) + +VACUUM FULL cmdata_zstd; +SELECT pg_column_compression(f1) FROM cmdata_zstd; + pg_column_compression +----------------------- + zstd +(1 row) + +RESET default_toast_compression; +-- check data is ok +SELECT length(f1) FROM cmdata_pglz; + length +-------- + 10000 + 36036 +(2 rows) + +SELECT length(f1) FROM cmdata_zstd; + length +-------- + 10040 +(1 row) + +SELECT length(f1) FROM cmmove1; + length +-------- + 10040 +(1 row) + +SELECT length(f1) FROM cmmove2; + length +-------- + 10040 +(1 row) + +SELECT length(f1) FROM cmmove3; + length +-------- + 10000 + 10040 +(2 rows) + +\set HIDE_TOAST_COMPRESSION true diff --git a/src/test/regress/expected/compression_zstd_1.out b/src/test/regress/expected/compression_zstd_1.out new file mode 100644 index 00000000000..5f07342fd51 --- /dev/null +++ b/src/test/regress/expected/compression_zstd_1.out @@ -0,0 +1,7 @@ +-- Tests for TOAST compression with zstd +SELECT NOT(enumvals @> '{zstd}') AS skip_test FROM pg_settings WHERE + name = 'default_toast_compression' \gset +\if :skip_test + \echo '*** skipping TOAST tests with zstd (not supported) ***' +*** skipping TOAST tests with zstd (not supported) *** + \quit diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 75063f87a4a..2b25b90ee3c 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -128,7 +128,7 @@ test: partition_join partition_prune reloptions hash_part indexing partition_agg # ---------- # Another group of parallel tests (compression) # ---------- -test: compression compression_lz4 compression_pglz cluster +test: compression compression_lz4 compression_pglz compression_zstd cluster # event_trigger depends on create_am and cannot run concurrently with # any test that runs DDL diff --git a/src/test/regress/sql/compression_zstd.sql b/src/test/regress/sql/compression_zstd.sql new file mode 100644 index 00000000000..a4130eff1e4 --- /dev/null +++ b/src/test/regress/sql/compression_zstd.sql @@ -0,0 +1,501 @@ +-- Tests for TOAST compression with zstd + +SELECT NOT(enumvals @> '{zstd}') AS skip_test FROM pg_settings WHERE + name = 'default_toast_compression' \gset +\if :skip_test + \echo '*** skipping TOAST tests with zstd (not supported) ***' + \quit +\endif + +CREATE SCHEMA zstd; +SET search_path TO zstd, public; + +\set HIDE_TOAST_COMPRESSION false + +-- Ensure we get stable results regardless of the installation's default. +-- We rely on this GUC value for a few tests. +SET default_toast_compression = 'pglz'; + +-- Helpers. Hash output is poorly compressible, so values built from it +-- stay large after compression and are stored out of line; a bytea of raw +-- hash bytes is not compressible at all. None of the checks below print +-- compressed sizes, which depend on the zstd version. +CREATE FUNCTION large_val_zstd() RETURNS TEXT LANGUAGE SQL AS +'select array_agg(fipshash(g::text))::text from generate_series(1, 256) g'; +CREATE FUNCTION incompressible_val() RETURNS bytea LANGUAGE SQL AS +$$ select decode(string_agg(fipshash(g::text), '' order by g), 'hex') + from generate_series(1, 200) g $$; + +---------------------------------------------------------------------- +-- DDL +---------------------------------------------------------------------- + +-- test creating table with compression method +CREATE TABLE cmdata_pglz(f1 text COMPRESSION pglz); +CREATE INDEX idx ON cmdata_pglz(f1); +INSERT INTO cmdata_pglz VALUES(repeat('1234567890', 1000)); +\d+ cmdata_pglz +CREATE TABLE cmdata_zstd(f1 TEXT COMPRESSION zstd); +INSERT INTO cmdata_zstd VALUES(repeat('1234567890', 1004)); +\d+ cmdata_zstd + +-- the method name is an identifier, so case does not matter +CREATE TABLE cmupper(f1 text COMPRESSION ZSTD); +\d+ cmupper +DROP TABLE cmupper; + +-- a domain over a varlena type can be compressed too +CREATE DOMAIN zstdtext AS text; +CREATE TABLE cmdomain(f1 zstdtext COMPRESSION zstd); +INSERT INTO cmdomain VALUES (repeat('1234567890', 1004)); +\d+ cmdomain +SELECT pg_column_compression(f1), length(f1) FROM cmdomain; +DROP TABLE cmdomain; +DROP DOMAIN zstdtext; + +-- errors: incompressible data type, unknown method +CREATE TABLE cmbad(f1 int COMPRESSION zstd); +CREATE TABLE cmbad(f1 text); +ALTER TABLE cmbad ALTER COLUMN f1 SET COMPRESSION zstd_nodict; +DROP TABLE cmbad; + +---------------------------------------------------------------------- +-- Compressed-in-line values +---------------------------------------------------------------------- + +-- verify stored compression method in the data, and that compression +-- did save space +SELECT pg_column_compression(f1), + pg_column_size(f1) < length(f1) AS smaller, + pg_column_toast_chunk_id(f1) IS NULL AS is_inline + FROM cmdata_zstd; + +-- decompress data slice +SELECT SUBSTR(f1, 200, 5) FROM cmdata_pglz; +SELECT SUBSTR(f1, 2000, 50) FROM cmdata_zstd; + +-- check that slice decompression agrees with full decompression, at the +-- start, in the middle and at the very end of the value +SELECT SUBSTR(f1, 1, 100) = SUBSTR(repeat('1234567890', 1004), 1, 100) AS ok_head, + SUBSTR(f1, 5000, 100) = SUBSTR(repeat('1234567890', 1004), 5000, 100) AS ok_mid, + SUBSTR(f1, 10036, 5) = SUBSTR(repeat('1234567890', 1004), 10036, 5) AS ok_tail, + length(f1) AS len + FROM cmdata_zstd; + +-- slices of one byte, past the end, and longer than the whole value (the +-- latter falls back to a full decompression) +SELECT left(f1, 1) AS first_char, + right(f1, 1) AS last_char, + SUBSTR(f1, 10040, 1) AS at_end, + SUBSTR(f1, 10041, 10) = '' AS past_end, + length(SUBSTR(f1, 1, 1000000)) AS full_len + FROM cmdata_zstd; + +---------------------------------------------------------------------- +-- Values that do not get compressed +---------------------------------------------------------------------- + +-- values below the TOAST threshold are stored as they are; NULL and the +-- empty string are fine too +CREATE TABLE cmshort(f1 text COMPRESSION zstd); +INSERT INTO cmshort VALUES (repeat('x', 100)), (''), (NULL); +SELECT pg_column_compression(f1), length(f1) FROM cmshort ORDER BY length(f1); +DROP TABLE cmshort; + +-- data that zstd cannot shrink is stored uncompressed, out of line if +-- needed, with no compression method recorded +CREATE TABLE cmincompressible(f1 bytea COMPRESSION zstd); +INSERT INTO cmincompressible SELECT incompressible_val(); +SELECT pg_column_compression(f1) IS NULL AS uncompressed, + pg_column_toast_chunk_id(f1) IS NOT NULL AS is_external, + pg_column_size(f1) = octet_length(f1) AS same_size, + length(f1) AS len, + f1 = incompressible_val() AS ok_full, + substr(f1, 3000, 40) = substr(incompressible_val(), 3000, 40) AS ok_slice + FROM cmincompressible; +DROP TABLE cmincompressible; + +---------------------------------------------------------------------- +-- Other varlena types +---------------------------------------------------------------------- + +-- binary data with zero and high bytes +CREATE TABLE cmbytea(f1 bytea COMPRESSION zstd); +INSERT INTO cmbytea VALUES (decode(repeat('00ff10', 2000), 'hex')); +SELECT pg_column_compression(f1), length(f1), + f1 = decode(repeat('00ff10', 2000), 'hex') AS ok_full, + get_byte(f1, 0) AS byte0, get_byte(f1, 1) AS byte1, + substr(f1, 3001, 3) = decode('00ff10', 'hex') AS ok_slice + FROM cmbytea; +DROP TABLE cmbytea; + +-- jsonb, compressed inline and out of line +CREATE TABLE cmjsonb(j jsonb COMPRESSION zstd); +INSERT INTO cmjsonb SELECT jsonb_build_object('k', repeat('v', 5000), 'n', 1); +INSERT INTO cmjsonb SELECT jsonb_object_agg(g::text, fipshash(g::text)) + FROM generate_series(1, 300) g; +SELECT pg_column_compression(j), + pg_column_toast_chunk_id(j) IS NOT NULL AS is_external, + length(j ->> 'k') AS klen, + j ->> 'n' AS n, + j ->> '150' = fipshash('150') AS ok_150 + FROM cmjsonb ORDER BY is_external; +DROP TABLE cmjsonb; + +-- arrays +CREATE TABLE cmarray(f1 text[] COMPRESSION zstd); +INSERT INTO cmarray SELECT array_agg(repeat('x', 100) || g ORDER BY g) + FROM generate_series(1, 200) g; +SELECT pg_column_compression(f1), array_length(f1, 1) AS n, + f1[1] = repeat('x', 100) || '1' AS ok_first, + f1[200] = repeat('x', 100) || '200' AS ok_last + FROM cmarray; +DROP TABLE cmarray; + +---------------------------------------------------------------------- +-- Moving data between tables and methods +---------------------------------------------------------------------- + +-- copy with table creation +SELECT * INTO cmmove1 FROM cmdata_zstd; +\d+ cmmove1 +SELECT pg_column_compression(f1) FROM cmmove1; + +-- test LIKE INCLUDING COMPRESSION. The GUC default_toast_compression +-- has no effect, the compression method is taken from the table being copied. +CREATE TABLE cmdata2 (LIKE cmdata_zstd INCLUDING COMPRESSION); +\d+ cmdata2 +DROP TABLE cmdata2; + +-- copy to existing table: a compressed datum keeps its own method, in both +-- directions, so a table can hold values compressed with different methods +CREATE TABLE cmmove3(f1 text COMPRESSION pglz); +INSERT INTO cmmove3 SELECT * FROM cmdata_pglz; +INSERT INTO cmmove3 SELECT * FROM cmdata_zstd; +SELECT pg_column_compression(f1) FROM cmmove3 ORDER BY 1; +INSERT INTO cmdata_zstd SELECT * FROM cmdata_pglz; +SELECT pg_column_compression(f1), count(*) FROM cmdata_zstd GROUP BY 1 ORDER BY 1; +DELETE FROM cmdata_zstd WHERE pg_column_compression(f1) = 'pglz'; + +-- update using datum from different table with zstd data. +CREATE TABLE cmmove2(f1 text COMPRESSION pglz); +INSERT INTO cmmove2 VALUES (repeat('1234567890', 1004)); +SELECT pg_column_compression(f1) FROM cmmove2; +UPDATE cmmove2 SET f1 = cmdata_zstd.f1 FROM cmdata_zstd; +SELECT pg_column_compression(f1) FROM cmmove2; + +---------------------------------------------------------------------- +-- Externally stored compressed values +---------------------------------------------------------------------- + +CREATE TABLE cmdata2 (f1 text COMPRESSION zstd); +INSERT INTO cmdata2 SELECT large_val_zstd() || repeat('a', 4000); +SELECT pg_column_compression(f1), pg_column_size(f1) < length(f1) AS smaller + FROM cmdata2; +SELECT SUBSTR(f1, 200, 5) FROM cmdata2; +SELECT SUBSTR(f1, 200, 5) = SUBSTR(large_val_zstd() || repeat('a', 4000), 200, 5) AS ok_slice + FROM cmdata2; +DROP TABLE cmdata2; + +-- test that both TOAST value ID widths carry the compression method through +-- an external TOAST pointer. The data here is poorly compressible on +-- purpose, so that the value is certain to be stored out of line. +CREATE TABLE cmdata_oid(f1 text COMPRESSION zstd) + WITH (toast_value_type = 'oid'); +CREATE TABLE cmdata_oid8(f1 text COMPRESSION zstd) + WITH (toast_value_type = 'oid8'); +INSERT INTO cmdata_oid SELECT large_val_zstd(); +INSERT INTO cmdata_oid8 SELECT large_val_zstd(); +-- confirm the values really are out of line, so that the checks below +-- exercise the external TOAST pointer rather than the inline header +SELECT pg_column_toast_chunk_id(f1) IS NOT NULL AS is_external, + pg_column_compression(f1) FROM cmdata_oid; +SELECT pg_column_toast_chunk_id(f1) IS NOT NULL AS is_external, + pg_column_compression(f1) FROM cmdata_oid8; +SELECT length(f1) = length(large_val_zstd()) AS ok_len, + f1 = large_val_zstd() AS ok_full, + SUBSTR(f1, 3000, 40) = SUBSTR(large_val_zstd(), 3000, 40) AS ok_slice + FROM cmdata_oid; +SELECT length(f1) = length(large_val_zstd()) AS ok_len, + f1 = large_val_zstd() AS ok_full, + SUBSTR(f1, 3000, 40) = SUBSTR(large_val_zstd(), 3000, 40) AS ok_slice + FROM cmdata_oid8; + +-- a value spanning many TOAST chunks, with slices from the front, from +-- near the end (which has to decompress almost everything), and longer +-- than the value +CREATE TABLE cmbig(f1 text COMPRESSION zstd); +INSERT INTO cmbig SELECT string_agg(fipshash(g::text) || repeat('-', 40), '' ORDER BY g) + FROM generate_series(1, 10000) g; +SELECT pg_column_compression(f1), + pg_column_toast_chunk_id(f1) IS NOT NULL AS is_external, + pg_column_size(f1) < length(f1) AS smaller, + length(f1) AS len, + left(f1, 32) = fipshash('1') AS ok_head, + SUBSTR(f1, 360001, 32) = fipshash('5001') AS ok_mid, + SUBSTR(f1, 719929, 32) = fipshash('10000') AS ok_tail, + length(SUBSTR(f1, 1, 2000000)) AS full_len + FROM cmbig; +DROP TABLE cmbig; + +-- a smaller toast_tuple_target makes even a small compressed value go out +-- of line +CREATE TABLE cmtarget(f1 text COMPRESSION zstd) WITH (toast_tuple_target = 128); +INSERT INTO cmtarget SELECT substr(large_val_zstd(), 1, 3000); +SELECT pg_column_compression(f1), + pg_column_toast_chunk_id(f1) IS NOT NULL AS is_external, + f1 = substr(large_val_zstd(), 1, 3000) AS ok_full + FROM cmtarget; +DROP TABLE cmtarget; + +-- an external zstd value copied into a table with a different chunk_id type +-- and compression setting keeps its compression, and is re-externalized +-- with a fresh TOAST pointer; try both chunk_id types as destination +CREATE TABLE cmmove_ext (f1 text COMPRESSION pglz) WITH (toast_value_type = 'oid'); +CREATE TABLE cmmove_ext8 (f1 text COMPRESSION pglz) WITH (toast_value_type = 'oid8'); +INSERT INTO cmmove_ext SELECT f1 FROM cmdata_oid8; +INSERT INTO cmmove_ext8 SELECT f1 FROM cmdata_oid; +SELECT pg_column_toast_chunk_id(f1) IS NOT NULL AS is_external, + pg_column_compression(f1), + f1 = large_val_zstd() AS ok_full + FROM cmmove_ext; +SELECT pg_column_toast_chunk_id(f1) IS NOT NULL AS is_external, + pg_column_compression(f1), + f1 = large_val_zstd() AS ok_full + FROM cmmove_ext8; +DROP TABLE cmmove_ext, cmmove_ext8; + +---------------------------------------------------------------------- +-- Storage modes +---------------------------------------------------------------------- + +-- EXTERNAL: stored out of line but never compressed, slices cross chunk +-- boundaries directly +CREATE TABLE cmstorage(f1 text COMPRESSION zstd); +ALTER TABLE cmstorage ALTER COLUMN f1 SET STORAGE EXTERNAL; +INSERT INTO cmstorage SELECT large_val_zstd(); +SELECT pg_column_compression(f1) IS NULL AS uncompressed, + pg_column_toast_chunk_id(f1) IS NOT NULL AS is_external, + pg_column_size(f1) = octet_length(f1) AS same_size, + SUBSTR(f1, 1990, 20) = SUBSTR(large_val_zstd(), 1990, 20) AS ok_slice + FROM cmstorage; + +-- MAIN: compressed, and kept in the tuple when that makes it fit +TRUNCATE cmstorage; +ALTER TABLE cmstorage ALTER COLUMN f1 SET STORAGE MAIN; +INSERT INTO cmstorage VALUES (repeat('1234567890', 1004)); +SELECT pg_column_compression(f1), + pg_column_toast_chunk_id(f1) IS NULL AS is_inline, + length(f1) AS len + FROM cmstorage; +DROP TABLE cmstorage; + +-- PLAIN: new values are never compressed. An external compressed datum +-- coming from another table is fetched and decompressed before being +-- stored, while a datum that is already compressed in line is stored as it +-- is, like for the other methods. +CREATE TABLE cmsmall(f1 text COMPRESSION zstd) WITH (toast_tuple_target = 128); +INSERT INTO cmsmall VALUES (repeat('1234567890', 300)); +INSERT INTO cmsmall SELECT substr(large_val_zstd(), 1, 3000); +SELECT pg_column_compression(f1), + pg_column_toast_chunk_id(f1) IS NOT NULL AS is_external + FROM cmsmall ORDER BY is_external; +CREATE TABLE cmplain(f1 text COMPRESSION zstd); +ALTER TABLE cmplain ALTER COLUMN f1 SET STORAGE PLAIN; +INSERT INTO cmplain VALUES (repeat('1234567890', 300)); +INSERT INTO cmplain SELECT f1 FROM cmsmall; +SELECT pg_column_compression(f1), length(f1), + f1 IN (repeat('1234567890', 300), substr(large_val_zstd(), 1, 3000)) AS ok_full + FROM cmplain ORDER BY pg_column_compression(f1) NULLS FIRST, length(f1); +DROP TABLE cmplain, cmsmall; + +---------------------------------------------------------------------- +-- Table rewrites and updates of external values +---------------------------------------------------------------------- + +CREATE TABLE cmrewrite(id int, f1 text COMPRESSION zstd) + WITH (toast_value_type = 'oid8'); +INSERT INTO cmrewrite VALUES (1, large_val_zstd()), (2, repeat('1234567890', 1004)); +CREATE INDEX cmrewrite_id ON cmrewrite(id); + +-- VACUUM FULL and CLUSTER preserve the values and do not recompress +VACUUM FULL cmrewrite; +SELECT id, pg_column_compression(f1), + pg_column_toast_chunk_id(f1) IS NOT NULL AS is_external, + f1 = CASE id WHEN 1 THEN large_val_zstd() ELSE repeat('1234567890', 1004) END AS ok_full + FROM cmrewrite ORDER BY id; +CLUSTER cmrewrite USING cmrewrite_id; +SELECT id, pg_column_compression(f1), + pg_column_toast_chunk_id(f1) IS NOT NULL AS is_external, + f1 = CASE id WHEN 1 THEN large_val_zstd() ELSE repeat('1234567890', 1004) END AS ok_full + FROM cmrewrite ORDER BY id; + +-- a column type change resets the column's compression method to the +-- default (as it does its storage), so the rewritten values follow the +-- default_toast_compression GUC; values that were external come back in +-- one piece +ALTER TABLE cmrewrite ALTER COLUMN f1 TYPE varchar(30000); +\d+ cmrewrite +SELECT id, pg_column_compression(f1), + f1 = CASE id WHEN 1 THEN large_val_zstd() ELSE repeat('1234567890', 1004) END AS ok_full + FROM cmrewrite ORDER BY id; +SET default_toast_compression = 'zstd'; +ALTER TABLE cmrewrite ALTER COLUMN f1 TYPE varchar(20000); +SELECT id, pg_column_compression(f1), + f1 = CASE id WHEN 1 THEN large_val_zstd() ELSE repeat('1234567890', 1004) END AS ok_full + FROM cmrewrite ORDER BY id; +SET default_toast_compression = 'pglz'; +DROP TABLE cmrewrite; + +-- an UPDATE that leaves the external value alone must keep the same TOAST +-- value rather than re-toasting it +ALTER TABLE cmdata_oid ADD COLUMN n int DEFAULT 0; +SELECT pg_column_toast_chunk_id(f1) AS chunk_before FROM cmdata_oid \gset +UPDATE cmdata_oid SET n = 1; +SELECT pg_column_toast_chunk_id(f1) = :'chunk_before' AS same_toast_value, + f1 = large_val_zstd() AS ok_full + FROM cmdata_oid; + +-- whereas an UPDATE of the value itself replaces the TOAST value; the old +-- chunks are gone from the TOAST table as soon as the update commits +SELECT reltoastrelid::regclass AS toastrel FROM pg_class + WHERE oid = 'cmdata_oid'::regclass \gset +UPDATE cmdata_oid SET f1 = f1 || 'x'; +SELECT pg_column_toast_chunk_id(f1) <> :'chunk_before' AS new_toast_value, + pg_column_compression(f1), + f1 = large_val_zstd() || 'x' AS ok_full + FROM cmdata_oid; +SELECT count(DISTINCT chunk_id) AS live_toast_values FROM :toastrel; + +-- deleting the row deletes its TOAST value, for both chunk_id types +DELETE FROM cmdata_oid; +SELECT count(*) AS chunks_left FROM :toastrel; +SELECT reltoastrelid::regclass AS toastrel FROM pg_class + WHERE oid = 'cmdata_oid8'::regclass \gset +DELETE FROM cmdata_oid8; +SELECT count(*) AS chunks_left FROM :toastrel; +DROP TABLE cmdata_oid, cmdata_oid8; + +---------------------------------------------------------------------- +-- Indexes +---------------------------------------------------------------------- + +-- a compressible key large enough to need compression to fit in an index +-- entry; the index must be usable to find the row +CREATE TABLE cmindex(f1 text COMPRESSION zstd); +CREATE INDEX cmindex_f1 ON cmindex(f1); +INSERT INTO cmindex VALUES (repeat('abcdefghij', 300)), (repeat('klmnopqrst', 300)); +SET enable_seqscan = off; +SET enable_bitmapscan = off; +SELECT length(f1), left(f1, 10) FROM cmindex WHERE f1 = repeat('klmnopqrst', 300); +RESET enable_seqscan; +RESET enable_bitmapscan; +DROP TABLE cmindex; + +-- test expression index +CREATE TABLE cmdata2 (f1 TEXT COMPRESSION pglz, f2 TEXT COMPRESSION zstd); +CREATE UNIQUE INDEX idx1 ON cmdata2 ((f1 || f2)); +INSERT INTO cmdata2 VALUES((SELECT array_agg(fipshash(g::TEXT))::TEXT FROM +generate_series(1, 50) g), VERSION()); +DROP TABLE cmdata2; + +---------------------------------------------------------------------- +-- Materialized views, partitions, inheritance, temporary tables +---------------------------------------------------------------------- + +-- test compression with materialized view +CREATE MATERIALIZED VIEW compressmv(x) AS SELECT * FROM cmdata_zstd; +\d+ compressmv +SELECT pg_column_compression(f1) FROM cmdata_zstd; +SELECT pg_column_compression(x) FROM compressmv; + +-- test compression with partition +CREATE TABLE cmpart(f1 text COMPRESSION zstd) PARTITION BY HASH(f1); +CREATE TABLE cmpart1 PARTITION OF cmpart FOR VALUES WITH (MODULUS 2, REMAINDER 0); +CREATE TABLE cmpart2(f1 text COMPRESSION pglz); + +ALTER TABLE cmpart ATTACH PARTITION cmpart2 FOR VALUES WITH (MODULUS 2, REMAINDER 1); +INSERT INTO cmpart VALUES (repeat('123456789', 1004)); +INSERT INTO cmpart VALUES (repeat('123456789', 4004)); +SELECT pg_column_compression(f1) FROM cmpart1; +SELECT pg_column_compression(f1) FROM cmpart2; + +-- test compression with inheritance +CREATE TABLE cminh() INHERITS(cmdata_pglz, cmdata_zstd); -- error +CREATE TABLE cminh(f1 TEXT COMPRESSION zstd) INHERITS(cmdata_pglz); -- error +CREATE TABLE cmdata3(f1 text); +CREATE TABLE cminh() INHERITS (cmdata_pglz, cmdata3); + +-- temporary table +CREATE TEMP TABLE cmtemp(f1 text COMPRESSION zstd); +INSERT INTO cmtemp SELECT large_val_zstd(); +SELECT pg_column_compression(f1), + pg_column_toast_chunk_id(f1) IS NOT NULL AS is_external, + f1 = large_val_zstd() AS ok_full + FROM cmtemp; +DROP TABLE cmtemp; + +---------------------------------------------------------------------- +-- default_toast_compression GUC +---------------------------------------------------------------------- + +-- test default_toast_compression GUC; the value is case-insensitive +SET default_toast_compression = 'ZSTD'; +SHOW default_toast_compression; +SET default_toast_compression = 'zstd'; + +-- the GUC drives the method for columns with no explicit COMPRESSION, so +-- the column keeps a default attcompression but the data comes out zstd +CREATE TABLE cmdata_default(f1 text); +INSERT INTO cmdata_default VALUES(repeat('1234567890', 1004)); +\d+ cmdata_default +SELECT pg_column_compression(f1) FROM cmdata_default; +SELECT f1 = repeat('1234567890', 1004) AS ok_full, + SUBSTR(f1, 4000, 20) = SUBSTR(repeat('1234567890', 1004), 4000, 20) AS ok_slice + FROM cmdata_default; + +-- SET COMPRESSION default makes a column follow the GUC again +ALTER TABLE cmdata_default ALTER COLUMN f1 SET COMPRESSION pglz; +INSERT INTO cmdata_default VALUES(repeat('1234567890', 1004)); +ALTER TABLE cmdata_default ALTER COLUMN f1 SET COMPRESSION default; +\d+ cmdata_default +INSERT INTO cmdata_default VALUES(repeat('1234567890', 1004)); +SELECT pg_column_compression(f1), count(*) FROM cmdata_default GROUP BY 1 ORDER BY 1; +DROP TABLE cmdata_default; + +-- test alter compression method +ALTER TABLE cmdata_pglz ALTER COLUMN f1 SET COMPRESSION zstd; +INSERT INTO cmdata_pglz VALUES (repeat('123456789', 4004)); +\d+ cmdata_pglz +SELECT pg_column_compression(f1) FROM cmdata_pglz; +ALTER TABLE cmdata_pglz ALTER COLUMN f1 SET COMPRESSION pglz; + +-- test alter compression method for materialized views +ALTER MATERIALIZED VIEW compressmv ALTER COLUMN x SET COMPRESSION zstd; +\d+ compressmv + +-- test alter compression method for partitioned tables +ALTER TABLE cmpart1 ALTER COLUMN f1 SET COMPRESSION pglz; +ALTER TABLE cmpart2 ALTER COLUMN f1 SET COMPRESSION zstd; + +-- new data should be compressed with the current compression method +INSERT INTO cmpart VALUES (repeat('123456789', 1004)); +INSERT INTO cmpart VALUES (repeat('123456789', 4004)); +SELECT pg_column_compression(f1) FROM cmpart1; +SELECT pg_column_compression(f1) FROM cmpart2; + +-- VACUUM FULL does not recompress +SELECT pg_column_compression(f1) FROM cmdata_zstd; +VACUUM FULL cmdata_zstd; +SELECT pg_column_compression(f1) FROM cmdata_zstd; + +RESET default_toast_compression; + +-- check data is ok +SELECT length(f1) FROM cmdata_pglz; +SELECT length(f1) FROM cmdata_zstd; +SELECT length(f1) FROM cmmove1; +SELECT length(f1) FROM cmmove2; +SELECT length(f1) FROM cmmove3; + +\set HIDE_TOAST_COMPRESSION true -- 2.54.0 (Apple Git-157)