From b3e624be3cd4ecba7ecb2e3ffdf4c32e27d0c1f6 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Thu, 15 Oct 2026 00:11:33 -0400 Subject: [PATCH v20261015 4/4] Add zstd compression dictionaries with a column-local lifecycle This adds a TOAST compression method, zstd_dict, that compresses with a zstd dictionary trained for the column. It builds on the extensible method encoding: a zstd_dict datum is the long form of the compressed header followed by the 4-byte OID of its dictionary. The difficulty with dictionaries was never compression, it was knowing when a dictionary can be dropped. Earlier proposals either kept every dictionary forever, or recorded in pg_depend every table a compressed value might have been copied into; both were found wanting (dictionaries that cannot be dropped accumulate, e.g. with tenants coming and going; dependencies spread to every table and cannot be maintained under RowExclusiveLock, and no list of "copy" commands is complete). This patch instead keeps dictionary-compressed values inside the column that owns their dictionary, which makes the lifecycle local: - Ownership. pg_toast_dictionary rows belong to one (relid, attnum), like pg_statistic rows, and are deleted with the column or table. No pg_depend entries. - Containment. heap_fill_tuple() sets HEAP_HASEXTERNAL for a tuple holding a zstd_dict datum, so every path that already makes a tuple self-contained before it can outlive its source (composite datums, tuplestores and holdable cursors, PL/pgSQL records and non-atomic variables, index and BRIN tuples) reaches the datum, and detoast_external_attr() decompresses it as it would fetch a TOAST pointer. When storing into a heap, toast_tuple_init() keeps a zstd_dict datum compressed only if the target column owns its dictionary and the dictionary is still current; otherwise it is decompressed and then compressed, if at all, with the target column's method. Copies of values compressed with any other method are unaffected. - Retirement. pg_train_toast_dictionary() makes a new dictionary current and marks the previous one non-current. The next table rewrite recompresses every value using a non-current dictionary (because toast_dictionary_keep() rejects it), so finish_heap_swap() deletes those dictionaries afterwards; likewise those of a column switched to another compression method. pg_upgrade keeps the heap files, so binary-upgrade dumps restore dictionaries with their OIDs. Training is an explicit call for now; doing it from ANALYZE's sample is the obvious next step once the lifecycle is agreed on. Discussion: https://postgr.es/m/CAFAfj_HX84EK4hyRYw50AOHOcdVi-+FFwAAPo7JHx4aShCvunQ@mail.gmail.com Discussion: https://postgr.es/m/CAFAfj_GGY4W81XjtYLvYOhqPtvfZbBNj=YRS6Mq_0SA9AqEwoQ@mail.gmail.com --- contrib/amcheck/verify_heapam.c | 1 + doc/src/sgml/catalogs.sgml | 95 +++ doc/src/sgml/func/func-admin.sgml | 23 + doc/src/sgml/ref/alter_table.sgml | 8 + doc/src/sgml/ref/create_table.sgml | 8 + src/backend/access/brin/brin_tuple.c | 3 +- src/backend/access/common/Makefile | 1 + src/backend/access/common/detoast.c | 24 + src/backend/access/common/heaptuple.c | 12 + src/backend/access/common/indextuple.c | 3 +- src/backend/access/common/meson.build | 1 + src/backend/access/common/toast_compression.c | 9 + src/backend/access/common/toast_dictionary.c | 669 ++++++++++++++++++ src/backend/access/common/toast_internals.c | 36 +- src/backend/access/heap/heaptoast.c | 7 +- src/backend/access/table/toast_helper.c | 25 +- src/backend/catalog/heap.c | 13 + src/backend/catalog/system_views.sql | 3 + src/backend/commands/repack.c | 8 + src/backend/executor/tstoreReceiver.c | 4 +- src/backend/utils/adt/expandedrecord.c | 16 +- src/backend/utils/adt/varlena.c | 3 + src/bin/pg_dump/pg_dump.c | 65 ++ src/bin/psql/describe.c | 5 +- src/bin/psql/tab-complete.in.c | 2 +- src/include/access/toast_compression.h | 16 + src/include/access/toast_dictionary.h | 41 ++ src/include/access/toast_internals.h | 2 + src/include/catalog/Makefile | 1 + src/include/catalog/meson.build | 1 + src/include/catalog/pg_proc.dat | 11 + src/include/catalog/pg_toast_dictionary.h | 65 ++ src/pl/plpgsql/src/pl_exec.c | 4 +- .../expected/compression_zstd_dict.out | 414 +++++++++++ .../expected/compression_zstd_dict_1.out | 10 + src/test/regress/expected/oidjoins.out | 1 + src/test/regress/parallel_schedule | 2 +- .../regress/sql/compression_zstd_dict.sql | 203 ++++++ src/tools/pgindent/typedefs.list | 4 + 39 files changed, 1802 insertions(+), 17 deletions(-) create mode 100644 src/backend/access/common/toast_dictionary.c create mode 100644 src/include/access/toast_dictionary.h create mode 100644 src/include/catalog/pg_toast_dictionary.h create mode 100644 src/test/regress/expected/compression_zstd_dict.out create mode 100644 src/test/regress/expected/compression_zstd_dict_1.out create mode 100644 src/test/regress/sql/compression_zstd_dict.sql diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c index 9828ba7e03d..6d86a383c8b 100644 --- a/contrib/amcheck/verify_heapam.c +++ b/contrib/amcheck/verify_heapam.c @@ -1832,6 +1832,7 @@ check_tuple_attribute(HeapCheckContext *ctx) case TOAST_PGLZ_COMPRESSION_ID: case TOAST_LZ4_COMPRESSION_ID: case TOAST_ZSTD_COMPRESSION_ID: + case TOAST_ZSTD_DICT_COMPRESSION_ID: valid = (toast_compression_id_needs_cmid_byte(cmid) == VARTAG_IS_ONDISK_LONG(va_tag_value)); break; diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 9e44bbf71d5..b8d4ca98fdf 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -325,6 +325,11 @@ tablespaces within this database cluster + + pg_toast_dictionary + compression dictionaries for TOAST + + pg_transform transforms (data type to procedural language conversions) @@ -8487,6 +8492,96 @@ SCRAM-SHA-256$<iteration count>:&l + + <structname>pg_toast_dictionary</structname> + + + pg_toast_dictionary + + + + The catalog pg_toast_dictionary stores the + compression dictionaries used by the zstd_dict TOAST + compression method. Each dictionary belongs to one column of one table; + it is created by pg_train_toast_dictionary and + removed when its column or table is dropped, or when a rewrite of the + table has recompressed every value that used it. Values compressed with + a dictionary are never stored outside the column that owns it. + + + + Since dictionaries are built from the contents of the column, this + catalog is not readable by the public. + + + + <structname>pg_toast_dictionary</structname> Columns + + + + + Column Type + + + Description + + + + + + + + oid oid + + + Dictionary identifier, as recorded in each value compressed with it + + + + + + tdrelid oid + (references pg_class.oid) + + + The table owning the dictionary + + + + + + tdattnum int2 + (references pg_attribute.attnum) + + + The column owning the dictionary + + + + + + tdcurrent bool + + + True for the dictionary used to compress new values; a column has at + most one. Other dictionaries of the column are kept only until the + next rewrite of the table. + + + + + + tddict bytea + + + Dictionary content, in the format of the zstd library + + + + +
+ + <structname>pg_transform</structname> diff --git a/doc/src/sgml/func/func-admin.sgml b/doc/src/sgml/func/func-admin.sgml index 64b0e7bb972..84ed7941412 100644 --- a/doc/src/sgml/func/func-admin.sgml +++ b/doc/src/sgml/func/func-admin.sgml @@ -1586,6 +1586,29 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset + + + + pg_train_toast_dictionary + + pg_train_toast_dictionary ( rel regclass, col name , dict_size integer DEFAULT 65536 ) + oid + + + Trains a compression dictionary of at most + dict_size bytes from the current values of + column col, which must use the + zstd_dict compression method, and makes it the + dictionary used to compress new values of that column. Returns the + dictionary's OID, or NULL if no dictionary could + be trained from the values found. The column's previous dictionary, + if any, remains in use for the values compressed with it until the + next rewrite of the table, such as VACUUM FULL, + which recompresses them and removes it. Only the table's owner can + call this function. + + + diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml index 004bfe58812..cfd7f28e7b7 100644 --- a/doc/src/sgml/ref/alter_table.sgml +++ b/doc/src/sgml/ref/alter_table.sgml @@ -475,6 +475,14 @@ WITH ( MODULUS numeric_literal, REM consulting the setting at the time of data insertion to determine the method to use. + + zstd_dict is zstd with a + compression dictionary trained for the column by + pg_train_toast_dictionary; until one is trained it + behaves like zstd. Values compressed with a + dictionary are decompressed when copied to any other column or table. + See pg_toast_dictionary. + diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index 7340c61c1b1..635c2ca53e8 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -375,6 +375,14 @@ WITH ( MODULUS numeric_literal, REM setting at the time of data insertion to determine the method to use. + + zstd_dict is zstd with a + compression dictionary trained for the column by + pg_train_toast_dictionary; until one is trained it + behaves like zstd. Values compressed with a + dictionary are decompressed when copied to any other column or table. + See pg_toast_dictionary. + diff --git a/src/backend/access/brin/brin_tuple.c b/src/backend/access/brin/brin_tuple.c index 091cf71f585..37be5d566c0 100644 --- a/src/backend/access/brin/brin_tuple.c +++ b/src/backend/access/brin/brin_tuple.c @@ -206,7 +206,8 @@ brin_form_tuple(BrinDesc *brdesc, BlockNumber blkno, BrinMemTuple *tuple, * even for range with non-NULL data? E.g. degenerate bloom filter * may be thrown away, etc. */ - if (VARATT_IS_EXTERNAL(DatumGetPointer(value))) + if (VARATT_IS_EXTERNAL(DatumGetPointer(value)) || + VARATT_IS_COMPRESSED_DICT(DatumGetPointer(value))) { value = PointerGetDatum(detoast_external_attr((varlena *) DatumGetPointer(value))); diff --git a/src/backend/access/common/Makefile b/src/backend/access/common/Makefile index e78de312659..6f3e0a796a9 100644 --- a/src/backend/access/common/Makefile +++ b/src/backend/access/common/Makefile @@ -27,6 +27,7 @@ OBJS = \ syncscan.o \ tidstore.o \ toast_compression.o \ + toast_dictionary.o \ toast_internals.o \ tupconvert.o \ tupdesc.o diff --git a/src/backend/access/common/detoast.c b/src/backend/access/common/detoast.c index 2818ad3848f..1d2e7c615f4 100644 --- a/src/backend/access/common/detoast.c +++ b/src/backend/access/common/detoast.c @@ -17,6 +17,7 @@ #include "access/table.h" #include "access/tableam.h" #include "access/toast_compression.h" +#include "access/toast_dictionary.h" #include "access/toast_internals.h" #include "common/int.h" #include "common/pg_lzcompress.h" @@ -53,6 +54,15 @@ detoast_external_attr(varlena *attr) * This is an external stored plain value */ result = toast_fetch_datum(attr); + + /* see below about values compressed with a dictionary */ + if (VARATT_IS_COMPRESSED_DICT(result)) + { + varlena *tmp = result; + + result = zstd_dict_decompress_datum(tmp); + pfree(tmp); + } } else if (VARATT_IS_EXTERNAL_INDIRECT(attr)) { @@ -91,6 +101,16 @@ detoast_external_attr(varlena *attr) result = (varlena *) palloc(resultsize); EOH_flatten_into(eoh, result, resultsize); } + else if (VARATT_IS_COMPRESSED_DICT(attr)) + { + /* + * A value compressed with a dictionary depends on a catalog row of + * the column it came from, so for anything that wants a value + * independent of its source it counts as external; see + * toast_dictionary.c. + */ + result = zstd_dict_decompress_datum(attr); + } else { /* @@ -501,6 +521,8 @@ toast_decompress_datum(varlena *attr) return lz4_decompress_datum(attr); case TOAST_ZSTD_COMPRESSION_ID: return zstd_decompress_datum(attr); + case TOAST_ZSTD_DICT_COMPRESSION_ID: + return zstd_dict_decompress_datum(attr); default: elog(ERROR, "invalid compression method id %d", cmid); return NULL; /* keep compiler quiet */ @@ -546,6 +568,8 @@ toast_decompress_datum_slice(varlena *attr, int32 slicelength) return lz4_decompress_datum_slice(attr, slicelength); case TOAST_ZSTD_COMPRESSION_ID: return zstd_decompress_datum_slice(attr, slicelength); + case TOAST_ZSTD_DICT_COMPRESSION_ID: + return zstd_dict_decompress_datum_slice(attr, slicelength); default: elog(ERROR, "invalid compression method id %d", cmid); return NULL; /* keep compiler quiet */ diff --git a/src/backend/access/common/heaptuple.c b/src/backend/access/common/heaptuple.c index f30346469ed..1489d3d7854 100644 --- a/src/backend/access/common/heaptuple.c +++ b/src/backend/access/common/heaptuple.c @@ -59,6 +59,7 @@ #include "access/heaptoast.h" #include "access/sysattr.h" +#include "access/toast_compression.h" #include "access/tupdesc_details.h" #include "common/hashfn.h" #include "utils/datum.h" @@ -365,6 +366,17 @@ fill_val(CompactAttribute *att, data = (char *) att_nominal_alignby(data, att->attalignby); data_length = VARSIZE(val); memcpy(data, val, data_length); + + /* + * A value compressed with a dictionary depends on catalog state + * of the column it belongs to, like a TOAST pointer does on its + * TOAST table, so flag it the same way. Code that makes a tuple + * self-contained (toast_flatten_tuple and friends) tests + * HEAP_HASEXTERNAL and then detoast_external_attr() on each + * field, which decompresses it. + */ + if (VARATT_IS_COMPRESSED_DICT(val)) + *infomask |= HEAP_HASEXTERNAL; } } else if (att->attlen == -2) diff --git a/src/backend/access/common/indextuple.c b/src/backend/access/common/indextuple.c index 60bba0a2145..6b4decf12e0 100644 --- a/src/backend/access/common/indextuple.c +++ b/src/backend/access/common/indextuple.c @@ -105,7 +105,8 @@ index_form_tuple_context(TupleDesc tupleDescriptor, * If value is stored EXTERNAL, must fetch it so we are not depending * on outside storage. This should be improved someday. */ - if (VARATT_IS_EXTERNAL(DatumGetPointer(values[i]))) + if (VARATT_IS_EXTERNAL(DatumGetPointer(values[i])) || + VARATT_IS_COMPRESSED_DICT(DatumGetPointer(values[i]))) { untoasted_values[i] = PointerGetDatum(detoast_external_attr((varlena *) diff --git a/src/backend/access/common/meson.build b/src/backend/access/common/meson.build index 35e89b5ea67..921f0c6f8b0 100644 --- a/src/backend/access/common/meson.build +++ b/src/backend/access/common/meson.build @@ -15,6 +15,7 @@ backend_sources += files( 'syncscan.c', 'tidstore.c', 'toast_compression.c', + 'toast_dictionary.c', 'toast_internals.c', 'tupconvert.c', 'tupdesc.c', diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c index 5c54f43db30..abe27b9d473 100644 --- a/src/backend/access/common/toast_compression.c +++ b/src/backend/access/common/toast_compression.c @@ -503,6 +503,13 @@ CompressionNameToMethod(const char *compression) #endif return TOAST_ZSTD_COMPRESSION; } + else if (strcmp(compression, "zstd_dict") == 0) + { +#ifndef USE_ZSTD + NO_COMPRESSION_SUPPORT("zstd"); +#endif + return TOAST_ZSTD_DICT_COMPRESSION; + } return InvalidCompressionMethod; } @@ -521,6 +528,8 @@ GetCompressionMethodName(char method) return "lz4"; case TOAST_ZSTD_COMPRESSION: return "zstd"; + case TOAST_ZSTD_DICT_COMPRESSION: + return "zstd_dict"; default: elog(ERROR, "invalid compression method %c", method); return NULL; /* keep compiler quiet */ diff --git a/src/backend/access/common/toast_dictionary.c b/src/backend/access/common/toast_dictionary.c new file mode 100644 index 00000000000..ecd1c07308e --- /dev/null +++ b/src/backend/access/common/toast_dictionary.c @@ -0,0 +1,669 @@ +/*------------------------------------------------------------------------- + * + * toast_dictionary.c + * Lifecycle of zstd compression dictionaries for TOAST. + * + * A dictionary-compressed datum cannot be read without its dictionary, so the + * hard part of dictionary support is knowing when a dictionary may be + * dropped. Designs discussed on pgsql-hackers either kept every dictionary + * forever or tracked, through pg_depend, every table a compressed value might + * have been copied into; both were rejected. This file keeps + * dictionary-compressed values inside the column that owns their dictionary + * instead, which makes the lifecycle local to that column: + * + * Ownership. A pg_toast_dictionary row belongs to one (relid, attnum), like + * a pg_statistic row. DROP TABLE and DROP COLUMN delete it through + * RemoveToastDictionaries(); nothing else ever references it. + * + * Containment. heap_fill_tuple() sets HEAP_HASEXTERNAL for a tuple holding + * a zstd_dict datum, as it does for TOAST pointers, so every path that + * already makes a tuple self-contained before it outlives its source + * (composite datums, tuplestores, holdable cursors, syscache entries, + * PL/pgSQL records) also reaches the datum. detoast_external_attr() then + * decompresses it, exactly as it fetches a TOAST pointer. When a datum is + * stored into a heap table, toast_tuple_init() keeps it compressed only if + * the target column owns its dictionary (toast_dictionary_keep()), and + * decompresses it otherwise, so it is recompressed, if at all, with the + * target column's own method. The same applies to index and BRIN tuples, + * which inline external values already. + * + * Retirement. Training a dictionary marks the previous one of the column + * non-current. A non-current dictionary stays readable until the table is + * rewritten (VACUUM FULL, CLUSTER, REPACK, rewriting ALTER TABLE): during the + * rewrite toast_dictionary_keep() rejects it, so every value using it is + * recompressed with the current one, and finish_heap_swap() then deletes it + * with toast_dictionary_retire(). + * + * The cost is only paid when a dictionary-compressed value crosses a column + * boundary. Copies of values compressed with any other method are unchanged. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/access/common/toast_dictionary.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#ifdef USE_ZSTD +#include +#include +#endif + +#include "access/detoast.h" +#include "access/genam.h" +#include "access/htup_details.h" +#include "access/table.h" +#include "access/tableam.h" +#include "access/toast_compression.h" +#include "access/toast_dictionary.h" +#include "catalog/catalog.h" +#include "catalog/indexing.h" +#include "catalog/objectaddress.h" +#include "catalog/pg_toast_dictionary.h" +#include "executor/tuptable.h" +#include "fmgr.h" +#include "lib/stringinfo.h" +#include "miscadmin.h" +#include "utils/acl.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/inval.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" +#include "utils/snapmgr.h" +#include "utils/syscache.h" + +#define NO_ZSTD_SUPPORT() \ + ereport(ERROR, \ + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), \ + errmsg("compression method %s not supported", "zstd_dict"), \ + errdetail("This functionality requires the server to be built with %s support.", "zstd"))) + +#define DICT_MIN_SIZE 256 +#define DICT_MAX_SIZE (1024 * 1024) +#define DICT_MAX_SAMPLES 10000 +#define DICT_SAMPLE_LIMIT (128 * 1024) +/* zstd recommends samples totalling about 100 times the dictionary size */ +#define DICT_SAMPLE_FACTOR 100 + +/* + * The relation owning the dictionaries of rel's columns. During a table + * rewrite, values are written into a transient heap that stands for the + * table being rewritten. + */ +static Oid +dict_owner_relid(Relation rel) +{ + if (OidIsValid(rel->rd_rel->relrewrite)) + return rel->rd_rel->relrewrite; + return RelationGetRelid(rel); +} + +/* + * Does relid.attnum (still) use zstd_dict? A dropped column does not: a + * rewrite has replaced its values with nulls. + */ +static bool +dict_column_uses_dict(Oid relid, AttrNumber attnum) +{ + HeapTuple tp; + Form_pg_attribute att; + bool result; + + tp = SearchSysCache2(ATTNUM, ObjectIdGetDatum(relid), + Int16GetDatum(attnum)); + if (!HeapTupleIsValid(tp)) + return false; + att = (Form_pg_attribute) GETSTRUCT(tp); + result = !att->attisdropped && + att->attcompression == TOAST_ZSTD_DICT_COMPRESSION; + ReleaseSysCache(tp); + return result; +} + +/* Look up one dictionary row of rel.attnum; current_only selects tdcurrent */ +static HeapTuple +dict_lookup(Oid relid, AttrNumber attnum, bool current_only, Relation drel, + SysScanDesc *scanp) +{ + ScanKeyData key[2]; + HeapTuple tup; + + ScanKeyInit(&key[0], Anum_pg_toast_dictionary_tdrelid, + BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); + ScanKeyInit(&key[1], Anum_pg_toast_dictionary_tdattnum, + BTEqualStrategyNumber, F_INT2EQ, Int16GetDatum(attnum)); + *scanp = systable_beginscan(drel, ToastDictionaryRelidAttnumIndexId, true, + NULL, attnum != 0 ? 2 : 1, key); + while (HeapTupleIsValid(tup = systable_getnext(*scanp))) + { + if (!current_only || + ((Form_pg_toast_dictionary) GETSTRUCT(tup))->tdcurrent) + return tup; + } + return NULL; +} + +/* + * Current dictionary of a column, or InvalidOid. + * ponytail: catalog index probe per compressed value; cache the OID in the + * relcache entry if it ever shows in a profile. + */ +Oid +toast_dictionary_current(Relation rel, AttrNumber attnum) +{ + Relation drel; + SysScanDesc scan; + HeapTuple tup; + Oid result = InvalidOid; + + drel = table_open(ToastDictionaryRelationId, AccessShareLock); + tup = dict_lookup(dict_owner_relid(rel), attnum, true, drel, &scan); + if (tup) + result = ((Form_pg_toast_dictionary) GETSTRUCT(tup))->oid; + systable_endscan(scan); + table_close(drel, AccessShareLock); + return result; +} + +/* Dictionary OID of a compressed-in-line zstd_dict datum */ +static Oid +dict_datum_get_id(const varlena *value) +{ + Oid dictid; + + if (VARSIZE(value) < VARHDRSZ_COMPRESSED_DICT) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg_internal("compressed zstd data is corrupt"))); + memcpy(&dictid, (const char *) value + VARHDRSZ_COMPRESSED_LONG, + sizeof(dictid)); + return dictid; +} + +/* + * May this compressed-in-line zstd_dict datum be stored as-is into + * rel.attnum? Only if that column owns the dictionary, the dictionary is + * still current, and the column still uses zstd_dict. Anything else is + * recompressed whenever it is written, which is what lets a table rewrite + * retire dictionaries. + */ +bool +toast_dictionary_keep(Relation rel, AttrNumber attnum, const varlena *value) +{ + HeapTuple tup; + bool result = false; + Oid relid = dict_owner_relid(rel); + + if (TupleDescAttr(RelationGetDescr(rel), attnum - 1)->attcompression != + TOAST_ZSTD_DICT_COMPRESSION) + return false; + + tup = SearchSysCache1(TOASTDICTOID, + ObjectIdGetDatum(dict_datum_get_id(value))); + if (HeapTupleIsValid(tup)) + { + Form_pg_toast_dictionary d = (Form_pg_toast_dictionary) GETSTRUCT(tup); + + result = (d->tdrelid == relid && d->tdattnum == attnum && + d->tdcurrent); + ReleaseSysCache(tup); + } + return result; +} + +#ifdef USE_ZSTD +/* + * Per-backend cache of digested dictionaries. Digesting builds the entropy + * tables and costs far more than one small value's compression, so the + * ZSTD_CDict/ZSTD_DDict objects are kept and dropped on invalidation of + * pg_toast_dictionary. + * ponytail: one entry per direction, enough for a scan of one column; a small + * hash table if workloads mixing many dictionaries thrash it. + */ +static Oid cdict_id = InvalidOid; +static ZSTD_CDict *cdict = NULL; +static Oid ddict_id = InvalidOid; +static ZSTD_DDict *ddict = NULL; +static ZSTD_CCtx *dict_cctx = NULL; +static ZSTD_DCtx *dict_dctx = NULL; + +static void +dict_cache_invalidate(Datum arg, SysCacheIdentifier cacheid, uint32 hashvalue) +{ + if (cdict) + ZSTD_freeCDict(cdict); + if (ddict) + ZSTD_freeDDict(ddict); + cdict = NULL; + ddict = NULL; + cdict_id = ddict_id = InvalidOid; +} + +static bytea * +dict_fetch(Oid dictid) +{ + static bool registered = false; + HeapTuple tup; + bytea *result; + + if (!registered) + { + CacheRegisterSyscacheCallback(TOASTDICTOID, dict_cache_invalidate, + (Datum) 0); + registered = true; + } + + tup = SearchSysCache1(TOASTDICTOID, ObjectIdGetDatum(dictid)); + if (!HeapTupleIsValid(tup)) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("compression dictionary %u does not exist", dictid))); + result = DatumGetByteaPCopy(SysCacheGetAttrNotNull(TOASTDICTOID, tup, + Anum_pg_toast_dictionary_tddict)); + ReleaseSysCache(tup); + return result; +} + +static ZSTD_CDict * +dict_get_cdict(Oid dictid) +{ + if (cdict_id != dictid) + { + bytea *d = dict_fetch(dictid); + + if (cdict) + ZSTD_freeCDict(cdict); + cdict_id = InvalidOid; + cdict = ZSTD_createCDict(VARDATA_ANY(d), VARSIZE_ANY_EXHDR(d), + ZSTD_CLEVEL_DEFAULT); + pfree(d); + if (cdict == NULL) + elog(ERROR, "could not load compression dictionary %u", dictid); + cdict_id = dictid; + } + if (dict_cctx == NULL && (dict_cctx = ZSTD_createCCtx()) == NULL) + elog(ERROR, "could not create zstd compression context"); + return cdict; +} + +static ZSTD_DDict * +dict_get_ddict(Oid dictid) +{ + if (ddict_id != dictid) + { + bytea *d = dict_fetch(dictid); + + if (ddict) + ZSTD_freeDDict(ddict); + ddict_id = InvalidOid; + ddict = ZSTD_createDDict(VARDATA_ANY(d), VARSIZE_ANY_EXHDR(d)); + pfree(d); + if (ddict == NULL) + elog(ERROR, "could not load compression dictionary %u", dictid); + ddict_id = dictid; + } + if (dict_dctx == NULL && (dict_dctx = ZSTD_createDCtx()) == NULL) + elog(ERROR, "could not create zstd decompression context"); + return ddict; +} +#endif /* USE_ZSTD */ + +/* + * Compress with the given dictionary. The datum is laid out with + * VARHDRSZ_COMPRESSED_DICT bytes of header; toast_compress_datum() fills in + * va_tcinfo and the method byte. + */ +varlena * +zstd_dict_compress_datum(const varlena *value, Oid dictid) +{ +#ifndef USE_ZSTD + NO_ZSTD_SUPPORT(); + return NULL; +#else + int32 valsize = VARSIZE_ANY_EXHDR(value); + size_t max_size = ZSTD_compressBound(valsize); + ZSTD_CDict *cd = dict_get_cdict(dictid); + varlena *tmp; + size_t len; + + tmp = (varlena *) palloc(max_size + VARHDRSZ_COMPRESSED_DICT); + len = ZSTD_compress_usingCDict(dict_cctx, + (char *) tmp + VARHDRSZ_COMPRESSED_DICT, + max_size, VARDATA_ANY(value), valsize, cd); + if (ZSTD_isError(len)) + elog(ERROR, "zstd compression failed: %s", ZSTD_getErrorName(len)); + if (len > (size_t) valsize) + { + pfree(tmp); + return NULL; + } + memcpy((char *) tmp + VARHDRSZ_COMPRESSED_LONG, &dictid, sizeof(dictid)); + SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_COMPRESSED_DICT); + return tmp; +#endif +} + +varlena * +zstd_dict_decompress_datum(const varlena *value) +{ +#ifndef USE_ZSTD + NO_ZSTD_SUPPORT(); + return NULL; +#else + Oid dictid = dict_datum_get_id(value); + uint32 extsize = VARDATA_COMPRESSED_GET_EXTSIZE(value); + ZSTD_DDict *dd = dict_get_ddict(dictid); + varlena *result; + size_t rawsize; + + result = (varlena *) palloc(extsize + VARHDRSZ); + rawsize = ZSTD_decompress_usingDDict(dict_dctx, VARDATA(result), extsize, + (const char *) value + VARHDRSZ_COMPRESSED_DICT, + VARSIZE(value) - VARHDRSZ_COMPRESSED_DICT, + dd); + if (ZSTD_isError(rawsize) || rawsize != extsize) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg_internal("compressed zstd data is corrupt"))); + SET_VARSIZE(result, rawsize + VARHDRSZ); + return result; +#endif +} + +/* + * Prefix decompression. For values below zstd's 128kB block size a prefix + * costs a full decode anyway (see zstd_decompress_datum_slice). + * ponytail: full decode for larger values too; use the streaming loop of + * zstd_decompress_datum_slice() with ZSTD_DCtx_refDDict() if prefixes of + * large dictionary-compressed values matter. + */ +varlena * +zstd_dict_decompress_datum_slice(const varlena *value, int32 slicelength) +{ + varlena *result = zstd_dict_decompress_datum(value); + + if (VARSIZE(result) - VARHDRSZ > slicelength) + SET_VARSIZE(result, slicelength + VARHDRSZ); + return result; +} + +/* + * Delete the dictionaries of a relation (attnum == 0) or of one column, next + * to RemoveStatistics(). The values that used them go away at the same time. + */ +void +RemoveToastDictionaries(Oid relid, AttrNumber attnum) +{ + Relation drel; + SysScanDesc scan; + HeapTuple tup; + + drel = table_open(ToastDictionaryRelationId, RowExclusiveLock); + tup = dict_lookup(relid, attnum, false, drel, &scan); + for (; HeapTupleIsValid(tup); tup = systable_getnext(scan)) + CatalogTupleDelete(drel, &tup->t_self); + systable_endscan(scan); + table_close(drel, RowExclusiveLock); +} + +/* + * After relid has been rewritten, no stored value references a dictionary + * that toast_dictionary_keep() rejects: non-current ones, and all of those + * of a column that no longer uses zstd_dict. The rewrite recompressed those + * values, so delete such dictionaries. Caller holds AccessExclusiveLock on + * relid. + */ +void +toast_dictionary_retire(Oid relid) +{ + Relation drel; + SysScanDesc scan; + HeapTuple tup; + + drel = table_open(ToastDictionaryRelationId, RowExclusiveLock); + tup = dict_lookup(relid, 0, false, drel, &scan); + for (; HeapTupleIsValid(tup); tup = systable_getnext(scan)) + { + Form_pg_toast_dictionary d = (Form_pg_toast_dictionary) GETSTRUCT(tup); + + if (!d->tdcurrent || !dict_column_uses_dict(relid, d->tdattnum)) + CatalogTupleDelete(drel, &tup->t_self); + } + systable_endscan(scan); + table_close(drel, RowExclusiveLock); +} + +/* + * Insert a dictionary for rel.attnum and make it current, retiring the + * previous current one. dictid may be given (binary upgrade) or chosen here. + */ +static Oid +dict_install(Relation rel, AttrNumber attnum, Oid dictid, bool current, + const void *dict, size_t dictlen) +{ + Relation drel; + SysScanDesc scan; + HeapTuple tup; + Datum values[Natts_pg_toast_dictionary]; + bool nulls[Natts_pg_toast_dictionary] = {0}; + bytea *dictval; + + drel = table_open(ToastDictionaryRelationId, RowExclusiveLock); + + if (current) + { + tup = dict_lookup(RelationGetRelid(rel), attnum, true, drel, &scan); + if (tup) + { + HeapTuple newtup = heap_copytuple(tup); + + ((Form_pg_toast_dictionary) GETSTRUCT(newtup))->tdcurrent = false; + CatalogTupleUpdate(drel, &newtup->t_self, newtup); + heap_freetuple(newtup); + } + systable_endscan(scan); + } + + if (!OidIsValid(dictid)) + dictid = GetNewOidWithIndex(drel, ToastDictionaryOidIndexId, + Anum_pg_toast_dictionary_oid); + dictval = (bytea *) palloc(dictlen + VARHDRSZ); + SET_VARSIZE(dictval, dictlen + VARHDRSZ); + memcpy(VARDATA(dictval), dict, dictlen); + + values[Anum_pg_toast_dictionary_oid - 1] = ObjectIdGetDatum(dictid); + values[Anum_pg_toast_dictionary_tdrelid - 1] = + ObjectIdGetDatum(RelationGetRelid(rel)); + values[Anum_pg_toast_dictionary_tdattnum - 1] = Int16GetDatum(attnum); + values[Anum_pg_toast_dictionary_tdcurrent - 1] = BoolGetDatum(current); + values[Anum_pg_toast_dictionary_tddict - 1] = PointerGetDatum(dictval); + tup = heap_form_tuple(RelationGetDescr(drel), values, nulls); + CatalogTupleInsert(drel, tup); + heap_freetuple(tup); + table_close(drel, RowExclusiveLock); + + /* let a later call in the same command see this one's changes */ + CommandCounterIncrement(); + + return dictid; +} + +#ifdef USE_ZSTD +/* Checks for pg_train_toast_dictionary(); returns attnum */ +static AttrNumber +dict_check_column(Relation rel, const char *colname) +{ + AttrNumber attnum; + Form_pg_attribute att; + + if (!object_ownercheck(RelationRelationId, RelationGetRelid(rel), + GetUserId())) + aclcheck_error(ACLCHECK_NOT_OWNER, + get_relkind_objtype(rel->rd_rel->relkind), + RelationGetRelationName(rel)); + if (rel->rd_rel->relkind != RELKIND_RELATION && + rel->rd_rel->relkind != RELKIND_MATVIEW) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("\"%s\" is not a table or materialized view", + RelationGetRelationName(rel)))); + if (IsCatalogRelation(rel)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot use compression dictionaries on a system catalog"))); + + attnum = get_attnum(RelationGetRelid(rel), colname); + if (attnum <= 0) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("column \"%s\" of relation \"%s\" does not exist", + colname, RelationGetRelationName(rel)))); + att = TupleDescAttr(RelationGetDescr(rel), attnum - 1); + if (att->attcompression != TOAST_ZSTD_DICT_COMPRESSION) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("column \"%s\" does not use compression method %s", + colname, "zstd_dict"))); + return attnum; +} +#endif /* USE_ZSTD */ + +/* + * pg_train_toast_dictionary(rel regclass, col name, dict_size int4) + * + * Train a zstd dictionary from the column's current values and make it the + * column's current dictionary. The previous one stays readable until the + * next rewrite of the table retires it. Returns the new dictionary's OID, or + * NULL if zstd could not train one from the values found. + * + * Takes ShareUpdateExclusiveLock, like ANALYZE: DML continues, concurrent + * training and DDL wait. + * + * ponytail: explicit call, first DICT_MAX_SAMPLES rows of a seqscan. + * Training from ANALYZE's block sample is the natural follow-up once the + * lifecycle is agreed on. + */ +Datum +pg_train_toast_dictionary(PG_FUNCTION_ARGS) +{ +#ifndef USE_ZSTD + NO_ZSTD_SUPPORT(); + PG_RETURN_NULL(); +#else + Oid relid = PG_GETARG_OID(0); + char *colname = NameStr(*PG_GETARG_NAME(1)); + int32 dict_size = PG_GETARG_INT32(2); + Relation rel; + AttrNumber attnum; + TableScanDesc scan; + TupleTableSlot *slot; + StringInfoData samples; + size_t *sizes; + int nsamples = 0; + void *dictbuf; + size_t dictlen; + Oid dictid; + + if (dict_size < DICT_MIN_SIZE || dict_size > DICT_MAX_SIZE) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("dictionary size must be between %d and %d bytes", + DICT_MIN_SIZE, DICT_MAX_SIZE))); + + rel = table_open(relid, ShareUpdateExclusiveLock); + attnum = dict_check_column(rel, colname); + + initStringInfo(&samples); + sizes = palloc_array(size_t, DICT_MAX_SAMPLES); + slot = table_slot_create(rel, NULL); + scan = table_beginscan(rel, GetActiveSnapshot(), 0, NULL, 0); + while (nsamples < DICT_MAX_SAMPLES && + table_scan_getnextslot(scan, ForwardScanDirection, slot)) + { + bool isnull; + Datum d = slot_getattr(slot, attnum, &isnull); + varlena *v; + size_t len; + + CHECK_FOR_INTERRUPTS(); + if (isnull) + continue; + v = detoast_attr_slice((varlena *) DatumGetPointer(d), 0, + DICT_SAMPLE_LIMIT); + len = VARSIZE_ANY_EXHDR(v); + if (samples.len + len > (size_t) dict_size * DICT_SAMPLE_FACTOR) + { + pfree(v); + break; + } + if (len >= 8) + { + appendBinaryStringInfo(&samples, VARDATA_ANY(v), len); + sizes[nsamples++] = len; + } + pfree(v); + } + table_endscan(scan); + ExecDropSingleTupleTableSlot(slot); + + dictbuf = palloc(dict_size); + dictlen = nsamples == 0 ? 0 : + ZDICT_trainFromBuffer(dictbuf, dict_size, samples.data, sizes, + nsamples); + if (nsamples == 0 || ZDICT_isError(dictlen)) + { + ereport(NOTICE, + (errmsg("could not train a compression dictionary for column \"%s\" from %d values", + colname, nsamples))); + table_close(rel, NoLock); + PG_RETURN_NULL(); + } + + dictid = dict_install(rel, attnum, InvalidOid, true, dictbuf, dictlen); + table_close(rel, NoLock); + + PG_RETURN_OID(dictid); +#endif +} + +/* + * binary_upgrade_add_toast_dictionary(rel regclass, attnum int2, dictid oid, + * current bool, dict bytea) + * + * pg_upgrade copies the heap files as they are, and with them the dictionary + * OIDs embedded in each datum, so dictionaries are restored with their OIDs. + */ +Datum +binary_upgrade_add_toast_dictionary(PG_FUNCTION_ARGS) +{ + Oid relid = PG_GETARG_OID(0); + AttrNumber attnum = PG_GETARG_INT16(1); + Oid dictid = PG_GETARG_OID(2); + bool current = PG_GETARG_BOOL(3); + bytea *dict = PG_GETARG_BYTEA_PP(4); + Relation rel; + + if (!IsBinaryUpgrade) + ereport(ERROR, + (errcode(ERRCODE_CANT_CHANGE_RUNTIME_PARAM), + errmsg("function can only be called when server is in binary upgrade mode"))); + + /* + * No dict_check_column() here: the column may have been switched to + * another compression method, or dropped, with old values still using + * these dictionaries until the table is rewritten. pg_dump restores + * physical column numbers, so attnum is stable. + */ + rel = table_open(relid, AccessExclusiveLock); + if (attnum <= 0 || attnum > RelationGetNumberOfAttributes(rel)) + elog(ERROR, "invalid attribute number %d for relation %u", + attnum, relid); + dict_install(rel, attnum, dictid, current, VARDATA_ANY(dict), + VARSIZE_ANY_EXHDR(dict)); + table_close(rel, NoLock); + + PG_RETURN_VOID(); +} diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c index d902b90f467..7f48f9b5f06 100644 --- a/src/backend/access/common/toast_internals.c +++ b/src/backend/access/common/toast_internals.c @@ -19,6 +19,7 @@ #include "access/heaptoast.h" #include "access/table.h" #include "access/toast_compression.h" +#include "access/toast_dictionary.h" #include "access/toast_internals.h" #include "access/xact.h" #include "catalog/catalog.h" @@ -80,6 +81,19 @@ toast_pointer_build(vartag_external tag, const void *ptr, */ Datum toast_compress_datum(Datum value, char cmethod) +{ + return toast_compress_datum_rel(value, cmethod, NULL, 0); +} + +/* + * Same, for a value about to be stored in column attnum of rel. Only this + * variant can use a column's dictionary: zstd_dict needs to know which + * column owns the dictionary, and falls back to plain zstd without one (no + * dictionary trained yet, or a caller such as an index that passes no rel). + */ +Datum +toast_compress_datum_rel(Datum value, char cmethod, Relation rel, + AttrNumber attnum) { varlena *tmp = NULL; int32 valsize; @@ -111,6 +125,25 @@ toast_compress_datum(Datum value, char cmethod) tmp = zstd_compress_datum((const varlena *) DatumGetPointer(value)); cmid = TOAST_ZSTD_COMPRESSION_ID; break; + case TOAST_ZSTD_DICT_COMPRESSION: + { + Oid dictid = InvalidOid; + + if (rel != NULL) + dictid = toast_dictionary_current(rel, attnum); + if (OidIsValid(dictid)) + { + tmp = zstd_dict_compress_datum((const varlena *) DatumGetPointer(value), + dictid); + cmid = TOAST_ZSTD_DICT_COMPRESSION_ID; + } + else + { + tmp = zstd_compress_datum((const varlena *) DatumGetPointer(value)); + cmid = TOAST_ZSTD_COMPRESSION_ID; + } + } + break; default: elog(ERROR, "invalid compression method %c", cmethod); } @@ -284,7 +317,8 @@ toast_save_datum(Relation rel, Datum value, /* set external size and compression method */ Assert(cmid == TOAST_PGLZ_COMPRESSION_ID || cmid == TOAST_LZ4_COMPRESSION_ID || - cmid == TOAST_ZSTD_COMPRESSION_ID); + cmid == TOAST_ZSTD_COMPRESSION_ID || + cmid == TOAST_ZSTD_DICT_COMPRESSION_ID); if (toast_compression_id_needs_cmid_byte(cmid)) { pointer_long = true; diff --git a/src/backend/access/heap/heaptoast.c b/src/backend/access/heap/heaptoast.c index b03d8492959..87b66530224 100644 --- a/src/backend/access/heap/heaptoast.c +++ b/src/backend/access/heap/heaptoast.c @@ -28,6 +28,7 @@ #include "access/genam.h" #include "access/heapam.h" #include "access/heaptoast.h" +#include "access/toast_compression.h" #include "access/toast_helper.h" #include "access/toast_internals.h" #include "utils/fmgroids.h" @@ -374,7 +375,8 @@ toast_flatten_tuple(HeapTuple tup, TupleDesc tupleDesc) varlena *new_value; new_value = (varlena *) DatumGetPointer(toast_values[i]); - if (VARATT_IS_EXTERNAL(new_value)) + if (VARATT_IS_EXTERNAL(new_value) || + VARATT_IS_COMPRESSED_DICT(new_value)) { new_value = detoast_external_attr(new_value); toast_values[i] = PointerGetDatum(new_value); @@ -589,7 +591,8 @@ toast_build_flattened_tuple(TupleDesc tupleDesc, varlena *new_value; new_value = (varlena *) DatumGetPointer(new_values[i]); - if (VARATT_IS_EXTERNAL(new_value)) + if (VARATT_IS_EXTERNAL(new_value) || + VARATT_IS_COMPRESSED_DICT(new_value)) { new_value = detoast_external_attr(new_value); new_values[i] = PointerGetDatum(new_value); diff --git a/src/backend/access/table/toast_helper.c b/src/backend/access/table/toast_helper.c index 2613d9dc095..4925841ae38 100644 --- a/src/backend/access/table/toast_helper.c +++ b/src/backend/access/table/toast_helper.c @@ -15,6 +15,8 @@ #include "postgres.h" #include "access/detoast.h" +#include "access/toast_compression.h" +#include "access/toast_dictionary.h" #include "access/toast_helper.h" #include "access/toast_internals.h" #include "catalog/pg_type_d.h" @@ -150,6 +152,26 @@ toast_tuple_init(ToastTupleContext *ttc) ttc->ttc_flags |= (TOAST_NEEDS_CHANGE | TOAST_NEEDS_FREE); } + /* + * A value compressed with a dictionary may only be stored in the + * column owning that dictionary, and only while the dictionary is + * current; otherwise decompress it, so that it is recompressed, + * if at all, with this column's own method. This is what keeps a + * dictionary's values inside its column (see toast_dictionary.c). + */ + if (VARATT_IS_COMPRESSED_DICT(new_value) && + !toast_dictionary_keep(ttc->ttc_rel, att->attnum, new_value)) + { + varlena *plain = zstd_dict_decompress_datum(new_value); + + if ((ttc->ttc_attr[i].tai_colflags & TOASTCOL_NEEDS_FREE) != 0) + pfree(new_value); + new_value = plain; + ttc->ttc_values[i] = PointerGetDatum(new_value); + ttc->ttc_attr[i].tai_colflags |= TOASTCOL_NEEDS_FREE; + ttc->ttc_flags |= (TOAST_NEEDS_CHANGE | TOAST_NEEDS_FREE); + } + /* * Remember the size of this attribute */ @@ -246,7 +268,8 @@ toast_tuple_try_compression(ToastTupleContext *ttc, int attribute) Datum new_value; ToastAttrInfo *attr = &ttc->ttc_attr[attribute]; - new_value = toast_compress_datum(*value, attr->tai_compression); + new_value = toast_compress_datum_rel(*value, attr->tai_compression, + ttc->ttc_rel, attribute + 1); if (DatumGetPointer(new_value) != NULL) { diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index f0a159c1b77..99f70dabe18 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -34,6 +34,7 @@ #include "access/relation.h" #include "access/table.h" #include "access/tableam.h" +#include "access/toast_dictionary.h" #include "catalog/binary_upgrade.h" #include "catalog/catalog.h" #include "catalog/heap.h" @@ -1787,6 +1788,13 @@ RemoveAttributeById(Oid relid, AttrNumber attnum) RemoveStatistics(relid, attnum); + /* + * The column's compression dictionaries are not removed here: its values + * stay in the heap until the next rewrite, and whole-row references and + * REPLICA IDENTITY FULL still read them. The rewrite nulls them and + * toast_dictionary_retire() then deletes the dictionaries. + */ + relation_close(rel, NoLock); } @@ -1939,6 +1947,11 @@ heap_drop_with_catalog(Oid relid) */ RemoveStatistics(relid, 0); + /* + * delete compression dictionaries + */ + RemoveToastDictionaries(relid, 0); + /* * delete attribute tuples */ diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index ddf801307eb..5d4af433d1c 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -277,6 +277,9 @@ CREATE VIEW pg_stats WITH (security_barrier) AS REVOKE ALL ON pg_statistic FROM public; +-- dictionaries are built from column contents, so protect them as such +REVOKE ALL ON pg_toast_dictionary FROM public; + CREATE VIEW pg_stats_ext WITH (security_barrier) AS SELECT cn.nspname AS schemaname, c.relname AS tablename, diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index a584e138ada..d219b87ced4 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -37,6 +37,7 @@ #include "access/multixact.h" #include "access/relscan.h" #include "access/tableam.h" +#include "access/toast_dictionary.h" #include "access/toast_internals.h" #include "access/transam.h" #include "access/xact.h" @@ -2152,6 +2153,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap, table_close(relRelation, RowExclusiveLock); } + /* + * Every value has been rewritten, and toast_tuple_init() recompressed any + * that used a non-current compression dictionary, so those dictionaries + * are no longer referenced and can go. + */ + toast_dictionary_retire(OIDOldHeap); + /* Destroy new heap with old filenumber */ object.classId = RelationRelationId; object.objectId = OIDNewHeap; diff --git a/src/backend/executor/tstoreReceiver.c b/src/backend/executor/tstoreReceiver.c index 8531d4ca432..f020b19ebf7 100644 --- a/src/backend/executor/tstoreReceiver.c +++ b/src/backend/executor/tstoreReceiver.c @@ -23,6 +23,7 @@ #include "postgres.h" #include "access/detoast.h" +#include "access/toast_compression.h" #include "access/tupconvert.h" #include "executor/tstoreReceiver.h" #include "varatt.h" @@ -159,7 +160,8 @@ tstoreReceiveSlot_detoast(TupleTableSlot *slot, DestReceiver *self) if (!attr->attisdropped && attr->attlen == -1 && !slot->tts_isnull[i]) { - if (VARATT_IS_EXTERNAL(DatumGetPointer(val))) + if (VARATT_IS_EXTERNAL(DatumGetPointer(val)) || + VARATT_IS_COMPRESSED_DICT(DatumGetPointer(val))) { val = PointerGetDatum(detoast_external_attr((varlena *) DatumGetPointer(val))); diff --git a/src/backend/utils/adt/expandedrecord.c b/src/backend/utils/adt/expandedrecord.c index 123792aa725..05897aa5d22 100644 --- a/src/backend/utils/adt/expandedrecord.c +++ b/src/backend/utils/adt/expandedrecord.c @@ -20,6 +20,7 @@ #include "access/detoast.h" #include "access/heaptoast.h" +#include "access/toast_compression.h" #include "access/htup_details.h" #include "catalog/heap.h" #include "catalog/pg_type.h" @@ -703,7 +704,8 @@ ER_get_flat_size(ExpandedObjectHeader *eohptr) if (!erh->dnulls[i] && !attr->attbyval && attr->attlen == -1 && - VARATT_IS_EXTERNAL(DatumGetPointer(erh->dvalues[i]))) + (VARATT_IS_EXTERNAL(DatumGetPointer(erh->dvalues[i])) || + VARATT_IS_COMPRESSED_DICT(DatumGetPointer(erh->dvalues[i])))) { /* * expanded_record_set_field_internal can do the actual work @@ -1155,7 +1157,8 @@ expanded_record_set_field_internal(ExpandedRecordHeader *erh, int fnumber, if (expand_external) { if (attr->attlen == -1 && - VARATT_IS_EXTERNAL(DatumGetPointer(newValue))) + (VARATT_IS_EXTERNAL(DatumGetPointer(newValue)) || + VARATT_IS_COMPRESSED_DICT(DatumGetPointer(newValue)))) { /* Detoasting should be done in short-lived context. */ oldcxt = MemoryContextSwitchTo(get_short_term_cxt(erh)); @@ -1185,7 +1188,8 @@ expanded_record_set_field_internal(ExpandedRecordHeader *erh, int fnumber, * by itself have made the value non-external.) */ if (attr->attlen == -1 && - VARATT_IS_EXTERNAL(DatumGetPointer(newValue))) + (VARATT_IS_EXTERNAL(DatumGetPointer(newValue)) || + VARATT_IS_COMPRESSED_DICT(DatumGetPointer(newValue)))) erh->flags |= ER_FLAG_HAVE_EXTERNAL; } @@ -1300,7 +1304,8 @@ expanded_record_set_fields(ExpandedRecordHeader *erh, { /* Is it an external toasted value? */ if (attr->attlen == -1 && - VARATT_IS_EXTERNAL(DatumGetPointer(newValue))) + (VARATT_IS_EXTERNAL(DatumGetPointer(newValue)) || + VARATT_IS_COMPRESSED_DICT(DatumGetPointer(newValue)))) { if (expand_external) { @@ -1312,7 +1317,8 @@ expanded_record_set_fields(ExpandedRecordHeader *erh, /* Just copy the value */ newValue = datumCopy(newValue, false, -1); /* If it's still external, remember that */ - if (VARATT_IS_EXTERNAL(DatumGetPointer(newValue))) + if (VARATT_IS_EXTERNAL(DatumGetPointer(newValue)) || + VARATT_IS_COMPRESSED_DICT(DatumGetPointer(newValue))) erh->flags |= ER_FLAG_HAVE_EXTERNAL; } } diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index 25ca69ce78f..03624495185 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -4243,6 +4243,9 @@ pg_column_compression(PG_FUNCTION_ARGS) case TOAST_ZSTD_COMPRESSION_ID: result = "zstd"; break; + case TOAST_ZSTD_DICT_COMPRESSION_ID: + result = "zstd_dict"; + break; default: elog(ERROR, "invalid compression method id %d", cmid); } diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index f2590c0809f..79bea1c50a0 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -390,6 +390,10 @@ static void binary_upgrade_set_type_oids_by_type_oid(Archive *fout, Oid pg_type_oid, bool force_array_type, bool include_multirange_type); +static void binary_upgrade_dump_toast_dictionaries(Archive *fout, + PQExpBuffer q, + const TableInfo *tbinfo, + const char *qualrelname); static void binary_upgrade_set_type_oids_by_rel(Archive *fout, PQExpBuffer upgrade_buffer, const TableInfo *tbinfo); @@ -5832,6 +5836,51 @@ binary_upgrade_set_type_oids_by_type_oid(Archive *fout, destroyPQExpBuffer(upgrade_query); } +/* + * Emit calls restoring the compression dictionaries of a table, with their + * OIDs, for pg_upgrade. Dictionaries are identified by physical column + * number, which binary upgrade preserves, so that those of dropped columns + * whose values are still on disk are carried over too. + * ponytail: one query per table; fold into getTableAttrs() if dumps of + * databases with many thousands of tables notice it. + */ +static void +binary_upgrade_dump_toast_dictionaries(Archive *fout, PQExpBuffer q, + const TableInfo *tbinfo, + const char *qualrelname) +{ + PQExpBuffer query; + PGresult *res; + + if (fout->remoteVersion < 200000) + return; + + query = createPQExpBuffer(); + appendPQExpBuffer(query, + "SELECT d.oid, d.tdattnum, d.tdcurrent, d.tddict " + "FROM pg_catalog.pg_toast_dictionary d " + "WHERE d.tdrelid = '%u'::pg_catalog.oid " + "ORDER BY d.oid", + tbinfo->dobj.catId.oid); + res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); + + for (int i = 0; i < PQntuples(res); i++) + { + appendPQExpBufferStr(q, "\n-- For binary upgrade, restore compression dictionary.\n"); + appendPQExpBufferStr(q, "SELECT pg_catalog.binary_upgrade_add_toast_dictionary("); + appendStringLiteralAH(q, qualrelname, fout); + appendPQExpBuffer(q, "::pg_catalog.regclass, %s::pg_catalog.int2, '%s'::pg_catalog.oid, %s, ", + PQgetvalue(res, i, 1), + PQgetvalue(res, i, 0), + PQgetvalue(res, i, 2)[0] == 't' ? "true" : "false"); + appendStringLiteralAH(q, PQgetvalue(res, i, 3), fout); + appendPQExpBufferStr(q, "::pg_catalog.bytea);\n"); + } + + PQclear(res); + destroyPQExpBuffer(query); +} + static void binary_upgrade_set_type_oids_by_rel(Archive *fout, PQExpBuffer upgrade_buffer, @@ -17835,6 +17884,9 @@ dumpTableSchema(Archive *fout, const TableInfo *tbinfo) case 'z': cmname = "zstd"; break; + case 'd': + cmname = "zstd_dict"; + break; default: cmname = NULL; break; @@ -17903,6 +17955,19 @@ dumpTableSchema(Archive *fout, const TableInfo *tbinfo) appendPQExpBuffer(q, "\nALTER TABLE ONLY %s FORCE ROW LEVEL SECURITY;\n", qualrelname); + /* + * pg_upgrade keeps the heap files, whose values embed the OIDs of the + * dictionaries they were compressed with, so carry the dictionaries over + * with their OIDs. This must not depend on the columns' current + * compression method: after SET COMPRESSION, existing values keep using + * the old dictionaries until the table is rewritten. A plain dump has no + * compressed values and needs no dictionaries. + */ + if (dopt->binary_upgrade && + (tbinfo->relkind == RELKIND_RELATION || + tbinfo->relkind == RELKIND_MATVIEW)) + binary_upgrade_dump_toast_dictionaries(fout, q, tbinfo, qualrelname); + if (dopt->binary_upgrade) binary_upgrade_extension_member(q, &tbinfo->dobj, reltypename, qrelname, diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 51940726069..c73775de4d8 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -2095,8 +2095,9 @@ describeOneTableDetails(const char *schemaname, printTableAddCell(&cont, (compression[0] == 'p' ? "pglz" : (compression[0] == 'l' ? "lz4" : (compression[0] == 'z' ? "zstd" : - (compression[0] == '\0' ? "" : - "???")))), + (compression[0] == 'd' ? "zstd_dict" : + (compression[0] == '\0' ? "" : + "???"))))), false, false); } diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c index 4e9eb47ceae..ae4d32901f5 100644 --- a/src/bin/psql/tab-complete.in.c +++ b/src/bin/psql/tab-complete.in.c @@ -2972,7 +2972,7 @@ match_previous_words(int pattern_id, /* ALTER TABLE ALTER [COLUMN] SET COMPRESSION */ else if (Matches("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET", "COMPRESSION") || Matches("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET", "COMPRESSION")) - COMPLETE_WITH("DEFAULT", "PGLZ", "LZ4", "ZSTD"); + COMPLETE_WITH("DEFAULT", "PGLZ", "LZ4", "ZSTD", "ZSTD_DICT"); /* ALTER TABLE ALTER [COLUMN] SET EXPRESSION */ else if (Matches("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET", "EXPRESSION") || Matches("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET", "EXPRESSION")) diff --git a/src/include/access/toast_compression.h b/src/include/access/toast_compression.h index 53d4f72257c..fc66e88210c 100644 --- a/src/include/access/toast_compression.h +++ b/src/include/access/toast_compression.h @@ -52,6 +52,7 @@ typedef enum ToastCompressionId TOAST_LZ4_COMPRESSION_ID = 1, TOAST_ZSTD_COMPRESSION_ID = 2, TOAST_INVALID_COMPRESSION_ID = 3, + TOAST_ZSTD_DICT_COMPRESSION_ID = 4, } ToastCompressionId; StaticAssertDecl(TOAST_INVALID_COMPRESSION_ID == VARLENA_COMPRESS_METHOD_LONG, @@ -73,6 +74,20 @@ toast_compression_id_needs_cmid_byte(ToastCompressionId cmid) cmid != TOAST_LZ4_COMPRESSION_ID); } +/* + * Is this a compressed-in-line datum that needs a compression dictionary to + * be read? Such a datum depends on catalog state of the column it belongs + * to, so code that makes values self-contained treats it like a TOAST + * pointer (see toast_dictionary.c). + */ +static inline bool +VARATT_IS_COMPRESSED_DICT(const void *PTR) +{ + return VARATT_IS_COMPRESSED(PTR) && + VARDATA_COMPRESSED_GET_COMPRESS_METHOD(PTR) == + TOAST_ZSTD_DICT_COMPRESSION_ID; +} + /* * Built-in compression methods. pg_attribute will store these in the * attcompression column. In attcompression, InvalidCompressionMethod @@ -81,6 +96,7 @@ toast_compression_id_needs_cmid_byte(ToastCompressionId cmid) #define TOAST_PGLZ_COMPRESSION 'p' #define TOAST_LZ4_COMPRESSION 'l' #define TOAST_ZSTD_COMPRESSION 'z' +#define TOAST_ZSTD_DICT_COMPRESSION 'd' #define InvalidCompressionMethod '\0' #define CompressionMethodIsValid(cm) ((cm) != InvalidCompressionMethod) diff --git a/src/include/access/toast_dictionary.h b/src/include/access/toast_dictionary.h new file mode 100644 index 00000000000..c2c16f5cf19 --- /dev/null +++ b/src/include/access/toast_dictionary.h @@ -0,0 +1,41 @@ +/*------------------------------------------------------------------------- + * + * toast_dictionary.h + * Lifecycle of zstd compression dictionaries for TOAST. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * + * src/include/access/toast_dictionary.h + * + *------------------------------------------------------------------------- + */ +#ifndef TOAST_DICTIONARY_H +#define TOAST_DICTIONARY_H + +#include "utils/relcache.h" +#include "varatt.h" + +/* + * A zstd_dict datum stores the dictionary OID right after the method byte + * of the long compressed header: + * + * va_header | va_tcinfo | va_cmid | dictid (4 bytes) | zstd frame + */ +#define VARHDRSZ_COMPRESSED_DICT (VARHDRSZ_COMPRESSED_LONG + sizeof(Oid)) + +/* Current dictionary of rel.attnum, or InvalidOid if none */ +extern Oid toast_dictionary_current(Relation rel, AttrNumber attnum); + +/* May this zstd_dict datum be stored as-is in rel.attnum? */ +extern bool toast_dictionary_keep(Relation rel, AttrNumber attnum, + const varlena *value); + +extern varlena *zstd_dict_compress_datum(const varlena *value, Oid dictid); +extern varlena *zstd_dict_decompress_datum(const varlena *value); +extern varlena *zstd_dict_decompress_datum_slice(const varlena *value, + int32 slicelength); + +extern void RemoveToastDictionaries(Oid relid, AttrNumber attnum); +extern void toast_dictionary_retire(Oid relid); + +#endif /* TOAST_DICTIONARY_H */ diff --git a/src/include/access/toast_internals.h b/src/include/access/toast_internals.h index a31b47f8df5..036b793fa23 100644 --- a/src/include/access/toast_internals.h +++ b/src/include/access/toast_internals.h @@ -18,6 +18,8 @@ #include "utils/snapshot.h" extern Datum toast_compress_datum(Datum value, char cmethod); +extern Datum toast_compress_datum_rel(Datum value, char cmethod, Relation rel, + AttrNumber attnum); extern Oid toast_get_valid_index(Oid toastoid, LOCKMODE lock); extern void toast_delete_datum(Relation rel, Datum value, bool is_speculative); diff --git a/src/include/catalog/Makefile b/src/include/catalog/Makefile index 444fc76eed6..49f65ec84b9 100644 --- a/src/include/catalog/Makefile +++ b/src/include/catalog/Makefile @@ -39,6 +39,7 @@ CATALOG_HEADERS := \ pg_statistic.h \ pg_statistic_ext.h \ pg_statistic_ext_data.h \ + pg_toast_dictionary.h \ pg_rewrite.h \ pg_trigger.h \ pg_event_trigger.h \ diff --git a/src/include/catalog/meson.build b/src/include/catalog/meson.build index bcc01c87c2e..8e111945a4e 100644 --- a/src/include/catalog/meson.build +++ b/src/include/catalog/meson.build @@ -26,6 +26,7 @@ catalog_headers = [ 'pg_statistic.h', 'pg_statistic_ext.h', 'pg_statistic_ext_data.h', + 'pg_toast_dictionary.h', 'pg_rewrite.h', 'pg_trigger.h', 'pg_event_trigger.h', diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 5e4c9e503f8..1b1323bea6c 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -7906,6 +7906,12 @@ { oid => '2121', descr => 'compression method for the compressed datum', proname => 'pg_column_compression', provolatile => 's', prorettype => 'text', proargtypes => 'any', prosrc => 'pg_column_compression' }, +{ oid => '9265', descr => 'train a TOAST compression dictionary for a column', + proname => 'pg_train_toast_dictionary', provolatile => 'v', + proparallel => 'u', prorettype => 'oid', + proargtypes => 'regclass name int4', + proargnames => '{rel,col,dict_size}', proargdefaults => '{65536}', + prosrc => 'pg_train_toast_dictionary' }, { oid => '6316', descr => 'chunk ID of on-disk TOASTed value', proname => 'pg_column_toast_chunk_id', provolatile => 's', prorettype => 'oid8', proargtypes => 'any', @@ -12019,6 +12025,11 @@ proname => 'binary_upgrade_set_missing_value', provolatile => 'v', proparallel => 'u', prorettype => 'void', proargtypes => 'oid text text', prosrc => 'binary_upgrade_set_missing_value' }, +{ oid => '9266', descr => 'for use by pg_upgrade', + proname => 'binary_upgrade_add_toast_dictionary', provolatile => 'v', + proparallel => 'u', prorettype => 'void', + proargtypes => 'regclass int2 oid bool bytea', + prosrc => 'binary_upgrade_add_toast_dictionary' }, { oid => '4545', descr => 'for use by pg_upgrade', proname => 'binary_upgrade_set_next_heap_relfilenode', provolatile => 'v', proparallel => 'u', prorettype => 'void', proargtypes => 'oid', diff --git a/src/include/catalog/pg_toast_dictionary.h b/src/include/catalog/pg_toast_dictionary.h new file mode 100644 index 00000000000..a2abc91ae9c --- /dev/null +++ b/src/include/catalog/pg_toast_dictionary.h @@ -0,0 +1,65 @@ +/*------------------------------------------------------------------------- + * + * pg_toast_dictionary.h + * definition of the "TOAST compression dictionary" system catalog + * (pg_toast_dictionary) + * + * Each row is one trained compression dictionary. A dictionary is owned by + * exactly one column of one table, just as a pg_statistic row is: it is + * deleted with that column or table, never shared, and needs no pg_depend + * entries. Values compressed with a dictionary are only ever stored in the + * column that owns it; see toast_dictionary.c for how that is enforced. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/catalog/pg_toast_dictionary.h + * + * NOTES + * The Catalog.pm module reads this file and derives schema + * information. + * + *------------------------------------------------------------------------- + */ +#ifndef PG_TOAST_DICTIONARY_H +#define PG_TOAST_DICTIONARY_H + +#include "catalog/genbki.h" +#include "catalog/pg_toast_dictionary_d.h" /* IWYU pragma: export */ + +/* ---------------- + * pg_toast_dictionary definition. cpp turns this into + * typedef struct FormData_pg_toast_dictionary + * ---------------- + */ +BEGIN_CATALOG_STRUCT + +CATALOG(pg_toast_dictionary,9260,ToastDictionaryRelationId) +{ + Oid oid; /* dictionary ID, as stored in each datum */ + Oid tdrelid BKI_LOOKUP(pg_class); /* owning relation */ + int16 tdattnum; /* owning column */ + bool tdcurrent; /* used to compress new values? */ + +#ifdef CATALOG_VARLEN /* variable-length fields start here */ + bytea tddict BKI_FORCE_NOT_NULL; /* zstd dictionary content */ +#endif +} FormData_pg_toast_dictionary; + +END_CATALOG_STRUCT + +/* ---------------- + * Form_pg_toast_dictionary corresponds to a pointer to a tuple with + * the format of pg_toast_dictionary relation. + * ---------------- + */ +typedef FormData_pg_toast_dictionary *Form_pg_toast_dictionary; + +DECLARE_TOAST(pg_toast_dictionary, 9261, 9262); + +DECLARE_UNIQUE_INDEX_PKEY(pg_toast_dictionary_oid_index, 9263, ToastDictionaryOidIndexId, pg_toast_dictionary, btree(oid oid_ops)); +DECLARE_INDEX(pg_toast_dictionary_relid_attnum_index, 9264, ToastDictionaryRelidAttnumIndexId, pg_toast_dictionary, btree(tdrelid oid_ops, tdattnum int2_ops)); + +MAKE_SYSCACHE(TOASTDICTOID, pg_toast_dictionary_oid_index, 16); + +#endif /* PG_TOAST_DICTIONARY_H */ diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c index bc1667f7a13..9e20379cc9f 100644 --- a/src/pl/plpgsql/src/pl_exec.c +++ b/src/pl/plpgsql/src/pl_exec.c @@ -18,6 +18,7 @@ #include #include "access/detoast.h" +#include "access/toast_compression.h" #include "access/htup_details.h" #include "access/tupconvert.h" #include "catalog/pg_proc.h" @@ -8909,7 +8910,8 @@ assign_simple_var(PLpgSQL_execstate *estate, PLpgSQL_var *var, * not a problem since all array entries are always detoasted.) */ if (!estate->atomic && !isnull && var->datatype->typlen == -1 && - VARATT_IS_EXTERNAL_NON_EXPANDED(DatumGetPointer(newvalue))) + (VARATT_IS_EXTERNAL_NON_EXPANDED(DatumGetPointer(newvalue)) || + VARATT_IS_COMPRESSED_DICT(DatumGetPointer(newvalue)))) { MemoryContext oldcxt; Datum detoasted; diff --git a/src/test/regress/expected/compression_zstd_dict.out b/src/test/regress/expected/compression_zstd_dict.out new file mode 100644 index 00000000000..4f39485dad7 --- /dev/null +++ b/src/test/regress/expected/compression_zstd_dict.out @@ -0,0 +1,414 @@ +-- Tests for zstd TOAST compression with dictionaries, and their lifecycle: +-- dictionaries belong to one column, values compressed with them never leave +-- that column in compressed form, a rewrite retires old dictionaries, and +-- dropping the column or table drops them. +SELECT NOT(enumvals @> '{zstd}') AS skip_test FROM pg_settings WHERE + name = 'default_toast_compression' \gset +\if :skip_test + \echo '*** skipping TOAST tests with zstd dictionaries (not supported) ***' + \quit +\endif +CREATE SCHEMA zstd_dict; +SET search_path TO zstd_dict, public; +\set HIDE_TOAST_COMPRESSION false +-- Many small, similar JSON documents: the case dictionaries are made for. +-- Each is below the TOAST threshold uncompressed only with a dictionary. +CREATE FUNCTION doc(i int) RETURNS text LANGUAGE SQL AS $$ + SELECT format('{"customer_id": %s, "status": "active", "region": "us-east", ' + '"plan": "enterprise", "features": ["sso", "audit", "backup"], ' + '"tags": ["%s", "priority-%s"], "notes": "%s"}', + i, md5(i::text), i % 5, repeat('lorem ipsum dolor sit amet ', 80)) +$$; +CREATE TABLE cmdict (id int, f1 text COMPRESSION zstd_dict); +\d+ cmdict + Table "zstd_dict.cmdict" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + id | integer | | | | plain | | | + f1 | text | | | | extended | zstd_dict | | + +-- before a dictionary is trained, zstd_dict compresses with plain zstd +INSERT INTO cmdict SELECT g, doc(g) FROM generate_series(1, 200) g; +SELECT pg_column_compression(f1), count(*) FROM cmdict GROUP BY 1; + pg_column_compression | count +-----------------------+------- + zstd | 200 +(1 row) + +-- only the owner can train; only for a zstd_dict column +CREATE ROLE regress_dict_user; +GRANT USAGE ON SCHEMA zstd_dict TO regress_dict_user; +GRANT SELECT ON cmdict TO regress_dict_user; +SET ROLE regress_dict_user; +SELECT pg_train_toast_dictionary('zstd_dict.cmdict', 'f1'); +ERROR: must be owner of table cmdict +RESET ROLE; +CREATE TABLE cmplain (f1 text COMPRESSION zstd); +SELECT pg_train_toast_dictionary('cmplain', 'f1'); +ERROR: column "f1" does not use compression method zstd_dict +SELECT pg_train_toast_dictionary('cmdict', 'nosuchcol'); +ERROR: column "nosuchcol" of relation "cmdict" does not exist +SELECT pg_train_toast_dictionary('cmdict', 'f1', 10); +ERROR: dictionary size must be between 256 and 1048576 bytes +-- the catalog is not readable by ordinary users +SET ROLE regress_dict_user; +SELECT count(*) FROM pg_toast_dictionary; +ERROR: permission denied for table pg_toast_dictionary +RESET ROLE; +SELECT pg_train_toast_dictionary('cmdict', 'f1', 4096) IS NOT NULL AS trained; + trained +--------- + t +(1 row) + +SELECT count(*) AS ndicts, count(*) FILTER (WHERE tdcurrent) AS ncurrent + FROM pg_toast_dictionary WHERE tdrelid = 'cmdict'::regclass; + ndicts | ncurrent +--------+---------- + 1 | 1 +(1 row) + +-- new values use the dictionary, and read back correctly, whole and sliced +INSERT INTO cmdict SELECT g, doc(g) FROM generate_series(201, 400) g; +SELECT pg_column_compression(f1), count(*) FROM cmdict GROUP BY 1 ORDER BY 1; + pg_column_compression | count +-----------------------+------- + zstd | 200 + zstd_dict | 200 +(2 rows) + +SELECT bool_and(f1 = doc(id)) AS all_equal FROM cmdict; + all_equal +----------- + t +(1 row) + +SELECT bool_and(substr(f1, 10, 30) = substr(doc(id), 10, 30)) AS slices_ok + FROM cmdict WHERE id > 200; + slices_ok +----------- + t +(1 row) + +-- the dictionary pays off: same data, smaller than without it +SELECT avg(pg_column_size(f1)) FILTER (WHERE pg_column_compression(f1) = 'zstd_dict') < + avg(pg_column_size(f1)) FILTER (WHERE pg_column_compression(f1) = 'zstd') + AS dict_smaller FROM cmdict; + dict_smaller +-------------- + t +(1 row) + +-- large values are compressed with the dictionary and stored out of line +INSERT INTO cmdict SELECT 1000 + g / 100, string_agg(doc(g), ',') + FROM generate_series(1, 299) g GROUP BY g / 100; +SELECT pg_column_compression(f1), pg_column_toast_chunk_id(f1) IS NOT NULL AS external + FROM cmdict WHERE id > 1000 LIMIT 1; + pg_column_compression | external +-----------------------+---------- + zstd_dict | t +(1 row) + +---------------------------------------------------------------------- +-- Containment: dictionary-compressed values do not leave their column +---------------------------------------------------------------------- +-- CTAS and INSERT ... SELECT into another table: the target gets values that +-- are not compressed with cmdict's dictionary +CREATE TABLE cmcopy AS SELECT * FROM cmdict; +SELECT count(*) FILTER (WHERE pg_column_compression(f1) = 'zstd_dict') AS dict_values + FROM cmcopy; + dict_values +------------- + 0 +(1 row) + +CREATE TABLE cmcopy2 (id int, f1 text COMPRESSION zstd_dict); +INSERT INTO cmcopy2 SELECT * FROM cmdict; +SELECT count(*) FILTER (WHERE pg_column_compression(f1) = 'zstd_dict') AS dict_values + FROM cmcopy2; + dict_values +------------- + 0 +(1 row) + +-- another column of the same table does not own the dictionary either +ALTER TABLE cmdict ADD COLUMN f2 text COMPRESSION zstd_dict; +UPDATE cmdict SET f2 = f1 WHERE id <= 400; +SELECT count(*) FILTER (WHERE pg_column_compression(f2) = 'zstd_dict') AS dict_values + FROM cmdict; + dict_values +------------- + 0 +(1 row) + +ALTER TABLE cmdict DROP COLUMN f2; +-- values nested in containers: rows, arrays, ranges of text are flattened +CREATE TYPE cmpair AS (a text, b text); +CREATE TABLE cmnest (p cmpair, arr text[]); +INSERT INTO cmnest SELECT ROW(f1, f1)::cmpair, ARRAY[f1, f1] + FROM cmdict WHERE id BETWEEN 201 AND 210; +SELECT pg_column_compression((p).a) AS in_row, pg_column_compression(arr[1]) AS in_array + FROM cmnest LIMIT 1; + in_row | in_array +--------+---------- + | +(1 row) + +-- a holdable cursor outlives the transaction; its rows must not depend on +-- the dictionary +BEGIN; +DECLARE c CURSOR WITH HOLD FOR SELECT id, f1 FROM cmdict WHERE id BETWEEN 201 AND 205 ORDER BY id; +COMMIT; +FETCH 1 FROM c; + id | f1 +-----+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + 201 | {"customer_id": 201, "status": "active", "region": "us-east", "plan": "enterprise", "features": ["sso", "audit", "backup"], "tags": ["757b505cfd34c64c85ca5b5690ee5293", "priority-1"], "notes": "lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet "} +(1 row) + +CLOSE c; +-- PL/pgSQL variable across COMMIT in a procedure +CREATE PROCEDURE cmproc() LANGUAGE plpgsql AS $$ +DECLARE v text; +BEGIN + SELECT f1 INTO v FROM cmdict WHERE id = 300; + COMMIT; + RAISE NOTICE 'length %, ok %', length(v), v = doc(300); +END $$; +CALL cmproc(); +NOTICE: length 2356, ok t +-- values written to the table itself are unaffected by any of this +SELECT bool_and(f1 = doc(id)) AS all_equal FROM cmdict WHERE id <= 400; + all_equal +----------- + t +(1 row) + +-- indexes: an expression index on a dictionary-compressed column works +CREATE INDEX cmdict_idx ON cmdict (left(f1, 40)); +CREATE INDEX cmdict_idx2 ON cmdict (f1) WHERE id = 300; +SELECT id FROM cmdict WHERE f1 = doc(300); + id +----- + 300 +(1 row) + +---------------------------------------------------------------------- +-- Retirement +---------------------------------------------------------------------- +-- retraining makes a new current dictionary; the old one stays readable +SELECT pg_train_toast_dictionary('cmdict', 'f1', 4096) IS NOT NULL AS trained; + trained +--------- + t +(1 row) + +SELECT count(*) AS ndicts, count(*) FILTER (WHERE tdcurrent) AS ncurrent + FROM pg_toast_dictionary WHERE tdrelid = 'cmdict'::regclass; + ndicts | ncurrent +--------+---------- + 2 | 1 +(1 row) + +SELECT bool_and(f1 = doc(id)) AS all_equal FROM cmdict WHERE id <= 400; + all_equal +----------- + t +(1 row) + +-- ordinary updates of other columns leave the value, and its old dictionary, +-- alone; updates of the column recompress with the current one +UPDATE cmdict SET id = id WHERE id = 250; +UPDATE cmdict SET f1 = f1 || '' WHERE id = 251; +-- a rewrite recompresses with the current dictionary and retires the rest +VACUUM FULL cmdict; +SELECT count(*) AS ndicts, count(*) FILTER (WHERE tdcurrent) AS ncurrent + FROM pg_toast_dictionary WHERE tdrelid = 'cmdict'::regclass; + ndicts | ncurrent +--------+---------- + 1 | 1 +(1 row) + +SELECT bool_and(f1 = doc(id)) AS all_equal FROM cmdict WHERE id <= 400; + all_equal +----------- + t +(1 row) + +SELECT pg_column_compression(f1), count(*) FROM cmdict WHERE id <= 400 GROUP BY 1 ORDER BY 1; + pg_column_compression | count +-----------------------+------- + zstd | 200 + zstd_dict | 200 +(2 rows) + +-- the same for CLUSTER and a rewriting ALTER TABLE +SELECT pg_train_toast_dictionary('cmdict', 'f1', 4096) IS NOT NULL AS trained; + trained +--------- + t +(1 row) + +CLUSTER cmdict USING cmdict_idx; +SELECT count(*) AS ndicts FROM pg_toast_dictionary WHERE tdrelid = 'cmdict'::regclass; + ndicts +-------- + 1 +(1 row) + +SELECT pg_train_toast_dictionary('cmdict', 'f1', 4096) IS NOT NULL AS trained; + trained +--------- + t +(1 row) + +ALTER TABLE cmdict ALTER COLUMN id TYPE bigint; +SELECT count(*) AS ndicts FROM pg_toast_dictionary WHERE tdrelid = 'cmdict'::regclass; + ndicts +-------- + 1 +(1 row) + +SELECT bool_and(f1 = doc(id::int)) AS all_equal FROM cmdict WHERE id <= 400; + all_equal +----------- + t +(1 row) + +-- switching the column away from zstd_dict: existing values stay readable +-- until the next rewrite, which recompresses them and drops the dictionary +ALTER TABLE cmdict ALTER COLUMN f1 SET COMPRESSION lz4; +SELECT bool_and(f1 = doc(id::int)) AS all_equal FROM cmdict WHERE id <= 400; + all_equal +----------- + t +(1 row) + +SELECT count(*) AS ndicts FROM pg_toast_dictionary WHERE tdrelid = 'cmdict'::regclass; + ndicts +-------- + 1 +(1 row) + +VACUUM FULL cmdict; +SELECT count(*) AS ndicts FROM pg_toast_dictionary WHERE tdrelid = 'cmdict'::regclass; + ndicts +-------- + 0 +(1 row) + +SELECT pg_column_compression(f1), count(*) FROM cmdict WHERE id <= 400 GROUP BY 1 ORDER BY 1; + pg_column_compression | count +-----------------------+------- + lz4 | 200 + zstd | 200 +(2 rows) + +SELECT bool_and(f1 = doc(id::int)) AS all_equal FROM cmdict WHERE id <= 400; + all_equal +----------- + t +(1 row) + +ALTER TABLE cmdict ALTER COLUMN f1 SET COMPRESSION zstd_dict; +SELECT pg_train_toast_dictionary('cmdict', 'f1', 4096) IS NOT NULL AS trained; + trained +--------- + t +(1 row) + +---------------------------------------------------------------------- +-- Cleanup with the owning objects +---------------------------------------------------------------------- +ALTER TABLE cmdict ADD COLUMN f3 text COMPRESSION zstd_dict; +UPDATE cmdict SET f3 = f1 WHERE id <= 400; +SELECT pg_train_toast_dictionary('cmdict', 'f3', 4096) IS NOT NULL AS trained; + trained +--------- + t +(1 row) + +SELECT tdattnum > 2 AS f3_dict, count(*) FROM pg_toast_dictionary + WHERE tdrelid = 'cmdict'::regclass GROUP BY 1 ORDER BY 1; + f3_dict | count +---------+------- + f | 1 + t | 1 +(2 rows) + +-- DROP COLUMN leaves the values on disk until a rewrite, and whole-row +-- references and REPLICA IDENTITY FULL still read them, so the dictionary +-- stays until then +ALTER TABLE cmdict DROP COLUMN f3; +SELECT count(*) AS ndicts FROM pg_toast_dictionary WHERE tdrelid = 'cmdict'::regclass; + ndicts +-------- + 2 +(1 row) + +SELECT count(*) AS wholerow FROM (SELECT t FROM cmdict t) s; + wholerow +---------- + 403 +(1 row) + +VACUUM FULL cmdict; +SELECT count(*) AS ndicts FROM pg_toast_dictionary WHERE tdrelid = 'cmdict'::regclass; + ndicts +-------- + 1 +(1 row) + +-- two trainings in one command: only the last one stays current +SELECT count(*) FROM (SELECT pg_train_toast_dictionary('cmdict', 'f1', 4096) + FROM generate_series(1, 2)) s; + count +------- + 2 +(1 row) + +SELECT count(*) AS ndicts, count(*) FILTER (WHERE tdcurrent) AS ncurrent + FROM pg_toast_dictionary WHERE tdrelid = 'cmdict'::regclass; + ndicts | ncurrent +--------+---------- + 3 | 1 +(1 row) + +-- the copies made above survive their source being dropped +SELECT 'cmdict'::regclass::oid AS cmdict_oid \gset +DROP TABLE cmdict; +SELECT count(*) AS ndicts FROM pg_toast_dictionary WHERE tdrelid = :cmdict_oid; + ndicts +-------- + 0 +(1 row) + +SELECT bool_and(f1 = doc(id)) AS copies_ok FROM cmcopy WHERE id <= 400; + copies_ok +----------- + t +(1 row) + +SELECT bool_and(f1 = doc(id)) AS copies_ok FROM cmcopy2 WHERE id <= 400; + copies_ok +----------- + t +(1 row) + +SELECT bool_and((p).a = arr[1]) AS nested_ok FROM cmnest; + nested_ok +----------- + t +(1 row) + +\set HIDE_TOAST_COMPRESSION true +RESET search_path; +DROP SCHEMA zstd_dict CASCADE; +NOTICE: drop cascades to 7 other objects +DETAIL: drop cascades to function zstd_dict.doc(integer) +drop cascades to table zstd_dict.cmplain +drop cascades to table zstd_dict.cmcopy +drop cascades to table zstd_dict.cmcopy2 +drop cascades to type zstd_dict.cmpair +drop cascades to table zstd_dict.cmnest +drop cascades to function zstd_dict.cmproc() +DROP ROLE regress_dict_user; diff --git a/src/test/regress/expected/compression_zstd_dict_1.out b/src/test/regress/expected/compression_zstd_dict_1.out new file mode 100644 index 00000000000..13f0f1c3c70 --- /dev/null +++ b/src/test/regress/expected/compression_zstd_dict_1.out @@ -0,0 +1,10 @@ +-- Tests for zstd TOAST compression with dictionaries, and their lifecycle: +-- dictionaries belong to one column, values compressed with them never leave +-- that column in compressed form, a rewrite retires old dictionaries, and +-- dropping the column or table drops them. +SELECT NOT(enumvals @> '{zstd}') AS skip_test FROM pg_settings WHERE + name = 'default_toast_compression' \gset +\if :skip_test + \echo '*** skipping TOAST tests with zstd dictionaries (not supported) ***' +*** skipping TOAST tests with zstd dictionaries (not supported) *** + \quit diff --git a/src/test/regress/expected/oidjoins.out b/src/test/regress/expected/oidjoins.out index 51b9608a668..cae38074ec7 100644 --- a/src/test/regress/expected/oidjoins.out +++ b/src/test/regress/expected/oidjoins.out @@ -168,6 +168,7 @@ NOTICE: checking pg_statistic_ext {stxnamespace} => pg_namespace {oid} NOTICE: checking pg_statistic_ext {stxowner} => pg_authid {oid} NOTICE: checking pg_statistic_ext {stxrelid,stxkeys} => pg_attribute {attrelid,attnum} NOTICE: checking pg_statistic_ext_data {stxoid} => pg_statistic_ext {oid} +NOTICE: checking pg_toast_dictionary {tdrelid} => pg_class {oid} NOTICE: checking pg_rewrite {ev_class} => pg_class {oid} NOTICE: checking pg_trigger {tgrelid} => pg_class {oid} NOTICE: checking pg_trigger {tgparentid} => pg_trigger {oid} diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 2b25b90ee3c..96cc3a7ebab 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -128,7 +128,7 @@ test: partition_join partition_prune reloptions hash_part indexing partition_agg # ---------- # Another group of parallel tests (compression) # ---------- -test: compression compression_lz4 compression_pglz compression_zstd cluster +test: compression compression_lz4 compression_pglz compression_zstd compression_zstd_dict cluster # event_trigger depends on create_am and cannot run concurrently with # any test that runs DDL diff --git a/src/test/regress/sql/compression_zstd_dict.sql b/src/test/regress/sql/compression_zstd_dict.sql new file mode 100644 index 00000000000..dc8a5cf9790 --- /dev/null +++ b/src/test/regress/sql/compression_zstd_dict.sql @@ -0,0 +1,203 @@ +-- Tests for zstd TOAST compression with dictionaries, and their lifecycle: +-- dictionaries belong to one column, values compressed with them never leave +-- that column in compressed form, a rewrite retires old dictionaries, and +-- dropping the column or table drops them. + +SELECT NOT(enumvals @> '{zstd}') AS skip_test FROM pg_settings WHERE + name = 'default_toast_compression' \gset +\if :skip_test + \echo '*** skipping TOAST tests with zstd dictionaries (not supported) ***' + \quit +\endif + +CREATE SCHEMA zstd_dict; +SET search_path TO zstd_dict, public; + +\set HIDE_TOAST_COMPRESSION false + +-- Many small, similar JSON documents: the case dictionaries are made for. +-- Each is below the TOAST threshold uncompressed only with a dictionary. +CREATE FUNCTION doc(i int) RETURNS text LANGUAGE SQL AS $$ + SELECT format('{"customer_id": %s, "status": "active", "region": "us-east", ' + '"plan": "enterprise", "features": ["sso", "audit", "backup"], ' + '"tags": ["%s", "priority-%s"], "notes": "%s"}', + i, md5(i::text), i % 5, repeat('lorem ipsum dolor sit amet ', 80)) +$$; + +CREATE TABLE cmdict (id int, f1 text COMPRESSION zstd_dict); +\d+ cmdict + +-- before a dictionary is trained, zstd_dict compresses with plain zstd +INSERT INTO cmdict SELECT g, doc(g) FROM generate_series(1, 200) g; +SELECT pg_column_compression(f1), count(*) FROM cmdict GROUP BY 1; + +-- only the owner can train; only for a zstd_dict column +CREATE ROLE regress_dict_user; +GRANT USAGE ON SCHEMA zstd_dict TO regress_dict_user; +GRANT SELECT ON cmdict TO regress_dict_user; +SET ROLE regress_dict_user; +SELECT pg_train_toast_dictionary('zstd_dict.cmdict', 'f1'); +RESET ROLE; +CREATE TABLE cmplain (f1 text COMPRESSION zstd); +SELECT pg_train_toast_dictionary('cmplain', 'f1'); +SELECT pg_train_toast_dictionary('cmdict', 'nosuchcol'); +SELECT pg_train_toast_dictionary('cmdict', 'f1', 10); +-- the catalog is not readable by ordinary users +SET ROLE regress_dict_user; +SELECT count(*) FROM pg_toast_dictionary; +RESET ROLE; + +SELECT pg_train_toast_dictionary('cmdict', 'f1', 4096) IS NOT NULL AS trained; +SELECT count(*) AS ndicts, count(*) FILTER (WHERE tdcurrent) AS ncurrent + FROM pg_toast_dictionary WHERE tdrelid = 'cmdict'::regclass; + +-- new values use the dictionary, and read back correctly, whole and sliced +INSERT INTO cmdict SELECT g, doc(g) FROM generate_series(201, 400) g; +SELECT pg_column_compression(f1), count(*) FROM cmdict GROUP BY 1 ORDER BY 1; +SELECT bool_and(f1 = doc(id)) AS all_equal FROM cmdict; +SELECT bool_and(substr(f1, 10, 30) = substr(doc(id), 10, 30)) AS slices_ok + FROM cmdict WHERE id > 200; +-- the dictionary pays off: same data, smaller than without it +SELECT avg(pg_column_size(f1)) FILTER (WHERE pg_column_compression(f1) = 'zstd_dict') < + avg(pg_column_size(f1)) FILTER (WHERE pg_column_compression(f1) = 'zstd') + AS dict_smaller FROM cmdict; + +-- large values are compressed with the dictionary and stored out of line +INSERT INTO cmdict SELECT 1000 + g / 100, string_agg(doc(g), ',') + FROM generate_series(1, 299) g GROUP BY g / 100; +SELECT pg_column_compression(f1), pg_column_toast_chunk_id(f1) IS NOT NULL AS external + FROM cmdict WHERE id > 1000 LIMIT 1; + +---------------------------------------------------------------------- +-- Containment: dictionary-compressed values do not leave their column +---------------------------------------------------------------------- + +-- CTAS and INSERT ... SELECT into another table: the target gets values that +-- are not compressed with cmdict's dictionary +CREATE TABLE cmcopy AS SELECT * FROM cmdict; +SELECT count(*) FILTER (WHERE pg_column_compression(f1) = 'zstd_dict') AS dict_values + FROM cmcopy; +CREATE TABLE cmcopy2 (id int, f1 text COMPRESSION zstd_dict); +INSERT INTO cmcopy2 SELECT * FROM cmdict; +SELECT count(*) FILTER (WHERE pg_column_compression(f1) = 'zstd_dict') AS dict_values + FROM cmcopy2; +-- another column of the same table does not own the dictionary either +ALTER TABLE cmdict ADD COLUMN f2 text COMPRESSION zstd_dict; +UPDATE cmdict SET f2 = f1 WHERE id <= 400; +SELECT count(*) FILTER (WHERE pg_column_compression(f2) = 'zstd_dict') AS dict_values + FROM cmdict; +ALTER TABLE cmdict DROP COLUMN f2; + +-- values nested in containers: rows, arrays, ranges of text are flattened +CREATE TYPE cmpair AS (a text, b text); +CREATE TABLE cmnest (p cmpair, arr text[]); +INSERT INTO cmnest SELECT ROW(f1, f1)::cmpair, ARRAY[f1, f1] + FROM cmdict WHERE id BETWEEN 201 AND 210; +SELECT pg_column_compression((p).a) AS in_row, pg_column_compression(arr[1]) AS in_array + FROM cmnest LIMIT 1; + +-- a holdable cursor outlives the transaction; its rows must not depend on +-- the dictionary +BEGIN; +DECLARE c CURSOR WITH HOLD FOR SELECT id, f1 FROM cmdict WHERE id BETWEEN 201 AND 205 ORDER BY id; +COMMIT; +FETCH 1 FROM c; +CLOSE c; + +-- PL/pgSQL variable across COMMIT in a procedure +CREATE PROCEDURE cmproc() LANGUAGE plpgsql AS $$ +DECLARE v text; +BEGIN + SELECT f1 INTO v FROM cmdict WHERE id = 300; + COMMIT; + RAISE NOTICE 'length %, ok %', length(v), v = doc(300); +END $$; +CALL cmproc(); + +-- values written to the table itself are unaffected by any of this +SELECT bool_and(f1 = doc(id)) AS all_equal FROM cmdict WHERE id <= 400; + +-- indexes: an expression index on a dictionary-compressed column works +CREATE INDEX cmdict_idx ON cmdict (left(f1, 40)); +CREATE INDEX cmdict_idx2 ON cmdict (f1) WHERE id = 300; +SELECT id FROM cmdict WHERE f1 = doc(300); + +---------------------------------------------------------------------- +-- Retirement +---------------------------------------------------------------------- + +-- retraining makes a new current dictionary; the old one stays readable +SELECT pg_train_toast_dictionary('cmdict', 'f1', 4096) IS NOT NULL AS trained; +SELECT count(*) AS ndicts, count(*) FILTER (WHERE tdcurrent) AS ncurrent + FROM pg_toast_dictionary WHERE tdrelid = 'cmdict'::regclass; +SELECT bool_and(f1 = doc(id)) AS all_equal FROM cmdict WHERE id <= 400; + +-- ordinary updates of other columns leave the value, and its old dictionary, +-- alone; updates of the column recompress with the current one +UPDATE cmdict SET id = id WHERE id = 250; +UPDATE cmdict SET f1 = f1 || '' WHERE id = 251; + +-- a rewrite recompresses with the current dictionary and retires the rest +VACUUM FULL cmdict; +SELECT count(*) AS ndicts, count(*) FILTER (WHERE tdcurrent) AS ncurrent + FROM pg_toast_dictionary WHERE tdrelid = 'cmdict'::regclass; +SELECT bool_and(f1 = doc(id)) AS all_equal FROM cmdict WHERE id <= 400; +SELECT pg_column_compression(f1), count(*) FROM cmdict WHERE id <= 400 GROUP BY 1 ORDER BY 1; + +-- the same for CLUSTER and a rewriting ALTER TABLE +SELECT pg_train_toast_dictionary('cmdict', 'f1', 4096) IS NOT NULL AS trained; +CLUSTER cmdict USING cmdict_idx; +SELECT count(*) AS ndicts FROM pg_toast_dictionary WHERE tdrelid = 'cmdict'::regclass; +SELECT pg_train_toast_dictionary('cmdict', 'f1', 4096) IS NOT NULL AS trained; +ALTER TABLE cmdict ALTER COLUMN id TYPE bigint; +SELECT count(*) AS ndicts FROM pg_toast_dictionary WHERE tdrelid = 'cmdict'::regclass; +SELECT bool_and(f1 = doc(id::int)) AS all_equal FROM cmdict WHERE id <= 400; + +-- switching the column away from zstd_dict: existing values stay readable +-- until the next rewrite, which recompresses them and drops the dictionary +ALTER TABLE cmdict ALTER COLUMN f1 SET COMPRESSION lz4; +SELECT bool_and(f1 = doc(id::int)) AS all_equal FROM cmdict WHERE id <= 400; +SELECT count(*) AS ndicts FROM pg_toast_dictionary WHERE tdrelid = 'cmdict'::regclass; +VACUUM FULL cmdict; +SELECT count(*) AS ndicts FROM pg_toast_dictionary WHERE tdrelid = 'cmdict'::regclass; +SELECT pg_column_compression(f1), count(*) FROM cmdict WHERE id <= 400 GROUP BY 1 ORDER BY 1; +SELECT bool_and(f1 = doc(id::int)) AS all_equal FROM cmdict WHERE id <= 400; +ALTER TABLE cmdict ALTER COLUMN f1 SET COMPRESSION zstd_dict; +SELECT pg_train_toast_dictionary('cmdict', 'f1', 4096) IS NOT NULL AS trained; + +---------------------------------------------------------------------- +-- Cleanup with the owning objects +---------------------------------------------------------------------- + +ALTER TABLE cmdict ADD COLUMN f3 text COMPRESSION zstd_dict; +UPDATE cmdict SET f3 = f1 WHERE id <= 400; +SELECT pg_train_toast_dictionary('cmdict', 'f3', 4096) IS NOT NULL AS trained; +SELECT tdattnum > 2 AS f3_dict, count(*) FROM pg_toast_dictionary + WHERE tdrelid = 'cmdict'::regclass GROUP BY 1 ORDER BY 1; +-- DROP COLUMN leaves the values on disk until a rewrite, and whole-row +-- references and REPLICA IDENTITY FULL still read them, so the dictionary +-- stays until then +ALTER TABLE cmdict DROP COLUMN f3; +SELECT count(*) AS ndicts FROM pg_toast_dictionary WHERE tdrelid = 'cmdict'::regclass; +SELECT count(*) AS wholerow FROM (SELECT t FROM cmdict t) s; +VACUUM FULL cmdict; +SELECT count(*) AS ndicts FROM pg_toast_dictionary WHERE tdrelid = 'cmdict'::regclass; + +-- two trainings in one command: only the last one stays current +SELECT count(*) FROM (SELECT pg_train_toast_dictionary('cmdict', 'f1', 4096) + FROM generate_series(1, 2)) s; +SELECT count(*) AS ndicts, count(*) FILTER (WHERE tdcurrent) AS ncurrent + FROM pg_toast_dictionary WHERE tdrelid = 'cmdict'::regclass; + +-- the copies made above survive their source being dropped +SELECT 'cmdict'::regclass::oid AS cmdict_oid \gset +DROP TABLE cmdict; +SELECT count(*) AS ndicts FROM pg_toast_dictionary WHERE tdrelid = :cmdict_oid; +SELECT bool_and(f1 = doc(id)) AS copies_ok FROM cmcopy WHERE id <= 400; +SELECT bool_and(f1 = doc(id)) AS copies_ok FROM cmcopy2 WHERE id <= 400; +SELECT bool_and((p).a = arr[1]) AS nested_ok FROM cmnest; + +\set HIDE_TOAST_COMPRESSION true +RESET search_path; +DROP SCHEMA zstd_dict CASCADE; +DROP ROLE regress_dict_user; diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 94aadd09b17..b6ae341f62f 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -943,6 +943,7 @@ FormData_pg_statistic_ext_data FormData_pg_subscription FormData_pg_subscription_rel FormData_pg_tablespace +FormData_pg_toast_dictionary FormData_pg_transform FormData_pg_trigger FormData_pg_ts_config @@ -1003,6 +1004,7 @@ Form_pg_statistic_ext_data Form_pg_subscription Form_pg_subscription_rel Form_pg_tablespace +Form_pg_toast_dictionary Form_pg_transform Form_pg_trigger Form_pg_ts_config @@ -3537,8 +3539,10 @@ YYLTYPE YYSTYPE YY_BUFFER_STATE ZSTD_CCtx +ZSTD_CDict ZSTD_CStream ZSTD_DCtx +ZSTD_DDict ZSTD_DStream ZSTD_cParameter ZSTD_inBuffer -- 2.54.0