From 88b0b4773afafb50173453fd243687b2b4a21d52 Mon Sep 17 00:00:00 2001 From: Dilip Kumar Date: Wed, 10 Feb 2021 16:29:55 +0530 Subject: [PATCH v24 10/10] Support compression methods options Dilip Kumar based on the patches from Ildus Kurbangaliev. Design input from Tomas Vondra and Robert Haas Reviewed by Robert Haas, Tomas Vondra, Alexander Korotkov and Justin Pryzby --- contrib/cmzlib/cmzlib.c | 75 +++++++++-- doc/src/sgml/ref/alter_table.sgml | 6 +- doc/src/sgml/ref/create_table.sgml | 6 +- src/backend/access/brin/brin_tuple.c | 5 +- src/backend/access/common/indextuple.c | 4 +- src/backend/access/common/reloptions.c | 64 ++++++++++ src/backend/access/common/toast_internals.c | 14 ++- src/backend/access/compression/compress_lz4.c | 75 ++++++++++- .../access/compression/compress_pglz.c | 116 +++++++++++++++--- src/backend/access/table/toast_helper.c | 8 +- src/backend/bootstrap/bootparse.y | 1 + src/backend/catalog/heap.c | 15 ++- src/backend/catalog/index.c | 43 +++++-- src/backend/catalog/toasting.c | 1 + src/backend/commands/cluster.c | 1 + src/backend/commands/compressioncmds.c | 79 +++++++++++- src/backend/commands/foreigncmds.c | 44 ------- src/backend/commands/tablecmds.c | 102 ++++++++++----- src/backend/nodes/copyfuncs.c | 1 + src/backend/nodes/equalfuncs.c | 1 + src/backend/nodes/outfuncs.c | 1 + src/backend/parser/gram.y | 16 ++- src/backend/parser/parse_utilcmd.c | 2 +- src/bin/pg_dump/pg_dump.c | 23 +++- src/bin/pg_dump/pg_dump.h | 1 + src/include/access/compressamapi.h | 18 ++- src/include/access/toast_helper.h | 2 + src/include/access/toast_internals.h | 2 +- src/include/catalog/heap.h | 2 + src/include/catalog/pg_attribute.h | 3 + src/include/commands/defrem.h | 9 +- src/include/nodes/parsenodes.h | 1 + src/test/regress/expected/compression.out | 52 ++++++++ src/test/regress/expected/compression_1.out | 55 +++++++++ src/test/regress/expected/misc_sanity.out | 3 +- src/test/regress/sql/compression.sql | 18 +++ 36 files changed, 728 insertions(+), 141 deletions(-) diff --git a/contrib/cmzlib/cmzlib.c b/contrib/cmzlib/cmzlib.c index 686a7c7e0d..d8c2865f21 100644 --- a/contrib/cmzlib/cmzlib.c +++ b/contrib/cmzlib/cmzlib.c @@ -14,6 +14,7 @@ #include "postgres.h" #include "access/compressamapi.h" #include "access/toast_internals.h" +#include "commands/defrem.h" #include "fmgr.h" #include "utils/builtins.h" @@ -45,6 +46,62 @@ typedef struct unsigned int dictlen; } zlib_state; +/* + * Check options if specified. All validation is located here so + * we don't need to do it again in cminitstate function. + */ +static void +zlib_cmcheck(List *options) +{ + ListCell *lc; + + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + + if (strcmp(def->defname, "level") == 0) + { + int8 level = pg_atoi(defGetString(def), sizeof(int8), 0); + + if (level < 0 || level > 9) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("unexpected value for zlib compression level: \"%s\"", + defGetString(def)), + errhint("expected value between 0 and 9") + )); + } + else + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_PARAMETER), + errmsg("unknown compression option for zlib: \"%s\"", def->defname))); + } +} + +static void * +zlib_cminitstate(List *options) +{ + zlib_state *state = NULL; + + state = palloc0(sizeof(zlib_state)); + state->level = Z_DEFAULT_COMPRESSION; + + if (list_length(options) > 0) + { + ListCell *lc; + + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + + if (strcmp(def->defname, "level") == 0) + state->level = pg_atoi(defGetString(def), sizeof(int), 0); + } + } + + return state; +} + /* * zlib_cmcompress - compression routine for zlib compression method * @@ -52,23 +109,21 @@ typedef struct * Returns the compressed varlena, or NULL if compression fails. */ static struct varlena * -zlib_cmcompress(const struct varlena *value, int32 header_size) +zlib_cmcompress(const struct varlena *value, int32 header_size, void *options) { - int32 valsize, - len; + int32 valsize, + len; struct varlena *tmp = NULL; - z_streamp zp; - int res; - zlib_state state; - - state.level = Z_DEFAULT_COMPRESSION; + z_streamp zp; + int res; + zlib_state *state = (zlib_state *) options; zp = (z_streamp) palloc(sizeof(z_stream)); zp->zalloc = Z_NULL; zp->zfree = Z_NULL; zp->opaque = Z_NULL; - if (deflateInit(zp, state.level) != Z_OK) + if (deflateInit(zp, state->level) != Z_OK) elog(ERROR, "could not initialize compression library: %s", zp->msg); valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value)); @@ -146,6 +201,8 @@ zlib_cmdecompress(const struct varlena *value, int32 header_size) const CompressionAmRoutine zlib_compress_methods = { .type = T_CompressionAmRoutine, + .datum_check = zlib_cmcheck, + .datum_initstate = zlib_cminitstate, .datum_compress = zlib_cmcompress, .datum_decompress = zlib_cmdecompress, .datum_decompress_slice = NULL}; diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml index 49c43df1c1..0f861e78f9 100644 --- a/doc/src/sgml/ref/alter_table.sgml +++ b/doc/src/sgml/ref/alter_table.sgml @@ -54,7 +54,7 @@ ALTER TABLE [ IF EXISTS ] name ALTER [ COLUMN ] column_name SET ( attribute_option = value [, ... ] ) ALTER [ COLUMN ] column_name RESET ( attribute_option [, ... ] ) ALTER [ COLUMN ] column_name SET STORAGE { PLAIN | EXTERNAL | EXTENDED | MAIN } - ALTER [ COLUMN ] column_name SET COMPRESSION compression_method [ PRESERVE (compression_preserve_list) | PRESERVE ALL ] + ALTER [ COLUMN ] column_name SET COMPRESSION compression_method [ WITH (compression_method_options) ] [ PRESERVE (compression_preserve_list) | PRESERVE ALL ] ADD table_constraint [ NOT VALID ] ADD table_constraint_using_index ALTER CONSTRAINT constraint_name [ DEFERRABLE | NOT DEFERRABLE ] [ INITIALLY DEFERRED | INITIALLY IMMEDIATE ] @@ -396,7 +396,9 @@ WITH ( MODULUS numeric_literal, REM it can be set to any available compression method. The supported buit-in compression methods are pglz and lz4. lz4 is available only if --with-lz4 - was used when building PostgreSQL. + was used when building PostgreSQL. If the + compression method has options they can be specified with the WITH + parameter. The PRESERVE list contains a list of compression methods used in the column and determines which of them may be kept. Without PRESERVE or if any of the pre-existing diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index 196352ef42..8529782d79 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -69,7 +69,7 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI GENERATED { ALWAYS | BY DEFAULT } AS IDENTITY [ ( sequence_options ) ] | UNIQUE index_parameters | PRIMARY KEY index_parameters | - COMPRESSION compression_method | + COMPRESSION compression_method [ WITH (compression_options) ] | REFERENCES reftable [ ( refcolumn ) ] [ MATCH FULL | MATCH PARTIAL | MATCH SIMPLE ] [ ON DELETE referential_action ] [ ON UPDATE referential_action ] } [ DEFERRABLE | NOT DEFERRABLE ] [ INITIALLY DEFERRED | INITIALLY IMMEDIATE ] @@ -994,7 +994,7 @@ WITH ( MODULUS numeric_literal, REM - COMPRESSION compression_method + COMPRESSION compression_method [ WITH (compression_am_options) ] The COMPRESSION clause sets the compression method @@ -1011,6 +1011,8 @@ WITH ( MODULUS numeric_literal, REM Compression methods can be created with or it can be set to any available compression method. + If the compression method has options they can be specified with the + WITH parameter. The default is pglz. diff --git a/src/backend/access/brin/brin_tuple.c b/src/backend/access/brin/brin_tuple.c index 0ab5712c71..76f824bc6f 100644 --- a/src/backend/access/brin/brin_tuple.c +++ b/src/backend/access/brin/brin_tuple.c @@ -38,6 +38,7 @@ #include "access/toast_internals.h" #include "access/tupdesc.h" #include "access/tupmacs.h" +#include "commands/defrem.h" #include "utils/datum.h" #include "utils/memutils.h" @@ -215,8 +216,10 @@ brin_form_tuple(BrinDesc *brdesc, BlockNumber blkno, BrinMemTuple *tuple, { Form_pg_attribute att = TupleDescAttr(brdesc->bd_tupdesc, keyno); + List *acoption = GetAttributeCompressionOptions(att); Datum cvalue = toast_compress_datum(value, - att->attcompression); + att->attcompression, + acoption); if (DatumGetPointer(cvalue) != NULL) { diff --git a/src/backend/access/common/indextuple.c b/src/backend/access/common/indextuple.c index 1d43d5d2ff..0d3307e94f 100644 --- a/src/backend/access/common/indextuple.c +++ b/src/backend/access/common/indextuple.c @@ -22,6 +22,7 @@ #include "access/htup_details.h" #include "access/itup.h" #include "access/toast_internals.h" +#include "commands/defrem.h" /* * This enables de-toasting of index entries. Needed until VACUUM is @@ -104,8 +105,9 @@ index_form_tuple(TupleDesc tupleDescriptor, (att->attstorage == TYPSTORAGE_EXTENDED || att->attstorage == TYPSTORAGE_MAIN)) { + List *acoption = GetAttributeCompressionOptions(att); Datum cvalue = toast_compress_datum(untoasted_values[i], - att->attcompression); + att->attcompression, acoption); if (DatumGetPointer(cvalue) != NULL) { diff --git a/src/backend/access/common/reloptions.c b/src/backend/access/common/reloptions.c index c687d3ee9e..2dda6c038b 100644 --- a/src/backend/access/common/reloptions.c +++ b/src/backend/access/common/reloptions.c @@ -2112,3 +2112,67 @@ AlterTableGetRelOptionsLockLevel(List *defList) return lockmode; } + +/* + * Convert a DefElem list to the text array format that is used in + * pg_foreign_data_wrapper, pg_foreign_server, pg_user_mapping, and + * pg_foreign_table. + * + * Returns the array in the form of a Datum, or PointerGetDatum(NULL) + * if the list is empty. + * + * Note: The array is usually stored to database without further + * processing, hence any validation should be done before this + * conversion. + */ +Datum +optionListToArray(List *options) +{ + ArrayBuildState *astate = NULL; + ListCell *cell; + + foreach(cell, options) + { + DefElem *def = lfirst(cell); + const char *value; + Size len; + text *t; + + value = defGetString(def); + len = VARHDRSZ + strlen(def->defname) + 1 + strlen(value); + t = palloc(len + 1); + SET_VARSIZE(t, len); + sprintf(VARDATA(t), "%s=%s", def->defname, value); + + astate = accumArrayResult(astate, PointerGetDatum(t), + false, TEXTOID, + CurrentMemoryContext); + } + + if (astate) + return makeArrayResult(astate, CurrentMemoryContext); + + return PointerGetDatum(NULL); +} + +/* + * Return human readable list of reloptions + */ +char * +formatRelOptions(List *options) +{ + StringInfoData buf; + ListCell *cell; + + initStringInfo(&buf); + + foreach(cell, options) + { + DefElem *def = (DefElem *) lfirst(cell); + + appendStringInfo(&buf, "%s%s=%s", buf.len > 0 ? ", " : "", + def->defname, defGetString(def)); + } + + return buf.data; +} diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c index a3539065d3..5ed3312f39 100644 --- a/src/backend/access/common/toast_internals.c +++ b/src/backend/access/common/toast_internals.c @@ -44,10 +44,11 @@ static bool toastid_valueid_exists(Oid toastrelid, Oid valueid); * ---------- */ Datum -toast_compress_datum(Datum value, Oid cmoid) +toast_compress_datum(Datum value, Oid cmoid, List *cmoptions) { struct varlena *tmp = NULL; int32 valsize; + void *options = NULL; bool isCustomCompression = false; const CompressionAmRoutine *cmroutine = NULL; @@ -71,11 +72,20 @@ toast_compress_datum(Datum value, Oid cmoid) break; } + if (cmroutine->datum_initstate) + options = cmroutine->datum_initstate(cmoptions); + /* Call the actual compression function */ tmp = cmroutine->datum_compress((const struct varlena *)value, isCustomCompression ? TOAST_CUSTOM_COMPRESS_HDRSZ : - TOAST_COMPRESS_HDRSZ); + TOAST_COMPRESS_HDRSZ, options); + if (options != NULL) + { + pfree(options); + list_free(cmoptions); + } + if (!tmp) return PointerGetDatum(NULL); diff --git a/src/backend/access/compression/compress_lz4.c b/src/backend/access/compression/compress_lz4.c index 2a3c162836..7c422fc534 100644 --- a/src/backend/access/compression/compress_lz4.c +++ b/src/backend/access/compression/compress_lz4.c @@ -17,9 +17,73 @@ #endif #include "access/compressamapi.h" +#include "commands/defrem.h" #include "fmgr.h" #include "utils/builtins.h" +/* + * Check options if specified. All validation is located here so + * we don't need to do it again in cminitstate function. + */ +static void +lz4_cmcheck(List *options) +{ +#ifndef HAVE_LIBLZ4 + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("not built with lz4 support"))); +#else + ListCell *lc; + + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + + /* + * We don't need to check the range for the acceleration parameter + * because the LZ4_compress_fast will automatically replace the + * values <=0 with LZ4_ACCELERATION_DEFAULT (currently == 1). And, + * Values > LZ4_ACCELERATION_MAX with LZ4_ACCELERATION_MAX + * (currently == 65537c). + */ + if (strcmp(def->defname, "acceleration") != 0) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_PARAMETER), + errmsg("unknown compression option for lz4: \"%s\"", def->defname))); + } +#endif +} + +static void * +lz4_cminitstate(List *options) +{ +#ifndef HAVE_LIBLZ4 + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("not built with lz4 support"))); +#else + int32 *acceleration = palloc(sizeof(int32)); + + /* initialize with the default acceleration */ + *acceleration = 1; + + if (list_length(options) > 0) + { + ListCell *lc; + + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + + if (strcmp(def->defname, "acceleration") == 0) + *acceleration = pg_atoi(defGetString(def), sizeof(int32), 0); + } + } + + return acceleration; +#endif +} + /* * lz4_cmcompress - compression routine for lz4 compression method * @@ -27,7 +91,7 @@ * compressed varlena, or NULL if compression fails. */ static struct varlena * -lz4_cmcompress(const struct varlena *value, int32 header_size) +lz4_cmcompress(const struct varlena *value, int32 header_size, void *options) { #ifndef HAVE_LIBLZ4 ereport(ERROR, @@ -37,6 +101,7 @@ lz4_cmcompress(const struct varlena *value, int32 header_size) int32 valsize; int32 len; int32 max_size; + int32 *acceleration = (int32 *) options; struct varlena *tmp = NULL; valsize = VARSIZE_ANY_EXHDR(value); @@ -48,9 +113,9 @@ lz4_cmcompress(const struct varlena *value, int32 header_size) max_size = LZ4_compressBound(valsize); tmp = (struct varlena *) palloc(max_size + header_size); - len = LZ4_compress_default(VARDATA_ANY(value), - (char *) tmp + header_size, - valsize, max_size); + len = LZ4_compress_fast(VARDATA_ANY(value), + (char *) tmp + header_size, + valsize, max_size, *acceleration); if (len <= 0) elog(ERROR, "could not compress data with lz4"); @@ -146,6 +211,8 @@ lz4_cmdecompress_slice(const struct varlena *value, int32 header_size, */ const CompressionAmRoutine lz4_compress_methods = { .type = T_CompressionAmRoutine, + .datum_check = lz4_cmcheck, + .datum_initstate = lz4_cminitstate, .datum_compress = lz4_cmcompress, .datum_decompress = lz4_cmdecompress, .datum_decompress_slice = lz4_cmdecompress_slice diff --git a/src/backend/access/compression/compress_pglz.c b/src/backend/access/compression/compress_pglz.c index 7f6e7429fe..1ca912831b 100644 --- a/src/backend/access/compression/compress_pglz.c +++ b/src/backend/access/compression/compress_pglz.c @@ -14,11 +14,93 @@ #include "postgres.h" #include "access/compressamapi.h" +#include "access/toast_internals.h" +#include "commands/defrem.h" #include "common/pg_lzcompress.h" #include "fmgr.h" #include "utils/builtins.h" +#define PGLZ_OPTIONS_COUNT 6 + +static char *PGLZ_options[PGLZ_OPTIONS_COUNT] = { + "min_input_size", + "max_input_size", + "min_comp_rate", + "first_success_by", + "match_size_good", + "match_size_drop" +}; + +/* + * Convert value from reloptions to int32, and report if it is not correct. + * Also checks parameter names + */ +static int32 +parse_option(char *name, char *value) +{ + int i; + + for (i = 0; i < PGLZ_OPTIONS_COUNT; i++) + { + if (strcmp(PGLZ_options[i], name) == 0) + return pg_atoi(value, 4, 0); + } + + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_PARAMETER), + errmsg("unknown compression option for pglz: \"%s\"", name))); +} + +/* + * Check PGLZ options if specified + */ +static void +pglz_cmcheck(List *options) +{ + ListCell *lc; + + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + + parse_option(def->defname, defGetString(def)); + } +} + +/* + * Configure PGLZ_Strategy struct for compression function + */ +static void * +pglz_cminitstate(List *options) +{ + ListCell *lc; + PGLZ_Strategy *strategy = palloc(sizeof(PGLZ_Strategy)); + + /* initialize with default strategy values */ + memcpy(strategy, PGLZ_strategy_default, sizeof(PGLZ_Strategy)); + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + int32 val = parse_option(def->defname, defGetString(def)); + + /* fill the strategy */ + if (strcmp(def->defname, "min_input_size") == 0) + strategy->min_input_size = val; + else if (strcmp(def->defname, "max_input_size") == 0) + strategy->max_input_size = val; + else if (strcmp(def->defname, "min_comp_rate") == 0) + strategy->min_comp_rate = val; + else if (strcmp(def->defname, "first_success_by") == 0) + strategy->first_success_by = val; + else if (strcmp(def->defname, "match_size_good") == 0) + strategy->match_size_good = val; + else if (strcmp(def->defname, "match_size_drop") == 0) + strategy->match_size_drop = val; + } + return (void *) strategy; +} + /* * pglz_cmcompress - compression routine for pglz compression method * @@ -26,42 +108,41 @@ * compressed varlena, or NULL if compression fails. */ static struct varlena * -pglz_cmcompress(const struct varlena *value, int32 header_size) +pglz_cmcompress(const struct varlena *value, int32 header_size, void *options) { int32 valsize, len; - struct varlena *tmp = NULL; + struct varlena *tmp = NULL; + PGLZ_Strategy *strategy; valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value)); + strategy = (PGLZ_Strategy *) options; /* * No point in wasting a palloc cycle if value size is outside the allowed * range for compression. */ - if (valsize < PGLZ_strategy_default->min_input_size || - valsize > PGLZ_strategy_default->max_input_size) + if (valsize < strategy->min_input_size || + valsize > strategy->max_input_size) return NULL; - /* - * Get maximum size of the compressed data that pglz compression may output - * and allocate the memory for holding the compressed data and the header. - */ tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) + header_size); - len = pglz_compress(VARDATA_ANY(value), + len = pglz_compress(VARDATA_ANY(DatumGetPointer(value)), valsize, (char *) tmp + header_size, - NULL); - if (len < 0) + strategy); + + if (len >= 0) { - pfree(tmp); - return NULL; + SET_VARSIZE_COMPRESSED(tmp, len + header_size); + return tmp; } - SET_VARSIZE_COMPRESSED(tmp, len + header_size); + pfree(tmp); - return tmp; + return NULL; } /* @@ -125,10 +206,11 @@ pglz_cmdecompress_slice(const struct varlena *value, int32 header_size, */ const CompressionAmRoutine pglz_compress_methods = { .type = T_CompressionAmRoutine, + .datum_check = pglz_cmcheck, + .datum_initstate = pglz_cminitstate, .datum_compress = pglz_cmcompress, .datum_decompress = pglz_cmdecompress, - .datum_decompress_slice = pglz_cmdecompress_slice -}; + .datum_decompress_slice = pglz_cmdecompress_slice}; /* pglz compression handler function */ Datum diff --git a/src/backend/access/table/toast_helper.c b/src/backend/access/table/toast_helper.c index 53f78f9c3e..c9d44c62fa 100644 --- a/src/backend/access/table/toast_helper.c +++ b/src/backend/access/table/toast_helper.c @@ -15,11 +15,13 @@ #include "postgres.h" #include "access/detoast.h" +#include "access/reloptions.h" #include "access/table.h" #include "access/toast_helper.h" #include "access/toast_internals.h" #include "catalog/pg_type_d.h" - +#include "commands/defrem.h" +#include "utils/syscache.h" /* * Prepare to TOAST a tuple. @@ -55,6 +57,7 @@ toast_tuple_init(ToastTupleContext *ttc) ttc->ttc_attr[i].tai_colflags = 0; ttc->ttc_attr[i].tai_oldexternal = NULL; ttc->ttc_attr[i].tai_compression = att->attcompression; + ttc->ttc_attr[i].tai_cmoptions = GetAttributeCompressionOptions(att); if (ttc->ttc_oldvalues != NULL) { @@ -230,7 +233,8 @@ toast_tuple_try_compression(ToastTupleContext *ttc, int attribute) Datum new_value; ToastAttrInfo *attr = &ttc->ttc_attr[attribute]; - new_value = toast_compress_datum(*value, attr->tai_compression); + new_value = toast_compress_datum(*value, attr->tai_compression, + attr->tai_cmoptions); if (DatumGetPointer(new_value) != NULL) { diff --git a/src/backend/bootstrap/bootparse.y b/src/backend/bootstrap/bootparse.y index 5fcd004e1b..a43e0e197c 100644 --- a/src/backend/bootstrap/bootparse.y +++ b/src/backend/bootstrap/bootparse.y @@ -239,6 +239,7 @@ Boot_CreateStmt: true, false, InvalidOid, + NULL, NULL); elog(DEBUG4, "relation created with OID %u", id); } diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index b53b6b50e6..18b04816f7 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -741,6 +741,7 @@ InsertPgAttributeTuples(Relation pg_attribute_rel, TupleDesc tupdesc, Oid new_rel_oid, Datum *attoptions, + Datum *attcmoptions, CatalogIndexState indstate) { TupleTableSlot **slot; @@ -796,6 +797,11 @@ InsertPgAttributeTuples(Relation pg_attribute_rel, else slot[slotCount]->tts_isnull[Anum_pg_attribute_attoptions - 1] = true; + if (attcmoptions && attcmoptions[natts] != (Datum) 0) + slot[slotCount]->tts_values[Anum_pg_attribute_attcmoptions - 1] = attcmoptions[natts]; + else + slot[slotCount]->tts_isnull[Anum_pg_attribute_attcmoptions - 1] = true; + /* start out with empty permissions and empty options */ slot[slotCount]->tts_isnull[Anum_pg_attribute_attacl - 1] = true; slot[slotCount]->tts_isnull[Anum_pg_attribute_attfdwoptions - 1] = true; @@ -843,6 +849,7 @@ InsertPgAttributeTuples(Relation pg_attribute_rel, static void AddNewAttributeTuples(Oid new_rel_oid, TupleDesc tupdesc, + Datum *acoption, char relkind) { Relation rel; @@ -861,7 +868,8 @@ AddNewAttributeTuples(Oid new_rel_oid, /* set stats detail level to a sane default */ for (int i = 0; i < natts; i++) tupdesc->attrs[i].attstattarget = -1; - InsertPgAttributeTuples(rel, tupdesc, new_rel_oid, NULL, indstate); + InsertPgAttributeTuples(rel, tupdesc, new_rel_oid, + NULL, acoption, indstate); /* add dependencies on their datatypes and collations */ for (int i = 0; i < natts; i++) @@ -893,7 +901,7 @@ AddNewAttributeTuples(Oid new_rel_oid, td = CreateTupleDesc(lengthof(SysAtt), (FormData_pg_attribute **) &SysAtt); - InsertPgAttributeTuples(rel, td, new_rel_oid, NULL, indstate); + InsertPgAttributeTuples(rel, td, new_rel_oid, NULL, NULL, indstate); FreeTupleDesc(td); } @@ -1160,6 +1168,7 @@ heap_create_with_catalog(const char *relname, bool allow_system_table_mods, bool is_internal, Oid relrewrite, + Datum *acoptions, ObjectAddress *typaddress) { Relation pg_class_desc; @@ -1418,7 +1427,7 @@ heap_create_with_catalog(const char *relname, /* * now add tuples to pg_attribute for the attributes in our new relation. */ - AddNewAttributeTuples(relid, new_rel_desc->rd_att, relkind); + AddNewAttributeTuples(relid, new_rel_desc->rd_att, acoptions, relkind); /* * Make a dependency link to force the relation to be deleted if its diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index d1d6bdf470..f5441ce24a 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -108,10 +108,12 @@ static TupleDesc ConstructTupleDescriptor(Relation heapRelation, List *indexColNames, Oid accessMethodObjectId, Oid *collationObjectId, - Oid *classObjectId); + Oid *classObjectId, + Datum *acoptions); static void InitializeAttributeOids(Relation indexRelation, int numatts, Oid indexoid); -static void AppendAttributeTuples(Relation indexRelation, Datum *attopts); +static void AppendAttributeTuples(Relation indexRelation, Datum *attopts, + Datum *attcmopts); static void UpdateIndexRelation(Oid indexoid, Oid heapoid, Oid parentIndexId, IndexInfo *indexInfo, @@ -273,7 +275,8 @@ ConstructTupleDescriptor(Relation heapRelation, List *indexColNames, Oid accessMethodObjectId, Oid *collationObjectId, - Oid *classObjectId) + Oid *classObjectId, + Datum *acoptions) { int numatts = indexInfo->ii_NumIndexAttrs; int numkeyatts = indexInfo->ii_NumIndexKeyAttrs; @@ -334,6 +337,9 @@ ConstructTupleDescriptor(Relation heapRelation, { /* Simple index column */ const FormData_pg_attribute *from; + HeapTuple attr_tuple; + Datum attcmoptions; + bool isNull; Assert(atnum > 0); /* should've been caught above */ @@ -350,6 +356,23 @@ ConstructTupleDescriptor(Relation heapRelation, to->attstorage = from->attstorage; to->attalign = from->attalign; to->attcompression = from->attcompression; + + attr_tuple = SearchSysCache2(ATTNUM, + ObjectIdGetDatum(from->attrelid), + Int16GetDatum(from->attnum)); + if (!HeapTupleIsValid(attr_tuple)) + elog(ERROR, "cache lookup failed for attribute %d of relation %u", + from->attnum, from->attrelid); + + attcmoptions = SysCacheGetAttr(ATTNUM, attr_tuple, + Anum_pg_attribute_attcmoptions, + &isNull); + if (isNull) + acoptions[i] = PointerGetDatum(NULL); + else + acoptions[i] = + PointerGetDatum(DatumGetArrayTypePCopy(attcmoptions)); + ReleaseSysCache(attr_tuple); } else { @@ -490,7 +513,7 @@ InitializeAttributeOids(Relation indexRelation, * ---------------------------------------------------------------- */ static void -AppendAttributeTuples(Relation indexRelation, Datum *attopts) +AppendAttributeTuples(Relation indexRelation, Datum *attopts, Datum *attcmopts) { Relation pg_attribute; CatalogIndexState indstate; @@ -508,7 +531,8 @@ AppendAttributeTuples(Relation indexRelation, Datum *attopts) */ indexTupDesc = RelationGetDescr(indexRelation); - InsertPgAttributeTuples(pg_attribute, indexTupDesc, InvalidOid, attopts, indstate); + InsertPgAttributeTuples(pg_attribute, indexTupDesc, InvalidOid, + attopts, attcmopts, indstate); CatalogCloseIndexes(indstate); @@ -721,6 +745,7 @@ index_create(Relation heapRelation, bool concurrent = (flags & INDEX_CREATE_CONCURRENT) != 0; bool partitioned = (flags & INDEX_CREATE_PARTITIONED) != 0; char relkind; + Datum *acoptions; TransactionId relfrozenxid; MultiXactId relminmxid; @@ -876,6 +901,8 @@ index_create(Relation heapRelation, indexRelationName, RelationGetRelationName(heapRelation)))); } + acoptions = (Datum *) palloc0(sizeof(Datum) * indexInfo->ii_NumIndexAttrs); + /* * construct tuple descriptor for index tuples */ @@ -884,7 +911,8 @@ index_create(Relation heapRelation, indexColNames, accessMethodObjectId, collationObjectId, - classObjectId); + classObjectId, + acoptions); /* * Allocate an OID for the index, unless we were told what to use. @@ -975,7 +1003,8 @@ index_create(Relation heapRelation, /* * append ATTRIBUTE tuples for the index */ - AppendAttributeTuples(indexRelation, indexInfo->ii_OpclassOptions); + AppendAttributeTuples(indexRelation, indexInfo->ii_OpclassOptions, + acoptions); /* ---------------- * update pg_index diff --git a/src/backend/catalog/toasting.c b/src/backend/catalog/toasting.c index a549481557..348b7d3a37 100644 --- a/src/backend/catalog/toasting.c +++ b/src/backend/catalog/toasting.c @@ -260,6 +260,7 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, true, true, InvalidOid, + NULL, NULL); Assert(toast_relid != InvalidOid); diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index 096a06f7b3..0ad946d802 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -699,6 +699,7 @@ make_new_heap(Oid OIDOldHeap, Oid NewTableSpace, char relpersistence, true, true, OIDOldHeap, + NULL, NULL); Assert(OIDNewHeap != InvalidOid); diff --git a/src/backend/commands/compressioncmds.c b/src/backend/commands/compressioncmds.c index fd6db24e7f..160d64ad32 100644 --- a/src/backend/commands/compressioncmds.c +++ b/src/backend/commands/compressioncmds.c @@ -29,6 +29,7 @@ #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" +#include "utils/syscache.h" /* * Get list of all supported compression methods for the given attribute. @@ -181,7 +182,7 @@ BinaryUpgradeAddPreserve(Form_pg_attribute att, List *preserve) */ Oid GetAttributeCompression(Form_pg_attribute att, ColumnCompression *compression, - bool *need_rewrite) + Datum *acoptions, bool *need_rewrite) { Oid cmoid; char typstorage = get_typstorage(att->atttypid); @@ -215,6 +216,20 @@ GetAttributeCompression(Form_pg_attribute att, ColumnCompression *compression, errmsg("not built with lz4 support"))); #endif + /* if compression options are given then check them */ + if (compression->options) + { + CompressionAmRoutine *routine = GetCompressionAmRoutineByAmId(cmoid); + + /* we need routine only to call cmcheck function */ + if (routine->datum_check != NULL) + routine->datum_check(compression->options); + + *acoptions = optionListToArray(compression->options); + } + else + *acoptions = PointerGetDatum(NULL); + /* * Determine if the column needs rewrite or not. Rewrite conditions: SET * COMPRESSION without PRESERVE - SET COMPRESSION with PRESERVE but not @@ -286,15 +301,71 @@ GetAttributeCompression(Form_pg_attribute att, ColumnCompression *compression, * Construct ColumnCompression node from the compression method oid. */ ColumnCompression * -MakeColumnCompression(Oid attcompression) +MakeColumnCompression(Form_pg_attribute att) { ColumnCompression *node; - if (!OidIsValid(attcompression)) + if (!OidIsValid(att->attcompression)) return NULL; node = makeNode(ColumnCompression); - node->cmname = get_am_name(attcompression); + node->cmname = get_am_name(att->attcompression); + node->options = GetAttributeCompressionOptions(att); return node; } + +/* + * Fetch atttribute's compression options + */ +List * +GetAttributeCompressionOptions(Form_pg_attribute att) +{ + HeapTuple attr_tuple; + Datum attcmoptions; + List *acoptions; + bool isNull; + + attr_tuple = SearchSysCache2(ATTNUM, + ObjectIdGetDatum(att->attrelid), + Int16GetDatum(att->attnum)); + if (!HeapTupleIsValid(attr_tuple)) + elog(ERROR, "cache lookup failed for attribute %d of relation %u", + att->attnum, att->attrelid); + + attcmoptions = SysCacheGetAttr(ATTNUM, attr_tuple, + Anum_pg_attribute_attcmoptions, + &isNull); + if (isNull) + acoptions = NULL; + else + acoptions = untransformRelOptions(attcmoptions); + + ReleaseSysCache(attr_tuple); + + return acoptions; +} + +/* + * Compare compression options for two columns. + */ +void +CheckCompressionMismatch(ColumnCompression *c1, ColumnCompression *c2, + const char *attributeName) +{ + if (strcmp(c1->cmname, c2->cmname)) + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("column \"%s\" has a compression method conflict", + attributeName), + errdetail("%s versus %s", c1->cmname, c2->cmname))); + + if (!equal(c1->options, c2->options)) + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("column \"%s\" has a compression options conflict", + attributeName), + errdetail("(%s) versus (%s)", + formatRelOptions(c1->options), + formatRelOptions(c2->options)))); +} diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c index eb7103fd3b..ae9ae2c51b 100644 --- a/src/backend/commands/foreigncmds.c +++ b/src/backend/commands/foreigncmds.c @@ -49,50 +49,6 @@ typedef struct /* Internal functions */ static void import_error_callback(void *arg); - -/* - * Convert a DefElem list to the text array format that is used in - * pg_foreign_data_wrapper, pg_foreign_server, pg_user_mapping, and - * pg_foreign_table. - * - * Returns the array in the form of a Datum, or PointerGetDatum(NULL) - * if the list is empty. - * - * Note: The array is usually stored to database without further - * processing, hence any validation should be done before this - * conversion. - */ -static Datum -optionListToArray(List *options) -{ - ArrayBuildState *astate = NULL; - ListCell *cell; - - foreach(cell, options) - { - DefElem *def = lfirst(cell); - const char *value; - Size len; - text *t; - - value = defGetString(def); - len = VARHDRSZ + strlen(def->defname) + 1 + strlen(value); - t = palloc(len + 1); - SET_VARSIZE(t, len); - sprintf(VARDATA(t), "%s=%s", def->defname, value); - - astate = accumArrayResult(astate, PointerGetDatum(t), - false, TEXTOID, - CurrentMemoryContext); - } - - if (astate) - return makeArrayResult(astate, CurrentMemoryContext); - - return PointerGetDatum(NULL); -} - - /* * Transform a list of DefElem into text array format. This is substantially * the same thing as optionListToArray(), except we recognize SET/ADD/DROP diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 2a1841c353..afb9f1190f 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -609,6 +609,7 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, LOCKMODE parentLockmode; const char *accessMethod = NULL; Oid accessMethodId = InvalidOid; + Datum *acoptions; /* * Truncate relname to appropriate length (probably a waste of time, as @@ -813,6 +814,8 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, cookedDefaults = NIL; attnum = 0; + acoptions = (Datum *) palloc0(sizeof(Datum) * descriptor->natts); + foreach(listptr, stmt->tableElts) { ColumnDef *colDef = lfirst(listptr); @@ -866,8 +869,9 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, if (relkind == RELKIND_RELATION || relkind == RELKIND_PARTITIONED_TABLE || relkind == RELKIND_MATVIEW) - attr->attcompression = - GetAttributeCompression(attr, colDef->compression, NULL); + attr->attcompression = GetAttributeCompression(attr, + colDef->compression, + &acoptions[attnum - 1], NULL); else attr->attcompression = InvalidOid; } @@ -921,8 +925,11 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, allowSystemTableMods, false, InvalidOid, + acoptions, typaddress); + pfree(acoptions); + /* * We must bump the command counter to make the newly-created relation * tuple visible for opening. @@ -2432,16 +2439,14 @@ MergeAttributes(List *schema, List *supers, char relpersistence, if (OidIsValid(attribute->attcompression)) { ColumnCompression *compression = - MakeColumnCompression(attribute->attcompression); + MakeColumnCompression(attribute); if (!def->compression) def->compression = compression; - else if (strcmp(def->compression->cmname, compression->cmname)) - ereport(ERROR, - (errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg("column \"%s\" has a compression method conflict", - attributeName), - errdetail("%s versus %s", def->compression->cmname, compression->cmname))); + else + CheckCompressionMismatch(def->compression, + compression, + attributeName); } def->inhcount++; @@ -2478,8 +2483,7 @@ MergeAttributes(List *schema, List *supers, char relpersistence, def->collOid = attribute->attcollation; def->constraints = NIL; def->location = -1; - def->compression = MakeColumnCompression( - attribute->attcompression); + def->compression = MakeColumnCompression(attribute); inhSchema = lappend(inhSchema, def); newattmap->attnums[parent_attno - 1] = ++child_attno; } @@ -2729,14 +2733,9 @@ MergeAttributes(List *schema, List *supers, char relpersistence, if (!def->compression) def->compression = newdef->compression; else if (newdef->compression) - { - if (strcmp(def->compression->cmname, newdef->compression->cmname)) - ereport(ERROR, - (errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg("column \"%s\" has a compression method conflict", - attributeName), - errdetail("%s versus %s", def->compression->cmname, newdef->compression->cmname))); - } + CheckCompressionMismatch(def->compression, + newdef->compression, + attributeName); /* Mark the column as locally defined */ def->is_local = true; @@ -6271,6 +6270,7 @@ ATExecAddColumn(List **wqueue, AlteredTableInfo *tab, Relation rel, AclResult aclresult; ObjectAddress address; TupleDesc tupdesc; + Datum acoptions = PointerGetDatum(NULL); FormData_pg_attribute *aattr[] = {&attribute}; /* At top level, permission check was done in ATPrepCmd, else do it */ @@ -6434,6 +6434,7 @@ ATExecAddColumn(List **wqueue, AlteredTableInfo *tab, Relation rel, rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) attribute.attcompression = GetAttributeCompression(&attribute, colDef->compression, + &acoptions, NULL); else attribute.attcompression = InvalidOid; @@ -6444,7 +6445,7 @@ ATExecAddColumn(List **wqueue, AlteredTableInfo *tab, Relation rel, tupdesc = CreateTupleDesc(lengthof(aattr), (FormData_pg_attribute **) &aattr); - InsertPgAttributeTuples(attrdesc, tupdesc, myrelid, NULL, NULL); + InsertPgAttributeTuples(attrdesc, tupdesc, myrelid, NULL, &acoptions, NULL); table_close(attrdesc, RowExclusiveLock); @@ -7840,13 +7841,20 @@ ATExecSetOptions(Relation rel, const char *colName, Node *options, * the respective input values are valid. */ static void -ApplyChangesToIndexes(Relation rel, Relation attrelation, AttrNumber attnum, - Oid newcompression, char newstorage, LOCKMODE lockmode) +ApplyChangesToIndexes(Relation rel, Relation attrel, AttrNumber attnum, + Oid newcompression, char newstorage, Datum acoptions, + LOCKMODE lockmode) { HeapTuple tuple; ListCell *lc; Form_pg_attribute attrtuple; + /* + * Compression option can only be valid if we are updating the compression + * method. + */ + Assert(DatumGetPointer(acoptions) == NULL || OidIsValid(newcompression)); + foreach(lc, RelationGetIndexList(rel)) { Oid indexoid = lfirst_oid(lc); @@ -7882,7 +7890,29 @@ ApplyChangesToIndexes(Relation rel, Relation attrelation, AttrNumber attnum, if (newstorage != '\0') attrtuple->attstorage = newstorage; - CatalogTupleUpdate(attrelation, &tuple->t_self, tuple); + if (DatumGetPointer(acoptions) != NULL) + { + Datum values[Natts_pg_attribute]; + bool nulls[Natts_pg_attribute]; + bool replace[Natts_pg_attribute]; + HeapTuple newtuple; + + /* Initialize buffers for new tuple values */ + memset(values, 0, sizeof(values)); + memset(nulls, false, sizeof(nulls)); + memset(replace, false, sizeof(replace)); + + values[Anum_pg_attribute_attcmoptions - 1] = acoptions; + replace[Anum_pg_attribute_attcmoptions - 1] = true; + + newtuple = heap_modify_tuple(tuple, RelationGetDescr(attrel), + values, nulls, replace); + CatalogTupleUpdate(attrel, &newtuple->t_self, newtuple); + + heap_freetuple(newtuple); + } + else + CatalogTupleUpdate(attrel, &tuple->t_self, tuple); InvokeObjectPostAlterHook(RelationRelationId, RelationGetRelid(rel), @@ -7973,7 +8003,7 @@ ATExecSetStorage(Relation rel, const char *colName, Node *newValue, LOCKMODE loc * matching behavior of index.c ConstructTupleDescriptor()). */ ApplyChangesToIndexes(rel, attrelation, attnum, InvalidOid, newstorage, - lockmode); + PointerGetDatum(NULL), lockmode); table_close(attrelation, RowExclusiveLock); @@ -15145,9 +15175,11 @@ ATExecSetCompression(AlteredTableInfo *tab, char typstorage; Oid cmoid; bool need_rewrite; + HeapTuple newtuple; Datum values[Natts_pg_attribute]; bool nulls[Natts_pg_attribute]; bool replace[Natts_pg_attribute]; + Datum acoptions; ObjectAddress address; attrel = table_open(AttributeRelationId, RowExclusiveLock); @@ -15182,7 +15214,8 @@ ATExecSetCompression(AlteredTableInfo *tab, memset(replace, false, sizeof(replace)); /* Get the attribute compression method. */ - cmoid = GetAttributeCompression(atttableform, compression, &need_rewrite); + cmoid = GetAttributeCompression(atttableform, compression, &acoptions, + &need_rewrite); if (atttableform->attcompression != cmoid) add_column_compression_dependency(atttableform->attrelid, @@ -15192,8 +15225,17 @@ ATExecSetCompression(AlteredTableInfo *tab, atttableform->attcompression = cmoid; - atttableform->attcompression = cmoid; - CatalogTupleUpdate(attrel, &tuple->t_self, tuple); + /* update an existing entry */ + if (acoptions) + { + values[Anum_pg_attribute_attcmoptions - 1] = acoptions; + replace[Anum_pg_attribute_attcmoptions - 1] = true; + newtuple = heap_modify_tuple(tuple, RelationGetDescr(attrel), + values, nulls, replace); + CatalogTupleUpdate(attrel, &newtuple->t_self, newtuple); + } + else + CatalogTupleUpdate(attrel, &tuple->t_self, tuple); InvokeObjectPostAlterHook(RelationRelationId, RelationGetRelid(rel), @@ -15201,8 +15243,10 @@ ATExecSetCompression(AlteredTableInfo *tab, ReleaseSysCache(tuple); - /* apply changes to the index column as well */ - ApplyChangesToIndexes(rel, attrel, attnum, cmoid, '\0', lockmode); + /* Apply the change to indexes as well */ + ApplyChangesToIndexes(rel, attrel, attnum, cmoid, '\0', acoptions, + lockmode); + table_close(attrel, RowExclusiveLock); /* make changes visible */ diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 6a11f8eb60..991340e2ad 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -2993,6 +2993,7 @@ _copyColumnCompression(const ColumnCompression *from) COPY_STRING_FIELD(cmname); COPY_SCALAR_FIELD(preserve_all); + COPY_NODE_FIELD(options); COPY_NODE_FIELD(preserve); return newnode; diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index 26a9b85974..57a2975da1 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -2624,6 +2624,7 @@ _equalColumnCompression(const ColumnCompression *a, const ColumnCompression *b) { COMPARE_STRING_FIELD(cmname); COMPARE_SCALAR_FIELD(preserve_all); + COMPARE_NODE_FIELD(options); COMPARE_NODE_FIELD(preserve); return true; diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index b584a58ba3..bc5c64df1e 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -2888,6 +2888,7 @@ _outColumnCompression(StringInfo str, const ColumnCompression *node) WRITE_STRING_FIELD(cmname); WRITE_BOOL_FIELD(preserve_all); + WRITE_NODE_FIELD(options); WRITE_NODE_FIELD(preserve); } diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index b22ce818cd..00fe3b7a5f 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -415,6 +415,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); relation_expr_list dostmt_opt_list transform_element_list transform_type_list TriggerTransitions TriggerReferencing + optCompressionParameters vacuum_relation_list opt_vacuum_relation_list drop_option_list @@ -3503,11 +3504,17 @@ compressionClause: COMPRESSION name { $$ = pstrdup($2); } ; +optCompressionParameters: + WITH '(' generic_option_list ')' { $$ = $3; } + | /*EMPTY*/ { $$ = NULL; } + ; + optColumnCompression: - compressionClause + compressionClause optCompressionParameters { ColumnCompression *n = makeNode(ColumnCompression); n->cmname = $1; + n->options = (List *) $2; n->preserve = NIL; $$ = (Node *) n; } @@ -3515,14 +3522,15 @@ optColumnCompression: ; alterColumnCompression: - compressionClause optCompressionPreserve + compressionClause optCompressionParameters optCompressionPreserve { ColumnCompression *n = makeNode(ColumnCompression); n->cmname = $1; - n->preserve = (List *) $2; + n->options = (List *) $2; + n->preserve = (List *) $3; $$ = (Node *) n; } - | compressionClause PRESERVE ALL + | compressionClause optCompressionParameters PRESERVE ALL { ColumnCompression *n = makeNode(ColumnCompression); n->cmname = $1; diff --git a/src/backend/parser/parse_utilcmd.c b/src/backend/parser/parse_utilcmd.c index 45f4724a13..3a33bce506 100644 --- a/src/backend/parser/parse_utilcmd.c +++ b/src/backend/parser/parse_utilcmd.c @@ -1086,7 +1086,7 @@ transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_cla /* Likewise, copy compression if requested */ if ((table_like_clause->options & CREATE_TABLE_LIKE_COMPRESSION) != 0 && OidIsValid(attribute->attcompression)) - def->compression = MakeColumnCompression(attribute->attcompression); + def->compression = MakeColumnCompression(attribute); else def->compression = NULL; diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 88fa7e1ed3..f9517549f9 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -8709,10 +8709,17 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) if (createWithCompression) appendPQExpBuffer(q, - "am.amname AS attcmname,\n"); + "am.amname AS attcmname,\n" + "pg_catalog.array_to_string(ARRAY(" + "SELECT pg_catalog.quote_ident(option_name) || " + "' ' || pg_catalog.quote_literal(option_value) " + "FROM pg_catalog.pg_options_to_table(a.attcmoptions) " + "ORDER BY option_name" + "), E',\n ') AS attcmoptions,\n"); else appendPQExpBuffer(q, - "NULL AS attcmname,\n"); + "NULL AS attcmname,\n" + "NULL AS attcmoptions,\n"); if (fout->remoteVersion >= 110000) appendPQExpBufferStr(q, @@ -8765,6 +8772,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) tbinfo->attfdwoptions = (char **) pg_malloc(ntups * sizeof(char *)); tbinfo->attmissingval = (char **) pg_malloc(ntups * sizeof(char *)); tbinfo->attcmnames = (char **) pg_malloc(ntups * sizeof(char *)); + tbinfo->attcmoptions = (char **) pg_malloc(ntups * sizeof(char *)); tbinfo->notnull = (bool *) pg_malloc(ntups * sizeof(bool)); tbinfo->inhNotNull = (bool *) pg_malloc(ntups * sizeof(bool)); tbinfo->attrdefs = (AttrDefInfo **) pg_malloc(ntups * sizeof(AttrDefInfo *)); @@ -8794,6 +8802,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) tbinfo->attfdwoptions[j] = pg_strdup(PQgetvalue(res, j, PQfnumber(res, "attfdwoptions"))); tbinfo->attmissingval[j] = pg_strdup(PQgetvalue(res, j, PQfnumber(res, "attmissingval"))); tbinfo->attcmnames[j] = pg_strdup(PQgetvalue(res, j, PQfnumber(res, "attcmname"))); + tbinfo->attcmoptions[j] = pg_strdup(PQgetvalue(res, j, PQfnumber(res, "attcmoptions"))); tbinfo->attrdefs[j] = NULL; /* fix below */ if (PQgetvalue(res, j, PQfnumber(res, "atthasdef"))[0] == 't') hasdefaults = true; @@ -15959,7 +15968,8 @@ dumpTableSchema(Archive *fout, const TableInfo *tbinfo) continue; has_non_default_compression = (tbinfo->attcmnames[j] && - (strcmp(tbinfo->attcmnames[j], "pglz") != 0)); + ((strcmp(tbinfo->attcmnames[j], "pglz") != 0) || + nonemptyReloptions(tbinfo->attcmoptions[j]))); /* Format properly if not first attr */ if (actual_atts == 0) @@ -16005,6 +16015,10 @@ dumpTableSchema(Archive *fout, const TableInfo *tbinfo) { appendPQExpBuffer(q, " COMPRESSION %s", tbinfo->attcmnames[j]); + + if (nonemptyReloptions(tbinfo->attcmoptions[j])) + appendPQExpBuffer(q, " WITH (%s)", + tbinfo->attcmoptions[j]); } if (print_default) @@ -16436,6 +16450,9 @@ dumpTableSchema(Archive *fout, const TableInfo *tbinfo) appendPQExpBuffer(q, "ALTER TABLE %s ALTER COLUMN %s\nSET COMPRESSION %s", qualrelname, fmtId(tbinfo->attnames[j]), tbinfo->attcmnames[j]); + if (nonemptyReloptions(tbinfo->attcmoptions[j])) + appendPQExpBuffer(q, "\nWITH (%s) ", tbinfo->attcmoptions[j]); + if (cminfo->nitems > 0) { appendPQExpBuffer(q, "\nPRESERVE ("); diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index a829528cd0..da47f3173f 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -327,6 +327,7 @@ typedef struct _tableInfo bool needs_override; /* has GENERATED ALWAYS AS IDENTITY */ char *amname; /* relation access method */ char **attcmnames; /* per-attribute current compression method */ + char **attcmoptions; /* per-attribute compression options */ struct _attrCompressionInfo **attcompression; /* per-attribute all compression data */ /* diff --git a/src/include/access/compressamapi.h b/src/include/access/compressamapi.h index bac7b47cfc..7f3346b2a6 100644 --- a/src/include/access/compressamapi.h +++ b/src/include/access/compressamapi.h @@ -51,8 +51,11 @@ extern Oid GetDefaultToastCompression(void); #define IsStorageCompressible(storage) ((storage) != TYPSTORAGE_PLAIN && \ (storage) != TYPSTORAGE_EXTERNAL) /* compression handler routines */ +typedef void (*cmcheck_function) (List *options); +typedef void *(*cminitstate_function) (List *options); typedef struct varlena *(*cmcompress_function) (const struct varlena *value, - int32 toast_header_size); + int32 toast_header_size, + void *options); typedef struct varlena *(*cmdecompress_function) (const struct varlena *value, int32 toast_header_size); typedef struct varlena *(*cmdecompress_slice_function) @@ -63,14 +66,25 @@ typedef struct varlena *(*cmdecompress_slice_function) /* * API struct for a compression AM. * + * 'cmcheck' - called when attribute is linking with compression method. + * This function should check compability of compression method with + * the attribute and its options. + * + * 'cminitstate' - called when CompressionAmOptions instance is created. + * Should return pointer to a memory in a caller memory context, or NULL. + * Could be used to pass some internal state between compression function + * calls, like internal structure for parsed compression options. + * * 'datum_compress' - varlena compression function. * 'datum_decompress' - varlena decompression function. * 'datum_decompress_slice' - varlena slice decompression functions. */ typedef struct CompressionAmRoutine { - NodeTag type; + NodeTag type; + cmcheck_function datum_check; /* can be NULL */ + cminitstate_function datum_initstate; /* can be NULL */ cmcompress_function datum_compress; cmdecompress_function datum_decompress; cmdecompress_slice_function datum_decompress_slice; diff --git a/src/include/access/toast_helper.h b/src/include/access/toast_helper.h index dca0bc37f3..fe8de405ac 100644 --- a/src/include/access/toast_helper.h +++ b/src/include/access/toast_helper.h @@ -15,6 +15,7 @@ #define TOAST_HELPER_H #include "utils/rel.h" +#include "nodes/pg_list.h" /* * Information about one column of a tuple being toasted. @@ -33,6 +34,7 @@ typedef struct int32 tai_size; uint8 tai_colflags; Oid tai_compression; + List *tai_cmoptions; } ToastAttrInfo; /* diff --git a/src/include/access/toast_internals.h b/src/include/access/toast_internals.h index ac28f9ed55..864de31739 100644 --- a/src/include/access/toast_internals.h +++ b/src/include/access/toast_internals.h @@ -54,7 +54,7 @@ typedef struct toast_compress_header_custom #define TOAST_COMPRESS_SET_CMOID(ptr, oid) \ (((toast_compress_header_custom *)(ptr))->cmoid = (oid)) -extern Datum toast_compress_datum(Datum value, Oid cmoid); +extern Datum toast_compress_datum(Datum value, Oid cmoid, List *cmoptions); extern Oid toast_get_valid_index(Oid toastoid, LOCKMODE lock); extern void toast_delete_datum(Relation rel, Datum value, bool is_speculative); diff --git a/src/include/catalog/heap.h b/src/include/catalog/heap.h index 6ce480b49c..5ecb6f3653 100644 --- a/src/include/catalog/heap.h +++ b/src/include/catalog/heap.h @@ -81,6 +81,7 @@ extern Oid heap_create_with_catalog(const char *relname, bool allow_system_table_mods, bool is_internal, Oid relrewrite, + Datum *acoptions, ObjectAddress *typaddress); extern void heap_drop_with_catalog(Oid relid); @@ -97,6 +98,7 @@ extern void InsertPgAttributeTuples(Relation pg_attribute_rel, TupleDesc tupdesc, Oid new_rel_oid, Datum *attoptions, + Datum *attcmoptions, CatalogIndexState indstate); extern void InsertPgClassTuple(Relation pg_class_desc, diff --git a/src/include/catalog/pg_attribute.h b/src/include/catalog/pg_attribute.h index 797e78b17c..7c74bc314e 100644 --- a/src/include/catalog/pg_attribute.h +++ b/src/include/catalog/pg_attribute.h @@ -178,6 +178,9 @@ CATALOG(pg_attribute,1249,AttributeRelationId) BKI_BOOTSTRAP BKI_ROWTYPE_OID(75, /* Column-level FDW options */ text attfdwoptions[1] BKI_DEFAULT(_null_); + /* current compression options */ + text attcmoptions[1] BKI_DEFAULT(_null_); + /* * Missing value for added columns. This is a one element array which lets * us store a value of the attribute type here. diff --git a/src/include/commands/defrem.h b/src/include/commands/defrem.h index bd53f9bb0f..8271e97dea 100644 --- a/src/include/commands/defrem.h +++ b/src/include/commands/defrem.h @@ -134,6 +134,8 @@ extern Datum transformGenericOptions(Oid catalogId, Datum oldOptions, List *options, Oid fdwvalidator); +extern Datum optionListToArray(List *options); +extern char *formatRelOptions(List *options); /* commands/amcmds.c */ extern ObjectAddress CreateAccessMethod(CreateAmStmt *stmt); @@ -146,9 +148,12 @@ extern char *get_am_name(Oid amOid); /* commands/compressioncmds.c */ extern Oid GetAttributeCompression(Form_pg_attribute att, ColumnCompression *compression, - bool *need_rewrite); -extern ColumnCompression *MakeColumnCompression(Oid atttcompression); + Datum *acoptions, bool *need_rewrite); +extern ColumnCompression *MakeColumnCompression(Form_pg_attribute att); extern bool IsCompressionSupported(Form_pg_attribute att, Oid cmoid); +extern List *GetAttributeCompressionOptions(Form_pg_attribute att); +extern void CheckCompressionMismatch(ColumnCompression *c1, ColumnCompression *c2, + const char *attributeName); /* support routines in commands/define.c */ diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index ce0913e18a..c3c878b80e 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -634,6 +634,7 @@ typedef struct ColumnCompression NodeTag type; char *cmname; bool preserve_all; + List *options; List *preserve; } ColumnCompression; diff --git a/src/test/regress/expected/compression.out b/src/test/regress/expected/compression.out index e52e272d39..4af72bdbdc 100644 --- a/src/test/regress/expected/compression.out +++ b/src/test/regress/expected/compression.out @@ -297,6 +297,58 @@ SELECT pg_column_compression(f1) FROM cmdata; Indexes: "idx" btree (f1) +-- compression options +CREATE TABLE cmdata3(f1 TEXT COMPRESSION pglz WITH (min_input_size '100')); +CREATE INDEX idx1 ON cmdata3(f1); +INSERT INTO cmdata3 VALUES(repeat('1234567890',1000)); +SELECT attcmoptions FROM pg_attribute WHERE attrelid = 'cmdata3'::regclass AND attname='f1'; + attcmoptions +---------------------- + {min_input_size=100} +(1 row) + +SELECT attcmoptions FROM pg_attribute WHERE attrelid = 'idx1'::regclass AND attname='f1'; + attcmoptions +---------------------- + {min_input_size=100} +(1 row) + +ALTER TABLE cmdata3 ALTER COLUMN f1 SET COMPRESSION lz4 WITH (acceleration '50'); +INSERT INTO cmdata3 VALUES(repeat('1234567890',1004)); +SELECT attcmoptions FROM pg_attribute WHERE attrelid = 'cmdata3'::regclass AND attname='f1'; + attcmoptions +------------------- + {acceleration=50} +(1 row) + +SELECT attcmoptions FROM pg_attribute WHERE attrelid = 'idx1'::regclass AND attname='f1'; + attcmoptions +------------------- + {acceleration=50} +(1 row) + +ALTER TABLE cmdata3 ALTER COLUMN f1 SET COMPRESSION pglz WITH (min_input_size '200') PRESERVE (lz4); +INSERT INTO cmdata3 VALUES(repeat('1234567890',1008)); +SELECT attcmoptions FROM pg_attribute WHERE attrelid = 'cmdata3'::regclass AND attname='f1'; + attcmoptions +---------------------- + {min_input_size=200} +(1 row) + +SELECT attcmoptions FROM pg_attribute WHERE attrelid = 'idx1'::regclass AND attname='f1'; + attcmoptions +---------------------- + {min_input_size=200} +(1 row) + +SELECT pg_column_compression(f1) FROM cmdata3; + pg_column_compression +----------------------- + lz4 + lz4 + pglz +(3 rows) + -- check data is ok SELECT length(f1) FROM cmdata; length diff --git a/src/test/regress/expected/compression_1.out b/src/test/regress/expected/compression_1.out index 36a5f8ba5e..40b0b33202 100644 --- a/src/test/regress/expected/compression_1.out +++ b/src/test/regress/expected/compression_1.out @@ -241,6 +241,61 @@ SELECT pg_column_compression(f1) FROM cmdata; pglz (2 rows) +-- compression options +CREATE TABLE cmdata3(f1 TEXT COMPRESSION pglz WITH (min_input_size '100')); +CREATE INDEX idx1 ON cmdata3(f1); +INSERT INTO cmdata3 VALUES(repeat('1234567890',1000)); +SELECT attcmoptions FROM pg_attribute WHERE attrelid = 'cmdata3'::regclass AND attname='f1'; + attcmoptions +---------------------- + {min_input_size=100} +(1 row) + +SELECT attcmoptions FROM pg_attribute WHERE attrelid = 'idx1'::regclass AND attname='f1'; + attcmoptions +---------------------- + {min_input_size=100} +(1 row) + +ALTER TABLE cmdata3 ALTER COLUMN f1 SET COMPRESSION lz4 WITH (acceleration '50'); +ERROR: not built with lz4 support +INSERT INTO cmdata3 VALUES(repeat('1234567890',1004)); +SELECT attcmoptions FROM pg_attribute WHERE attrelid = 'cmdata3'::regclass AND attname='f1'; + attcmoptions +---------------------- + {min_input_size=100} +(1 row) + +SELECT attcmoptions FROM pg_attribute WHERE attrelid = 'idx1'::regclass AND attname='f1'; + attcmoptions +---------------------- + {min_input_size=100} +(1 row) + +ALTER TABLE cmdata3 ALTER COLUMN f1 SET COMPRESSION pglz WITH (min_input_size '200') PRESERVE (lz4); +ERROR: "lz4" compression access method cannot be preserved +HINT: use "pg_column_compression" function for list of compression methods +INSERT INTO cmdata3 VALUES(repeat('1234567890',1008)); +SELECT attcmoptions FROM pg_attribute WHERE attrelid = 'cmdata3'::regclass AND attname='f1'; + attcmoptions +---------------------- + {min_input_size=100} +(1 row) + +SELECT attcmoptions FROM pg_attribute WHERE attrelid = 'idx1'::regclass AND attname='f1'; + attcmoptions +---------------------- + {min_input_size=100} +(1 row) + +SELECT pg_column_compression(f1) FROM cmdata3; + pg_column_compression +----------------------- + pglz + pglz + pglz +(3 rows) + -- check data is ok SELECT length(f1) FROM cmdata; length diff --git a/src/test/regress/expected/misc_sanity.out b/src/test/regress/expected/misc_sanity.out index 9ebe28a78d..4e3b0c13c1 100644 --- a/src/test/regress/expected/misc_sanity.out +++ b/src/test/regress/expected/misc_sanity.out @@ -99,6 +99,7 @@ ORDER BY 1, 2; relname | attname | atttypid -------------------------+---------------+-------------- pg_attribute | attacl | aclitem[] + pg_attribute | attcmoptions | text[] pg_attribute | attfdwoptions | text[] pg_attribute | attmissingval | anyarray pg_attribute | attoptions | text[] @@ -109,7 +110,7 @@ ORDER BY 1, 2; pg_index | indpred | pg_node_tree pg_largeobject | data | bytea pg_largeobject_metadata | lomacl | aclitem[] -(11 rows) +(12 rows) -- system catalogs without primary keys -- diff --git a/src/test/regress/sql/compression.sql b/src/test/regress/sql/compression.sql index 9d2e72b784..17320c9062 100644 --- a/src/test/regress/sql/compression.sql +++ b/src/test/regress/sql/compression.sql @@ -115,6 +115,24 @@ ALTER TABLE cmdata ALTER COLUMN f1 SET COMPRESSION lz4 PRESERVE (pglz2); SELECT pg_column_compression(f1) FROM cmdata; \d+ cmdata +-- compression options +CREATE TABLE cmdata3(f1 TEXT COMPRESSION pglz WITH (min_input_size '100')); +CREATE INDEX idx1 ON cmdata3(f1); +INSERT INTO cmdata3 VALUES(repeat('1234567890',1000)); +SELECT attcmoptions FROM pg_attribute WHERE attrelid = 'cmdata3'::regclass AND attname='f1'; +SELECT attcmoptions FROM pg_attribute WHERE attrelid = 'idx1'::regclass AND attname='f1'; + +ALTER TABLE cmdata3 ALTER COLUMN f1 SET COMPRESSION lz4 WITH (acceleration '50'); +INSERT INTO cmdata3 VALUES(repeat('1234567890',1004)); +SELECT attcmoptions FROM pg_attribute WHERE attrelid = 'cmdata3'::regclass AND attname='f1'; +SELECT attcmoptions FROM pg_attribute WHERE attrelid = 'idx1'::regclass AND attname='f1'; + +ALTER TABLE cmdata3 ALTER COLUMN f1 SET COMPRESSION pglz WITH (min_input_size '200') PRESERVE (lz4); +INSERT INTO cmdata3 VALUES(repeat('1234567890',1008)); +SELECT attcmoptions FROM pg_attribute WHERE attrelid = 'cmdata3'::regclass AND attname='f1'; +SELECT attcmoptions FROM pg_attribute WHERE attrelid = 'idx1'::regclass AND attname='f1'; +SELECT pg_column_compression(f1) FROM cmdata3; + -- check data is ok SELECT length(f1) FROM cmdata; SELECT length(f1) FROM cmdata1; -- 2.17.0