From b93d6f436fab78905251f3dea592b101951b6154 Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Mon, 18 Jun 2018 15:57:13 +0300 Subject: [PATCH 6/8] Add psql, pg_dump and pg_upgrade support Signed-off-by: Ildus Kurbangaliev --- src/backend/commands/compressioncmds.c | 80 ++++++--- src/backend/commands/tablecmds.c | 14 +- src/backend/utils/adt/pg_upgrade_support.c | 10 ++ src/bin/pg_dump/pg_backup.h | 2 + src/bin/pg_dump/pg_dump.c | 195 ++++++++++++++++++++- src/bin/pg_dump/pg_dump.h | 17 ++ src/bin/pg_dump/pg_dumpall.c | 5 + src/bin/pg_dump/pg_restore.c | 3 + src/bin/pg_dump/t/002_pg_dump.pl | 95 ++++++++++ src/bin/psql/describe.c | 42 +++++ src/bin/psql/tab-complete.c | 5 +- src/include/catalog/binary_upgrade.h | 2 + src/include/catalog/pg_proc.dat | 4 + 13 files changed, 432 insertions(+), 42 deletions(-) diff --git a/src/backend/commands/compressioncmds.c b/src/backend/commands/compressioncmds.c index 7841c7700a..89ffa227b0 100644 --- a/src/backend/commands/compressioncmds.c +++ b/src/backend/commands/compressioncmds.c @@ -36,6 +36,9 @@ #include "utils/syscache.h" #include "utils/snapmgr.h" +/* Set by pg_upgrade_support functions */ +Oid binary_upgrade_next_attr_compression_oid = InvalidOid; + /* * When conditions of compression satisfies one if builtin attribute * compresssion tuples the compressed attribute will be linked to @@ -129,11 +132,12 @@ lookup_attribute_compression(Oid attrelid, AttrNumber attnum, tup_amoid; Datum values[Natts_pg_attr_compression]; bool nulls[Natts_pg_attr_compression]; + char *amname; heap_deform_tuple(tuple, RelationGetDescr(rel), values, nulls); acoid = DatumGetObjectId(values[Anum_pg_attr_compression_acoid - 1]); - tup_amoid = get_am_oid( - NameStr(*DatumGetName(values[Anum_pg_attr_compression_acname - 1])), false); + amname = NameStr(*DatumGetName(values[Anum_pg_attr_compression_acname - 1])); + tup_amoid = get_am_oid(amname, false); if (previous_amoids) *previous_amoids = list_append_unique_oid(*previous_amoids, tup_amoid); @@ -150,17 +154,15 @@ lookup_attribute_compression(Oid attrelid, AttrNumber attnum, if (DatumGetPointer(acoptions) == NULL) result = acoid; } - else + else if (DatumGetPointer(acoptions) != NULL) { bool equal; /* check if arrays for WITH options are equal */ equal = DatumGetBool(CallerFInfoFunctionCall2( - array_eq, - &arrayeq_info, - InvalidOid, - acoptions, - values[Anum_pg_attr_compression_acoptions - 1])); + array_eq, &arrayeq_info, InvalidOid, acoptions, + values[Anum_pg_attr_compression_acoptions - 1])); + if (equal) result = acoid; } @@ -227,6 +229,16 @@ CreateAttributeCompression(Form_pg_attribute att, /* Try to find builtin compression first */ acoid = lookup_attribute_compression(0, 0, amoid, arropt, NULL); + /* no rewrite by default */ + if (need_rewrite != NULL) + *need_rewrite = false; + + if (IsBinaryUpgrade) + { + /* Skip the rewrite checks and searching of identical compression */ + goto add_tuple; + } + /* * attrelid will be invalid on CREATE TABLE, no need for table rewrite * check. @@ -252,16 +264,10 @@ CreateAttributeCompression(Form_pg_attribute att, */ if (need_rewrite != NULL) { - /* no rewrite by default */ - *need_rewrite = false; - Assert(preserved_amoids != NULL); if (compression->preserve == NIL) - { - Assert(!IsBinaryUpgrade); *need_rewrite = true; - } else { ListCell *cell; @@ -294,7 +300,7 @@ CreateAttributeCompression(Form_pg_attribute att, * In binary upgrade list will not be free since it contains * Oid of builtin compression access method. */ - if (!IsBinaryUpgrade && list_length(previous_amoids) != 0) + if (list_length(previous_amoids) != 0) *need_rewrite = true; } } @@ -303,9 +309,6 @@ CreateAttributeCompression(Form_pg_attribute att, list_free(previous_amoids); } - if (IsBinaryUpgrade && !OidIsValid(acoid)) - elog(ERROR, "could not restore attribute compression data"); - /* Return Oid if we already found identical compression on this column */ if (OidIsValid(acoid)) { @@ -315,6 +318,7 @@ CreateAttributeCompression(Form_pg_attribute att, return acoid; } +add_tuple: /* Initialize buffers for new tuple values */ memset(values, 0, sizeof(values)); memset(nulls, false, sizeof(nulls)); @@ -323,13 +327,27 @@ CreateAttributeCompression(Form_pg_attribute att, rel = heap_open(AttrCompressionRelationId, RowExclusiveLock); - acoid = GetNewOidWithIndex(rel, AttrCompressionIndexId, - Anum_pg_attr_compression_acoid); + if (IsBinaryUpgrade) + { + /* acoid should be found in some cases */ + if (binary_upgrade_next_attr_compression_oid < FirstNormalObjectId && + (!OidIsValid(acoid) || binary_upgrade_next_attr_compression_oid != acoid)) + elog(ERROR, "could not link to built-in attribute compression"); + + acoid = binary_upgrade_next_attr_compression_oid; + } + else + { + acoid = GetNewOidWithIndex(rel, AttrCompressionIndexId, + Anum_pg_attr_compression_acoid); + + } + if (acoid < FirstNormalObjectId) { - /* this is database initialization */ + /* this is built-in attribute compression */ heap_close(rel, RowExclusiveLock); - return DefaultCompressionOid; + return acoid; } /* we need routine only to call cmcheck function */ @@ -390,8 +408,8 @@ RemoveAttributeCompression(Oid acoid) /* * CleanupAttributeCompression * - * Remove entries in pg_attr_compression except current attribute compression - * and related with specified list of access methods. + * Remove entries in pg_attr_compression of the column except current + * attribute compression and related with specified list of access methods. */ void CleanupAttributeCompression(Oid relid, AttrNumber attnum, List *keepAmOids) @@ -419,9 +437,7 @@ CleanupAttributeCompression(Oid relid, AttrNumber attnum, List *keepAmOids) ReleaseSysCache(attrtuple); Assert(relid > 0 && attnum > 0); - - if (IsBinaryUpgrade) - goto builtin_removal; + Assert(!IsBinaryUpgrade); rel = heap_open(AttrCompressionRelationId, RowExclusiveLock); @@ -438,7 +454,10 @@ CleanupAttributeCompression(Oid relid, AttrNumber attnum, List *keepAmOids) scan = systable_beginscan(rel, AttrCompressionRelidAttnumIndexId, true, NULL, 2, key); - /* Remove attribute compression tuples and collect removed Oids to list */ + /* + * Remove attribute compression tuples and collect removed Oids + * to list. + */ while (HeapTupleIsValid(tuple = systable_getnext(scan))) { Form_pg_attr_compression acform; @@ -460,7 +479,10 @@ CleanupAttributeCompression(Oid relid, AttrNumber attnum, List *keepAmOids) systable_endscan(scan); heap_close(rel, RowExclusiveLock); - /* Now remove dependencies */ + /* + * Now remove dependencies between attribute compression (dependent) + * and column. + */ rel = heap_open(DependRelationId, RowExclusiveLock); foreach(lc, removed) { diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 185c9650c7..b3ff97a5cb 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -798,10 +798,10 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, if (colDef->identity) attr->attidentity = colDef->identity; - if (relkind == RELKIND_RELATION || relkind == RELKIND_PARTITIONED_TABLE) + if (!IsBinaryUpgrade && + (relkind == RELKIND_RELATION || relkind == RELKIND_PARTITIONED_TABLE)) attr->attcompression = CreateAttributeCompression(attr, - colDef->compression, - NULL, NULL); + colDef->compression, NULL, NULL); else attr->attcompression = InvalidOid; } @@ -13696,14 +13696,6 @@ ATExecSetCompression(AlteredTableInfo *tab, /* make changes visible */ CommandCounterIncrement(); - /* - * Normally cleanup is done in rewrite but in binary upgrade we should do - * it explicitly. - */ - if (IsBinaryUpgrade) - CleanupAttributeCompression(RelationGetRelid(rel), - attnum, preserved_amoids); - ObjectAddressSet(address, AttrCompressionRelationId, acoid); return address; } diff --git a/src/backend/utils/adt/pg_upgrade_support.c b/src/backend/utils/adt/pg_upgrade_support.c index 99db5ba389..0e81e70e09 100644 --- a/src/backend/utils/adt/pg_upgrade_support.c +++ b/src/backend/utils/adt/pg_upgrade_support.c @@ -116,6 +116,16 @@ binary_upgrade_set_next_pg_authid_oid(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +Datum +binary_upgrade_set_next_attr_compression_oid(PG_FUNCTION_ARGS) +{ + Oid acoid = PG_GETARG_OID(0); + + CHECK_IS_BINARY_UPGRADE; + binary_upgrade_next_attr_compression_oid = acoid; + PG_RETURN_VOID(); +} + Datum binary_upgrade_create_empty_extension(PG_FUNCTION_ARGS) { diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 7ab27391fb..2f97e244c7 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -78,6 +78,7 @@ typedef struct _restoreOptions int no_publications; /* Skip publication entries */ int no_security_labels; /* Skip security label entries */ int no_subscriptions; /* Skip subscription entries */ + int no_compression_methods; /* Skip compression methods */ int strict_names; const char *filename; @@ -150,6 +151,7 @@ typedef struct _dumpOptions int no_security_labels; int no_publications; int no_subscriptions; + int no_compression_methods; int no_synchronized_snapshots; int no_unlogged_table_data; int serializable_deferrable; diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 4c98ae4d7f..2e32d4e126 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -41,11 +41,13 @@ #include "getopt_long.h" #include "access/attnum.h" +#include "access/cmapi.h" #include "access/sysattr.h" #include "access/transam.h" #include "catalog/pg_aggregate_d.h" #include "catalog/pg_am_d.h" #include "catalog/pg_attribute_d.h" +#include "catalog/pg_attr_compression_d.h" #include "catalog/pg_cast_d.h" #include "catalog/pg_class_d.h" #include "catalog/pg_default_acl_d.h" @@ -389,6 +391,7 @@ main(int argc, char **argv) {"no-synchronized-snapshots", no_argument, &dopt.no_synchronized_snapshots, 1}, {"no-unlogged-table-data", no_argument, &dopt.no_unlogged_table_data, 1}, {"no-subscriptions", no_argument, &dopt.no_subscriptions, 1}, + {"no-compression-methods", no_argument, &dopt.no_compression_methods, 1}, {"no-sync", no_argument, NULL, 7}, {"on-conflict-do-nothing", no_argument, &dopt.do_nothing, 1}, {"rows-per-insert", required_argument, NULL, 10}, @@ -885,6 +888,8 @@ main(int argc, char **argv) * We rely on dependency information to help us determine a safe order, so * the initial sort is mostly for cosmetic purposes: we sort by name to * ensure that logically identical schemas will dump identically. + * + * If we do a parallel dump, we want the largest tables to go first. */ sortDumpableObjectsByTypeName(dobjs, numObjs); @@ -8227,9 +8232,12 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) int i_attcollation; int i_attfdwoptions; int i_attmissingval; + int i_attcmoptions; + int i_attcmname; PGresult *res; int ntups; bool hasdefaults; + bool createWithCompression; for (i = 0; i < numTables; i++) { @@ -8272,6 +8280,23 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) "a.attislocal,\n" "pg_catalog.format_type(t.oid, a.atttypmod) AS atttypname,\n"); + createWithCompression = (!dopt->binary_upgrade && fout->remoteVersion >= 120000); + + if (createWithCompression) + appendPQExpBuffer(q, + "pg_catalog.array_to_string(ARRAY(" + "SELECT pg_catalog.quote_ident(option_name) || " + "' ' || pg_catalog.quote_literal(option_value) " + "FROM pg_catalog.pg_options_to_table(c.acoptions) " + "ORDER BY option_name" + "), E',\n ') AS attcmoptions,\n" + "c.acname AS attcmname,\n"); + else + appendPQExpBuffer(q, + "NULL AS attcmoptions,\n" + "NULL AS attcmname,\n"); + + if (fout->remoteVersion >= 110000) appendPQExpBuffer(q, "CASE WHEN a.atthasmissing AND NOT a.attisdropped " @@ -8324,7 +8349,13 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) /* need left join here to not fail on dropped columns ... */ appendPQExpBuffer(q, "FROM pg_catalog.pg_attribute a LEFT JOIN pg_catalog.pg_type t " - "ON a.atttypid = t.oid\n" + "ON a.atttypid = t.oid\n"); + + if (createWithCompression) + appendPQExpBuffer(q, "LEFT JOIN pg_catalog.pg_attr_compression c " + "ON a.attcompression = c.acoid\n"); + + appendPQExpBuffer(q, "WHERE a.attrelid = '%u'::pg_catalog.oid " "AND a.attnum > 0::pg_catalog.int2\n" "ORDER BY a.attnum", @@ -8352,6 +8383,8 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) i_attcollation = PQfnumber(res, "attcollation"); i_attfdwoptions = PQfnumber(res, "attfdwoptions"); i_attmissingval = PQfnumber(res, "attmissingval"); + i_attcmname = PQfnumber(res, "attcmname"); + i_attcmoptions = PQfnumber(res, "attcmoptions"); tbinfo->numatts = ntups; tbinfo->attnames = (char **) pg_malloc(ntups * sizeof(char *)); @@ -8369,9 +8402,12 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) tbinfo->attcollation = (Oid *) pg_malloc(ntups * sizeof(Oid)); tbinfo->attfdwoptions = (char **) pg_malloc(ntups * sizeof(char *)); tbinfo->attmissingval = (char **) pg_malloc(ntups * sizeof(char *)); + tbinfo->attcmoptions = (char **) pg_malloc(ntups * sizeof(char *)); + tbinfo->attcmnames = (char **) pg_malloc(ntups * sizeof(char *)); tbinfo->notnull = (bool *) pg_malloc(ntups * sizeof(bool)); tbinfo->inhNotNull = (bool *) pg_malloc(ntups * sizeof(bool)); tbinfo->attrdefs = (AttrDefInfo **) pg_malloc(ntups * sizeof(AttrDefInfo *)); + tbinfo->attcompression = NULL; hasdefaults = false; for (j = 0; j < ntups; j++) @@ -8397,6 +8433,8 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) tbinfo->attcollation[j] = atooid(PQgetvalue(res, j, i_attcollation)); tbinfo->attfdwoptions[j] = pg_strdup(PQgetvalue(res, j, i_attfdwoptions)); tbinfo->attmissingval[j] = pg_strdup(PQgetvalue(res, j, i_attmissingval)); + tbinfo->attcmoptions[j] = pg_strdup(PQgetvalue(res, j, i_attcmoptions)); + tbinfo->attcmnames[j] = pg_strdup(PQgetvalue(res, j, i_attcmname)); tbinfo->attrdefs[j] = NULL; /* fix below */ if (PQgetvalue(res, j, i_atthasdef)[0] == 't') hasdefaults = true; @@ -8614,6 +8652,104 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) } PQclear(res); } + + /* + * Get compression info + */ + if (fout->remoteVersion >= 120000 && dopt->binary_upgrade) + { + int i_acname; + int i_acoid; + int i_parsedoptions; + int i_curattnum; + int start; + + if (g_verbose) + write_msg(NULL, "finding compression info for table \"%s.%s\"\n", + tbinfo->dobj.namespace->dobj.name, + tbinfo->dobj.name); + + tbinfo->attcompression = pg_malloc0(tbinfo->numatts * sizeof(AttrCompressionInfo *)); + + resetPQExpBuffer(q); + appendPQExpBuffer(q, + "SELECT attrelid::pg_catalog.regclass AS relname, attname," + " (CASE WHEN deptype = 'i' THEN refobjsubid ELSE objsubid END) AS curattnum," + " (CASE WHEN deptype = 'n' THEN attcompression = refobjid" + " ELSE attcompression = objid END) AS iscurrent," + " acname, acoid," + " (CASE WHEN acoptions IS NOT NULL" + " THEN pg_catalog.array_to_string(ARRAY(" + " SELECT pg_catalog.quote_ident(option_name) || " + " ' ' || pg_catalog.quote_literal(option_value) " + " FROM pg_catalog.pg_options_to_table(acoptions) " + " ORDER BY option_name" + " ), E',\n ')" + " ELSE NULL END) AS parsedoptions " + " FROM pg_depend d" + " JOIN pg_attribute a ON" + " (classid = 'pg_class'::pg_catalog.regclass::pg_catalog.oid AND a.attrelid = d.objid" + " AND a.attnum = d.objsubid AND d.deptype = 'n'" + " AND d.refclassid = 'pg_attr_compression'::pg_catalog.regclass::pg_catalog.oid)" + " OR (d.refclassid = 'pg_class'::pg_catalog.regclass::pg_catalog.oid" + " AND d.refobjid = a.attrelid" + " AND d.refobjsubid = a.attnum AND d.deptype = 'i'" + " AND d.classid = 'pg_attr_compression'::pg_catalog.regclass::pg_catalog.oid)" + " JOIN pg_attr_compression c ON" + " (d.deptype = 'i' AND d.objid = c.acoid AND a.attnum = c.acattnum" + " AND a.attrelid = c.acrelid) OR" + " (d.deptype = 'n' AND d.refobjid = c.acoid AND c.acattnum = 0" + " AND c.acrelid = 0)" + " WHERE (deptype = 'n' AND d.objid = %d) OR (deptype = 'i' AND d.refobjid = %d)" + " ORDER BY curattnum, iscurrent;", + tbinfo->dobj.catId.oid, tbinfo->dobj.catId.oid); + + res = ExecuteSqlQuery(fout, q->data, PGRES_TUPLES_OK); + ntups = PQntuples(res); + + if (ntups > 0) + { + int k; + + i_acname = PQfnumber(res, "acname"); + i_acoid = PQfnumber(res, "acoid"); + i_parsedoptions = PQfnumber(res, "parsedoptions"); + i_curattnum = PQfnumber(res, "curattnum"); + + start = 0; + + for (j = 0; j < ntups; j++) + { + int attnum = atoi(PQgetvalue(res, j, i_curattnum)); + + if ((j == ntups - 1) || atoi(PQgetvalue(res, j + 1, i_curattnum)) != attnum) + { + AttrCompressionInfo *cminfo = pg_malloc(sizeof(AttrCompressionInfo)); + + cminfo->nitems = j - start + 1; + cminfo->items = pg_malloc(sizeof(AttrCompressionItem *) * cminfo->nitems); + + for (k = start; k < start + cminfo->nitems; k++) + { + AttrCompressionItem *cmitem = pg_malloc0(sizeof(AttrCompressionItem)); + + cmitem->acname = pg_strdup(PQgetvalue(res, k, i_acname)); + cmitem->acoid = atooid(PQgetvalue(res, k, i_acoid)); + + if (!PQgetisnull(res, k, i_parsedoptions)) + cmitem->parsedoptions = pg_strdup(PQgetvalue(res, k, i_parsedoptions)); + + cminfo->items[k - start] = cmitem; + } + + tbinfo->attcompression[attnum - 1] = cminfo; + start = j + 1; /* start from next */ + } + } + } + + PQclear(res); + } } destroyPQExpBuffer(q); @@ -12729,6 +12865,8 @@ dumpAccessMethod(Archive *fout, AccessMethodInfo *aminfo) break; case AMTYPE_TABLE: appendPQExpBuffer(q, "TYPE TABLE "); + case AMTYPE_COMPRESSION: + appendPQExpBuffer(q, "TYPE COMPRESSION "); break; default: write_msg(NULL, "WARNING: invalid type \"%c\" of access method \"%s\"\n", @@ -15650,6 +15788,14 @@ dumpTableSchema(Archive *fout, TableInfo *tbinfo) (!tbinfo->inhNotNull[j] || dopt->binary_upgrade)); + /* + * Compression will require a record in + * pg_attr_compression + */ + bool has_custom_compression = (tbinfo->attcmnames[j] && + ((strcmp(tbinfo->attcmnames[j], "pglz") != 0) || + nonemptyReloptions(tbinfo->attcmoptions[j]))); + /* * Skip column if fully defined by reloftype or the * partition parent. @@ -15708,6 +15854,25 @@ dumpTableSchema(Archive *fout, TableInfo *tbinfo) fmtQualifiedDumpable(coll)); } + /* + * Compression + * + * In binary-upgrade mode, compression is assigned by + * ALTER. Even if we're skipping compression the attribute + * will get default compression. It's the task for ALTER + * command to restore compression info. + */ + if (!dopt->no_compression_methods && !dopt->binary_upgrade && + tbinfo->attcmnames[j] && strlen(tbinfo->attcmnames[j]) && + has_custom_compression) + { + appendPQExpBuffer(q, " COMPRESSION %s", + tbinfo->attcmnames[j]); + if (nonemptyReloptions(tbinfo->attcmoptions[j])) + appendPQExpBuffer(q, " WITH (%s)", + tbinfo->attcmoptions[j]); + } + if (has_default) appendPQExpBuffer(q, " DEFAULT %s", tbinfo->attrdefs[j]->adef_expr); @@ -16123,6 +16288,34 @@ dumpTableSchema(Archive *fout, TableInfo *tbinfo) appendPQExpBuffer(q, "OPTIONS (\n %s\n);\n", tbinfo->attfdwoptions[j]); } + + /* + * Dump per-column compression options + */ + if (tbinfo->attcompression && tbinfo->attcompression[j]) + { + AttrCompressionInfo *cminfo = tbinfo->attcompression[j]; + + if (cminfo->nitems) + appendPQExpBuffer(q, "\n-- For binary upgrade, recreate compression metadata on column %s\n", + fmtId(tbinfo->attnames[j])); + + for (int i = 0; i < cminfo->nitems; i++) + { + AttrCompressionItem *item = cminfo->items[i]; + + appendPQExpBuffer(q, + "SELECT binary_upgrade_set_next_attr_compression_oid('%d'::pg_catalog.oid);\n", + item->acoid); + appendPQExpBuffer(q, "ALTER TABLE %s ALTER COLUMN %s\nSET COMPRESSION %s", + qualrelname, fmtId(tbinfo->attnames[j]), item->acname); + + if (item->parsedoptions) + appendPQExpBuffer(q, "\nWITH (%s);\n", item->parsedoptions); + else + appendPQExpBuffer(q, ";\n"); + } + } } if (ftoptions) diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index 2e1b90acd0..810d001e84 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -326,6 +326,10 @@ typedef struct _tableInfo bool needs_override; /* has GENERATED ALWAYS AS IDENTITY */ char *amname; /* relation access method */ + char **attcmoptions; /* per-attribute current compression options */ + char **attcmnames; /* per-attribute current compression method names */ + struct _attrCompressionInfo **attcompression; /* per-attribute all compression data */ + /* * Stuff computed only for dumpable tables. */ @@ -347,6 +351,19 @@ typedef struct _attrDefInfo bool separate; /* true if must dump as separate item */ } AttrDefInfo; +typedef struct _attrCompressionItem +{ + Oid acoid; /* attribute compression oid */ + char *acname; /* compression access method name */ + char *parsedoptions; /* WITH options */ +} AttrCompressionItem; + +typedef struct _attrCompressionInfo +{ + int nitems; + AttrCompressionItem **items; +} AttrCompressionInfo; + typedef struct _tableDataInfo { DumpableObject dobj; diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index a86965e670..f09925a85d 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -76,6 +76,7 @@ static int no_comments = 0; static int no_publications = 0; static int no_security_labels = 0; static int no_subscriptions = 0; +static int no_compression_methods = 0; static int no_unlogged_table_data = 0; static int no_role_passwords = 0; static int server_version; @@ -143,6 +144,7 @@ main(int argc, char *argv[]) {"no-role-passwords", no_argument, &no_role_passwords, 1}, {"no-security-labels", no_argument, &no_security_labels, 1}, {"no-subscriptions", no_argument, &no_subscriptions, 1}, + {"no-compression-methods", no_argument, &no_compression_methods, 1}, {"no-sync", no_argument, NULL, 4}, {"no-unlogged-table-data", no_argument, &no_unlogged_table_data, 1}, {"on-conflict-do-nothing", no_argument, &on_conflict_do_nothing, 1}, @@ -432,6 +434,8 @@ main(int argc, char *argv[]) appendPQExpBufferStr(pgdumpopts, " --no-security-labels"); if (no_subscriptions) appendPQExpBufferStr(pgdumpopts, " --no-subscriptions"); + if (no_compression_methods) + appendPQExpBufferStr(pgdumpopts, " --no-compression-methods"); if (no_unlogged_table_data) appendPQExpBufferStr(pgdumpopts, " --no-unlogged-table-data"); if (on_conflict_do_nothing) @@ -656,6 +660,7 @@ help(void) printf(_(" --no-role-passwords do not dump passwords for roles\n")); printf(_(" --no-security-labels do not dump security label assignments\n")); printf(_(" --no-subscriptions do not dump subscriptions\n")); + printf(_(" --no-compression-methods do not dump compression methods\n")); printf(_(" --no-sync do not wait for changes to be written safely to disk\n")); printf(_(" --no-tablespaces do not dump tablespace assignments\n")); printf(_(" --no-unlogged-table-data do not dump unlogged table data\n")); diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c index 428e040acb..03f84933c6 100644 --- a/src/bin/pg_dump/pg_restore.c +++ b/src/bin/pg_dump/pg_restore.c @@ -75,6 +75,7 @@ main(int argc, char **argv) static int no_publications = 0; static int no_security_labels = 0; static int no_subscriptions = 0; + static int no_compression_methods = 0; static int strict_names = 0; struct option cmdopts[] = { @@ -124,6 +125,7 @@ main(int argc, char **argv) {"no-publications", no_argument, &no_publications, 1}, {"no-security-labels", no_argument, &no_security_labels, 1}, {"no-subscriptions", no_argument, &no_subscriptions, 1}, + {"no-compression-methods", no_argument, &no_compression_methods, 1}, {NULL, 0, NULL, 0} }; @@ -375,6 +377,7 @@ main(int argc, char **argv) opts->no_publications = no_publications; opts->no_security_labels = no_security_labels; opts->no_subscriptions = no_subscriptions; + opts->no_compression_methods = no_compression_methods; if (if_exists && !opts->dropSchema) { diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index de6895122e..87a397303f 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -651,6 +651,43 @@ my %tests = ( }, }, + # compression data in binary upgrade mode + 'ALTER TABLE test_table_compression ALTER COLUMN ... SET COMPRESSION' => { + all_runs => 1, + catch_all => 'ALTER TABLE ... commands', + regexp => qr/^ + \QCREATE TABLE dump_test.test_table_compression (\E\n + \s+\Qcol1 text,\E\n + \s+\Qcol2 text,\E\n + \s+\Qcol3 text,\E\n + \s+\Qcol4 text\E\n + \); + .* + \QSELECT binary_upgrade_set_next_attr_compression_oid('\E\d+\Q'::pg_catalog.oid);\E\n + \QALTER TABLE dump_test.test_table_compression ALTER COLUMN col1\E\n + \QSET COMPRESSION pglz;\E\n + .* + \QSELECT binary_upgrade_set_next_attr_compression_oid('\E\d+\Q'::pg_catalog.oid);\E\n + \QALTER TABLE dump_test.test_table_compression ALTER COLUMN col2\E\n + \QSET COMPRESSION pglz2;\E\n + .* + \QSELECT binary_upgrade_set_next_attr_compression_oid('\E\d+\Q'::pg_catalog.oid);\E\n + \QALTER TABLE dump_test.test_table_compression ALTER COLUMN col3\E\n + \QSET COMPRESSION pglz\E\n + \QWITH (min_input_size '1000');\E\n + .* + \QSELECT binary_upgrade_set_next_attr_compression_oid('\E\d+\Q'::pg_catalog.oid);\E\n + \QALTER TABLE dump_test.test_table_compression ALTER COLUMN col4\E\n + \QSET COMPRESSION pglz2\E\n + \QWITH (min_input_size '1000');\E\n + \QSELECT binary_upgrade_set_next_attr_compression_oid('\E\d+\Q'::pg_catalog.oid);\E\n + \QALTER TABLE dump_test.test_table_compression ALTER COLUMN col4\E\n + \QSET COMPRESSION pglz2\E\n + \QWITH (min_input_size '2000');\E\n + /xms, + like => { binary_upgrade => 1, }, + }, + 'ALTER TABLE ONLY test_table ALTER COLUMN col1 SET STATISTICS 90' => { create_order => 93, create_sql => @@ -1367,6 +1404,17 @@ my %tests = ( like => { %full_runs, section_pre_data => 1, }, }, + 'CREATE ACCESS METHOD pglz2' => { + all_runs => 1, + catch_all => 'CREATE ... commands', + create_order => 52, + create_sql => + 'CREATE ACCESS METHOD pglz2 TYPE COMPRESSION HANDLER pglzhandler;', + regexp => + qr/CREATE ACCESS METHOD pglz2 TYPE COMPRESSION HANDLER pglzhandler;/m, + like => { %full_runs, section_pre_data => 1, }, + }, + 'CREATE COLLATION test0 FROM "C"' => { create_order => 76, create_sql => 'CREATE COLLATION test0 FROM "C";', @@ -2414,6 +2462,53 @@ my %tests = ( unlike => { exclude_dump_test_schema => 1, }, }, + 'CREATE TABLE test_table_compression' => { + create_order => 55, + create_sql => 'CREATE TABLE dump_test.test_table_compression ( + col1 text, + col2 text COMPRESSION pglz2, + col3 text COMPRESSION pglz WITH (min_input_size \'1000\'), + col4 text COMPRESSION pglz2 WITH (min_input_size \'1000\') + );', + regexp => qr/^ + \QCREATE TABLE dump_test.test_table_compression (\E\n + \s+\Qcol1 text,\E\n + \s+\Qcol2 text COMPRESSION pglz2,\E\n + \s+\Qcol3 text COMPRESSION pglz WITH (min_input_size '1000'),\E\n + \s+\Qcol4 text COMPRESSION pglz2 WITH (min_input_size '2000')\E\n + \); + /xm, + like => + { %full_runs, %dump_test_schema_runs, section_pre_data => 1, }, + unlike => { + binary_upgrade => 1, + exclude_dump_test_schema => 1, + }, + }, + + 'ALTER TABLE test_table_compression' => { + create_order => 56, + create_sql => 'ALTER TABLE dump_test.test_table_compression + ALTER COLUMN col4 + SET COMPRESSION pglz2 + WITH (min_input_size \'2000\') + PRESERVE (pglz2);', + regexp => qr/^ + \QCREATE TABLE dump_test.test_table_compression (\E\n + \s+\Qcol1 text,\E\n + \s+\Qcol2 text COMPRESSION pglz2,\E\n + \s+\Qcol3 text COMPRESSION pglz WITH (min_input_size '1000'),\E\n + \s+\Qcol4 text COMPRESSION pglz2 WITH (min_input_size '2000')\E\n + \); + /xm, + like => + { %full_runs, %dump_test_schema_runs, section_pre_data => 1, }, + unlike => { + binary_upgrade => 1, + exclude_dump_test_schema => 1, + }, + }, + 'CREATE STATISTICS extended_stats_no_options' => { create_order => 97, create_sql => 'CREATE STATISTICS dump_test.test_ext_stats_no_options diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 779e48437c..428f839d03 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -1467,6 +1467,7 @@ describeOneTableDetails(const char *schemaname, fdwopts_col = -1, attstorage_col = -1, attstattarget_col = -1, + attcompression_col = -1, attdescr_col = -1; int numrows; struct @@ -1859,6 +1860,24 @@ describeOneTableDetails(const char *schemaname, appendPQExpBufferStr(&buf, ",\n a.attstorage"); attstorage_col = cols++; + /* compresssion info */ + if (pset.sversion >= 120000 && + (tableinfo.relkind == RELKIND_RELATION || + tableinfo.relkind == RELKIND_PARTITIONED_TABLE)) + { + appendPQExpBufferStr(&buf, ",\n CASE WHEN attcompression = 0 THEN NULL ELSE " + " (SELECT c.acname || " + " (CASE WHEN acoptions IS NULL " + " THEN '' " + " ELSE '(' || array_to_string(ARRAY(SELECT quote_ident(option_name) || ' ' || quote_literal(option_value)" + " FROM pg_options_to_table(acoptions)), ', ') || ')'" + " END) " + " FROM pg_catalog.pg_attr_compression c " + " WHERE c.acoid = a.attcompression) " + " END AS attcmname"); + attcompression_col = cols++; + } + /* stats target, if relevant to relkind */ if (tableinfo.relkind == RELKIND_RELATION || tableinfo.relkind == RELKIND_INDEX || @@ -1985,6 +2004,8 @@ describeOneTableDetails(const char *schemaname, headers[cols++] = gettext_noop("FDW options"); if (attstorage_col >= 0) headers[cols++] = gettext_noop("Storage"); + if (attcompression_col >= 0) + headers[cols++] = gettext_noop("Compression"); if (attstattarget_col >= 0) headers[cols++] = gettext_noop("Stats target"); if (attdescr_col >= 0) @@ -2056,6 +2077,27 @@ describeOneTableDetails(const char *schemaname, false, false); } + /* Column compression. */ + if (attcompression_col >= 0) + { + bool mustfree = false; + const int trunclen = 100; + char *val = PQgetvalue(res, i, attcompression_col); + + /* truncate the options if they're too long */ + if (strlen(val) > trunclen + 3) + { + char *trunc = pg_malloc0(trunclen + 4); + strncpy(trunc, val, trunclen); + strncpy(trunc + trunclen, "...", 4); + + val = trunc; + mustfree = true; + } + + printTableAddCell(&cont, val, false, mustfree); + } + /* Statistics target, if the relkind supports this feature */ if (attstattarget_col >= 0) printTableAddCell(&cont, PQgetvalue(res, i, attstattarget_col), diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 10ae21cc61..19ef736b98 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -1919,11 +1919,14 @@ psql_completion(const char *text, int start, int end) /* ALTER TABLE ALTER [COLUMN] SET */ else if (Matches("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET") || Matches("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET")) - COMPLETE_WITH("(", "DEFAULT", "NOT NULL", "STATISTICS", "STORAGE"); + COMPLETE_WITH("(", "COMPRESSION", "DEFAULT", "NOT NULL", "STATISTICS", "STORAGE"); /* ALTER TABLE ALTER [COLUMN] SET ( */ else if (Matches("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET", "(") || Matches("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET", "(")) COMPLETE_WITH("n_distinct", "n_distinct_inherited"); + else if (Matches("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET", "COMPRESSION", MatchAny) || + Matches("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET", "COMPRESSION", MatchAny)) + COMPLETE_WITH("WITH (", "PRESERVE ("); /* ALTER TABLE ALTER [COLUMN] SET STORAGE */ else if (Matches("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET", "STORAGE") || Matches("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET", "STORAGE")) diff --git a/src/include/catalog/binary_upgrade.h b/src/include/catalog/binary_upgrade.h index 2927b7a4d3..00f91e4b90 100644 --- a/src/include/catalog/binary_upgrade.h +++ b/src/include/catalog/binary_upgrade.h @@ -25,6 +25,8 @@ extern PGDLLIMPORT Oid binary_upgrade_next_toast_pg_class_oid; extern PGDLLIMPORT Oid binary_upgrade_next_pg_enum_oid; extern PGDLLIMPORT Oid binary_upgrade_next_pg_authid_oid; +extern PGDLLIMPORT Oid binary_upgrade_next_attr_compression_oid; + extern PGDLLIMPORT bool binary_upgrade_record_init_privs; #endif /* BINARY_UPGRADE_H */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 2dce2c087d..8ca3f53b96 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -9981,6 +9981,10 @@ proname => 'binary_upgrade_set_missing_value', provolatile => 'v', proparallel => 'u', prorettype => 'void', proargtypes => 'oid text text', prosrc => 'binary_upgrade_set_missing_value' }, +{ oid => '4012', descr => 'for use by pg_upgrade', + proname => 'binary_upgrade_set_next_attr_compression_oid', provolatile => 'v', + proparallel => 'r', prorettype => 'void', proargtypes => 'oid', + prosrc => 'binary_upgrade_set_next_attr_compression_oid' }, # conversion functions { oid => '4300', -- 2.21.0