From d451e6192f26ddc14cfe58f1c6668bbbe93f1a25 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Thu, 19 Jun 2025 11:51:52 +0900 Subject: [PATCH v2 11/13] Add support for TOAST table types in pg_dump and pg_restore This includes the possibility to perform binary upgrades with TOAST table types applied to a new cluster, relying on SET commands based on default_toast_type to apply one type of TOAST table or the other. Some tests are included, this is a pretty mechanical change. Dump format is bumped to 1.17 due to the addition of the TOAST table type in the custom format. --- src/bin/pg_dump/pg_backup.h | 2 + src/bin/pg_dump/pg_backup_archiver.c | 69 +++++++++++++++++++++++++++- src/bin/pg_dump/pg_backup_archiver.h | 6 ++- src/bin/pg_dump/pg_dump.c | 21 +++++++++ src/bin/pg_dump/pg_dump.h | 1 + src/bin/pg_dump/pg_dumpall.c | 5 ++ src/bin/pg_dump/pg_restore.c | 4 ++ src/bin/pg_dump/t/002_pg_dump.pl | 35 ++++++++++++++ doc/src/sgml/ref/pg_dump.sgml | 12 +++++ doc/src/sgml/ref/pg_dumpall.sgml | 12 +++++ doc/src/sgml/ref/pg_restore.sgml | 12 +++++ 11 files changed, 177 insertions(+), 2 deletions(-) diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index af0007fb6d2f..84dccdb0eed7 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -99,6 +99,7 @@ typedef struct _restoreOptions int noOwner; /* Don't try to match original object owner */ int noTableAm; /* Don't issue table-AM-related commands */ int noTablespace; /* Don't issue tablespace-related commands */ + int noToastType; /* Don't issue TOAST-type-related commands */ int disable_triggers; /* disable triggers during data-only * restore */ int use_setsessauth; /* Use SET SESSION AUTHORIZATION commands @@ -192,6 +193,7 @@ typedef struct _dumpOptions int disable_triggers; int outputNoTableAm; int outputNoTablespaces; + int outputNoToastType; int use_setsessauth; int enable_row_security; int load_via_partition_root; diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 197c1295d93f..574e62d53a3e 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -183,6 +183,7 @@ dumpOptionsFromRestoreOptions(RestoreOptions *ropt) dopt->outputNoOwner = ropt->noOwner; dopt->outputNoTableAm = ropt->noTableAm; dopt->outputNoTablespaces = ropt->noTablespace; + dopt->outputNoToastType = ropt->noToastType; dopt->disable_triggers = ropt->disable_triggers; dopt->use_setsessauth = ropt->use_setsessauth; dopt->disable_dollar_quoting = ropt->disable_dollar_quoting; @@ -1247,6 +1248,7 @@ ArchiveEntry(Archive *AHX, CatalogId catalogId, DumpId dumpId, newToc->namespace = opts->namespace ? pg_strdup(opts->namespace) : NULL; newToc->tablespace = opts->tablespace ? pg_strdup(opts->tablespace) : NULL; newToc->tableam = opts->tableam ? pg_strdup(opts->tableam) : NULL; + newToc->toasttype = opts->toasttype ? pg_strdup(opts->toasttype) : NULL; newToc->relkind = opts->relkind; newToc->owner = opts->owner ? pg_strdup(opts->owner) : NULL; newToc->desc = pg_strdup(opts->description); @@ -2407,6 +2409,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->currSchema = NULL; /* ditto */ AH->currTablespace = NULL; /* ditto */ AH->currTableAm = NULL; /* ditto */ + AH->currToastType = NULL; /* ditto */ AH->toc = (TocEntry *) pg_malloc0(sizeof(TocEntry)); @@ -2674,6 +2677,7 @@ WriteToc(ArchiveHandle *AH) WriteStr(AH, te->tablespace); WriteStr(AH, te->tableam); WriteInt(AH, te->relkind); + WriteStr(AH, te->toasttype); WriteStr(AH, te->owner); WriteStr(AH, "false"); @@ -2782,6 +2786,9 @@ ReadToc(ArchiveHandle *AH) if (AH->version >= K_VERS_1_16) te->relkind = ReadInt(AH); + if (AH->version >= K_VERS_1_17) + te->toasttype = ReadStr(AH); + te->owner = ReadStr(AH); is_supported = true; if (AH->version < K_VERS_1_9) @@ -3468,6 +3475,9 @@ _reconnectToDB(ArchiveHandle *AH, const char *dbname) free(AH->currTablespace); AH->currTablespace = NULL; + free(AH->currToastType); + AH->currToastType = NULL; + /* re-establish fixed state */ _doSetFixedOutputState(AH); } @@ -3673,6 +3683,56 @@ _selectTableAccessMethod(ArchiveHandle *AH, const char *tableam) AH->currTableAm = pg_strdup(want); } + +/* + * Set the proper default_toast_type value for the table. + */ +static void +_selectToastType(ArchiveHandle *AH, const char *toasttype) +{ + RestoreOptions *ropt = AH->public.ropt; + PQExpBuffer cmd; + const char *want, + *have; + + /* do nothing in --no-toast-type mode */ + if (ropt->noToastType) + return; + + have = AH->currToastType; + want = toasttype; + + if (!want) + return; + + if (have && strcmp(want, have) == 0) + return; + + cmd = createPQExpBuffer(); + + appendPQExpBuffer(cmd, "SET default_toast_type = %s;", fmtId(toasttype)); + + if (RestoringToDB(AH)) + { + PGresult *res; + + res = PQexec(AH->connection, cmd->data); + + if (!res || PQresultStatus(res) != PGRES_COMMAND_OK) + warn_or_exit_horribly(AH, + "could not set \"default_toast_type\": %s", + PQerrorMessage(AH->connection)); + PQclear(res); + } + else + ahprintf(AH, "%s\n\n", cmd->data); + + destroyPQExpBuffer(cmd); + + free(AH->currToastType); + AH->currToastType = pg_strdup(want); +} + /* * Set the proper default table access method for a table without storage. * Currently, this is required only for partitioned tables with a table AM. @@ -3828,13 +3888,16 @@ _printTocEntry(ArchiveHandle *AH, TocEntry *te, const char *pfx) * Select owner, schema, tablespace and default AM as necessary. The * default access method for partitioned tables is handled after * generating the object definition, as it requires an ALTER command - * rather than SET. + * rather than SET. Partitioned tables do not have TOAST tables. */ _becomeOwner(AH, te); _selectOutputSchema(AH, te->namespace); _selectTablespace(AH, te->tablespace); if (te->relkind != RELKIND_PARTITIONED_TABLE) + { _selectTableAccessMethod(AH, te->tableam); + _selectToastType(AH, te->toasttype); + } /* Emit header comment for item */ if (!AH->noTocComments) @@ -4393,6 +4456,8 @@ restore_toc_entries_prefork(ArchiveHandle *AH, TocEntry *pending_list) AH->currTablespace = NULL; free(AH->currTableAm); AH->currTableAm = NULL; + free(AH->currToastType); + AH->currToastType = NULL; } /* @@ -5130,6 +5195,7 @@ CloneArchive(ArchiveHandle *AH) clone->currSchema = NULL; clone->currTableAm = NULL; clone->currTablespace = NULL; + clone->currToastType = NULL; /* savedPassword must be local in case we change it while connecting */ if (clone->savedPassword) @@ -5189,6 +5255,7 @@ DeCloneArchive(ArchiveHandle *AH) free(AH->currSchema); free(AH->currTablespace); free(AH->currTableAm); + free(AH->currToastType); free(AH->savedPassword); free(AH); diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index 365073b3eae4..cc7aa46b483a 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -71,10 +71,11 @@ #define K_VERS_1_16 MAKE_ARCHIVE_VERSION(1, 16, 0) /* BLOB METADATA entries * and multiple BLOBS, * relkind */ +#define K_VERS_1_17 MAKE_ARCHIVE_VERSION(1, 17, 0) /* TOAST type */ /* Current archive version number (the format we can output) */ #define K_VERS_MAJOR 1 -#define K_VERS_MINOR 16 +#define K_VERS_MINOR 17 #define K_VERS_REV 0 #define K_VERS_SELF MAKE_ARCHIVE_VERSION(K_VERS_MAJOR, K_VERS_MINOR, K_VERS_REV) @@ -325,6 +326,7 @@ struct _archiveHandle char *currSchema; /* current schema, or NULL */ char *currTablespace; /* current tablespace, or NULL */ char *currTableAm; /* current table access method, or NULL */ + char *currToastType; /* current TOAST type, or NULL */ /* in --transaction-size mode, this counts objects emitted in cur xact */ int txnCount; @@ -359,6 +361,7 @@ struct _tocEntry char *tablespace; /* null if not in a tablespace; empty string * means use database default */ char *tableam; /* table access method, only for TABLE tags */ + char *toasttype; /* TOAST table type, only for TABLE tags */ char relkind; /* relation kind, only for TABLE tags */ char *owner; char *desc; @@ -405,6 +408,7 @@ typedef struct _archiveOpts const char *namespace; const char *tablespace; const char *tableam; + const char *toasttype; char relkind; const char *owner; const char *description; diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1937997ea674..c8542aa85abe 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -499,6 +499,7 @@ main(int argc, char **argv) {"lock-wait-timeout", required_argument, NULL, 2}, {"no-table-access-method", no_argument, &dopt.outputNoTableAm, 1}, {"no-tablespaces", no_argument, &dopt.outputNoTablespaces, 1}, + {"no-toast-type", no_argument, &dopt.outputNoToastType, 1}, {"quote-all-identifiers", no_argument, "e_all_identifiers, 1}, {"load-via-partition-root", no_argument, &dopt.load_via_partition_root, 1}, {"role", required_argument, NULL, 3}, @@ -1186,6 +1187,7 @@ main(int argc, char **argv) ropt->noOwner = dopt.outputNoOwner; ropt->noTableAm = dopt.outputNoTableAm; ropt->noTablespace = dopt.outputNoTablespaces; + ropt->noToastType = dopt.outputNoToastType; ropt->disable_triggers = dopt.disable_triggers; ropt->use_setsessauth = dopt.use_setsessauth; ropt->disable_dollar_quoting = dopt.disable_dollar_quoting; @@ -1308,6 +1310,7 @@ help(const char *progname) printf(_(" --no-table-access-method do not dump table access methods\n")); printf(_(" --no-tablespaces do not dump tablespace assignments\n")); printf(_(" --no-toast-compression do not dump TOAST compression methods\n")); + printf(_(" --no-toast-type do not dump TOAST table type\n")); printf(_(" --no-unlogged-table-data do not dump unlogged table data\n")); printf(_(" --on-conflict-do-nothing add ON CONFLICT DO NOTHING to INSERT commands\n")); printf(_(" --quote-all-identifiers quote all identifiers, even if not key words\n")); @@ -6993,6 +6996,7 @@ getTables(Archive *fout, int *numTables) int i_relfrozenxid; int i_toastfrozenxid; int i_toastoid; + int i_toasttype; int i_relminmxid; int i_toastminmxid; int i_reloptions; @@ -7047,6 +7051,14 @@ getTables(Archive *fout, int *numTables) "ELSE 0 END AS foreignserver, " "c.relfrozenxid, tc.relfrozenxid AS tfrozenxid, " "tc.oid AS toid, " + "CASE WHEN c.reltoastrelid <> 0 THEN " + " (SELECT CASE " + " WHEN a.atttypid::regtype = 'oid'::regtype THEN 'oid'::text " + " WHEN a.atttypid::regtype = 'bigint'::regtype THEN 'int8'::text " + " ELSE NULL END" + " FROM pg_attribute AS a " + " WHERE a.attrelid = tc.oid AND a.attname = 'chunk_id') " + " ELSE NULL END AS toasttype, " "tc.relpages AS toastpages, " "tc.reloptions AS toast_reloptions, " "d.refobjid AS owning_tab, " @@ -7217,6 +7229,7 @@ getTables(Archive *fout, int *numTables) i_relfrozenxid = PQfnumber(res, "relfrozenxid"); i_toastfrozenxid = PQfnumber(res, "tfrozenxid"); i_toastoid = PQfnumber(res, "toid"); + i_toasttype = PQfnumber(res, "toasttype"); i_relminmxid = PQfnumber(res, "relminmxid"); i_toastminmxid = PQfnumber(res, "tminmxid"); i_reloptions = PQfnumber(res, "reloptions"); @@ -7295,6 +7308,10 @@ getTables(Archive *fout, int *numTables) tblinfo[i].frozenxid = atooid(PQgetvalue(res, i, i_relfrozenxid)); tblinfo[i].toast_frozenxid = atooid(PQgetvalue(res, i, i_toastfrozenxid)); tblinfo[i].toast_oid = atooid(PQgetvalue(res, i, i_toastoid)); + if (PQgetisnull(res, i, i_toasttype)) + tblinfo[i].toast_type = NULL; + else + tblinfo[i].toast_type = pg_strdup(PQgetvalue(res, i, i_toasttype)); tblinfo[i].minmxid = atooid(PQgetvalue(res, i, i_relminmxid)); tblinfo[i].toast_minmxid = atooid(PQgetvalue(res, i, i_toastminmxid)); tblinfo[i].reloptions = pg_strdup(PQgetvalue(res, i, i_reloptions)); @@ -17667,6 +17684,7 @@ dumpTableSchema(Archive *fout, const TableInfo *tbinfo) { char *tablespace = NULL; char *tableam = NULL; + char *toasttype = NULL; /* * _selectTablespace() relies on tablespace-enabled objects in the @@ -17681,12 +17699,15 @@ dumpTableSchema(Archive *fout, const TableInfo *tbinfo) if (RELKIND_HAS_TABLE_AM(tbinfo->relkind) || tbinfo->relkind == RELKIND_PARTITIONED_TABLE) tableam = tbinfo->amname; + if (OidIsValid(tbinfo->toast_oid)) + toasttype = tbinfo->toast_type; ArchiveEntry(fout, tbinfo->dobj.catId, tbinfo->dobj.dumpId, ARCHIVE_OPTS(.tag = tbinfo->dobj.name, .namespace = tbinfo->dobj.namespace->dobj.name, .tablespace = tablespace, .tableam = tableam, + .toasttype = toasttype, .relkind = tbinfo->relkind, .owner = tbinfo->rolname, .description = reltypename, diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index 39eef1d6617f..1e6627067fc7 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -318,6 +318,7 @@ typedef struct _tableInfo uint32 frozenxid; /* table's relfrozenxid */ uint32 minmxid; /* table's relminmxid */ Oid toast_oid; /* toast table's OID, or 0 if none */ + char *toast_type; /* toast table type, or NULL if none */ uint32 toast_frozenxid; /* toast table's relfrozenxid, if any */ uint32 toast_minmxid; /* toast table's relminmxid */ int ncheck; /* # of CHECK expressions */ diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index 3cbcad65c5fb..1e52c79ce120 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -95,6 +95,7 @@ static int if_exists = 0; static int inserts = 0; static int no_table_access_method = 0; static int no_tablespaces = 0; +static int no_toast_type = 0; static int use_setsessauth = 0; static int no_comments = 0; static int no_policies = 0; @@ -167,6 +168,7 @@ main(int argc, char *argv[]) {"lock-wait-timeout", required_argument, NULL, 2}, {"no-table-access-method", no_argument, &no_table_access_method, 1}, {"no-tablespaces", no_argument, &no_tablespaces, 1}, + {"no-toast-type", no_argument, &no_tablespaces, 1}, {"quote-all-identifiers", no_argument, "e_all_identifiers, 1}, {"load-via-partition-root", no_argument, &load_via_partition_root, 1}, {"role", required_argument, NULL, 3}, @@ -471,6 +473,8 @@ main(int argc, char *argv[]) appendPQExpBufferStr(pgdumpopts, " --no-table-access-method"); if (no_tablespaces) appendPQExpBufferStr(pgdumpopts, " --no-tablespaces"); + if (no_toast_type) + appendPQExpBufferStr(pgdumpopts, " --no-toast-type"); if (quote_all_identifiers) appendPQExpBufferStr(pgdumpopts, " --quote-all-identifiers"); if (load_via_partition_root) @@ -745,6 +749,7 @@ help(void) printf(_(" --no-table-access-method do not dump table access methods\n")); printf(_(" --no-tablespaces do not dump tablespace assignments\n")); printf(_(" --no-toast-compression do not dump TOAST compression methods\n")); + printf(_(" --no-toast-type do not dump TOAST table types\n")); printf(_(" --no-unlogged-table-data do not dump unlogged table data\n")); printf(_(" --on-conflict-do-nothing add ON CONFLICT DO NOTHING to INSERT commands\n")); printf(_(" --quote-all-identifiers quote all identifiers, even if not key words\n")); diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c index 6ef789cb06d6..610103ae4b99 100644 --- a/src/bin/pg_dump/pg_restore.c +++ b/src/bin/pg_dump/pg_restore.c @@ -100,6 +100,7 @@ main(int argc, char **argv) static int no_data_for_failed_tables = 0; static int outputNoTableAm = 0; static int outputNoTablespaces = 0; + static int outputNoToastType = 0; static int use_setsessauth = 0; static int no_comments = 0; static int no_data = 0; @@ -156,6 +157,7 @@ main(int argc, char **argv) {"no-data-for-failed-tables", no_argument, &no_data_for_failed_tables, 1}, {"no-table-access-method", no_argument, &outputNoTableAm, 1}, {"no-tablespaces", no_argument, &outputNoTablespaces, 1}, + {"no-toast-type", no_argument, &outputNoToastType, 1}, {"role", required_argument, NULL, 2}, {"section", required_argument, NULL, 3}, {"strict-names", no_argument, &strict_names, 1}, @@ -461,6 +463,7 @@ main(int argc, char **argv) opts->noDataForFailedTables = no_data_for_failed_tables; opts->noTableAm = outputNoTableAm; opts->noTablespace = outputNoTablespaces; + opts->noToastType = outputNoToastType; opts->use_setsessauth = use_setsessauth; opts->no_comments = no_comments; opts->no_policies = no_policies; @@ -704,6 +707,7 @@ usage(const char *progname) printf(_(" --no-subscriptions do not restore subscriptions\n")); printf(_(" --no-table-access-method do not restore table access methods\n")); printf(_(" --no-tablespaces do not restore tablespace assignments\n")); + printf(_(" --no-toast-type do not restore TOAST table types\n")); printf(_(" --section=SECTION restore named section (pre-data, data, or post-data)\n")); printf(_(" --statistics-only restore only the statistics, not schema or data\n")); printf(_(" --strict-names require table and/or schema include patterns to\n" diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index 2485d8f360e5..b49e1324114c 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -659,6 +659,15 @@ my %pgdump_runs = ( 'postgres', ], }, + no_toast_type => { + dump_cmd => [ + 'pg_dump', '--no-sync', + '--file' => "$tempdir/no_toast_type.sql", + '--no-toast-type', + '--with-statistics', + 'postgres', + ], + }, only_dump_test_schema => { dump_cmd => [ 'pg_dump', '--no-sync', @@ -881,6 +890,7 @@ my %full_runs = ( no_privs => 1, no_statistics => 1, no_table_access_method => 1, + no_toast_type => 1, pg_dumpall_dbprivs => 1, pg_dumpall_exclude => 1, schema_only => 1, @@ -4913,6 +4923,31 @@ my %tests = ( }, }, + # Test the case of multiple TOAST table types. + 'CREATE TABLE regress_toast_type' => { + create_order => 13, + create_sql => ' + SET default_toast_type = int8; + CREATE TABLE dump_test.regress_toast_type_int8 (col1 text); + SET default_toast_type = oid; + CREATE TABLE dump_test.regress_toast_type_oid (col1 text); + RESET default_toast_type;', + regexp => qr/^ + \QSET default_toast_type = int8;\E + (\n(?!SET[^;]+;)[^\n]*)* + \n\QCREATE TABLE dump_test.regress_toast_type_int8 (\E + \n\s+\Qcol1 text\E + \n\);/xm, + like => { + %full_runs, %dump_test_schema_runs, section_pre_data => 1, + }, + unlike => { + exclude_dump_test_schema => 1, + no_toast_type => 1, + only_dump_measurement => 1, + }, + }, + # # TABLE and MATVIEW stats will end up in SECTION_DATA. # INDEX stats (expression columns only) will end up in SECTION_POST_DATA. diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index 2ae084b5fa6f..d0efc5955505 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -1208,6 +1208,18 @@ PostgreSQL documentation + + + + + Do not output commands to set TOAST table + types. + With this option, all TOAST tables will be + restored with the default type. + + + + diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml index 8ca68da5a556..6e5a8beded5d 100644 --- a/doc/src/sgml/ref/pg_dumpall.sgml +++ b/doc/src/sgml/ref/pg_dumpall.sgml @@ -633,6 +633,18 @@ exclude database PATTERN + + + + + Do not output commands to set TOAST table + types. + With this option, all TOAST tables will be + restored with the default type. + + + + diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml index b649bd3a5ae0..fb64315d8aa6 100644 --- a/doc/src/sgml/ref/pg_restore.sgml +++ b/doc/src/sgml/ref/pg_restore.sgml @@ -842,6 +842,18 @@ PostgreSQL documentation + + + + + Do not output commands to select TOAST table + types. + With this option, all TOAST tables will be + created with whichever type is the default during restore. + + + + -- 2.50.0