From f70db23d24c295efa0e632a2b685d0c30141120e Mon Sep 17 00:00:00 2001 From: Mahendra Singh Thalor Date: Thu, 27 Aug 2026 11:51:43 +0530 Subject: [PATCH 4/4] Add non-text output formats to pg_dumpall Teach pg_dumpall to write custom, directory, and tar archive output, in addition to its existing plain-text SQL script format, and teach pg_restore to restore a whole cluster from such an archive. Design ------ pg_dumpall gains a --format/-F option (plain, custom, directory, tar; plain remains the default). For a non-plain format, --file names a directory rather than a file. That directory holds: - toc.glo, a custom-format archive of global objects: roles, tablespaces, and the DROP statements needed to remove databases, roles, and tablespaces that already exist on the target before recreating them. - map.dat, a plain-text list mapping each database's OID to its name, so pg_restore knows what to restore and in what order. Lines can be commented out with '#' to skip a database. - a "databases" subdirectory holding one regular pg_dump archive per database, named by OID, produced by invoking pg_dump as a subprocess for each one. Each global object gets a single archive entry built the same way pg_dump.c builds its own entries: the SQL is assembled once and handed to ArchiveEntry() with both a .createStmt and a .dropStmt, and the archiver's normal restore-time drop pass (gated on --clean) decides whether the drop actually runs. Roles, tablespaces, and the per-database drop entries are cluster-wide objects with no owning database or schema, so the archiver's selective-restore logic (_tocEntryRequired) and its createDB-mode drop restriction both carry a small carve-out recognizing them, ensuring they survive filters like --schema and --no-tablespaces and are not gated on -C the way an ordinary DATABASE entry is. pg_restore gains matching support for this layout: -g/--globals-only to restore only the global objects, --no-globals to skip them, and --exclude-database=PATTERN to skip specific databases, whether restoring live (-d) or to a script file. Restoring a full archive requires -C/--create, since each database needs to be created before its own contents are restored into it; databases are restored one per pg_dump sub-archive, optionally in parallel via -j, while global objects are always restored first and serially. Other implementation notes: map.dat is fsynced like the rest of the archive; the per-database archive's file name is shell-quoted with appendShellString() rather than hand-rolled quoting; and COMMENT and SECURITY LABEL entries for roles and tablespaces record a dependency on their parent object, as pg_dump.c's own entries do. Tests in t/007_pg_dumpall.pl cover each output format, globals-only and no-globals restores, excluding databases both at dump time and restore time (via a live connection and via the static database list), --clean and --clean --if-exists semantics for roles, tablespaces, and databases, restoring live into a fresh cluster (including a second restore with --clean --if-exists to confirm objects are actually dropped and recreated rather than skipped), selective restore with --schema, and parallel restore. --- doc/src/sgml/ref/pg_dumpall.sgml | 134 +++- doc/src/sgml/ref/pg_restore.sgml | 123 +++- src/bin/pg_dump/meson.build | 1 + src/bin/pg_dump/parallel.c | 14 + src/bin/pg_dump/pg_backup.h | 2 +- src/bin/pg_dump/pg_backup_archiver.c | 71 +- src/bin/pg_dump/pg_backup_archiver.h | 1 + src/bin/pg_dump/pg_backup_tar.c | 2 +- src/bin/pg_dump/pg_dump.c | 2 +- src/bin/pg_dump/pg_dumpall.c | 978 +++++++++++++++++++++------ src/bin/pg_dump/pg_restore.c | 670 +++++++++++++++++- src/bin/pg_dump/t/001_basic.pl | 76 ++- src/bin/pg_dump/t/007_pg_dumpall.pl | 862 +++++++++++++++++++++++ src/tools/pgindent/typedefs.list | 1 + 14 files changed, 2661 insertions(+), 276 deletions(-) create mode 100644 src/bin/pg_dump/t/007_pg_dumpall.pl diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml index 238c87c13f5..481e6925f25 100644 --- a/doc/src/sgml/ref/pg_dumpall.sgml +++ b/doc/src/sgml/ref/pg_dumpall.sgml @@ -16,7 +16,10 @@ PostgreSQL documentation pg_dumpall - extract a PostgreSQL database cluster into a script file + + + export a PostgreSQL database cluster as an SQL script or to other formats + @@ -33,14 +36,33 @@ PostgreSQL documentation pg_dumpall is a utility for writing out (dumping) all PostgreSQL databases - of a cluster into one script file. The script file contains - SQL commands that can be used as input to to restore the databases. It does this by + of a cluster into an SQL script file or an archive. It does this by calling for each database in the cluster. + The output contains SQL commands that can be used + as input to or + to restore the databases. pg_dumpall also dumps global objects that are common to all databases, namely database roles, tablespaces, and privilege grants for configuration parameters. (pg_dump does not save these objects.) + The only parts of a database cluster's state that + are not included in the default output + of pg_dumpall are the configuration files + and any database parameter setting changes made with + . + + + + If the output format is a + plain text SQL script, it will be written to the standard output. Use the + / option or shell operators to + redirect it into a file. + + + + If another output format is selected, the archive will be placed in a + directory named using the / + option, which is required in this case. @@ -51,12 +73,6 @@ PostgreSQL documentation allowed to add roles and create databases. - - The SQL script will be written to the standard output. Use the - / option or shell operators to - redirect it into a file. - - pg_dumpall needs to connect several times to the PostgreSQL server (once per @@ -131,16 +147,92 @@ PostgreSQL documentation Send output to the specified file. If this is omitted, the standard output is used. + This option can only be omitted when is plain. + + + + + + Specify the format of dump files. In plain format, all the dump data is + sent in a single text stream. This is the default. + + In all other modes, pg_dumpall first creates two files, + toc.glo and map.dat, in the directory + specified by . + The first file contains global data (roles and tablespaces) in custom format. The second + contains a mapping between database OIDs and names. These files are used by + pg_restore. Data for individual databases is placed in + the databases subdirectory, named using the database's OID. + + + + p + plain + + + Output a plain-text SQL script file (the default). + + + + + + c + custom + + + Output a custom-format archive for each database, + suitable for input into pg_restore. The archive + will be named dboid.dmp where dboid is the + oid of the database. + + + + + + d + directory + + + Output directory-format archives for each database, + suitable for input into pg_restore. The directory + will have database oid as its name. + + + + + + t + tar + + + Output a tar-format archive for each database, + suitable for input into pg_restore. The archive + will be named dboid.tar where dboid is the + oid of the database. + + + + + + See for details on how the + various non-plain-text archive formats work. + + + + + Dump only global objects (roles and tablespaces), no databases. + Note: for non-text dump formats, + and cannot be used together. @@ -935,13 +1027,21 @@ exclude database PATTERN Examples - To dump all databases: - + To dump all databases in plain text format (the default): $ pg_dumpall > db.out + + To dump all databases using other formats: + +$ pg_dumpall --format=directory -f db.out +$ pg_dumpall --format=custom -f db.out +$ pg_dumpall --format=tar -f db.out + + + To restore database(s) from this file, you can use: @@ -955,6 +1055,16 @@ exclude database PATTERN the script will attempt to drop other databases immediately, and that will fail for the database you are connected to. + + + If the dump was taken in a non-plain-text format, use + pg_restore to restore the databases: + +$ pg_restore db.out -d postgres -C + + This will restore all databases. To restore only some databases, use + the option to skip those not wanted. + diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml index b6c5299c36e..e46116b5c8a 100644 --- a/doc/src/sgml/ref/pg_restore.sgml +++ b/doc/src/sgml/ref/pg_restore.sgml @@ -18,8 +18,9 @@ PostgreSQL documentation pg_restore - restore a PostgreSQL database from an - archive file created by pg_dump + restore PostgreSQL databases from archives + created by pg_dump or + pg_dumpall @@ -38,13 +39,14 @@ PostgreSQL documentation pg_restore is a utility for restoring a - PostgreSQL database from an archive - created by in one of the non-plain-text + PostgreSQL database or cluster from an archive + created by or + in one of the non-plain-text formats. It will issue the commands necessary to reconstruct the - database to the state it was in at the time it was saved. The - archive files also allow pg_restore to + database or cluster to the state it was in at the time it was saved. The + archives also allow pg_restore to be selective about what is restored, or even to reorder the items - prior to being restored. The archive files are designed to be + prior to being restored. The archive formats are designed to be portable across architectures. @@ -52,14 +54,38 @@ PostgreSQL documentation pg_restore can operate in two modes. If a database name is specified, pg_restore connects to that database and restores archive contents directly into - the database. Otherwise, a script containing the SQL - commands necessary to rebuild the database is created and written + the database. + When restoring from a dump made by pg_dumpall, + each database will be created and then the restoration will be run in that + database. + + + + Otherwise, when a database name is not specified, a script containing the SQL + commands necessary to rebuild the database or cluster is created and written to a file or standard output. This script output is equivalent to - the plain text output format of pg_dump. + the plain text output format of pg_dump or + pg_dumpall. + + + Some of the options controlling the output are therefore analogous to pg_dump options. + + A non-plain-text archive made using pg_dumpall + is a directory containing a toc.glo file with global + objects (roles and tablespaces), a map.dat file + listing the databases, and a subdirectory for each database containing + its archive. When restoring such an archive, + pg_restore first restores global objects from + toc.glo, then processes each database listed in + map.dat. Lines in map.dat can + be commented out with # to skip restoring specific + databases. + + Obviously, pg_restore cannot restore information that is not present in the archive file. For instance, if the @@ -152,6 +178,8 @@ PostgreSQL documentation commands that mention this database. Access privileges for the database itself are also restored, unless is specified. + is required when restoring multiple databases + from a non-plain-text archive made using pg_dumpall. @@ -252,6 +280,29 @@ PostgreSQL documentation + + + + + + Restore only global objects (roles and tablespaces), no databases. + + + This option is only relevant when restoring from a non-plain-text archive made using pg_dumpall. + Note: cannot be used with + , + , + , + , + , + , + , + , or + . + + + + @@ -588,6 +639,28 @@ PostgreSQL documentation + + + + + Do not restore databases whose name matches + pattern. + Multiple patterns can be excluded by writing multiple + switches. The + pattern parameter is + interpreted as a pattern according to the same rules used by + psql's \d + commands (see ), + so multiple databases can also be excluded by writing wildcard + characters in the pattern. When using wildcards, be careful to + quote the pattern if needed to prevent shell wildcard expansion. + + + This option is only relevant when restoring from a non-plain-text archive made using pg_dumpall. + + + + @@ -723,6 +796,21 @@ PostgreSQL documentation + + + + + Do not restore global objects (roles and tablespaces). When + / is not specified, + databases that do not already exist on the target server are skipped. + + + This option is only relevant when restoring from a non-plain-text + archive made using pg_dumpall. + + + + @@ -1148,6 +1236,21 @@ CREATE DATABASE foo WITH TEMPLATE template0; + + + The following options cannot be used when restoring from a non-plain-text + archive made using pg_dumpall: + , + , + , + , + , and + . + Also, if the option is used, it must + include . + + + diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build index 79bd5036841..7c9a475963b 100644 --- a/src/bin/pg_dump/meson.build +++ b/src/bin/pg_dump/meson.build @@ -103,6 +103,7 @@ tests += { 't/004_pg_dump_parallel.pl', 't/005_pg_dump_filterfile.pl', 't/006_pg_dump_compress.pl', + 't/007_pg_dumpall.pl', 't/010_dump_connstr.pl', ], }, diff --git a/src/bin/pg_dump/parallel.c b/src/bin/pg_dump/parallel.c index d6bdb60977c..745d665a6dd 100644 --- a/src/bin/pg_dump/parallel.c +++ b/src/bin/pg_dump/parallel.c @@ -333,6 +333,20 @@ on_exit_close_archive(Archive *AHX) on_exit_nicely(archive_close_connection, &shutdown_info); } +/* + * Update the archive handle in the on_exit callback registered by + * on_exit_close_archive(). When pg_restore processes a pg_dumpall archive + * containing multiple databases, each database is restored from a separate + * archive. After closing one archive and opening the next, we update the + * shutdown_info to reference the new archive handle so the cleanup callback + * will close the correct archive on exit. + */ +void +replace_on_exit_close_archive(Archive *AHX) +{ + shutdown_info.AHX = AHX; +} + /* * on_exit_nicely handler for shutting down database connections and * worker processes cleanly. diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index c7bdda1deed..28e7ff6fa16 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -313,7 +313,7 @@ extern void SetArchiveOptions(Archive *AH, DumpOptions *dopt, RestoreOptions *ro extern void ProcessArchiveRestoreOptions(Archive *AHX); -extern void RestoreArchive(Archive *AHX); +extern void RestoreArchive(Archive *AHX, bool append_data); /* Open an existing archive */ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 4c3e05e10db..9c277742ebe 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -87,7 +87,7 @@ static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); static void SetOutput(ArchiveHandle *AH, const char *filename, - const pg_compress_specification compression_spec); + const pg_compress_specification compression_spec, bool append_data); static CompressFileHandle *SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, CompressFileHandle *savedOutput); @@ -340,9 +340,14 @@ ProcessArchiveRestoreOptions(Archive *AHX) StrictNamesCheck(ropt); } -/* Public */ +/* + * RestoreArchive + * + * If append_data is set, then append data into file as we are restoring dump + * of multiple databases which was taken by pg_dumpall. + */ void -RestoreArchive(Archive *AHX) +RestoreArchive(Archive *AHX, bool append_data) { ArchiveHandle *AH = (ArchiveHandle *) AHX; RestoreOptions *ropt = AH->public.ropt; @@ -459,7 +464,7 @@ RestoreArchive(Archive *AHX) */ sav = SaveOutput(AH); if (ropt->filename || ropt->compression_spec.algorithm != PG_COMPRESSION_NONE) - SetOutput(AH, ropt->filename, ropt->compression_spec); + SetOutput(AH, ropt->filename, ropt->compression_spec, append_data); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -516,11 +521,20 @@ RestoreArchive(Archive *AHX) * because at this point we're connected to the wrong database. * (The DATABASE PROPERTIES entry, if any, should be treated like * the DATABASE entry.) + * + * ROLE, ROLE PROPERTIES, TABLESPACE, and DROP_GLOBAL entries are + * exempt: they are cluster-wide, not scoped to any database, so + * dropping them is safe regardless of which database we're + * connected to. */ if (ropt->createDB) { if (strcmp(te->desc, "DATABASE") != 0 && - strcmp(te->desc, "DATABASE PROPERTIES") != 0) + strcmp(te->desc, "DATABASE PROPERTIES") != 0 && + strcmp(te->desc, "ROLE") != 0 && + strcmp(te->desc, "ROLE PROPERTIES") != 0 && + strcmp(te->desc, "TABLESPACE") != 0 && + strcmp(te->desc, "DROP_GLOBAL") != 0) continue; } @@ -1317,7 +1331,7 @@ PrintTOCSummary(Archive *AHX) sav = SaveOutput(AH); if (ropt->filename) - SetOutput(AH, ropt->filename, out_compression_spec); + SetOutput(AH, ropt->filename, out_compression_spec, false); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1692,11 +1706,15 @@ archprintf(Archive *AH, const char *fmt, ...) /******************************* * Stuff below here should be 'private' to the archiver routines + * + * If append_data is set, then append data into file as we are restoring dump + * of multiple databases which was taken by pg_dumpall. *******************************/ static void SetOutput(ArchiveHandle *AH, const char *filename, - const pg_compress_specification compression_spec) + const pg_compress_specification compression_spec, + bool append_data) { CompressFileHandle *CFH; const char *mode; @@ -1716,7 +1734,7 @@ SetOutput(ArchiveHandle *AH, const char *filename, else fn = fileno(stdout); - if (AH->mode == archModeAppend) + if (append_data || AH->mode == archModeAppend) mode = PG_BINARY_A; else mode = PG_BINARY_W; @@ -3028,6 +3046,24 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH) return 0; } + /* In --no-tablespaces mode, skip creating tablespaces outright. */ + if (ropt->noTablespace && strcmp(te->desc, "TABLESPACE") == 0) + return 0; + + /* + * Global object TOC entries (e.g., ROLEs or TABLESPACEs) must not be + * ignored. They have no te->namespace, so without this, the + * --schema/--exclude-schema checks below would exclude them entirely. + * DROP_GLOBAL (pg_dumpall's cross-database drops) is included here + * rather than above with DATABASE/DATABASE PROPERTIES because it must + * be gated on dropSchema, not createDB. + */ + if (strcmp(te->desc, "ROLE") == 0 || + strcmp(te->desc, "ROLE PROPERTIES") == 0 || + strcmp(te->desc, "TABLESPACE") == 0 || + strcmp(te->desc, "DROP_GLOBAL") == 0) + return REQ_SCHEMA; + /* * Process exclusions that affect certain classes of TOC entries. */ @@ -3063,6 +3099,14 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH) if (ropt->no_subscriptions && strncmp(te->tag, "SUBSCRIPTION", strlen("SUBSCRIPTION")) == 0) return 0; + + /* + * Comments on global objects (ROLEs or TABLESPACEs) should not be + * skipped, since global objects themselves are never skipped. + */ + if (strncmp(te->tag, "ROLE", strlen("ROLE")) == 0 || + strncmp(te->tag, "TABLESPACE", strlen("TABLESPACE")) == 0) + return REQ_SCHEMA; } /* @@ -3092,6 +3136,14 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH) if (ropt->no_subscriptions && strncmp(te->tag, "SUBSCRIPTION", strlen("SUBSCRIPTION")) == 0) return 0; + + /* + * Security labels on global objects (ROLEs or TABLESPACEs) should not + * be skipped, since global objects themselves are never skipped. + */ + if (strncmp(te->tag, "ROLE", strlen("ROLE")) == 0 || + strncmp(te->tag, "TABLESPACE", strlen("TABLESPACE")) == 0) + return REQ_SCHEMA; } /* If it's a subscription, maybe ignore it */ @@ -3908,8 +3960,11 @@ _getObjectDescription(PQExpBuffer buf, const TocEntry *te) strcmp(type, "CONSTRAINT") == 0 || strcmp(type, "DATABASE PROPERTIES") == 0 || strcmp(type, "DEFAULT") == 0 || + strcmp(type, "DROP_GLOBAL") == 0 || strcmp(type, "FK CONSTRAINT") == 0 || strcmp(type, "INDEX") == 0 || + strcmp(type, "ROLE") == 0 || + strcmp(type, "ROLE PROPERTIES") == 0 || strcmp(type, "RULE") == 0 || strcmp(type, "TRIGGER") == 0 || strcmp(type, "ROW SECURITY") == 0 || diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index c1528d78853..1218bf6a6a1 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -394,6 +394,7 @@ struct _tocEntry extern int parallel_restore(ArchiveHandle *AH, TocEntry *te); extern void on_exit_close_archive(Archive *AHX); +extern void replace_on_exit_close_archive(Archive *AHX); extern void warn_or_exit_horribly(ArchiveHandle *AH, const char *fmt, ...) pg_attribute_printf(2, 3); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 542fdb453fb..31056da95bf 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -826,7 +826,7 @@ _CloseArchive(ArchiveHandle *AH) savVerbose = AH->public.verbose; AH->public.verbose = 0; - RestoreArchive((Archive *) AH); + RestoreArchive((Archive *) AH, false); SetArchiveOptions((Archive *) AH, savDopt, savRopt); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 0511c49606e..98607906cd7 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -1279,7 +1279,7 @@ main(int argc, char **argv) * right now. */ if (plainText) - RestoreArchive(fout); + RestoreArchive(fout, false); CloseArchive(fout); diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index aee76bb4761..d1714cb7447 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -1,13 +1,20 @@ /*------------------------------------------------------------------------- * * pg_dumpall.c + * pg_dumpall dumps all databases and global objects (roles and + * tablespaces) from a PostgreSQL cluster. + * + * For text format output, globals are written directly and pg_dump is + * invoked for each database, with all output going to stdout or a file. + * + * For non-text formats (custom, directory, tar), a directory is created + * containing a toc.glo file with global objects, a map.dat file mapping + * database OIDs to names, and a databases/ subdirectory with individual + * pg_dump archives for each database. * * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * pg_dumpall forces all pg_dump output to be text, since it also outputs - * text into the same output stream. - * * src/bin/pg_dump/pg_dumpall.c * *------------------------------------------------------------------------- @@ -31,6 +38,7 @@ #include "fe_utils/string_utils.h" #include "filter.h" #include "getopt_long.h" +#include "pg_backup_archiver.h" /* version string we expect back from pg_dump */ #define PGDUMP_VERSIONSTR "pg_dump (PostgreSQL) " PG_VERSION "\n" @@ -58,30 +66,35 @@ typedef struct static void help(void); -static void dropRoles(PGconn *conn); +static void dropDBsText(PGconn *conn); +static void dropTablespacesText(PGconn *conn); +static void dropRolesText(PGconn *conn); static void dumpRoles(PGconn *conn); static void dumpRoleMembership(PGconn *conn); static void dumpRoleGUCPrivs(PGconn *conn); -static void dropTablespaces(PGconn *conn); static void dumpTablespaces(PGconn *conn); -static void dropDBs(PGconn *conn); static void dumpUserConfig(PGconn *conn, const char *username); static void dumpDatabases(PGconn *conn); static void dumpTimestamp(const char *msg); -static int runPgDump(const char *dbname, const char *create_opts); +static int runPgDump(const char *dbname, const char *create_opts, char *dbfile); static void buildShSecLabels(PGconn *conn, const char *catalog_name, Oid objectId, const char *objtype, const char *objname, PQExpBuffer buffer); static void executeCommand(PGconn *conn, const char *query); +static void check_for_invalid_global_names(PGconn *conn, + SimpleStringList *excluded_names); static void expand_dbname_patterns(PGconn *conn, SimpleStringList *patterns, SimpleStringList *names); static void read_dumpall_filters(const char *filename, SimpleStringList *pattern); +static ArchiveFormat parseDumpFormat(const char *format); +static DumpId createDumpId(void); static char pg_dump_bin[MAXPGPATH]; static PQExpBuffer pgdumpopts; static const char *connstr = ""; static bool output_clean = false; +static bool data_only = false; static bool skip_acls = false; static bool verbose = false; static bool dosync = true; @@ -124,6 +137,10 @@ static SimpleStringList database_exclude_patterns = {NULL, NULL}; static SimpleStringList database_exclude_names = {NULL, NULL}; static char *restrict_key; +static Archive *fout = NULL; +static DumpId lastDumpId = 0; +static ArchiveFormat archDumpFormat = archNull; +static const CatalogId nilCatalogId = {0, 0}; int main(int argc, char *argv[]) @@ -149,6 +166,7 @@ main(int argc, char *argv[]) {"password", no_argument, NULL, 'W'}, {"no-privileges", no_argument, NULL, 'x'}, {"no-acl", no_argument, NULL, 'x'}, + {"format", required_argument, NULL, 'F'}, /* * the following options don't have an equivalent short option letter @@ -198,8 +216,8 @@ main(int argc, char *argv[]) char *pgdb = NULL; char *use_role = NULL; const char *dumpencoding = NULL; + const char *format_name = "p"; trivalue prompt_password = TRI_DEFAULT; - bool data_only = false; bool globals_only = false; bool roles_only = false; bool schema_only = false; @@ -209,6 +227,7 @@ main(int argc, char *argv[]) int c, ret; int optindex; + DumpOptions dopt; pg_logging_init(argv[0]); pg_logging_set_level(PG_LOG_WARNING); @@ -246,8 +265,9 @@ main(int argc, char *argv[]) } pgdumpopts = createPQExpBuffer(); + InitDumpOptions(&dopt); - while ((c = getopt_long(argc, argv, "acd:E:f:gh:l:Op:rsS:tU:vwWx", long_options, &optindex)) != -1) + while ((c = getopt_long(argc, argv, "acd:E:f:F:gh:l:Op:rsS:tU:vwWx", long_options, &optindex)) != -1) { switch (c) { @@ -271,9 +291,12 @@ main(int argc, char *argv[]) break; case 'f': + /* Not put into pgdumpopts; runPgDump() needs its own -f. */ filename = pg_strdup(optarg); - appendPQExpBufferStr(pgdumpopts, " -f "); - appendShellString(pgdumpopts, filename); + break; + + case 'F': + format_name = pg_strdup(optarg); break; case 'g': @@ -316,6 +339,7 @@ main(int argc, char *argv[]) case 'U': pguser = pg_strdup(optarg); + dopt.cparams.username = pg_strdup(optarg); break; case 'v': @@ -440,6 +464,32 @@ main(int argc, char *argv[]) pg_fatal("option %s requires option %s", "--if-exists", "-c/--clean"); + /* Get format for dump. */ + archDumpFormat = parseDumpFormat(format_name); + + /* + * If a non-plain format is specified, a file name is also required as the + * path to the main directory. + */ + if (archDumpFormat != archNull && + (!filename || strcmp(filename, "") == 0)) + { + pg_log_error("option %s=d|c|t requires option %s", + "-F/--format", "-f/--file"); + pg_log_error_hint("Try \"%s --help\" for more information.", progname); + exit_nicely(1); + } + + /* restrict-key is only supported with --format=plain */ + if (archDumpFormat != archNull && restrict_key) + pg_fatal("option %s can only be used with %s=plain", + "--restrict-key", "--format"); + + /* --clean and -g/--globals-only cannot be used together in non-text dump */ + if (archDumpFormat != archNull && output_clean && globals_only) + pg_fatal("options %s and %s cannot be used together in non-text dump", + "--clean", "-g/--globals-only"); + /* * If password values are not required in the dump, switch to using * pg_roles which is equally useful, just more likely to have unrestricted @@ -500,6 +550,27 @@ main(int argc, char *argv[]) if (sequence_data) appendPQExpBufferStr(pgdumpopts, " --sequence-data"); + /* + * Open the output file if required, otherwise use stdout. If required, + * then create new directory. + */ + if (archDumpFormat != archNull) + { + Assert(filename); + + /* Create new directory or accept the empty existing directory. */ + create_or_open_dir(filename); + } + else if (filename) + { + OPF = fopen(filename, PG_BINARY_W); + if (!OPF) + pg_fatal("could not open output file \"%s\": %m", + filename); + } + else + OPF = stdout; + /* * If you don't provide a restrict key, one will be appointed for you. */ @@ -549,19 +620,6 @@ main(int argc, char *argv[]) expand_dbname_patterns(conn, &database_exclude_patterns, &database_exclude_names); - /* - * Open the output file if required, otherwise use stdout - */ - if (filename) - { - OPF = fopen(filename, PG_BINARY_W); - if (!OPF) - pg_fatal("could not open output file \"%s\": %m", - filename); - } - else - OPF = stdout; - /* * Set the client encoding if requested. */ @@ -599,60 +657,150 @@ main(int argc, char *argv[]) if (quote_all_identifiers) executeCommand(conn, "SET quote_all_identifiers = true"); - fprintf(OPF, "--\n-- PostgreSQL database cluster dump\n--\n\n"); - if (verbose) - dumpTimestamp("Started on"); + /* Create an archive file for global commands. */ + if (archDumpFormat != archNull) + { + PQExpBuffer qry = createPQExpBuffer(); + char global_path[MAXPGPATH]; + const char *encname; + pg_compress_specification compression_spec = {0}; - /* - * Enter restricted mode to block any unexpected psql meta-commands. A - * malicious source might try to inject a variety of things via bogus - * responses to queries. While we cannot prevent such sources from - * affecting the destination at restore time, we can block psql - * meta-commands so that the client machine that runs psql with the dump - * output remains unaffected. - */ - fprintf(OPF, "\\restrict %s\n\n", restrict_key); + /* + * Check that no global object names contain newlines or carriage + * returns, which would break the map.dat file format. This is only + * needed for servers older than v19, which started prohibiting such + * names. + */ + if (server_version < 190000) + check_for_invalid_global_names(conn, &database_exclude_names); - /* - * We used to emit \connect postgres here, but that served no purpose - * other than to break things for installations without a postgres - * database. Everything we're restoring here is a global, so whichever - * database we're connected to at the moment is fine. - */ + /* Set file path for global sql commands. */ + snprintf(global_path, MAXPGPATH, "%s/toc.glo", filename); + + /* Open the output file */ + fout = CreateArchive(global_path, archCustom, compression_spec, + dosync, archModeWrite, NULL, DATA_DIR_SYNC_METHOD_FSYNC); + + /* Make dump options accessible right away */ + SetArchiveOptions(fout, &dopt, NULL); + + ((ArchiveHandle *) fout)->connection = conn; + ((ArchiveHandle *) fout)->public.numWorkers = 1; + + /* Register the cleanup hook */ + on_exit_close_archive(fout); + + /* Let the archiver know how noisy to be */ + fout->verbose = verbose; + + /* + * We allow the server to be back to 9.2, and up to any minor release + * of our own major version. (See also version check in + * pg_dumpall.c.) + */ + fout->minRemoteVersion = 90200; + fout->maxRemoteVersion = (PG_VERSION_NUM / 100) * 100 + 99; + fout->numWorkers = 1; + + /* Dump default_transaction_read_only. */ + appendPQExpBufferStr(qry, "SET default_transaction_read_only = off;\n\n"); + ArchiveEntry(fout, + nilCatalogId, /* catalog ID */ + createDumpId(), /* dump ID */ + ARCHIVE_OPTS(.tag = "default_transaction_read_only", + .description = "default_transaction_read_only", + .section = SECTION_PRE_DATA, + .createStmt = qry->data)); + resetPQExpBuffer(qry); + + /* Put the correct encoding into the archive */ + encname = pg_encoding_to_char(encoding); + + appendPQExpBufferStr(qry, "SET client_encoding = "); + appendStringLiteralAH(qry, encname, fout); + appendPQExpBufferStr(qry, ";\n"); + ArchiveEntry(fout, + nilCatalogId, /* catalog ID */ + createDumpId(), /* dump ID */ + ARCHIVE_OPTS(.tag = "ENCODING", + .description = "ENCODING", + .section = SECTION_PRE_DATA, + .createStmt = qry->data)); + resetPQExpBuffer(qry); + + /* Put the correct escape string behavior into the archive. */ + appendPQExpBufferStr(qry, "SET standard_conforming_strings = 'on';\n"); + ArchiveEntry(fout, + nilCatalogId, /* catalog ID */ + createDumpId(), /* dump ID */ + ARCHIVE_OPTS(.tag = "STDSTRINGS", + .description = "STDSTRINGS", + .section = SECTION_PRE_DATA, + .createStmt = qry->data)); + destroyPQExpBuffer(qry); + } + else + { + fprintf(OPF, "--\n-- PostgreSQL database cluster dump\n--\n\n"); - /* Restore will need to write to the target cluster */ - fprintf(OPF, "SET default_transaction_read_only = off;\n\n"); + if (verbose) + dumpTimestamp("Started on"); - /* Replicate encoding and standard_conforming_strings in output */ - fprintf(OPF, "SET client_encoding = '%s';\n", - pg_encoding_to_char(encoding)); - fprintf(OPF, "SET standard_conforming_strings = on;\n"); - fprintf(OPF, "\n"); + /* + * Enter restricted mode to block any unexpected psql meta-commands. A + * malicious source might try to inject a variety of things via bogus + * responses to queries. While we cannot prevent such sources from + * affecting the destination at restore time, we can block psql + * meta-commands so that the client machine that runs psql with the + * dump output remains unaffected. + */ + fprintf(OPF, "\\restrict %s\n\n", restrict_key); + + /* + * We used to emit \connect postgres here, but that served no purpose + * other than to break things for installations without a postgres + * database. Everything we're restoring here is a global, so + * whichever database we're connected to at the moment is fine. + */ + + /* Restore will need to write to the target cluster */ + fprintf(OPF, "SET default_transaction_read_only = off;\n\n"); + + /* Replicate encoding and standard_conforming_strings in output */ + fprintf(OPF, "SET client_encoding = '%s';\n", + pg_encoding_to_char(encoding)); + fprintf(OPF, "SET standard_conforming_strings = on;\n"); + fprintf(OPF, "\n"); + } if (!data_only && !statistics_only && !no_schema) { /* - * If asked to --clean, do that first. We can avoid detailed - * dependency analysis because databases never depend on each other, - * and tablespaces never depend on each other. Roles could have - * grants to each other, but DROP ROLE will clean those up silently. + * For plain-text output with --clean, print the Drop + * databases/Drop tablespaces/Drop roles sections first, ahead of + * any creates (see dropDBsText() for why the order matters). For + * non-plain formats, dumpRoles()/dumpTablespaces()/dumpDatabases() + * instead attach a drop statement to each archive entry, and the + * archiver decides at restore time whether to use it. + * + * For non-text formats, pg_dumpall unconditionally process --clean + * option. In contrast, pg_restore only applies it if the user + * explicitly provides the flag. This discrepancy resolves corner + * cases where pg_restore requires cleanup instructions that may be + * missing from a standard pg_dumpall output. */ - if (output_clean) + if (archDumpFormat == archNull && output_clean) { if (!globals_only && !roles_only && !tablespaces_only) - dropDBs(conn); + dropDBsText(conn); if (!roles_only && !no_tablespaces) - dropTablespaces(conn); + dropTablespacesText(conn); if (!tablespaces_only) - dropRoles(conn); + dropRolesText(conn); } - /* - * Now create objects as requested. Be careful that option logic here - * is the same as for drops above. - */ if (!tablespaces_only) { /* Dump roles (users) */ @@ -671,28 +819,45 @@ main(int argc, char *argv[]) dumpTablespaces(conn); } - /* - * Exit restricted mode just before dumping the databases. pg_dump will - * handle entering restricted mode again as appropriate. - */ - fprintf(OPF, "\\unrestrict %s\n\n", restrict_key); + if (archDumpFormat == archNull) + { + /* + * Exit restricted mode just before dumping the databases. pg_dump + * will handle entering restricted mode again as appropriate. + */ + fprintf(OPF, "\\unrestrict %s\n\n", restrict_key); + } if (!globals_only && !roles_only && !tablespaces_only) dumpDatabases(conn); - PQfinish(conn); + if (archDumpFormat == archNull) + { + PQfinish(conn); - if (verbose) - dumpTimestamp("Completed on"); - fprintf(OPF, "--\n-- PostgreSQL database cluster dump complete\n--\n\n"); + if (verbose) + dumpTimestamp("Completed on"); + fprintf(OPF, "--\n-- PostgreSQL database cluster dump complete\n--\n\n"); - if (filename) + if (filename) + { + fclose(OPF); + + /* sync the resulting file, errors are not fatal */ + if (dosync) + (void) fsync_fname(filename, false); + } + } + else { - fclose(OPF); + RestoreOptions *ropt; - /* sync the resulting file, errors are not fatal */ - if (dosync) - (void) fsync_fname(filename, false); + ropt = NewRestoreOptions(); + SetArchiveOptions(fout, &dopt, ropt); + + /* Mark which entries should be output */ + ProcessArchiveRestoreOptions(fout); + CloseArchive(fout); } exit_nicely(0); @@ -702,12 +867,14 @@ main(int argc, char *argv[]) static void help(void) { - printf(_("%s exports a PostgreSQL database cluster as an SQL script.\n\n"), progname); + printf(_("%s exports a PostgreSQL database cluster as an SQL script or to other formats.\n\n"), progname); printf(_("Usage:\n")); printf(_(" %s [OPTION]...\n"), progname); printf(_("\nGeneral options:\n")); printf(_(" -f, --file=FILENAME output file name\n")); + printf(_(" -F, --format=c|d|t|p output file format (custom, directory, tar,\n" + " plain text (default))\n")); printf(_(" -v, --verbose verbose mode\n")); printf(_(" -V, --version output version information, then exit\n")); printf(_(" --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n")); @@ -776,16 +943,96 @@ help(void) /* - * Drop roles + * Print the "Drop databases"/"Drop tablespaces"/"Drop roles" plain-text + * sections, in that order, ahead of any "create" sections. The order + * matters: a database may depend on a tablespace or role, so those must + * go first. Non-plain formats need no such helper; the archiver's + * restore-time drop pass already runs all drops before any creates, + * regardless of registration order. + */ +static void +dropDBsText(PGconn *conn) +{ + PGresult *res; + int i; + + Assert(archDumpFormat == archNull); + + res = executeQuery(conn, + "SELECT datname " + "FROM pg_database d " + "WHERE datallowconn AND datconnlimit != -2 " + "ORDER BY datname"); + + if (PQntuples(res) > 0) + fprintf(OPF, "--\n-- Drop databases (except postgres and template1)\n--\n\n"); + + for (i = 0; i < PQntuples(res); i++) + { + char *dbname = PQgetvalue(res, i, 0); + + if (strcmp(dbname, "template1") != 0 && + strcmp(dbname, "template0") != 0 && + strcmp(dbname, "postgres") != 0) + fprintf(OPF, "DROP DATABASE %s%s;\n", + if_exists ? "IF EXISTS " : "", + fmtId(dbname)); + } + + PQclear(res); + + if (PQntuples(res) > 0) + fprintf(OPF, "\n\n"); +} + + +/* + * Drop tablespaces (see dropDBsText() for why this is plain-text only). + */ +static void +dropTablespacesText(PGconn *conn) +{ + PGresult *res; + int i; + + Assert(archDumpFormat == archNull); + + res = executeQuery(conn, "SELECT spcname " + "FROM pg_catalog.pg_tablespace " + "WHERE spcname !~ '^pg_' " + "ORDER BY 1"); + + if (PQntuples(res) > 0) + fprintf(OPF, "--\n-- Drop tablespaces\n--\n\n"); + + for (i = 0; i < PQntuples(res); i++) + { + char *spcname = PQgetvalue(res, i, 0); + + fprintf(OPF, "DROP TABLESPACE %s%s;\n", + if_exists ? "IF EXISTS " : "", + fmtId(spcname)); + } + + PQclear(res); + + if (PQntuples(res) > 0) + fprintf(OPF, "\n\n"); +} + + +/* + * Drop roles (see dropDBsText() for why this is plain-text only). */ static void -dropRoles(PGconn *conn) +dropRolesText(PGconn *conn) { PQExpBuffer buf = createPQExpBuffer(); PGresult *res; - int i_rolname; int i; + Assert(archDumpFormat == archNull); + printfPQExpBuffer(buf, "SELECT rolname " "FROM %s " @@ -793,17 +1040,14 @@ dropRoles(PGconn *conn) "ORDER BY 1", role_catalog); res = executeQuery(conn, buf->data); - - i_rolname = PQfnumber(res, "rolname"); + destroyPQExpBuffer(buf); if (PQntuples(res) > 0) fprintf(OPF, "--\n-- Drop roles\n--\n\n"); for (i = 0; i < PQntuples(res); i++) { - const char *rolename; - - rolename = PQgetvalue(res, i, i_rolname); + char *rolename = PQgetvalue(res, i, 0); fprintf(OPF, "DROP ROLE %s%s;\n", if_exists ? "IF EXISTS " : "", @@ -811,18 +1055,21 @@ dropRoles(PGconn *conn) } PQclear(res); - destroyPQExpBuffer(buf); - fprintf(OPF, "\n\n"); + if (PQntuples(res) > 0) + fprintf(OPF, "\n\n"); } /* - * Dump roles + * Drop and dump roles. */ static void dumpRoles(PGconn *conn) { PQExpBuffer buf = createPQExpBuffer(); + PQExpBuffer delQry = createPQExpBuffer(); + PQExpBuffer comment_buf = createPQExpBuffer(); + PQExpBuffer seclabel_buf = createPQExpBuffer(); PGresult *res; int i_oid, i_rolname, @@ -872,7 +1119,7 @@ dumpRoles(PGconn *conn) i_rolcomment = PQfnumber(res, "rolcomment"); i_is_current_user = PQfnumber(res, "is_current_user"); - if (PQntuples(res) > 0) + if (PQntuples(res) > 0 && archDumpFormat == archNull) fprintf(OPF, "--\n-- Roles\n--\n\n"); for (i = 0; i < PQntuples(res); i++) @@ -891,6 +1138,11 @@ dumpRoles(PGconn *conn) } resetPQExpBuffer(buf); + resetPQExpBuffer(delQry); + resetPQExpBuffer(comment_buf); + resetPQExpBuffer(seclabel_buf); + + appendPQExpBuffer(delQry, "DROP ROLE %s;\n", fmtId(rolename)); if (binary_upgrade) { @@ -967,17 +1219,59 @@ dumpRoles(PGconn *conn) if (!no_comments && !PQgetisnull(res, i, i_rolcomment)) { - appendPQExpBuffer(buf, "COMMENT ON ROLE %s IS ", fmtId(rolename)); - appendStringLiteralConn(buf, PQgetvalue(res, i, i_rolcomment), conn); - appendPQExpBufferStr(buf, ";\n"); + appendPQExpBuffer(comment_buf, "COMMENT ON ROLE %s IS ", fmtId(rolename)); + appendStringLiteralConn(comment_buf, PQgetvalue(res, i, i_rolcomment), conn); + appendPQExpBufferStr(comment_buf, ";\n"); } if (!no_security_labels) buildShSecLabels(conn, "pg_authid", auth_oid, "ROLE", rolename, - buf); + seclabel_buf); - fprintf(OPF, "%s", buf->data); + if (archDumpFormat == archNull) + { + fprintf(OPF, "%s", buf->data); + fprintf(OPF, "%s", comment_buf->data); + + if (seclabel_buf->data[0] != '\0') + fprintf(OPF, "%s", seclabel_buf->data); + } + else + { + char *tag = psprintf("ROLE %s", fmtId(rolename)); + DumpId roleDumpId = createDumpId(); + + ArchiveEntry(fout, + nilCatalogId, /* catalog ID */ + roleDumpId, /* dump ID */ + ARCHIVE_OPTS(.tag = tag, + .description = "ROLE", + .section = SECTION_PRE_DATA, + .createStmt = buf->data, + .dropStmt = delQry->data)); + if (comment_buf->data[0] != '\0') + ArchiveEntry(fout, + nilCatalogId, /* catalog ID */ + createDumpId(), /* dump ID */ + ARCHIVE_OPTS(.tag = tag, + .description = "COMMENT", + .section = SECTION_PRE_DATA, + .createStmt = comment_buf->data, + .deps = &roleDumpId, + .nDeps = 1)); + + if (seclabel_buf->data[0] != '\0') + ArchiveEntry(fout, + nilCatalogId, /* catalog ID */ + createDumpId(), /* dump ID */ + ARCHIVE_OPTS(.tag = tag, + .description = "SECURITY LABEL", + .section = SECTION_PRE_DATA, + .createStmt = seclabel_buf->data, + .deps = &roleDumpId, + .nDeps = 1)); + } } /* @@ -985,7 +1279,7 @@ dumpRoles(PGconn *conn) * We do it this way because config settings for roles could mention the * names of other roles. */ - if (PQntuples(res) > 0) + if (PQntuples(res) > 0 && archDumpFormat == archNull) fprintf(OPF, "\n--\n-- User Configurations\n--\n"); for (i = 0; i < PQntuples(res); i++) @@ -993,9 +1287,13 @@ dumpRoles(PGconn *conn) PQclear(res); - fprintf(OPF, "\n\n"); + if (archDumpFormat == archNull) + fprintf(OPF, "\n\n"); destroyPQExpBuffer(buf); + destroyPQExpBuffer(delQry); + destroyPQExpBuffer(comment_buf); + destroyPQExpBuffer(seclabel_buf); } @@ -1009,6 +1307,7 @@ static void dumpRoleMembership(PGconn *conn) { PQExpBuffer buf = createPQExpBuffer(); + PQExpBuffer querybuf = createPQExpBuffer(); PQExpBuffer optbuf = createPQExpBuffer(); PGresult *res; int start = 0, @@ -1071,7 +1370,7 @@ dumpRoleMembership(PGconn *conn) i_inherit_option = PQfnumber(res, "inherit_option"); i_set_option = PQfnumber(res, "set_option"); - if (PQntuples(res) > 0) + if (PQntuples(res) > 0 && archDumpFormat == archNull) fprintf(OPF, "--\n-- Role memberships\n--\n\n"); /* @@ -1218,8 +1517,9 @@ dumpRoleMembership(PGconn *conn) /* Generate the actual GRANT statement. */ resetPQExpBuffer(optbuf); - fprintf(OPF, "GRANT %s", fmtId(role)); - fprintf(OPF, " TO %s", fmtId(member)); + resetPQExpBuffer(querybuf); + appendPQExpBuffer(querybuf, "GRANT %s", fmtId(role)); + appendPQExpBuffer(querybuf, " TO %s", fmtId(member)); if (*admin_option == 't') appendPQExpBufferStr(optbuf, "ADMIN OPTION"); if (dump_grant_options) @@ -1240,10 +1540,21 @@ dumpRoleMembership(PGconn *conn) appendPQExpBufferStr(optbuf, "SET FALSE"); } if (optbuf->data[0] != '\0') - fprintf(OPF, " WITH %s", optbuf->data); + appendPQExpBuffer(querybuf, " WITH %s", optbuf->data); if (dump_this_grantor) - fprintf(OPF, " GRANTED BY %s", fmtId(grantor)); - fprintf(OPF, ";\n"); + appendPQExpBuffer(querybuf, " GRANTED BY %s", fmtId(grantor)); + appendPQExpBufferStr(querybuf, ";\n"); + + if (archDumpFormat == archNull) + fprintf(OPF, "%s", querybuf->data); + else + ArchiveEntry(fout, + nilCatalogId, /* catalog ID */ + createDumpId(), /* dump ID */ + ARCHIVE_OPTS(.tag = psprintf("ROLE %s", fmtId(role)), + .description = "ROLE PROPERTIES", + .section = SECTION_PRE_DATA, + .createStmt = querybuf->data)); } } @@ -1254,8 +1565,11 @@ dumpRoleMembership(PGconn *conn) PQclear(res); destroyPQExpBuffer(buf); + destroyPQExpBuffer(querybuf); + destroyPQExpBuffer(optbuf); - fprintf(OPF, "\n\n"); + if (archDumpFormat == archNull) + fprintf(OPF, "\n\n"); } @@ -1282,7 +1596,7 @@ dumpRoleGUCPrivs(PGconn *conn) "FROM pg_catalog.pg_parameter_acl " "ORDER BY 1"); - if (PQntuples(res) > 0) + if (PQntuples(res) > 0 && archDumpFormat == archNull) fprintf(OPF, "--\n-- Role privileges on configuration parameters\n--\n\n"); for (i = 0; i < PQntuples(res); i++) @@ -1307,59 +1621,38 @@ dumpRoleGUCPrivs(PGconn *conn) exit_nicely(1); } - fprintf(OPF, "%s", buf->data); + if (archDumpFormat == archNull) + fprintf(OPF, "%s", buf->data); + else + ArchiveEntry(fout, + nilCatalogId, /* catalog ID */ + createDumpId(), /* dump ID */ + ARCHIVE_OPTS(.tag = psprintf("ROLE %s", fmtId(parowner)), + .description = "ROLE PROPERTIES", + .section = SECTION_PRE_DATA, + .createStmt = buf->data)); pg_free(fparname); destroyPQExpBuffer(buf); } PQclear(res); - fprintf(OPF, "\n\n"); -} - - -/* - * Drop tablespaces. - */ -static void -dropTablespaces(PGconn *conn) -{ - PGresult *res; - int i; - - /* - * Get all tablespaces except built-in ones (which we assume are named - * pg_xxx) - */ - res = executeQuery(conn, "SELECT spcname " - "FROM pg_catalog.pg_tablespace " - "WHERE spcname !~ '^pg_' " - "ORDER BY 1"); - if (PQntuples(res) > 0) - fprintf(OPF, "--\n-- Drop tablespaces\n--\n\n"); - - for (i = 0; i < PQntuples(res); i++) - { - char *spcname = PQgetvalue(res, i, 0); - - fprintf(OPF, "DROP TABLESPACE %s%s;\n", - if_exists ? "IF EXISTS " : "", - fmtId(spcname)); - } - - PQclear(res); - - fprintf(OPF, "\n\n"); + if (archDumpFormat == archNull) + fprintf(OPF, "\n\n"); } + /* - * Dump tablespaces. + * Drop and dump tablespaces. */ static void dumpTablespaces(PGconn *conn) { PGresult *res; + PQExpBuffer delQry = createPQExpBuffer(); + PQExpBuffer comment_buf = createPQExpBuffer(); + PQExpBuffer seclabel_buf = createPQExpBuffer(); int i; /* @@ -1376,7 +1669,7 @@ dumpTablespaces(PGconn *conn) "WHERE spcname !~ '^pg_' " "ORDER BY 1"); - if (PQntuples(res) > 0) + if (PQntuples(res) > 0 && archDumpFormat == archNull) fprintf(OPF, "--\n-- Tablespaces\n--\n\n"); for (i = 0; i < PQntuples(res); i++) @@ -1395,6 +1688,12 @@ dumpTablespaces(PGconn *conn) /* needed for buildACLCommands() */ fspcname = pg_strdup(fmtId(spcname)); + resetPQExpBuffer(delQry); + resetPQExpBuffer(comment_buf); + resetPQExpBuffer(seclabel_buf); + + appendPQExpBuffer(delQry, "DROP TABLESPACE %s;\n", fspcname); + if (binary_upgrade) { appendPQExpBufferStr(buf, "\n-- For binary upgrade, must preserve pg_tablespace oid\n"); @@ -1436,71 +1735,74 @@ dumpTablespaces(PGconn *conn) if (!no_comments && spccomment && spccomment[0] != '\0') { - appendPQExpBuffer(buf, "COMMENT ON TABLESPACE %s IS ", fspcname); - appendStringLiteralConn(buf, spccomment, conn); - appendPQExpBufferStr(buf, ";\n"); + appendPQExpBuffer(comment_buf, "COMMENT ON TABLESPACE %s IS ", fspcname); + appendStringLiteralConn(comment_buf, spccomment, conn); + appendPQExpBufferStr(comment_buf, ";\n"); } if (!no_security_labels) buildShSecLabels(conn, "pg_tablespace", spcoid, "TABLESPACE", spcname, - buf); - - fprintf(OPF, "%s", buf->data); - - pg_free(fspcname); - destroyPQExpBuffer(buf); - } - - PQclear(res); - fprintf(OPF, "\n\n"); -} - + seclabel_buf); -/* - * Dump commands to drop each database. - */ -static void -dropDBs(PGconn *conn) -{ - PGresult *res; - int i; - - /* - * Skip databases marked not datallowconn, since we'd be unable to connect - * to them anyway. This must agree with dumpDatabases(). - */ - res = executeQuery(conn, - "SELECT datname " - "FROM pg_database d " - "WHERE datallowconn AND datconnlimit != -2 " - "ORDER BY datname"); + if (archDumpFormat == archNull) + { + fprintf(OPF, "%s", buf->data); - if (PQntuples(res) > 0) - fprintf(OPF, "--\n-- Drop databases (except postgres and template1)\n--\n\n"); + if (comment_buf->data[0] != '\0') + fprintf(OPF, "%s", comment_buf->data); - for (i = 0; i < PQntuples(res); i++) - { - char *dbname = PQgetvalue(res, i, 0); - - /* - * Skip "postgres" and "template1"; dumpDatabases() will deal with - * them specially. Also, be sure to skip "template0", even if for - * some reason it's not marked !datallowconn. - */ - if (strcmp(dbname, "template1") != 0 && - strcmp(dbname, "template0") != 0 && - strcmp(dbname, "postgres") != 0) + if (seclabel_buf->data[0] != '\0') + fprintf(OPF, "%s", seclabel_buf->data); + } + else { - fprintf(OPF, "DROP DATABASE %s%s;\n", - if_exists ? "IF EXISTS " : "", - fmtId(dbname)); + char *tag = psprintf("TABLESPACE %s", fmtId(fspcname)); + DumpId spcDumpId = createDumpId(); + + ArchiveEntry(fout, + nilCatalogId, /* catalog ID */ + spcDumpId, /* dump ID */ + ARCHIVE_OPTS(.tag = tag, + .description = "TABLESPACE", + .section = SECTION_PRE_DATA, + .createStmt = buf->data, + .dropStmt = delQry->data)); + + if (comment_buf->data[0] != '\0') + ArchiveEntry(fout, + nilCatalogId, /* catalog ID */ + createDumpId(), /* dump ID */ + ARCHIVE_OPTS(.tag = tag, + .description = "COMMENT", + .section = SECTION_PRE_DATA, + .createStmt = comment_buf->data, + .deps = &spcDumpId, + .nDeps = 1)); + + if (seclabel_buf->data[0] != '\0') + ArchiveEntry(fout, + nilCatalogId, /* catalog ID */ + createDumpId(), /* dump ID */ + ARCHIVE_OPTS(.tag = tag, + .description = "SECURITY LABEL", + .section = SECTION_PRE_DATA, + .createStmt = seclabel_buf->data, + .deps = &spcDumpId, + .nDeps = 1)); } + + pg_free(fspcname); + destroyPQExpBuffer(buf); } PQclear(res); + destroyPQExpBuffer(delQry); + destroyPQExpBuffer(comment_buf); + destroyPQExpBuffer(seclabel_buf); - fprintf(OPF, "\n\n"); + if (archDumpFormat == archNull) + fprintf(OPF, "\n\n"); } @@ -1522,7 +1824,7 @@ dumpUserConfig(PGconn *conn, const char *username) res = executeQuery(conn, buf->data); - if (PQntuples(res) > 0) + if (PQntuples(res) > 0 && archDumpFormat == archNull) { char *sanitized; @@ -1537,7 +1839,17 @@ dumpUserConfig(PGconn *conn, const char *username) makeAlterConfigCommand(conn, PQgetvalue(res, i, 0), "ROLE", username, NULL, NULL, buf); - fprintf(OPF, "%s", buf->data); + + if (archDumpFormat == archNull) + fprintf(OPF, "%s", buf->data); + else + ArchiveEntry(fout, + nilCatalogId, /* catalog ID */ + createDumpId(), /* dump ID */ + ARCHIVE_OPTS(.tag = psprintf("ROLE %s", fmtId(username)), + .description = "ROLE PROPERTIES", + .section = SECTION_PRE_DATA, + .createStmt = buf->data)); } PQclear(res); @@ -1606,11 +1918,15 @@ static void dumpDatabases(PGconn *conn) { PGresult *res; + PQExpBuffer delQry = createPQExpBuffer(); int i; + char db_subdir[MAXPGPATH]; + char dbfilepath[MAXPGPATH]; + FILE *map_file = NULL; /* * Skip databases marked not datallowconn, since we'd be unable to connect - * to them anyway. This must agree with dropDBs(). + * to them anyway. * * We arrange for template1 to be processed first, then we process other * DBs in alphabetical order. If we just did them all alphabetically, we @@ -1620,19 +1936,55 @@ dumpDatabases(PGconn *conn) * doesn't have some failure mode with --clean. */ res = executeQuery(conn, - "SELECT datname " + "SELECT datname, oid " "FROM pg_database d " "WHERE datallowconn AND datconnlimit != -2 " "ORDER BY (datname <> 'template1'), datname"); - if (PQntuples(res) > 0) + if (PQntuples(res) > 0 && archDumpFormat == archNull) fprintf(OPF, "--\n-- Databases\n--\n\n"); + /* + * If directory/tar/custom format is specified, create a subdirectory + * under the main directory and each database dump file or subdirectory + * will be created in that subdirectory by pg_dump. + */ + if (archDumpFormat != archNull) + { + char map_file_path[MAXPGPATH]; + + snprintf(db_subdir, MAXPGPATH, "%s/databases", filename); + + /* Create a subdirectory with 'databases' name under main directory. */ + if (mkdir(db_subdir, pg_dir_create_mode) != 0) + pg_fatal("could not create directory \"%s\": %m", db_subdir); + + snprintf(map_file_path, MAXPGPATH, "%s/map.dat", filename); + + /* Create a map file (to store dboid and dbname) */ + map_file = fopen(map_file_path, PG_BINARY_W); + if (!map_file) + pg_fatal("could not open file \"%s\": %m", map_file_path); + + fprintf(map_file, + "#################################################################\n" + "# map.dat\n" + "#\n" + "# This file maps oids to database names\n" + "#\n" + "# pg_restore will restore all the databases listed here, unless\n" + "# otherwise excluded. You can also inhibit restoration of a\n" + "# database by removing the line or commenting out the line with\n" + "# a # mark.\n" + "#################################################################\n"); + } + for (i = 0; i < PQntuples(res); i++) { char *dbname = PQgetvalue(res, i, 0); char *sanitized; - const char *create_opts; + char *oid = PQgetvalue(res, i, 1); + const char *create_opts = ""; int ret; /* Skip template0, even if it's not marked !datallowconn. */ @@ -1649,13 +2001,16 @@ dumpDatabases(PGconn *conn) pg_log_info("dumping database \"%s\"", dbname); sanitized = sanitize_line(dbname, true); - fprintf(OPF, "--\n-- Database \"%s\" dump\n--\n\n", sanitized); + + if (archDumpFormat == archNull) + fprintf(OPF, "--\n-- Database \"%s\" dump\n--\n\n", sanitized); + free(sanitized); /* * We assume that "template1" and "postgres" already exist in the - * target installation. dropDBs() won't have removed them, for fear - * of removing the DB the restore script is initially connected to. If + * target installation and are never dropped above, for fear of + * dropping the DB the restore script is initially connected to. If * --clean was specified, tell pg_dump to drop and recreate them; * otherwise we'll merely restore their contents. Other databases * should simply be created. @@ -1664,24 +2019,54 @@ dumpDatabases(PGconn *conn) { if (output_clean) create_opts = "--clean --create"; + /* Since pg_dump won't emit a \connect command, we must */ + else if (archDumpFormat == archNull) + fprintf(OPF, "\\connect %s\n\n", dbname); else - { create_opts = ""; - /* Since pg_dump won't emit a \connect command, we must */ - fprintf(OPF, "\\connect %s\n\n", dbname); - } } else create_opts = "--create"; - if (filename) + if (filename && archDumpFormat == archNull) fclose(OPF); - ret = runPgDump(dbname, create_opts); + if (archDumpFormat != archNull) + { + /* Compute this database's own archive file path. */ + if (archDumpFormat == archCustom) + snprintf(dbfilepath, MAXPGPATH, "%s/%s.dmp", db_subdir, oid); + else if (archDumpFormat == archTar) + snprintf(dbfilepath, MAXPGPATH, "%s/%s.tar", db_subdir, oid); + else + snprintf(dbfilepath, MAXPGPATH, "%s/%s", db_subdir, oid); + + /* Put one line entry for dboid and dbname in map file. */ + fprintf(map_file, "%s %s\n", oid, dbname); + + /* Let the archiver drop this database first, if --clean. */ + if (!data_only && !statistics_only && !no_schema && + strcmp(dbname, "template1") != 0 && strcmp(dbname, "postgres") != 0) + { + resetPQExpBuffer(delQry); + appendPQExpBuffer(delQry, "DROP DATABASE IF EXISTS %s;\n", + fmtId(dbname)); + + ArchiveEntry(fout, + nilCatalogId, /* catalog ID */ + createDumpId(), /* dump ID */ + ARCHIVE_OPTS(.tag = psprintf("DATABASE %s", fmtId(dbname)), + .description = "DROP_GLOBAL", + .section = SECTION_PRE_DATA, + .dropStmt = delQry->data)); + } + } + + ret = runPgDump(dbname, create_opts, dbfilepath); if (ret != 0) pg_fatal("pg_dump failed on database \"%s\", exiting", dbname); - if (filename) + if (filename && archDumpFormat == archNull) { OPF = fopen(filename, PG_BINARY_A); if (!OPF) @@ -1690,7 +2075,22 @@ dumpDatabases(PGconn *conn) } } + /* Close map file */ + if (archDumpFormat != archNull) + { + char map_file_path[MAXPGPATH]; + + fclose(map_file); + + if (dosync) + { + snprintf(map_file_path, MAXPGPATH, "%s/map.dat", filename); + (void) fsync_fname(map_file_path, false); + } + } + PQclear(res); + destroyPQExpBuffer(delQry); } @@ -1699,7 +2099,7 @@ dumpDatabases(PGconn *conn) * Run pg_dump on dbname, with specified options. */ static int -runPgDump(const char *dbname, const char *create_opts) +runPgDump(const char *dbname, const char *create_opts, char *dbfile) { PQExpBufferData connstrbuf; PQExpBufferData cmd; @@ -1708,17 +2108,41 @@ runPgDump(const char *dbname, const char *create_opts) initPQExpBuffer(&connstrbuf); initPQExpBuffer(&cmd); - printfPQExpBuffer(&cmd, "\"%s\" %s %s", pg_dump_bin, - pgdumpopts->data, create_opts); - /* - * If we have a filename, use the undocumented plain-append pg_dump - * format. + * If this is not a plain format dump, append the file name and dump + * format to the pg_dump command to produce an archive-format dump. */ - if (filename) - appendPQExpBufferStr(&cmd, " -Fa "); + if (archDumpFormat != archNull) + { + printfPQExpBuffer(&cmd, "\"%s\" %s -f ", pg_dump_bin, pgdumpopts->data); + appendShellString(&cmd, dbfile); + appendPQExpBuffer(&cmd, " %s", create_opts); + + if (archDumpFormat == archDirectory) + appendPQExpBufferStr(&cmd, " --format=directory "); + else if (archDumpFormat == archCustom) + appendPQExpBufferStr(&cmd, " --format=custom "); + else if (archDumpFormat == archTar) + appendPQExpBufferStr(&cmd, " --format=tar "); + } else - appendPQExpBufferStr(&cmd, " -Fp "); + { + printfPQExpBuffer(&cmd, "\"%s\" %s %s", pg_dump_bin, + pgdumpopts->data, create_opts); + + /* + * If we have a filename, use the undocumented plain-append pg_dump + * format. + */ + if (filename) + { + appendPQExpBufferStr(&cmd, " -Fa -f "); + appendShellString(&cmd, filename); + appendPQExpBufferChar(&cmd, ' '); + } + else + appendPQExpBufferStr(&cmd, " -Fp "); + } /* * Append the database name to the already-constructed stem of connection @@ -1792,6 +2216,76 @@ executeCommand(PGconn *conn, const char *query) } +/* + * check_for_invalid_global_names + * + * Check that no database, role, or tablespace name contains a newline or + * carriage return character. Such characters in database names would break + * the map.dat file format used for non-plain-text dumps. Role and tablespace + * names are also checked because such characters were forbidden starting in + * v19. + * + * Excluded databases are skipped since they won't appear in map.dat. + */ +static void +check_for_invalid_global_names(PGconn *conn, + SimpleStringList *excluded_names) +{ + PGresult *res; + int i; + PQExpBuffer names; + int count = 0; + + res = executeQuery(conn, + "SELECT datname AS objname, 'database' AS objtype " + "FROM pg_catalog.pg_database " + "WHERE datallowconn AND datconnlimit != -2 " + "UNION ALL " + "SELECT rolname AS objname, 'role' AS objtype " + "FROM pg_catalog.pg_roles " + "UNION ALL " + "SELECT spcname AS objname, 'tablespace' AS objtype " + "FROM pg_catalog.pg_tablespace"); + + names = createPQExpBuffer(); + + for (i = 0; i < PQntuples(res); i++) + { + char *objname = PQgetvalue(res, i, 0); + char *objtype = PQgetvalue(res, i, 1); + + /* Skip excluded databases since they won't be in map.dat */ + if (strcmp(objtype, "database") == 0 && + simple_string_list_member(excluded_names, objname)) + continue; + + if (strpbrk(objname, "\n\r")) + { + appendPQExpBuffer(names, " %s: \"", objtype); + for (char *p = objname; *p; p++) + { + if (*p == '\n') + appendPQExpBufferStr(names, "\\n"); + else if (*p == '\r') + appendPQExpBufferStr(names, "\\r"); + else + appendPQExpBufferChar(names, *p); + } + appendPQExpBufferStr(names, "\"\n"); + count++; + } + } + + PQclear(res); + + if (count > 0) + pg_fatal("database, role, or tablespace names contain a newline or carriage return character, which is not supported in non-plain-text dumps:\n%s", + names->data); + + destroyPQExpBuffer(names); +} + + /* * dumpTimestamp */ @@ -1863,3 +2357,51 @@ read_dumpall_filters(const char *filename, SimpleStringList *pattern) filter_free(&fstate); } + +/* + * parseDumpFormat + * + * This will validate dump formats. + */ +static ArchiveFormat +parseDumpFormat(const char *format) +{ + if (pg_strcasecmp(format, "c") == 0) + return archCustom; + else if (pg_strcasecmp(format, "custom") == 0) + return archCustom; + else if (pg_strcasecmp(format, "d") == 0) + return archDirectory; + else if (pg_strcasecmp(format, "directory") == 0) + return archDirectory; + else if (pg_strcasecmp(format, "p") == 0) + return archNull; + else if (pg_strcasecmp(format, "plain") == 0) + return archNull; + else if (pg_strcasecmp(format, "t") == 0) + return archTar; + else if (pg_strcasecmp(format, "tar") == 0) + return archTar; + else + pg_fatal("unrecognized output format \"%s\"; please specify \"c\", \"d\", \"p\", or \"t\"", + format); + + return archUnknown; +} + +/* + * createDumpId + * + * Return the next dumpId. + * + * This is a local, file-scope copy of pg_dump.c's createDumpId() rather + * than a shared one: that function's home, common.c, is compiled only + * into the pg_dump binary (pg_dump_sources in meson.build), not into + * pg_dump_common, the static library pg_dumpall and pg_restore link + * against, so it isn't available here to call. + */ +static DumpId +createDumpId(void) +{ + return ++lastDumpId; +} diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c index c9c358a3657..a655f50848b 100644 --- a/src/bin/pg_dump/pg_restore.c +++ b/src/bin/pg_dump/pg_restore.c @@ -2,7 +2,7 @@ * * pg_restore.c * pg_restore is an utility extracting postgres database definitions - * from a backup archive created by pg_dump using the archiver + * from a backup archive created by pg_dump/pg_dumpall using the archiver * interface. * * pg_restore will read the backup archive and @@ -41,12 +41,16 @@ #include "postgres_fe.h" #include +#include #ifdef HAVE_TERMIOS_H #include #endif +#include "common/string.h" +#include "connectdb.h" #include "dumputils.h" #include "fe_utils/option_utils.h" +#include "fe_utils/string_utils.h" #include "filter.h" #include "getopt_long.h" #include "parallel.h" @@ -54,18 +58,41 @@ static void usage(const char *progname); static void read_restore_filters(const char *filename, RestoreOptions *opts); +static bool file_exists_in_directory(const char *dir, const char *filename); +static int restore_one_database(const char *inputFileSpec, RestoreOptions *opts, + int numWorkers, bool append_data); +static int restore_global_objects(const char *inputFileSpec, RestoreOptions *opts); + +static int restore_all_databases(const char *inputFileSpec, + SimpleStringList db_exclude_patterns, RestoreOptions *opts, int numWorkers); +static int mark_excluded_databases(PGconn *conn, + SimplePtrList *dbname_oid_list, + SimpleStringList db_exclude_patterns); +static int get_dbname_oid_list_from_map_file(const char *dumpdirpath, + SimplePtrList *dbname_oid_list); + +/* + * Stores a database OID and the corresponding name. + */ +typedef struct DbOidName +{ + Oid oid; + char str[FLEXIBLE_ARRAY_MEMBER]; /* null-terminated string here */ +} DbOidName; + int main(int argc, char **argv) { RestoreOptions *opts; int c; - int exit_code; int numWorkers = 1; - Archive *AH; char *inputFileSpec; bool data_only = false; bool schema_only = false; + int n_errors = 0; + bool globals_only = false; + SimpleStringList db_exclude_patterns = {NULL, NULL}; static int disable_triggers = 0; static int enable_row_security = 0; static int if_exists = 0; @@ -80,6 +107,7 @@ main(int argc, char **argv) static int no_schema = 0; static int no_security_labels = 0; static int no_statistics = 0; + static int no_globals = 0; static int no_subscriptions = 0; static int strict_names = 0; static int statistics_only = 0; @@ -89,6 +117,7 @@ main(int argc, char **argv) {"clean", 0, NULL, 'c'}, {"create", 0, NULL, 'C'}, {"data-only", 0, NULL, 'a'}, + {"globals-only", 0, NULL, 'g'}, {"dbname", 1, NULL, 'd'}, {"exit-on-error", 0, NULL, 'e'}, {"exclude-schema", 1, NULL, 'N'}, @@ -136,12 +165,14 @@ main(int argc, char **argv) {"no-publications", no_argument, &no_publications, 1}, {"no-schema", no_argument, &no_schema, 1}, {"no-security-labels", no_argument, &no_security_labels, 1}, + {"no-globals", no_argument, &no_globals, 1}, {"no-subscriptions", no_argument, &no_subscriptions, 1}, {"no-statistics", no_argument, &no_statistics, 1}, {"statistics", no_argument, &with_statistics, 1}, {"statistics-only", no_argument, &statistics_only, 1}, {"filter", required_argument, NULL, 4}, {"restrict-key", required_argument, NULL, 6}, + {"exclude-database", required_argument, NULL, 7}, {NULL, 0, NULL, 0} }; @@ -170,7 +201,7 @@ main(int argc, char **argv) } } - while ((c = getopt_long(argc, argv, "acCd:ef:F:h:I:j:lL:n:N:Op:P:RsS:t:T:U:vwWx1", + while ((c = getopt_long(argc, argv, "acCd:ef:F:gh:I:j:lL:n:N:Op:P:RsS:t:T:U:vwWx1", cmdopts, NULL)) != -1) { switch (c) @@ -197,11 +228,14 @@ main(int argc, char **argv) if (strlen(optarg) != 0) opts->formatName = pg_strdup(optarg); break; + case 'g': + /* restore only global sql commands. */ + globals_only = true; + break; case 'h': if (strlen(optarg) != 0) opts->cparams.pghost = pg_strdup(optarg); break; - case 'j': /* number of restore jobs */ if (!option_parse_int(optarg, "-j/--jobs", 1, PG_MAX_JOBS, @@ -321,6 +355,10 @@ main(int argc, char **argv) opts->restrict_key = pg_strdup(optarg); break; + case 7: /* database patterns to skip */ + simple_string_list_append(&db_exclude_patterns, optarg); + break; + default: /* getopt_long already emitted a complaint */ pg_log_error_hint("Try \"%s --help\" for more information.", progname); @@ -347,6 +385,10 @@ main(int argc, char **argv) if (!opts->cparams.dbname && !opts->filename && !opts->tocSummary) pg_fatal("one of -d/--dbname and -f/--file must be specified"); + /* --exclude-database and --globals-only are incompatible */ + check_mut_excl_opts(db_exclude_patterns.head, "--exclude-database", + globals_only, "-g/--globals-only"); + /* Should get at most one of -d and -f, else user is confused */ check_mut_excl_opts(opts->cparams.dbname, "-d/--dbname", opts->filename, "-f/--file"); @@ -375,9 +417,17 @@ main(int argc, char **argv) data_only, "-a/--data-only", statistics_only, "--statistics-only"); + /* -g/--globals-only is incompatible with the other *-only options */ + check_mut_excl_opts(data_only, "-a/--data-only", + globals_only, "-g/--globals-only", + schema_only, "-s/--schema-only", + statistics_only, "--statistics-only"); + /* --no-* and *-only for same thing are incompatible */ check_mut_excl_opts(data_only, "-a/--data-only", no_data, "--no-data"); + check_mut_excl_opts(globals_only, "-g/--globals-only", + no_globals, "--no-globals"); check_mut_excl_opts(schema_only, "-s/--schema-only", no_schema, "--no-schema"); check_mut_excl_opts(statistics_only, "--statistics-only", @@ -388,17 +438,27 @@ main(int argc, char **argv) no_statistics, "--no-statistics"); /* --statistics is incompatible with *-only (except --statistics-only) */ - check_mut_excl_opts(data_only, "-a/--data-only", - schema_only, "-s/--schema-only", - with_statistics, "--statistics"); + check_mut_excl_opts(with_statistics, "--statistics", + data_only, "-a/--data-only", + globals_only, "-g/--globals-only", + schema_only, "-s/--schema-only"); /* --clean is incompatible with --data-only */ check_mut_excl_opts(opts->dropSchema, "-c/--clean", data_only, "-a/--data-only"); - check_mut_excl_opts(opts->single_txn, "-1/--single-transaction", + /* + * --globals-only, --single-transaction, and --transaction-size are + * incompatible. + */ + check_mut_excl_opts(globals_only, "-g/--globals-only", + opts->single_txn, "-1/--single-transaction", opts->txn_size > 0, "--transaction-size"); + /* --exit-on-error and --globals-only are incompatible */ + check_mut_excl_opts(opts->exit_on_error, "--exit-on-error", + globals_only, "-g/--globals-only"); + /* * -C is not compatible with -1, because we can't create a database inside * a transaction block. @@ -463,6 +523,182 @@ main(int argc, char **argv) opts->formatName); } + /* + * If toc.glo file is present, then restore all the databases from + * map.dat, but skip restoring those matching --exclude-database patterns. + */ + if (inputFileSpec != NULL && + (file_exists_in_directory(inputFileSpec, "toc.glo"))) + { + char global_path[MAXPGPATH]; + RestoreOptions *tmpopts = pg_malloc0_object(RestoreOptions); + + opts->format = archUnknown; + + memcpy(tmpopts, opts, sizeof(RestoreOptions)); + + /* + * Can only use --list or --use-list options with a single database + * dump. + */ + if (opts->tocSummary) + pg_fatal("option %s cannot be used when restoring an archive created by pg_dumpall", + "-l/--list"); + if (opts->tocFile) + pg_fatal("option %s cannot be used when restoring an archive created by pg_dumpall", + "-L/--use-list"); + + if (opts->strict_names) + pg_fatal("option %s cannot be used when restoring an archive created by pg_dumpall", + "--strict-names"); + if (globals_only && opts->dropSchema) + pg_fatal("options %s and %s cannot be used together when restoring an archive created by pg_dumpall", + "--clean", "-g/--globals-only"); + + if (no_schema) + pg_fatal("option %s cannot be used when restoring an archive created by pg_dumpall", + "--no-schema"); + + if (data_only) + pg_fatal("option %s cannot be used when restoring an archive created by pg_dumpall", + "-a/--data-only"); + + if (statistics_only) + pg_fatal("option %s cannot be used when restoring an archive created by pg_dumpall", + "--statistics-only"); + + if (!(opts->dumpSections & DUMP_PRE_DATA)) + pg_fatal("option %s cannot exclude %s when restoring a pg_dumpall archive", + "--section", "pre-data"); + + /* + * To restore from a pg_dumpall archive, -C (create database) option + * must be specified unless we are only restoring globals or we are + * skipping globals. + */ + if (!no_globals && !globals_only && opts->createDB != 1) + { + pg_log_error("option %s must be specified when restoring an archive created by pg_dumpall", + "-C/--create"); + pg_log_error_hint("Try \"%s --help\" for more information.", progname); + pg_log_error_hint("Individual databases can be restored using their specific archives."); + exit_nicely(1); + } + + /* + * Restore global objects, even if --exclude-database results in zero + * databases to process. If 'globals-only' is set, skip restoring the + * individual databases afterward. + */ + snprintf(global_path, MAXPGPATH, "%s/toc.glo", inputFileSpec); + + if (!no_globals) + n_errors = restore_global_objects(global_path, tmpopts); + else + pg_log_info("skipping restore of global objects because %s was specified", + "--no-globals"); + + if (globals_only) + pg_log_info("database restoring skipped because option %s was specified", + "-g/--globals-only"); + else + { + /* Now restore all the databases from map.dat */ + n_errors += restore_all_databases(inputFileSpec, db_exclude_patterns, + opts, numWorkers); + } + + /* Free db pattern list. */ + simple_string_list_destroy(&db_exclude_patterns); + pg_free(tmpopts); + } + else + { + if (db_exclude_patterns.head != NULL) + { + simple_string_list_destroy(&db_exclude_patterns); + pg_fatal("option %s can be used only when restoring an archive created by pg_dumpall", + "--exclude-database"); + } + + if (globals_only) + pg_fatal("option %s can be used only when restoring an archive created by pg_dumpall", + "-g/--globals-only"); + + /* Process if toc.glo file does not exist. */ + n_errors = restore_one_database(inputFileSpec, opts, numWorkers, false); + } + + /* Done, print a summary of ignored errors during restore. */ + if (n_errors) + { + pg_log_warning("errors ignored on restore: %d", n_errors); + return 1; + } + + return 0; +} + +/* + * restore_global_objects + * + * This restores all global objects. + */ +static int +restore_global_objects(const char *inputFileSpec, RestoreOptions *opts) +{ + Archive *AH; + int nerror = 0; + + /* Set format as custom so that toc.glo file can be read. */ + opts->format = archCustom; + opts->txn_size = 0; + + AH = OpenArchive(inputFileSpec, opts->format); + + SetArchiveOptions(AH, NULL, opts); + + on_exit_close_archive(AH); + + /* Let the archiver know how noisy to be */ + AH->verbose = opts->verbose; + + /* + * Don't output TOC entry comments when restoring globals; they're + * purely informational and just add noise ahead of the SQL. + */ + ((ArchiveHandle *) AH)->noTocComments = 1; + + AH->exit_on_error = false; + + /* Parallel execution is not supported for global object restoration. */ + AH->numWorkers = 1; + + ProcessArchiveRestoreOptions(AH); + RestoreArchive(AH, false); + + nerror = AH->n_errors; + + /* AH may be freed in CloseArchive? */ + CloseArchive(AH); + + return nerror; +} + +/* + * restore_one_database + * + * This will restore one database using toc.dat file. + * + * returns the number of errors while doing restore. + */ +static int +restore_one_database(const char *inputFileSpec, RestoreOptions *opts, + int numWorkers, bool append_data) +{ + Archive *AH; + int n_errors; + AH = OpenArchive(inputFileSpec, opts->format); SetArchiveOptions(AH, NULL, opts); @@ -470,9 +706,15 @@ main(int argc, char **argv) /* * We don't have a connection yet but that doesn't matter. The connection * is initialized to NULL and if we terminate through exit_nicely() while - * it's still NULL, the cleanup function will just be a no-op. + * it's still NULL, the cleanup function will just be a no-op. If we are + * restoring multiple databases, then only update AH handle for cleanup as + * the previous entry was already in the array and we had closed previous + * connection, so we can use the same array slot. */ - on_exit_close_archive(AH); + if (!append_data) + on_exit_close_archive(AH); + else + replace_on_exit_close_archive(AH); /* Let the archiver know how noisy to be */ AH->verbose = opts->verbose; @@ -492,25 +734,21 @@ main(int argc, char **argv) else { ProcessArchiveRestoreOptions(AH); - RestoreArchive(AH); + RestoreArchive(AH, append_data); } - /* done, print a summary of ignored errors */ - if (AH->n_errors) - pg_log_warning("errors ignored on restore: %d", AH->n_errors); + n_errors = AH->n_errors; /* AH may be freed in CloseArchive? */ - exit_code = AH->n_errors ? 1 : 0; - CloseArchive(AH); - return exit_code; + return n_errors; } static void usage(const char *progname) { - printf(_("%s restores a PostgreSQL database from an archive created by pg_dump.\n\n"), progname); + printf(_("%s restores PostgreSQL databases from archives created by pg_dump or pg_dumpall.\n\n"), progname); printf(_("Usage:\n")); printf(_(" %s [OPTION]... [FILE]\n"), progname); @@ -528,6 +766,7 @@ usage(const char *progname) printf(_(" -c, --clean clean (drop) database objects before recreating\n")); printf(_(" -C, --create create the target database\n")); printf(_(" -e, --exit-on-error exit on error, default is to continue\n")); + printf(_(" -g, --globals-only restore only global objects, no databases\n")); printf(_(" -I, --index=NAME restore named index\n")); printf(_(" -j, --jobs=NUM use this many parallel jobs to restore\n")); printf(_(" -L, --use-list=FILENAME use table of contents from this file for\n" @@ -544,6 +783,7 @@ usage(const char *progname) printf(_(" -1, --single-transaction restore as a single transaction\n")); printf(_(" --disable-triggers disable triggers during data-only restore\n")); printf(_(" --enable-row-security enable row security\n")); + printf(_(" --exclude-database=PATTERN do not restore the specified database(s)\n")); printf(_(" --filter=FILENAME restore or skip objects based on expressions\n" " in FILENAME\n")); printf(_(" --if-exists use IF EXISTS when dropping objects\n")); @@ -551,6 +791,7 @@ usage(const char *progname) printf(_(" --no-data do not restore data\n")); printf(_(" --no-data-for-failed-tables do not restore data of tables that could not be\n" " created\n")); + printf(_(" --no-globals do not restore global objects (roles and tablespaces)\n")); printf(_(" --no-policies do not restore row security policies\n")); printf(_(" --no-publications do not restore publications\n")); printf(_(" --no-schema do not restore schema\n")); @@ -579,8 +820,8 @@ usage(const char *progname) printf(_(" --role=ROLENAME do SET ROLE before restore\n")); printf(_("\n" - "The options -I, -n, -N, -P, -t, -T, and --section can be combined and specified\n" - "multiple times to select multiple objects.\n")); + "The options -I, -n, -N, -P, -t, -T, --section, and --exclude-database can be\n" + "combined and specified multiple times to select multiple objects.\n")); printf(_("\nIf no input file name is supplied, then standard input is used.\n\n")); printf(_("Report bugs to <%s>.\n"), PACKAGE_BUGREPORT); printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL); @@ -685,3 +926,390 @@ read_restore_filters(const char *filename, RestoreOptions *opts) filter_free(&fstate); } + +/* + * file_exists_in_directory + * + * Returns true if the file exists in the given directory. + */ +static bool +file_exists_in_directory(const char *dir, const char *filename) +{ + struct stat st; + char buf[MAXPGPATH]; + + if (snprintf(buf, MAXPGPATH, "%s/%s", dir, filename) >= MAXPGPATH) + pg_fatal("directory name too long: \"%s\"", dir); + + return (stat(buf, &st) == 0 && S_ISREG(st.st_mode)); +} + +/* + * mark_excluded_databases + * + * This will mark for skipping any entries from dbname_oid_list that pattern match an + * entry in the db_exclude_patterns list. + * + * Returns the number of databases to be restored. + */ +static int +mark_excluded_databases(PGconn *conn, + SimplePtrList *dbname_oid_list, + SimpleStringList db_exclude_patterns) +{ + int count_db = 0; + PQExpBuffer query; + PQExpBuffer db_lit; + PGresult *res; + + query = createPQExpBuffer(); + db_lit = createPQExpBuffer(); + + /* + * Process one by one all dbnames and if specified to skip restoring, then + * remove dbname from list. + */ + for (SimplePtrListCell *db_cell = dbname_oid_list->head; + db_cell; db_cell = db_cell->next) + { + DbOidName *dbidname = (DbOidName *) db_cell->ptr; + bool skip_db_restore = false; + + resetPQExpBuffer(query); + resetPQExpBuffer(db_lit); + + appendStringLiteralConn(db_lit, dbidname->str, conn); + + for (SimpleStringListCell *pat_cell = db_exclude_patterns.head; pat_cell; pat_cell = pat_cell->next) + { + /* + * If there is an exact match then we don't need to try a pattern + * match + */ + if (pg_strcasecmp(dbidname->str, pat_cell->val) == 0) + skip_db_restore = true; + /* Otherwise, try a pattern match if there is a connection */ + else + { + int dotcnt; + + appendPQExpBufferStr(query, "SELECT 1 "); + processSQLNamePattern(conn, query, pat_cell->val, false, + false, NULL, db_lit->data, + NULL, NULL, NULL, &dotcnt); + + if (dotcnt > 0) + { + pg_log_error("improper qualified name (too many dotted names): %s", + dbidname->str); + PQfinish(conn); + exit_nicely(1); + } + + res = executeQuery(conn, query->data); + + if (PQntuples(res)) + { + skip_db_restore = true; + pg_log_info("database name \"%s\" matches --exclude-database pattern \"%s\"", dbidname->str, pat_cell->val); + } + + PQclear(res); + resetPQExpBuffer(query); + } + + if (skip_db_restore) + break; + } + + /* + * Mark db to be skipped or increment the counter of dbs to be + * restored + */ + if (skip_db_restore) + { + pg_log_info("excluding database \"%s\"", dbidname->str); + dbidname->oid = InvalidOid; + } + else + count_db++; + } + + destroyPQExpBuffer(query); + destroyPQExpBuffer(db_lit); + + return count_db; +} + +/* + * get_dbname_oid_list_from_map_file + * + * Open map.dat file and read line by line and then prepare a list of database + * names and corresponding db_oid. + * + * Returns the total number of database names in the map.dat file. + */ +static int +get_dbname_oid_list_from_map_file(const char *dumpdirpath, + SimplePtrList *dbname_oid_list) +{ + StringInfoData linebuf; + FILE *pfile; + char map_file_path[MAXPGPATH]; + int count = 0; + + /* + * If there is no map.dat file in the dump, then return from here as there + * is no database to restore. + */ + if (!file_exists_in_directory(dumpdirpath, "map.dat")) + { + pg_log_info("database restoring is skipped because file \"%s\" does not exist in directory \"%s\"", "map.dat", dumpdirpath); + return 0; + } + + snprintf(map_file_path, MAXPGPATH, "%s/map.dat", dumpdirpath); + + /* Open map.dat file. */ + pfile = fopen(map_file_path, PG_BINARY_R); + + if (pfile == NULL) + pg_fatal("could not open file \"%s\": %m", map_file_path); + + initStringInfo(&linebuf); + + /* Append all the dbname/db_oid combinations to the list. */ + while (pg_get_line_buf(pfile, &linebuf)) + { + Oid db_oid = InvalidOid; + char *dbname; + DbOidName *dbidname; + int namelen; + char *p = linebuf.data; + + /* look for the dboid. */ + while (isdigit((unsigned char) *p)) + p++; + + /* ignore lines that don't begin with a digit */ + if (p == linebuf.data) + continue; + + if (*p == ' ') + { + sscanf(linebuf.data, "%u", &db_oid); + p++; + } + + /* dbname is the rest of the line */ + dbname = p; + namelen = strlen(dbname); + + /* Strip trailing newline */ + if (namelen > 0 && dbname[namelen - 1] == '\n') + dbname[--namelen] = '\0'; + + /* Report error and exit if the file has any corrupted data. */ + if (!OidIsValid(db_oid) || namelen < 1) + pg_fatal("invalid entry in file \"%s\" on line %d", map_file_path, + count + 1); + + dbidname = pg_malloc(offsetof(DbOidName, str) + namelen + 1); + dbidname->oid = db_oid; + strlcpy(dbidname->str, dbname, namelen + 1); + + pg_log_info("found database \"%s\" (OID: %u) in file \"%s\"", + dbidname->str, db_oid, map_file_path); + + simple_ptr_list_append(dbname_oid_list, dbidname); + count++; + } + + /* Close map.dat file. */ + fclose(pfile); + + pfree(linebuf.data); + + return count; +} + +/* + * restore_all_databases + * + * This will restore the databases whose dumps are present in the + * "databases" subdirectory, per the map.dat file mapping. + * + * This will skip restoring for databases that are specified with the + * --exclude-database option. + * + * Returns the number of errors encountered while restoring. + */ +static int +restore_all_databases(const char *inputFileSpec, + SimpleStringList db_exclude_patterns, RestoreOptions *opts, + int numWorkers) +{ + SimplePtrList dbname_oid_list = {NULL, NULL}; + int num_db_restore = 0; + int num_total_db; + int n_errors_total = 0; + char *connected_db = NULL; + PGconn *conn = NULL; + RestoreOptions *original_opts = pg_malloc0_object(RestoreOptions); + RestoreOptions *tmpopts = pg_malloc0_object(RestoreOptions); + + memcpy(original_opts, opts, sizeof(RestoreOptions)); + + /* Save db name to reuse it for all the database. */ + if (opts->cparams.dbname) + connected_db = opts->cparams.dbname; + + num_total_db = get_dbname_oid_list_from_map_file(inputFileSpec, &dbname_oid_list); + + pg_log_info(ngettext("found %d database name in \"%s\"", + "found %d database names in \"%s\"", + num_total_db), + num_total_db, "map.dat"); + + /* + * If exclude-patterns is given, connect to the database to process them. + */ + if (db_exclude_patterns.head != NULL) + { + if (opts->cparams.dbname) + { + conn = ConnectDatabase(opts->cparams.dbname, NULL, opts->cparams.pghost, + opts->cparams.pgport, opts->cparams.username, TRI_DEFAULT, + false, progname, NULL, NULL, NULL, NULL); + + if (!conn) + pg_fatal("could not connect to database \"%s\"", opts->cparams.dbname); + } + + if (!conn) + { + pg_log_info("trying to connect to database \"%s\"", "postgres"); + + conn = ConnectDatabase("postgres", NULL, opts->cparams.pghost, + opts->cparams.pgport, opts->cparams.username, TRI_DEFAULT, + false, progname, NULL, NULL, NULL, NULL); + + /* Try with template1. */ + if (!conn) + { + pg_log_info("trying to connect to database \"%s\"", "template1"); + + conn = ConnectDatabase("template1", NULL, opts->cparams.pghost, + opts->cparams.pgport, opts->cparams.username, TRI_DEFAULT, + false, progname, NULL, NULL, NULL, NULL); + if (!conn) + { + pg_log_error("could not connect to databases \"postgres\" or \"template1\"\n" + "Please specify an alternative database."); + pg_log_error_hint("Try \"%s --help\" for more information.", progname); + exit_nicely(1); + } + } + } + + /* Filter the db list according to the exclude patterns. */ + num_db_restore = mark_excluded_databases(conn, &dbname_oid_list, + db_exclude_patterns); + PQfinish(conn); + } + else + num_db_restore = num_total_db; + + /* Exit if no db needs to be restored. */ + if (num_db_restore == 0) + { + pg_log_info(ngettext("no database needs restoring out of %d database", + "no database needs restoring out of %d databases", num_total_db), + num_total_db); + pg_free(original_opts); + pg_free(tmpopts); + return 0; + } + + pg_log_info("need to restore %d databases out of %d databases", num_db_restore, num_total_db); + + /* + * We have a list of databases to restore after processing the + * exclude-database switch(es). Now we can restore them one by one. + */ + for (SimplePtrListCell *db_cell = dbname_oid_list.head; + db_cell; db_cell = db_cell->next) + { + DbOidName *dbidname = (DbOidName *) db_cell->ptr; + char db_archive_path[MAXPGPATH]; + char databases_dir[MAXPGPATH]; + char dbfilename[MAXPGPATH]; + int n_errors; + + /* ignore dbs marked for skipping */ + if (dbidname->oid == InvalidOid) + continue; + + /* + * Since pg_backup_archiver.c may modify RestoreOptions during the + * previous restore, we must provide a fresh copy of the original + * "opts" for each call to restore_one_database. + */ + memcpy(tmpopts, original_opts, sizeof(RestoreOptions)); + + snprintf(databases_dir, MAXPGPATH, "%s/databases", inputFileSpec); + + /* + * Look for the database dump file/dir. If there is an {oid}.tar or + * {oid}.dmp file, use it. Otherwise try to use a directory called + * {oid} + */ + snprintf(dbfilename, MAXPGPATH, "%u.tar", dbidname->oid); + if (file_exists_in_directory(databases_dir, dbfilename)) + snprintf(db_archive_path, MAXPGPATH, "%s/databases/%u.tar", inputFileSpec, dbidname->oid); + else + { + snprintf(dbfilename, MAXPGPATH, "%u.dmp", dbidname->oid); + + if (file_exists_in_directory(databases_dir, dbfilename)) + snprintf(db_archive_path, MAXPGPATH, "%s/databases/%u.dmp", inputFileSpec, dbidname->oid); + else + snprintf(db_archive_path, MAXPGPATH, "%s/databases/%u", inputFileSpec, dbidname->oid); + } + + pg_log_info("restoring database \"%s\"", dbidname->str); + + /* + * With --create, connect to the anchor database and let CREATE + * DATABASE fail if it already exists; without it, connect directly + * to the target and let that fail if it doesn't exist yet. + */ + if (tmpopts->cparams.dbname) + { + if (tmpopts->createDB) + tmpopts->cparams.dbname = connected_db; + else + tmpopts->cparams.dbname = dbidname->str; + } + + /* Restore the single database. */ + n_errors = restore_one_database(db_archive_path, tmpopts, numWorkers, true); + + n_errors_total += n_errors; + + /* Print a summary of ignored errors during single database restore. */ + if (n_errors) + pg_log_warning("errors ignored on database \"%s\" restore: %d", dbidname->str, n_errors); + } + + /* Log number of processed databases. */ + pg_log_info("number of restored databases is %d", num_db_restore); + + /* Free dbname and dboid list. */ + simple_ptr_list_destroy(&dbname_oid_list); + + pg_free(original_opts); + pg_free(tmpopts); + + return n_errors_total; +} diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl index d89ba4a4ceb..c3dce6d0ba5 100644 --- a/src/bin/pg_dump/t/001_basic.pl +++ b/src/bin/pg_dump/t/001_basic.pl @@ -253,13 +253,13 @@ command_fails_like( command_fails_like( [ 'pg_dumpall', '-a', '--no-data' ], qr/\Qpg_dumpall: error: options -a\/--data-only and --no-data cannot be used together\E/, - 'pg_dumpall: options -a\/--data-only and --no-data cannot be used together' + 'pg_dumpall: options -a/--data-only and --no-data cannot be used together' ); command_fails_like( [ 'pg_dumpall', '-s', '--no-schema' ], qr/\Qpg_dumpall: error: options -s\/--schema-only and --no-schema cannot be used together\E/, - 'pg_dumpall: options -s\/--schema-only and --no-schema cannot be used together' + 'pg_dumpall: options -s/--schema-only and --no-schema cannot be used together' ); command_fails_like( @@ -271,13 +271,81 @@ command_fails_like( command_fails_like( [ 'pg_dumpall', '--statistics', '--no-statistics' ], qr/\Qpg_dumpall: error: options --statistics and --no-statistics cannot be used together\E/, - 'pg_dumpall: options --statistics-only and --no-statistics cannot be used together' + 'pg_dumpall: options --statistics and --no-statistics cannot be used together' ); command_fails_like( [ 'pg_dumpall', '--statistics', '--tablespaces-only' ], qr/\Qpg_dumpall: error: options --statistics and -t\/--tablespaces-only cannot be used together\E/, - 'pg_dumpall: options --statistics and -t\/--tablespaces-only cannot be used together' + 'pg_dumpall: options --statistics and -t/--tablespaces-only cannot be used together' +); + +command_fails_like( + [ 'pg_dumpall', '--format', 'x' ], + qr/\Qpg_dumpall: error: unrecognized output format "x";\E/, + 'pg_dumpall: unrecognized output format'); + +command_fails_like( + [ 'pg_dumpall', '--format', 'd', '--restrict-key=uu', '-f dumpfile' ], + qr/\Qpg_dumpall: error: option --restrict-key can only be used with --format=plain\E/, + 'pg_dumpall: --restrict-key can only be used with plain dump format'); + +command_fails_like( + [ + 'pg_dumpall', '--format', 'd', '--globals-only', + '--clean', '-f', 'dumpfile' + ], + qr/\Qpg_dumpall: error: options --clean and -g\/--globals-only cannot be used together in non-text dump\E/, + 'pg_dumpall: --clean and -g/--globals-only cannot be used together in non-text dump' +); + +command_fails_like( + [ 'pg_dumpall', '--format', 'd' ], + qr/\Qpg_dumpall: error: option -F\/--format=d|c|t requires option -f\/--file\E/, + 'pg_dumpall: non-plain format requires --file option'); + +command_fails_like( + [ 'pg_restore', '--exclude-database=foo', '--globals-only', '-d', 'xxx' ], + qr/\Qpg_restore: error: options --exclude-database and -g\/--globals-only cannot be used together\E/, + 'pg_restore: options --exclude-database and -g/--globals-only cannot be used together' +); + +command_fails_like( + [ 'pg_restore', '--data-only', '--globals-only', '-d', 'xxx' ], + qr/\Qpg_restore: error: options -a\/--data-only and -g\/--globals-only cannot be used together\E/, + 'pg_restore: error: options -a/--data-only and -g/--globals-only cannot be used together' +); + +command_fails_like( + [ 'pg_restore', '--schema-only', '--globals-only', '-d', 'xxx' ], + qr/\Qpg_restore: error: options -g\/--globals-only and -s\/--schema-only cannot be used together\E/, + 'pg_restore: error: options -g/--globals-only and -s/--schema-only cannot be used together' ); +command_fails_like( + [ 'pg_restore', '--statistics-only', '--globals-only', '-d', 'xxx' ], + qr/\Qpg_restore: error: options -g\/--globals-only and --statistics-only cannot be used together\E/, + 'pg_restore: error: options -g/--globals-only and --statistics-only cannot be used together' +); + +command_fails_like( + [ 'pg_restore', '--exclude-database=foo', '-d', 'xxx', 'dumpdir' ], + qr/\Qpg_restore: error: option --exclude-database can be used only when restoring an archive created by pg_dumpall\E/, + 'pg_restore: option --exclude-database can be used only when restoring an archive created by pg_dumpall' +); + +command_fails_like( + [ 'pg_restore', '--globals-only', '-d', 'xxx', 'dumpdir' ], + qr/\Qpg_restore: error: option -g\/--globals-only can be used only when restoring an archive created by pg_dumpall\E/, + 'pg_restore: option -g/--globals-only can be used only when restoring an archive created by pg_dumpall' +); + +command_fails_like( + [ + 'pg_restore', '--globals-only', '--no-globals', '-d', 'xxx', + 'dumpdir' + ], + qr/\Qpg_restore: error: options -g\/--globals-only and --no-globals cannot be used together\E/, + 'pg_restore: options -g/--globals-only and --no-globals cannot be used together' +); done_testing(); diff --git a/src/bin/pg_dump/t/007_pg_dumpall.pl b/src/bin/pg_dump/t/007_pg_dumpall.pl new file mode 100644 index 00000000000..debaf1e770b --- /dev/null +++ b/src/bin/pg_dump/t/007_pg_dumpall.pl @@ -0,0 +1,862 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $tempdir = PostgreSQL::Test::Utils::tempdir; +my $run_db = 'postgres'; +my $sep = $windows_os ? "\\" : "/"; + +# Tablespace locations used by "restore_tablespace" test case. +my $tablespace1 = "${tempdir}${sep}tbl1"; +my $tablespace2 = "${tempdir}${sep}tbl2"; +mkdir($tablespace1) || die "mkdir $tablespace1 $!"; +mkdir($tablespace2) || die "mkdir $tablespace2 $!"; + +# escape tablespace locations on Windows. +my $tablespace2_orig = $tablespace2; +$tablespace1 = $windows_os ? ($tablespace1 =~ s/\\/\\\\/gr) : $tablespace1; +$tablespace2 = $windows_os ? ($tablespace2 =~ s/\\/\\\\/gr) : $tablespace2; + +# Where pg_dumpall will be executed. +my $node = PostgreSQL::Test::Cluster->new('node'); +$node->init; +$node->start; + + +############################################################### +# Definition of the pg_dumpall test cases to run. +# +# Each of these test cases are named and those names are used for fail +# reporting and also to save the dump and restore information needed for the +# test to assert. +# +# The "setup_sql" is a psql valid script that contains SQL commands to execute +# before of actually execute the tests. The setups are all executed before of +# any test execution. +# +# The "dump_cmd" and "restore_cmd" are the commands that will be executed. The +# "restore_cmd" must have the --file flag to save the restore output so that we +# can assert on it. +# +# The "like" and "unlike" is a regexp that is used to match the pg_restore +# output. It must have at least one of then filled per test cases but it also +# can have both. See "excluding_databases" test case for example. +my %pgdumpall_runs = ( + # Test 1: dump and restore roles, including a role's password and its + # attributes/connection limit. + restore_roles => { + setup_sql => ' + CREATE ROLE dumpall WITH ENCRYPTED PASSWORD \'admin\' SUPERUSER; + CREATE ROLE dumpall2 WITH REPLICATION CONNECTION LIMIT 10;', + dump_cmd => [ + 'pg_dumpall', + '--format' => 'directory', + '--file' => "$tempdir/restore_roles", + ], + restore_cmd => [ + 'pg_restore', '-C', + '--format' => 'directory', + '--file' => "$tempdir/restore_roles.sql", + "$tempdir/restore_roles", + ], + like => qr/ + \s*\QCREATE ROLE dumpall2;\E + \s*\QALTER ROLE dumpall2 WITH NOSUPERUSER INHERIT NOCREATEROLE NOCREATEDB NOLOGIN REPLICATION NOBYPASSRLS CONNECTION LIMIT 10;\E + /xm + }, + + # Test 2: dump and restore a tablespace, including a non-default storage + # parameter. + restore_tablespace => { + setup_sql => " + CREATE ROLE tap; + CREATE TABLESPACE tbl1 OWNER tap LOCATION '$tablespace1'; + CREATE TABLESPACE tbl2 OWNER tap LOCATION '$tablespace2' WITH (seq_page_cost=1.0);", + dump_cmd => [ + 'pg_dumpall', + '--format' => 'directory', + '--file' => "$tempdir/restore_tablespace", + ], + restore_cmd => [ + 'pg_restore', '-C', + '--format' => 'directory', + '--file' => "$tempdir/restore_tablespace.sql", + "$tempdir/restore_tablespace", + ], + # Match "E" as optional since it is added on LOCATION when running on + # Windows. + like => qr/^ + \n\QCREATE TABLESPACE tbl2 OWNER tap LOCATION \E(?:E)?\Q'$tablespace2_orig';\E + \n\QALTER TABLESPACE tbl2 SET (seq_page_cost=1.0);\E + /xm, + }, + + # Test 3: dump and restore GRANTs on a table, sequence, function, schema, + # database, and role membership. + restore_grants => { + setup_sql => " + CREATE DATABASE tapgrantsdb; + CREATE SCHEMA private; + CREATE SEQUENCE serial START 101; + CREATE FUNCTION fn() RETURNS void AS \$\$ + BEGIN + END; + \$\$ LANGUAGE plpgsql; + CREATE ROLE super; + CREATE ROLE grant1; + CREATE ROLE grant2; + CREATE ROLE grant3; + CREATE ROLE grant4; + CREATE ROLE grant5; + CREATE ROLE grant6; + CREATE ROLE grant7; + CREATE ROLE grant8; + + CREATE TABLE t (id int); + INSERT INTO t VALUES (1), (2), (3), (4); + + GRANT SELECT ON TABLE t TO grant1; + GRANT INSERT ON TABLE t TO grant2; + GRANT ALL PRIVILEGES ON TABLE t to grant3; + GRANT CONNECT, CREATE ON DATABASE tapgrantsdb TO grant4; + GRANT USAGE, CREATE ON SCHEMA private TO grant5; + GRANT USAGE, SELECT, UPDATE ON SEQUENCE serial TO grant6; + GRANT super TO grant7; + GRANT EXECUTE ON FUNCTION fn() TO grant8; + ", + dump_cmd => [ + 'pg_dumpall', + '--format' => 'directory', + '--file' => "$tempdir/restore_grants", + ], + restore_cmd => [ + 'pg_restore', '-C', + '--format' => 'directory', + '--file' => "$tempdir/restore_grants.sql", + "$tempdir/restore_grants", + ], + like => qr/^ + \n\QGRANT ALL ON SCHEMA private TO grant5;\E + (.*\n)* + \n\QGRANT ALL ON FUNCTION public.fn() TO grant8;\E + (.*\n)* + \n\QGRANT ALL ON SEQUENCE public.serial TO grant6;\E + (.*\n)* + \n\QGRANT SELECT ON TABLE public.t TO grant1;\E + \n\QGRANT INSERT ON TABLE public.t TO grant2;\E + \n\QGRANT ALL ON TABLE public.t TO grant3;\E + (.*\n)* + \n\QGRANT CREATE,CONNECT ON DATABASE tapgrantsdb TO grant4;\E + /xm, + }, + + # Test 4: --exclude-database at dump time and at restore time (from the + # static map.dat database list) each skip their own database. + excluding_databases => { + setup_sql => 'CREATE DATABASE db1; + \c db1 + CREATE TABLE t1 (id int); + INSERT INTO t1 VALUES (1), (2), (3), (4); + CREATE TABLE t2 (id int); + INSERT INTO t2 VALUES (1), (2), (3), (4); + + CREATE DATABASE db2; + \c db2 + CREATE TABLE t3 (id int); + INSERT INTO t3 VALUES (1), (2), (3), (4); + CREATE TABLE t4 (id int); + INSERT INTO t4 VALUES (1), (2), (3), (4); + + CREATE DATABASE dbex3; + \c dbex3 + CREATE TABLE t5 (id int); + INSERT INTO t5 VALUES (1), (2), (3), (4); + CREATE TABLE t6 (id int); + INSERT INTO t6 VALUES (1), (2), (3), (4); + + CREATE DATABASE dbex4; + \c dbex4 + CREATE TABLE t7 (id int); + INSERT INTO t7 VALUES (1), (2), (3), (4); + CREATE TABLE t8 (id int); + INSERT INTO t8 VALUES (1), (2), (3), (4); + + CREATE DATABASE db5; + \c db5 + CREATE TABLE t9 (id int); + INSERT INTO t9 VALUES (1), (2), (3), (4); + CREATE TABLE t10 (id int); + INSERT INTO t10 VALUES (1), (2), (3), (4); + ', + dump_cmd => [ + 'pg_dumpall', + '--format' => 'directory', + '--file' => "$tempdir/excluding_databases", + '--exclude-database' => 'dbex*', + ], + restore_cmd => [ + 'pg_restore', '-C', + '--format' => 'directory', + '--file' => "$tempdir/excluding_databases.sql", + '--exclude-database' => 'db5', + "$tempdir/excluding_databases", + ], + like => qr/^ + \n\QCREATE DATABASE db1\E + (.*\n)* + \n\QCREATE TABLE public.t1 (\E + (.*\n)* + \n\QCREATE TABLE public.t2 (\E + (.*\n)* + \n\QCREATE DATABASE db2\E + (.*\n)* + \n\QCREATE TABLE public.t3 (\E + (.*\n)* + \n\QCREATE TABLE public.t4 (/xm, + unlike => qr/^ + \n\QCREATE DATABASE dbex3\E + (.*\n)* + \n\QCREATE TABLE public.t5 (\E + (.*\n)* + \n\QCREATE TABLE public.t6 (\E + (.*\n)* + \n\QCREATE DATABASE dbex4\E + (.*\n)* + \n\QCREATE TABLE public.t7 (\E + (.*\n)* + \n\QCREATE TABLE public.t8 (\E + \n\QCREATE DATABASE db5\E + (.*\n)* + \n\QCREATE TABLE public.t9 (\E + (.*\n)* + \n\QCREATE TABLE public.t10 (\E + /xm, + }, + + # Test 5: dump and restore using --format=directory. + format_directory => { + setup_sql => "CREATE TABLE format_directory(a int, b boolean, c text); + INSERT INTO format_directory VALUES (1, true, 'name1'), (2, false, 'name2');", + dump_cmd => [ + 'pg_dumpall', + '--format' => 'directory', + '--file' => "$tempdir/format_directory", + ], + restore_cmd => [ + 'pg_restore', '-C', + '--format' => 'directory', + '--file' => "$tempdir/format_directory.sql", + "$tempdir/format_directory", + ], + like => qr/^\n\QCOPY public.format_directory (a, b, c) FROM stdin;/xm + }, + + # Test 6: dump and restore using --format=tar. + format_tar => { + setup_sql => "CREATE TABLE format_tar(a int, b boolean, c text); + INSERT INTO format_tar VALUES (1, false, 'name3'), (2, true, 'name4');", + dump_cmd => [ + 'pg_dumpall', + '--format' => 'tar', + '--file' => "$tempdir/format_tar", + ], + restore_cmd => [ + 'pg_restore', '-C', + '--format' => 'tar', + '--file' => "$tempdir/format_tar.sql", + "$tempdir/format_tar", + ], + like => qr/^\n\QCOPY public.format_tar (a, b, c) FROM stdin;/xm + }, + + # Test 7: dump and restore using --format=custom. + format_custom => { + setup_sql => "CREATE TABLE format_custom(a int, b boolean, c text); + INSERT INTO format_custom VALUES (1, false, 'name5'), (2, true, 'name6');", + dump_cmd => [ + 'pg_dumpall', + '--format' => 'custom', + '--file' => "$tempdir/format_custom", + ], + restore_cmd => [ + 'pg_restore', '-C', + '--format' => 'custom', + '--file' => "$tempdir/format_custom.sql", + "$tempdir/format_custom", + ], + like => qr/^ \n\QCOPY public.format_custom (a, b, c) FROM stdin;/xm + }, + + # Test 8: pg_dumpall --globals-only + pg_restore --globals-only restores + # just the role, not the accompanying table data. + dump_globals_only => { + setup_sql => "CREATE TABLE format_dir(a int, b boolean, c text); + INSERT INTO format_dir VALUES (1, false, 'name5'), (2, true, 'name6');", + dump_cmd => [ + 'pg_dumpall', + '--format' => 'directory', + '--globals-only', + '--file' => "$tempdir/dump_globals_only", + ], + restore_cmd => [ + 'pg_restore', '-C', '--globals-only', + '--format' => 'directory', + '--file' => "$tempdir/dump_globals_only.sql", + "$tempdir/dump_globals_only", + ], + like => qr/ + ^\s*\QCREATE ROLE dumpall;\E\s*\n + /xm + }, + + # Test 9: pg_restore --no-globals restores table data but skips the role. + restore_no_globals => { + setup_sql => "CREATE TABLE no_globals_test(a int, b text); + INSERT INTO no_globals_test VALUES (1, 'hello'), (2, 'world');", + dump_cmd => [ + 'pg_dumpall', + '--format' => 'directory', + '--file' => "$tempdir/restore_no_globals", + ], + restore_cmd => [ + 'pg_restore', '-C', '--no-globals', + '--format' => 'directory', + '--file' => "$tempdir/restore_no_globals.sql", + "$tempdir/restore_no_globals", + ], + like => qr/^\n\QCOPY public.no_globals_test (a, b) FROM stdin;\E/xm, + unlike => qr/^\QCREATE ROLE dumpall;\E/xm, + },); + +# First execute the setup_sql +foreach my $run (sort keys %pgdumpall_runs) +{ + if ($pgdumpall_runs{$run}->{setup_sql}) + { + $node->safe_psql($run_db, $pgdumpall_runs{$run}->{setup_sql}); + } +} + +# Execute the tests +foreach my $run (sort keys %pgdumpall_runs) +{ + # Create a new target cluster to pg_restore each test case run so that we + # don't need to take care of the cleanup from the target cluster after each + # run. + my $target_node = PostgreSQL::Test::Cluster->new("target_$run"); + $target_node->init; + $target_node->start; + + # Dumpall from node cluster. + $node->command_ok(\@{ $pgdumpall_runs{$run}->{dump_cmd} }, + "$run: pg_dumpall runs"); + + # Restore the dump on "target_node" cluster. + my @restore_cmd = ( + @{ $pgdumpall_runs{$run}->{restore_cmd} }, + '--host', $target_node->host, '--port', $target_node->port); + + my ($stdout, $stderr) = run_command(\@restore_cmd); + + # pg_restore --file output file. + my $output_file = slurp_file("$tempdir/${run}.sql"); + + if ( !($pgdumpall_runs{$run}->{like}) + && !($pgdumpall_runs{$run}->{unlike})) + { + die "missing \"like\" or \"unlike\" in test \"$run\""; + } + + if ($pgdumpall_runs{$run}->{like}) + { + like($output_file, $pgdumpall_runs{$run}->{like}, "should dump $run"); + } + + if ($pgdumpall_runs{$run}->{unlike}) + { + unlike( + $output_file, + $pgdumpall_runs{$run}->{unlike}, + "should not dump $run"); + } + + $target_node->stop; + $target_node->clean_node; +} + +############################################################### +# Further positive test cases exercising specific behaviors of the +# non-text pg_dumpall archive format. + +# Test 10: verify map.dat preamble exists +my $map_dat_content = slurp_file("$tempdir/format_directory/map.dat"); +like( + $map_dat_content, + qr/^# map\.dat\n.*# This file maps oids to database names/ms, + 'map.dat contains expected preamble'); + +# Test 11: verify commenting out a line in map.dat skips that database +$node->safe_psql( + $run_db, 'CREATE DATABASE comment_test_db; +\c comment_test_db +CREATE TABLE comment_test_table (id int);'); + +$node->command_ok( + [ + 'pg_dumpall', + '--format' => 'directory', + '--file' => "$tempdir/comment_test", + ], + 'pg_dumpall for comment test'); + +# Modify map.dat to comment out the comment_test_db entry +my $map_content = slurp_file("$tempdir/comment_test/map.dat"); +$map_content =~ s/^(\d+ comment_test_db)$/# $1/m; +open(my $fh, '>', "$tempdir/comment_test/map.dat") + or die "Cannot open map.dat: $!"; +print $fh $map_content; +close($fh); + +# Create a target node and restore - commented db should be skipped +my $target_comment = PostgreSQL::Test::Cluster->new("target_comment"); +$target_comment->init; +$target_comment->start; + +$node->command_ok( + [ + 'pg_restore', '-C', + '--format' => 'directory', + '--file' => "$tempdir/comment_test_restore.sql", + '--host', $target_comment->host, + '--port', $target_comment->port, + "$tempdir/comment_test", + ], + 'pg_restore with commented out database in map.dat'); + +my $restore_output = slurp_file("$tempdir/comment_test_restore.sql"); +unlike( + $restore_output, + qr/CREATE DATABASE comment_test_db/, + 'commented out database in map.dat is not restored'); + +# Test 12: --clean without --if-exists should not add IF EXISTS (no +# auto-implication). DROP DATABASE always includes IF EXISTS, unlike +# ROLE/TABLESPACE. +$node->command_ok( + [ + 'pg_restore', '-C', + '--format' => 'custom', + '--clean', + '--file' => "$tempdir/clean_test.sql", + "$tempdir/format_custom", + ], + 'pg_restore with --clean on pg_dumpall archive'); + +my $clean_output = slurp_file("$tempdir/clean_test.sql"); +like( + $clean_output, + qr/DROP ROLE(?! IF EXISTS)/, + '--clean without --if-exists: DROP ROLE without IF EXISTS in output'); +like( + $clean_output, + qr/DROP TABLESPACE(?! IF EXISTS)/, + '--clean without --if-exists: DROP TABLESPACE without IF EXISTS in output' +); +like( + $clean_output, + qr/DROP DATABASE IF EXISTS/, + '--clean without --if-exists: DROP DATABASE IF EXISTS in output'); + +# Test 13: --clean --if-exists adds IF EXISTS to all object types. +$node->command_ok( + [ + 'pg_restore', '-C', + '--format' => 'custom', + '--clean', + '--if-exists', + '--file' => "$tempdir/clean_if_exists_test.sql", + "$tempdir/format_custom", + ], + 'pg_restore with --clean --if-exists on pg_dumpall archive'); + +my $clean_if_exists_output = slurp_file("$tempdir/clean_if_exists_test.sql"); +like( + $clean_if_exists_output, + qr/DROP ROLE IF EXISTS/, + '--clean --if-exists: DROP ROLE IF EXISTS in output'); +like( + $clean_if_exists_output, + qr/DROP TABLESPACE IF EXISTS/, + '--clean --if-exists: DROP TABLESPACE IF EXISTS in output'); +like( + $clean_if_exists_output, + qr/DROP DATABASE IF EXISTS/, + '--clean --if-exists: DROP DATABASE IF EXISTS in output'); + +# Test 14: live (-d) restore of a full archive into a fresh cluster, twice, +# with --clean --if-exists the second time, must drop and recreate the +# role/tablespace/database rather than silently skip the drops. +my $liverestore_tablespace = "$tempdir/liverestore_tbl"; +mkdir $liverestore_tablespace + or die "unable to create $liverestore_tablespace"; + +$node->safe_psql( + $run_db, " + CREATE ROLE liverestore_role; + CREATE TABLESPACE liverestore_tbl OWNER liverestore_role LOCATION '$liverestore_tablespace'; + CREATE DATABASE liverestore_db TABLESPACE liverestore_tbl; + "); + +$node->command_ok( + [ 'pg_dumpall', '--format' => 'custom', '--file' => "$tempdir/liverestore" ], + 'pg_dumpall for live restore test'); + +# Free the tablespace path for the target cluster below to reuse. +$node->safe_psql($run_db, + "DROP DATABASE liverestore_db; DROP TABLESPACE liverestore_tbl;"); + +my $target_liverestore = PostgreSQL::Test::Cluster->new("target_liverestore"); +$target_liverestore->init; +$target_liverestore->start; + +my @liverestore_cmd = ( + 'pg_restore', '-C', + '--format' => 'custom', + '-d' => 'postgres', + '--host' => $target_liverestore->host, + '--port' => $target_liverestore->port, + "$tempdir/liverestore"); + +my $liverestore_exists_re = qr/ + \Qrole "liverestore_role" already exists\E | + \Qtablespace "liverestore_tbl" already exists\E | + \Qdatabase "liverestore_db" already exists\E +/x; + +sub check_liverestore_objects +{ + my ($label) = @_; + is($target_liverestore->safe_psql('postgres', + "SELECT count(*) FROM pg_roles WHERE rolname = 'liverestore_role'"), + 1, "$label: role present"); + is($target_liverestore->safe_psql('postgres', + "SELECT count(*) FROM pg_tablespace WHERE spcname = 'liverestore_tbl'"), + 1, "$label: tablespace present"); + is($target_liverestore->safe_psql('postgres', + "SELECT count(*) FROM pg_database WHERE datname = 'liverestore_db'"), + 1, "$label: database present"); +} + +my (undef, $liverestore_stderr1) = run_command(\@liverestore_cmd); +unlike($liverestore_stderr1, $liverestore_exists_re, + 'first live restore: no unexpected "already exists" errors'); +check_liverestore_objects('first live restore'); + +my (undef, $liverestore_stderr2) = + run_command([ @liverestore_cmd, '--clean', '--if-exists' ]); +unlike($liverestore_stderr2, $liverestore_exists_re, + 'second live restore: no unexpected "already exists" errors'); +check_liverestore_objects('second live restore'); + +$target_liverestore->stop; +$target_liverestore->clean_node; + +# Test 15: --schema must still restore global objects (roles, tablespaces), +# even though they have no schema and would otherwise be excluded by it. +$node->command_ok( + [ + 'pg_restore', '-C', + '--format' => 'custom', + '--schema' => 'nonexistent_schema', + '--file' => "$tempdir/schema_filter_test.sql", + "$tempdir/format_custom", + ], + 'pg_restore with --schema on pg_dumpall archive'); + +my $schema_filter_output = slurp_file("$tempdir/schema_filter_test.sql"); +like($schema_filter_output, qr/CREATE ROLE dumpall2;/, + '--schema filter: global ROLE objects are still restored'); +like($schema_filter_output, qr/CREATE TABLESPACE tbl2 /, + '--schema filter: global TABLESPACE objects are still restored'); +unlike($schema_filter_output, qr/CREATE TABLE public\.format_custom/, + '--schema filter: non-matching schema-scoped objects are excluded'); + +# Test 16: --exclude-database over a live connection (-d) exercises the SQL +# pattern-matching path, distinct from the static map.dat-parsing path +# used when restoring to a script file with no connection. +my $target_liveexclude = PostgreSQL::Test::Cluster->new("target_liveexclude"); +$target_liveexclude->init; +$target_liveexclude->start; + +run_command( + [ + 'pg_restore', '-C', + '--format' => 'directory', + '-d' => 'postgres', + '--host' => $target_liveexclude->host, + '--port' => $target_liveexclude->port, + '--exclude-database' => 'db2', + "$tempdir/excluding_databases", + ]); + +is($target_liveexclude->safe_psql('postgres', + "SELECT count(*) FROM pg_database WHERE datname = 'db1'"), + 1, 'live --exclude-database: non-excluded database was restored'); +is($target_liveexclude->safe_psql('postgres', + "SELECT count(*) FROM pg_database WHERE datname = 'db2'"), + 0, 'live --exclude-database: excluded database was not restored'); + +$target_liveexclude->stop; +$target_liveexclude->clean_node; + +# Test 17: parallel (-j) restore, combined with --no-tablespaces. +my $target_parallel = PostgreSQL::Test::Cluster->new("target_parallel"); +$target_parallel->init; +$target_parallel->start; + +run_command( + [ + 'pg_restore', '-C', + '--format' => 'directory', + '-d' => 'postgres', + '--host' => $target_parallel->host, + '--port' => $target_parallel->port, + '--jobs' => 2, + '--no-tablespaces', + "$tempdir/excluding_databases", + ]); + +is($target_parallel->safe_psql('postgres', + "SELECT count(*) FROM pg_database WHERE datname = 'db1'"), + 1, 'parallel restore: database was created'); +is($target_parallel->safe_psql('postgres', + "SELECT count(*) FROM pg_tablespace WHERE spcname = 'tbl2'"), + 0, 'parallel restore with --no-tablespaces: tablespace was not created'); + +$target_parallel->stop; +$target_parallel->clean_node; + +############################################################### +# Negative test cases: dump using pg_dumpall, then check that pg_restore +# rejects invalid option combinations against the resulting archive. + +# Test 18: report an error when -C is not used in pg_restore with dump of pg_dumpall +$node->command_fails_like( + [ + 'pg_restore', + "$tempdir/format_custom", + '--format' => 'custom', + '--file' => "$tempdir/error_test.sql", + ], + qr/\Qpg_restore: error: option -C\/--create must be specified when restoring an archive created by pg_dumpall\E/, + 'When -C is not used in pg_restore with dump of pg_dumpall'); + +# Test 19: report an error when \l/--list option is used with dump of pg_dumpall +$node->command_fails_like( + [ + 'pg_restore', + "$tempdir/format_custom", '-C', + '--format' => 'custom', + '--list', + '--file' => "$tempdir/error_test.sql", + ], + qr/\Qpg_restore: error: option -l\/--list cannot be used when restoring an archive created by pg_dumpall\E/, + 'When --list is used in pg_restore with dump of pg_dumpall'); + +# Test 20: report an error when -L/--use-list option is used with dump of pg_dumpall +$node->command_fails_like( + [ + 'pg_restore', + "$tempdir/format_custom", '-C', + '--format' => 'custom', + '--use-list' => 'use', + '--file' => "$tempdir/error_test.sql", + ], + qr/\Qpg_restore: error: option -L\/--use-list cannot be used when restoring an archive created by pg_dumpall\E/, + 'When -L/--use-list is used in pg_restore with dump of pg_dumpall'); + +# Test 21: report an error when --strict-names option is used with dump of pg_dumpall +$node->command_fails_like( + [ + 'pg_restore', + "$tempdir/format_custom", '-C', + '--format' => 'custom', + '--strict-names', + '--file' => "$tempdir/error_test.sql", + ], + qr/\Qpg_restore: error: option --strict-names cannot be used when restoring an archive created by pg_dumpall\E/, + 'When --strict-names is used in pg_restore with dump of pg_dumpall'); + +# Test 22: report an error when --clean and -g/--globals-only are used in pg_restore with dump of pg_dumpall +$node->command_fails_like( + [ + 'pg_restore', + "$tempdir/format_custom", '-C', + '--format' => 'custom', + '--clean', + '--globals-only', + '--file' => "$tempdir/error_test.sql", + ], + qr/\Qpg_restore: error: options --clean and -g\/--globals-only cannot be used together when restoring an archive created by pg_dumpall\E/, + 'When --clean and -g/--globals-only are used in pg_restore with dump of pg_dumpall' +); + +# Test 23: report an error when non-exist database is given with -d option +$node->command_fails_like( + [ + 'pg_restore', + "$tempdir/format_custom", '-C', + '--format' => 'custom', + '-d' => 'dbpq', + ], + qr/\QFATAL: database "dbpq" does not exist\E/, + 'When non-existent database is given with -d option in pg_restore with dump of pg_dumpall' +); + +# Test 24: report an error when --no-schema is used with dump of pg_dumpall +$node->command_fails_like( + [ + 'pg_restore', + "$tempdir/format_custom", '-C', + '--format' => 'custom', + '--no-schema', + '--file' => "$tempdir/error_test.sql", + ], + qr/\Qpg_restore: error: option --no-schema cannot be used when restoring an archive created by pg_dumpall\E/, + 'When --no-schema is used in pg_restore with dump of pg_dumpall'); + +# Test 25: report an error when --data-only is used with dump of pg_dumpall +$node->command_fails_like( + [ + 'pg_restore', + "$tempdir/format_custom", '-C', + '--format' => 'custom', + '--data-only', + '--file' => "$tempdir/error_test.sql", + ], + qr/\Qpg_restore: error: option -a\/--data-only cannot be used when restoring an archive created by pg_dumpall\E/, + 'When --data-only is used in pg_restore with dump of pg_dumpall'); + +# Test 26: report an error when --statistics-only is used with dump of pg_dumpall +$node->command_fails_like( + [ + 'pg_restore', + "$tempdir/format_custom", '-C', + '--format' => 'custom', + '--statistics-only', + '--file' => "$tempdir/error_test.sql", + ], + qr/\Qpg_restore: error: option --statistics-only cannot be used when restoring an archive created by pg_dumpall\E/, + 'When --statistics-only is used in pg_restore with dump of pg_dumpall'); + +# Test 27: report an error when --section excludes pre-data with dump of pg_dumpall +$node->command_fails_like( + [ + 'pg_restore', + "$tempdir/format_custom", '-C', + '--format' => 'custom', + '--section' => 'post-data', + '--file' => "$tempdir/error_test.sql", + ], + qr/\Qpg_restore: error: option --section cannot exclude pre-data when restoring a pg_dumpall archive\E/, + 'When --section=post-data is used in pg_restore with dump of pg_dumpall'); + +# Test 28: report an error when --globals-only and --data-only are used together +$node->command_fails_like( + [ + 'pg_restore', + "$tempdir/format_custom", '-C', + '--format' => 'custom', + '--globals-only', + '--data-only', + '--file' => "$tempdir/error_test.sql", + ], + qr/\Qpg_restore: error: options -a\/--data-only and -g\/--globals-only cannot be used together\E/, + 'When --globals-only and --data-only are used together'); + +# Test 29: report an error when --globals-only and --schema-only are used together +$node->command_fails_like( + [ + 'pg_restore', + "$tempdir/format_custom", '-C', + '--format' => 'custom', + '--globals-only', + '--schema-only', + '--file' => "$tempdir/error_test.sql", + ], + qr/\Qpg_restore: error: options -g\/--globals-only and -s\/--schema-only cannot be used together\E/, + 'When --globals-only and --schema-only are used together'); + +# Test 30: report an error when --globals-only and --statistics-only are used together +$node->command_fails_like( + [ + 'pg_restore', + "$tempdir/format_custom", '-C', + '--format' => 'custom', + '--globals-only', + '--statistics-only', + '--file' => "$tempdir/error_test.sql", + ], + qr/\Qpg_restore: error: options -g\/--globals-only and --statistics-only cannot be used together\E/, + 'When --globals-only and --statistics-only are used together'); + +# Test 31: report an error when --globals-only and --statistics are used together +$node->command_fails_like( + [ + 'pg_restore', + "$tempdir/format_custom", '-C', + '--format' => 'custom', + '--globals-only', + '--statistics', + '--file' => "$tempdir/error_test.sql", + ], + qr/\Qpg_restore: error: options --statistics and -g\/--globals-only cannot be used together\E/, + 'When --globals-only and --statistics are used together'); + +# Test 32: report an error when --globals-only and --exit-on-error are used together +$node->command_fails_like( + [ + 'pg_restore', + "$tempdir/format_custom", '-C', + '--format' => 'custom', + '--globals-only', + '--exit-on-error', + '--file' => "$tempdir/error_test.sql", + ], + qr/\Qpg_restore: error: options --exit-on-error and -g\/--globals-only cannot be used together\E/, + 'When --globals-only and --exit-on-error are used together'); + +# Test 33: report an error when --globals-only and --single-transaction are used together +$node->command_fails_like( + [ + 'pg_restore', + "$tempdir/format_custom", '-C', + '--format' => 'custom', + '--globals-only', + '--single-transaction', + '--file' => "$tempdir/error_test.sql", + ], + qr/\Qpg_restore: error: options -g\/--globals-only and -1\/--single-transaction cannot be used together\E/, + 'When --globals-only and --single-transaction are used together'); + +# Test 34: report an error when --globals-only and --transaction-size are used together +$node->command_fails_like( + [ + 'pg_restore', + "$tempdir/format_custom", '-C', + '--format' => 'custom', + '--globals-only', + '--transaction-size' => '100', + '--file' => "$tempdir/error_test.sql", + ], + qr/\Qpg_restore: error: options -g\/--globals-only and --transaction-size cannot be used together\E/, + 'When --globals-only and --transaction-size are used together'); + +$node->stop('fast'); + +done_testing(); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 59cf40b5bb0..6dacbaea1b3 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -652,6 +652,7 @@ DatumTupleFields DbInfo DbInfoArr DbLocaleInfo +DbOidName DdlOptType DdlOption DeClonePtrType -- 2.52.0