From c012e1e1149d04abc39bb4099fe1e18a4cd2ca2d Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Mon, 18 Feb 2019 12:23:37 +0300 Subject: [PATCH v3 2/2] Options to use restore_command with pg_rewind --- doc/src/sgml/ref/pg_rewind.sgml | 30 ++++- src/bin/pg_rewind/Makefile | 2 +- src/bin/pg_rewind/parsexlog.c | 163 +++++++++++++++++++++++++- src/bin/pg_rewind/pg_rewind.c | 100 +++++++++++++++- src/bin/pg_rewind/pg_rewind.h | 10 +- src/bin/pg_rewind/t/001_basic.pl | 4 +- src/bin/pg_rewind/t/002_databases.pl | 4 +- src/bin/pg_rewind/t/003_extrafiles.pl | 4 +- src/bin/pg_rewind/t/RewindTest.pm | 93 ++++++++++++++- 9 files changed, 388 insertions(+), 22 deletions(-) diff --git a/doc/src/sgml/ref/pg_rewind.sgml b/doc/src/sgml/ref/pg_rewind.sgml index 53a64ee29e..0c2441afa7 100644 --- a/doc/src/sgml/ref/pg_rewind.sgml +++ b/doc/src/sgml/ref/pg_rewind.sgml @@ -67,8 +67,10 @@ PostgreSQL documentation ancestor. In the typical failover scenario where the target cluster was shut down soon after the divergence, this is not a problem, but if the target cluster ran for a long time after the divergence, the old WAL - files might no longer be present. In that case, they can be manually - copied from the WAL archive to the pg_wal directory, or + files might no longer be present. In that case, they can be automatically + copied by pg_rewind from the WAL archive to the + pg_wal directory if either -r or + -R option is specified, or fetched on startup by configuring or . The use of pg_rewind is not limited to failover, e.g. a standby @@ -200,6 +202,30 @@ PostgreSQL documentation + + + + + + Use restore_command in the postgresql.conf to + retreive missing in the target pg_wal directory + WAL files from the WAL archive. + + + + + + + + + + Specifies the restore_command to use for retrieval of the missing + in the target pg_wal directory WAL files from + the WAL archive. + + + + diff --git a/src/bin/pg_rewind/Makefile b/src/bin/pg_rewind/Makefile index 04f3b8f520..65ebf3c78c 100644 --- a/src/bin/pg_rewind/Makefile +++ b/src/bin/pg_rewind/Makefile @@ -15,7 +15,7 @@ subdir = src/bin/pg_rewind top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -override CPPFLAGS := -I$(libpq_srcdir) -DFRONTEND $(CPPFLAGS) +override CPPFLAGS := -I. -I$(srcdir) -I$(libpq_srcdir) -DFRONTEND $(CPPFLAGS) LDFLAGS_INTERNAL += $(libpq_pgport) OBJS = pg_rewind.o parsexlog.o xlogreader.o datapagemap.o timeline.o \ diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index e19c265cbb..35c75cd8b9 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -12,6 +12,7 @@ #include "postgres_fe.h" #include +#include #include "pg_rewind.h" #include "filemap.h" @@ -45,6 +46,7 @@ static char xlogfpath[MAXPGPATH]; typedef struct XLogPageReadPrivate { const char *datadir; + const char *restoreCommand; int tliIndex; } XLogPageReadPrivate; @@ -53,6 +55,9 @@ static int SimpleXLogPageRead(XLogReaderState *xlogreader, int reqLen, XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *pageTLI); +static int RestoreArchivedWAL(const char *path, const char *xlogfname, + off_t expectedSize, const char *restoreCommand); + /* * Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline * index 'tliIndex' in target timeline history, until 'endpoint'. Make note of @@ -60,7 +65,7 @@ static int SimpleXLogPageRead(XLogReaderState *xlogreader, */ void extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, - XLogRecPtr endpoint) + XLogRecPtr endpoint, const char *restore_command) { XLogRecord *record; XLogReaderState *xlogreader; @@ -69,6 +74,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, private.datadir = datadir; private.tliIndex = tliIndex; + private.restoreCommand = restore_command; xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead, &private); if (xlogreader == NULL) @@ -156,7 +162,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex) void findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli, - XLogRecPtr *lastchkptredo) + XLogRecPtr *lastchkptredo, const char *restoreCommand) { /* Walk backwards, starting from the given record */ XLogRecord *record; @@ -181,6 +187,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, private.datadir = datadir; private.tliIndex = tliIndex; + private.restoreCommand = restoreCommand; xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead, &private); if (xlogreader == NULL) @@ -285,15 +292,37 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, XLogFileName(xlogfname, targetHistory[private->tliIndex].tli, xlogreadsegno, WalSegSz); - snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s", private->datadir, xlogfname); + snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s", + private->datadir, xlogfname); xlogreadfd = open(xlogfpath, O_RDONLY | PG_BINARY, 0); if (xlogreadfd < 0) { - printf(_("could not open file \"%s\": %s\n"), xlogfpath, - strerror(errno)); - return -1; + /* + * If we have no restore_command to execute, then exit. + */ + if (private->restoreCommand == NULL) + { + printf(_("could not open file \"%s\": %s\n"), xlogfpath, + strerror(errno)); + return -1; + } + + /* + * Since we have restore_command to execute, then try to retreive + * missing WAL file from the archive. + */ + xlogreadfd = RestoreArchivedWAL(private->datadir, + xlogfname, + WalSegSz, + private->restoreCommand); + + if (xlogreadfd < 0) + return -1; + else + pg_log(PG_DEBUG, "using restored from archive version of file \"%s\"\n", + xlogfpath); } } @@ -409,3 +438,125 @@ extractPageInfo(XLogReaderState *record) process_block_change(forknum, rnode, blkno); } } + +/* + * Attempt to retrieve the specified file from off-line archival storage. + * If successful return true. + * + * For fixed-size files, the caller may pass the expected size as an + * additional crosscheck on successful recovery. If the file size is not + * known, set expectedSize = 0. + * + * This is a simplified and adapted to frontend version of + * RestoreArchivedFile function from transam/xlogarchive.c + */ +static int +RestoreArchivedWAL(const char *path, const char *xlogfname, + off_t expectedSize, const char *restoreCommand) +{ + char xlogpath[MAXPGPATH]; + char xlogRestoreCmd[MAXPGPATH]; + char *dp; + char *endp; + const char *sp; + int rc, + xlogfd; + struct stat stat_buf; + + snprintf(xlogpath, MAXPGPATH, "%s/" XLOGDIR "/%s", path, xlogfname); + + /* + * Construct the command to be executed. + */ + dp = xlogRestoreCmd; + endp = xlogRestoreCmd + MAXPGPATH - 1; + *endp = '\0'; + + for (sp = restoreCommand; *sp; sp++) + { + if (*sp == '%') + { + switch (sp[1]) + { + case 'p': + /* %p: relative path of target file */ + sp++; + StrNCpy(dp, xlogpath, endp - dp); + make_native_path(dp); + dp += strlen(dp); + break; + case 'f': + /* %f: filename of desired file */ + sp++; + StrNCpy(dp, xlogfname, endp - dp); + dp += strlen(dp); + break; + case 'r': + /* %r: filename of last restartpoint */ + pg_fatal("restore_command with %%r cannot be used during rewind process.\n"); + break; + case '%': + /* convert %% to a single % */ + sp++; + if (dp < endp) + *dp++ = *sp; + break; + default: + /* otherwise treat the % as not special */ + if (dp < endp) + *dp++ = *sp; + break; + } + } + else + { + if (dp < endp) + *dp++ = *sp; + } + } + *dp = '\0'; + + /* + * Execute restore_command, which should copy + * the missing WAL file from archival storage. + */ + rc = system(xlogRestoreCmd); + + if (rc == 0) + { + /* + * Command apparently succeeded, but let's make sure the file is + * really there now and has the correct size. + */ + if (stat(xlogpath, &stat_buf) == 0) + { + if (expectedSize > 0 && stat_buf.st_size != expectedSize) + { + printf(_("archive file \"%s\" has wrong size: %lu instead of %lu, %s"), + xlogfname, (unsigned long) stat_buf.st_size, + (unsigned long) expectedSize, strerror(errno)); + } + else + { + xlogfd = open(xlogpath, O_RDONLY | PG_BINARY, 0); + + if (xlogfd < 0) + printf(_("could not open restored from archive file \"%s\": %s\n"), + xlogpath, strerror(errno)); + else + return xlogfd; + } + } + else + { + /* Stat failed */ + printf(_("could not stat file \"%s\": %s"), + xlogpath, strerror(errno)); + } + } + + printf(_("could not restore file \"%s\" from archive: %s\n"), + xlogfname, strerror(errno)); + + return -1; +} diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 7ccde5c87f..5abfeec26c 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -26,6 +26,7 @@ #include "catalog/pg_control.h" #include "common/file_perm.h" #include "common/file_utils.h" +#include "common/guc-file.h" #include "common/restricted_token.h" #include "getopt_long.h" #include "storage/bufpage.h" @@ -52,11 +53,13 @@ int WalSegSz; char *datadir_target = NULL; char *datadir_source = NULL; char *connstr_source = NULL; +char *restore_command = NULL; bool debug = false; bool showprogress = false; bool dry_run = false; bool do_sync = true; +bool restore_wals = false; /* Target history */ TimeLineHistoryEntry *targetHistory; @@ -75,6 +78,9 @@ usage(const char *progname) printf(_(" -N, --no-sync do not wait for changes to be written\n")); printf(_(" safely to disk\n")); printf(_(" -P, --progress write progress messages\n")); + printf(_(" -r, --use-postgresql-conf use restore_command in the postgresql.conf to\n")); + printf(_(" retreive WALs from archive\n")); + printf(_(" -R, --restore-command=COMMAND restore_command\n")); printf(_(" --debug write a lot of debug messages\n")); printf(_(" -V, --version output version information, then exit\n")); printf(_(" -?, --help show this help, then exit\n")); @@ -94,9 +100,12 @@ main(int argc, char **argv) {"dry-run", no_argument, NULL, 'n'}, {"no-sync", no_argument, NULL, 'N'}, {"progress", no_argument, NULL, 'P'}, + {"use-postgresql-conf", no_argument, NULL, 'r'}, + {"restore-command", required_argument, NULL, 'R'}, {"debug", no_argument, NULL, 3}, {NULL, 0, NULL, 0} }; + char recfile_fullpath[MAXPGPATH]; int option_index; int c; XLogRecPtr divergerec; @@ -129,7 +138,7 @@ main(int argc, char **argv) } } - while ((c = getopt_long(argc, argv, "D:nNP", long_options, &option_index)) != -1) + while ((c = getopt_long(argc, argv, "D:nNPR:r", long_options, &option_index)) != -1) { switch (c) { @@ -141,6 +150,10 @@ main(int argc, char **argv) showprogress = true; break; + case 'r': + restore_wals = true; + break; + case 'n': dry_run = true; break; @@ -157,6 +170,10 @@ main(int argc, char **argv) datadir_target = pg_strdup(optarg); break; + case 'R': + restore_command = pg_strdup(optarg); + break; + case 1: /* --source-pgdata */ datadir_source = pg_strdup(optarg); break; @@ -223,6 +240,80 @@ main(int argc, char **argv) umask(pg_mode_mask); + if (restore_command != NULL) + { + if (restore_wals) + { + fprintf(stderr, _("%s: conflicting options: both -r and -R are specified\n"), + progname); + fprintf(stderr, _("You must run %s with either -r/--use-postgresql-conf or -R/--restore-command.\n"), + progname); + exit(1); + } + + pg_log(PG_DEBUG, "using command line restore_command=\'%s\'.\n", restore_command); + } + else if (restore_wals) + { + FILE *conf_file; + + /* + * Look for configuration file in the target data directory and + * try to get restore_command from there. + */ + snprintf(recfile_fullpath, sizeof(recfile_fullpath), "%s/%s", datadir_target, RESTORE_COMMAND_FILE); + conf_file = fopen(recfile_fullpath, "r"); + + if (conf_file == NULL) + { + fprintf(stderr, _("%s: option -r/--use-postgresql-conf is specified, but postgresql.conf is absent in the target directory\n"), + progname); + fprintf(stderr, _("You have to add postgresql.conf or pass restore_command with -R/--restore-command option.\n")); + exit(1); + } + else + { + ConfigVariable *item, + *head = NULL, + *tail = NULL; + bool config_is_parsed; + + /* + * We pass a fullpath to the configuration file as calling_file here, since + * parser will use its parent directory as base for all further includes + * if any exist. + */ + config_is_parsed = ParseConfigFile(RESTORE_COMMAND_FILE, true, + recfile_fullpath, 0, 0, + PG_WARNING, &head, &tail); + fclose(conf_file); + + if (config_is_parsed) + { + for (item = head; item; item = item->next) + { + if (strcmp(item->name, "restore_command") == 0) + { + if (restore_command != NULL) + { + pfree(restore_command); + restore_command = NULL; + } + restore_command = pstrdup(item->value); + pg_log(PG_DEBUG, "using restore_command=\'%s\' from %s.\n", restore_command, RESTORE_COMMAND_FILE); + } + } + + if (restore_command == NULL) + pg_fatal("could not find restore_command in %s file %s\n", RESTORE_COMMAND_FILE, recfile_fullpath); + } + else + pg_fatal("could not parse %s file %s\n", RESTORE_COMMAND_FILE, recfile_fullpath); + + FreeConfigVariables(head); + } + } + /* Connect to remote server */ if (connstr_source) libpqConnect(connstr_source); @@ -294,9 +385,8 @@ main(int argc, char **argv) exit(0); } - findLastCheckpoint(datadir_target, divergerec, - lastcommontliIndex, - &chkptrec, &chkpttli, &chkptredo); + findLastCheckpoint(datadir_target, divergerec, lastcommontliIndex, + &chkptrec, &chkpttli, &chkptredo, restore_command); printf(_("rewinding from last common checkpoint at %X/%X on timeline %u\n"), (uint32) (chkptrec >> 32), (uint32) chkptrec, chkpttli); @@ -319,7 +409,7 @@ main(int argc, char **argv) */ pg_log(PG_PROGRESS, "reading WAL in target\n"); extractPageMap(datadir_target, chkptrec, lastcommontliIndex, - ControlFile_target.checkPoint); + ControlFile_target.checkPoint, restore_command); filemap_finalize(); if (showprogress) diff --git a/src/bin/pg_rewind/pg_rewind.h b/src/bin/pg_rewind/pg_rewind.h index 83b2898b8b..23b9526f5d 100644 --- a/src/bin/pg_rewind/pg_rewind.h +++ b/src/bin/pg_rewind/pg_rewind.h @@ -14,9 +14,12 @@ #include "datapagemap.h" #include "access/timeline.h" +#include "catalog/pg_control.h" #include "storage/block.h" #include "storage/relfilenode.h" +#define RESTORE_COMMAND_FILE "postgresql.conf" + /* Configuration options */ extern char *datadir_target; extern char *datadir_source; @@ -32,11 +35,10 @@ extern int targetNentries; /* in parsexlog.c */ extern void extractPageMap(const char *datadir, XLogRecPtr startpoint, - int tliIndex, XLogRecPtr endpoint); + int tliIndex, XLogRecPtr endpoint, const char *restoreCommand); extern void findLastCheckpoint(const char *datadir, XLogRecPtr searchptr, - int tliIndex, - XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli, - XLogRecPtr *lastchkptredo); + int tliIndex, XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli, + XLogRecPtr *lastchkptredo, const char *restoreCommand); extern XLogRecPtr readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex); diff --git a/src/bin/pg_rewind/t/001_basic.pl b/src/bin/pg_rewind/t/001_basic.pl index 115192170e..8a6fa33016 100644 --- a/src/bin/pg_rewind/t/001_basic.pl +++ b/src/bin/pg_rewind/t/001_basic.pl @@ -1,7 +1,7 @@ use strict; use warnings; use TestLib; -use Test::More tests => 10; +use Test::More tests => 20; use FindBin; use lib $FindBin::RealBin; @@ -106,5 +106,7 @@ in master, before promotion # Run the test in both modes run_test('local'); run_test('remote'); +run_test('archive'); +run_test('archive_conf'); exit(0); diff --git a/src/bin/pg_rewind/t/002_databases.pl b/src/bin/pg_rewind/t/002_databases.pl index 6dc05720a1..99ea821cfe 100644 --- a/src/bin/pg_rewind/t/002_databases.pl +++ b/src/bin/pg_rewind/t/002_databases.pl @@ -1,7 +1,7 @@ use strict; use warnings; use TestLib; -use Test::More tests => 6; +use Test::More tests => 12; use FindBin; use lib $FindBin::RealBin; @@ -62,5 +62,7 @@ template1 # Run the test in both modes. run_test('local'); run_test('remote'); +run_test('archive'); +run_test('archive_conf'); exit(0); diff --git a/src/bin/pg_rewind/t/003_extrafiles.pl b/src/bin/pg_rewind/t/003_extrafiles.pl index c4040bd562..24cec256de 100644 --- a/src/bin/pg_rewind/t/003_extrafiles.pl +++ b/src/bin/pg_rewind/t/003_extrafiles.pl @@ -3,7 +3,7 @@ use strict; use warnings; use TestLib; -use Test::More tests => 4; +use Test::More tests => 8; use File::Find; @@ -90,5 +90,7 @@ sub run_test # Run the test in both modes. run_test('local'); run_test('remote'); +run_test('archive'); +run_test('archive_conf'); exit(0); diff --git a/src/bin/pg_rewind/t/RewindTest.pm b/src/bin/pg_rewind/t/RewindTest.pm index 85cae7e47b..79bd6434b4 100644 --- a/src/bin/pg_rewind/t/RewindTest.pm +++ b/src/bin/pg_rewind/t/RewindTest.pm @@ -39,7 +39,9 @@ use Carp; use Config; use Exporter 'import'; use File::Copy; -use File::Path qw(rmtree); +use File::Glob ':bsd_glob'; +use File::Path qw(remove_tree make_path); +use File::Spec::Functions qw(catpath catfile); use IPC::Run qw(run); use PostgresNode; use TestLib; @@ -249,6 +251,95 @@ sub run_pg_rewind ], 'pg_rewind remote'); } + elsif ($test_mode eq "archive") + { + + # Do rewind using a local pgdata as source and + # specified directory with target WALs archive. + my $wals_archive_dir = catpath(${TestLib::tmp_check}, 'master_wals_archive'); + my @wal_files = bsd_glob catpath($master_pgdata, 'pg_wal', '0000000*'); + my $restore_command; + + remove_tree($wals_archive_dir); + make_path($wals_archive_dir) or die; + + # Move all old master WAL files to the archive. + # Old master should be stopped at this point. + foreach my $wal_file (@wal_files) + { + move($wal_file, "$wals_archive_dir") or die; + } + + if ($windows_os) + { + $restore_command = "copy $wals_archive_dir\\\%f \%p"; + } + else + { + $restore_command = "cp $wals_archive_dir/\%f \%p"; + } + + # Stop the new master and be ready to perform the rewind. + $node_standby->stop; + command_ok( + [ + 'pg_rewind', + "--debug", + "--source-pgdata=$standby_pgdata", + "--target-pgdata=$master_pgdata", + "--no-sync", + "--restore-command=$restore_command" + ], + 'pg_rewind archive'); + } + elsif ($test_mode eq "archive_conf") + { + + # Do rewind using a local pgdata as source and + # specified directory with target WALs archive. + my $wals_archive_dir = catpath(${TestLib::tmp_check}, 'master_wals_archive'); + my @wal_files = bsd_glob catpath($master_pgdata, 'pg_wal', '0000000*'); + my $master_conf_path = catfile($master_pgdata, 'postgresql.conf'); + my $restore_command; + + remove_tree($wals_archive_dir); + make_path($wals_archive_dir) or die; + + # Move all old master WAL files to the archive. + # Old master should be stopped at this point. + foreach my $wal_file (@wal_files) + { + move($wal_file, "$wals_archive_dir") or die; + } + + if ($windows_os) + { + $restore_command = "copy $wals_archive_dir\\\%f \%p"; + } + else + { + $restore_command = "cp $wals_archive_dir/\%f \%p"; + } + + # Stop the new master and be ready to perform the rewind. + $node_standby->stop; + + # Add restore_command to postgresql.conf of target cluster. + open(my $conf_fd, ">>", $master_conf_path) or die; + print $conf_fd "\nrestore_command='$restore_command'"; + close $conf_fd; + + command_ok( + [ + 'pg_rewind', + "--debug", + "--source-pgdata=$standby_pgdata", + "--target-pgdata=$master_pgdata", + "--no-sync", + "-r" + ], + 'pg_rewind archive_conf'); + } else { -- 2.17.1