diff -r 699924885324 -r 8c43292e0012 doc/src/sgml/ref/pg_rewind.sgml
--- a/doc/src/sgml/ref/pg_rewind.sgml Sun Sep 03 11:12:29 2017 -0400
+++ b/doc/src/sgml/ref/pg_rewind.sgml Fri Oct 27 18:47:42 2017 +0200
@@ -48,16 +48,26 @@
- The result is equivalent to replacing the target data directory with the
- source one. Only changed blocks from relation files are copied;
- all other files are copied in full, including configuration files. The
- advantage of pg_rewind> over taking a new base backup, or
- tools like rsync>, is that pg_rewind> does
- not require reading through unchanged blocks in the cluster. This makes
+ The result is equivalent to replacing the data-related files in the target
+ data directory with the source one. Only changed blocks from relation files
+ are copied; all other files relating to control or WAL information are copied
+ in full. The advantage of pg_rewind> over taking a new base
+ backup, or tools like rsync>, is that pg_rewind>
+ does not require reading through unchanged blocks in the cluster. This makes
it a lot faster when the database is large and only a small
fraction of blocks differ between the clusters.
+
+ A second advantage is predictability. pg_rewind> is aware of
+ what directories are relevant to restoring replication and which ones are not.
+ The result is that you get something of a guaranteed state at the end. Log
+ files on the source are unlikely to clobber those on the client. The
+ postgresql.conf.auto> is unlikely to be copied over. Replication
+ slot information is removed (and must be added again), and so forth. Your
+ system is as it had been before, but the data is synchronized from the master.
+
+
pg_rewind> examines the timeline histories of the source
and target clusters to determine the point where they diverged, and
@@ -84,7 +94,9 @@
pg_rewind> session, it must be made available when the
target server is started. This can be done by creating a
recovery.conf> file in the target data directory with a
- suitable restore_command>.
+ suitable restore_command>. Alternatively replication slots can
+ be set just before promotion to ensure that the wal files have not been
+ removed.
@@ -206,7 +218,7 @@
The basic idea is to copy all file system-level changes from the source
- cluster to the target cluster:
+ cluster to the target cluster if they implicate the data stored:
@@ -229,9 +241,7 @@
- Copy all other files such as pg_xact and
- configuration files from the source cluster to the target cluster
- (everything except the relation files).
+ Copy wal-related files such as those in pg_xact.
diff -r 699924885324 -r 8c43292e0012 src/bin/pg_rewind/RewindTest.pm
--- a/src/bin/pg_rewind/RewindTest.pm Sun Sep 03 11:12:29 2017 -0400
+++ b/src/bin/pg_rewind/RewindTest.pm Fri Oct 27 18:47:42 2017 +0200
@@ -1,4 +1,5 @@
package RewindTest;
+use Carp::Always;
# Test driver for pg_rewind. Each test consists of a cycle where a new cluster
# is first created with initdb, and a streaming replication standby is set up
diff -r 699924885324 -r 8c43292e0012 src/bin/pg_rewind/copy_fetch.c
--- a/src/bin/pg_rewind/copy_fetch.c Sun Sep 03 11:12:29 2017 -0400
+++ b/src/bin/pg_rewind/copy_fetch.c Fri Oct 27 18:47:42 2017 +0200
@@ -26,6 +26,30 @@
static void recurse_dir(const char *datadir, const char *path,
process_file_callback_t callback);
+/* List of directories to synchronize:
+ * base data dirs (and ablespaces)
+ * wal/transaction data
+ * and that is it.
+ *
+ * This array is null-terminated to make
+ * it easy to expand
+ */
+
+const char *rewind_dirs[] = {
+ "base",
+ "global",
+ "pg_commit_ts",
+ "pg_logical",
+ "pg_multixact",
+ "pg_serial",
+ "pg_subtrans",
+ "pg_tblspc",
+ "pg_twophase",
+ "pg_wal",
+ "pg_xact",
+ NULL
+};
+
static void execute_pagemap(datapagemap_t *pagemap, const char *path);
/*
@@ -38,6 +62,15 @@
recurse_dir(datadir, NULL, callback);
}
+void
+traverse_rewinddirs(const char *datadir, process_file_callback_t callback)
+{
+ int i;
+ for(i = 0; rewind_dirs[i] != NULL; i++){
+ recurse_dir(datadir, rewind_dirs[i], callback);
+ }
+}
+
/*
* recursive part of traverse_datadir
*
diff -r 699924885324 -r 8c43292e0012 src/bin/pg_rewind/fetch.c
--- a/src/bin/pg_rewind/fetch.c Sun Sep 03 11:12:29 2017 -0400
+++ b/src/bin/pg_rewind/fetch.c Fri Oct 27 18:47:42 2017 +0200
@@ -28,7 +28,7 @@
fetchSourceFileList(void)
{
if (datadir_source)
- traverse_datadir(datadir_source, &process_source_file);
+ traverse_rewinddirs(datadir_source, &process_source_file);
else
libpqProcessFileList();
}
diff -r 699924885324 -r 8c43292e0012 src/bin/pg_rewind/fetch.h
--- a/src/bin/pg_rewind/fetch.h Sun Sep 03 11:12:29 2017 -0400
+++ b/src/bin/pg_rewind/fetch.h Fri Oct 27 18:47:42 2017 +0200
@@ -36,9 +36,11 @@
extern XLogRecPtr libpqGetCurrentXlogInsertLocation(void);
/* in copy_fetch.c */
+extern char *rewind_dirs[];
extern void copy_executeFileMap(filemap_t *map);
typedef void (*process_file_callback_t) (const char *path, file_type_t type, size_t size, const char *link_target);
extern void traverse_datadir(const char *datadir, process_file_callback_t callback);
+extern void traverse_rewinddirs(const char *datadir, process_file_callback_t callback);
#endif /* FETCH_H */
diff -r 699924885324 -r 8c43292e0012 src/bin/pg_rewind/filemap.c
--- a/src/bin/pg_rewind/filemap.c Sun Sep 03 11:12:29 2017 -0400
+++ b/src/bin/pg_rewind/filemap.c Fri Oct 27 18:47:42 2017 +0200
@@ -95,6 +95,7 @@
if (strstr(path, "/" PG_TEMP_FILES_DIR "/") != NULL)
return;
+
/*
* sanity check: a filename that looks like a data file better be a
* regular file
diff -r 699924885324 -r 8c43292e0012 src/bin/pg_rewind/libpq_fetch.c
--- a/src/bin/pg_rewind/libpq_fetch.c Sun Sep 03 11:12:29 2017 -0400
+++ b/src/bin/pg_rewind/libpq_fetch.c Fri Oct 27 18:47:42 2017 +0200
@@ -150,7 +150,7 @@
{
PGresult *res;
const char *sql;
- int i;
+ int i, p;
/*
* Create a recursive directory listing of the whole data directory.
@@ -166,7 +166,8 @@
"WITH RECURSIVE files (path, filename, size, isdir) AS (\n"
" SELECT '' AS path, filename, size, isdir FROM\n"
" (SELECT pg_ls_dir('.', true, false) AS filename) AS fn,\n"
- " pg_stat_file(fn.filename, true) AS this\n"
+ " LATERAL pg_stat_file(fn.filename, true) AS this\n"
+ " WHERE filename = $1\n"
" UNION ALL\n"
" SELECT parent.path || parent.filename || '/' AS path,\n"
" fn, this.size, this.isdir\n"
@@ -180,44 +181,48 @@
"FROM files\n"
"LEFT OUTER JOIN pg_tablespace ON files.path = 'pg_tblspc/'\n"
" AND oid::text = files.filename\n";
- res = PQexec(conn, sql);
+ for (p = 0; rewind_dirs[p] != NULL; ++p)
+ {
+ const char *paths[1];
+ paths[0] = rewind_dirs[p];
+ res = PQexecParams(conn, sql, 1, NULL, paths, NULL, NULL, 0);
- if (PQresultStatus(res) != PGRES_TUPLES_OK)
- pg_fatal("could not fetch file list: %s",
+ if (PQresultStatus(res) != PGRES_TUPLES_OK)
+ pg_fatal("could not fetch file list: %s",
PQresultErrorMessage(res));
- /* sanity check the result set */
- if (PQnfields(res) != 4)
- pg_fatal("unexpected result set while fetching file list\n");
+ /* sanity check the result set */
+ if (PQnfields(res) != 4)
+ pg_fatal("unexpected result set while fetching file list\n");
- /* Read result to local variables */
- for (i = 0; i < PQntuples(res); i++)
- {
- char *path = PQgetvalue(res, i, 0);
- int64 filesize = atol(PQgetvalue(res, i, 1));
- bool isdir = (strcmp(PQgetvalue(res, i, 2), "t") == 0);
- char *link_target = PQgetvalue(res, i, 3);
- file_type_t type;
-
- if (PQgetisnull(res, 0, 1))
+ /* Read result to local variables */
+ for (i = 0; i < PQntuples(res); i++)
{
- /*
- * The file was removed from the server while the query was
- * running. Ignore it.
- */
- continue;
- }
+ char *path = PQgetvalue(res, i, 0);
+ int64 filesize = atol(PQgetvalue(res, i, 1));
+ bool isdir = (strcmp(PQgetvalue(res, i, 2), "t") == 0);
+ char *link_target = PQgetvalue(res, i, 3);
+ file_type_t type;
- if (link_target[0])
- type = FILE_TYPE_SYMLINK;
- else if (isdir)
- type = FILE_TYPE_DIRECTORY;
- else
- type = FILE_TYPE_REGULAR;
+ if (PQgetisnull(res, 0, 1))
+ {
+ /*
+ * The file was removed from the server while the query was
+ * running. Ignore it.
+ */
+ continue;
+ }
- process_source_file(path, type, filesize, link_target);
+ if (link_target[0])
+ type = FILE_TYPE_SYMLINK;
+ else if (isdir)
+ type = FILE_TYPE_DIRECTORY;
+ else
+ type = FILE_TYPE_REGULAR;
+ process_source_file(path, type, filesize, link_target);
+ }
+ PQclear(res);
}
- PQclear(res);
}
/*
diff -r 699924885324 -r 8c43292e0012 src/bin/pg_rewind/pg_rewind.c
--- a/src/bin/pg_rewind/pg_rewind.c Sun Sep 03 11:12:29 2017 -0400
+++ b/src/bin/pg_rewind/pg_rewind.c Fri Oct 27 18:47:42 2017 +0200
@@ -286,7 +286,7 @@
pg_log(PG_PROGRESS, "reading source file list\n");
fetchSourceFileList();
pg_log(PG_PROGRESS, "reading target file list\n");
- traverse_datadir(datadir_target, &process_target_file);
+ traverse_rewinddirs(datadir_target, &process_target_file);
/*
* Read the target WAL from last checkpoint before the point of fork, to
diff -r 699924885324 -r 8c43292e0012 src/bin/pg_rewind/t/001_basic.pl
--- a/src/bin/pg_rewind/t/001_basic.pl Sun Sep 03 11:12:29 2017 -0400
+++ b/src/bin/pg_rewind/t/001_basic.pl Fri Oct 27 18:47:42 2017 +0200
@@ -8,7 +8,6 @@
sub run_test
{
my $test_mode = shift;
-
RewindTest::setup_cluster();
RewindTest::start_master();
diff -r 699924885324 -r 8c43292e0012 src/bin/pg_rewind/t/003_extrafiles.pl
--- a/src/bin/pg_rewind/t/003_extrafiles.pl Sun Sep 03 11:12:29 2017 -0400
+++ b/src/bin/pg_rewind/t/003_extrafiles.pl Fri Oct 27 18:47:42 2017 +0200
@@ -71,12 +71,12 @@
"$test_master_datadir/tst_both_dir/both_file2",
"$test_master_datadir/tst_both_dir/both_subdir",
"$test_master_datadir/tst_both_dir/both_subdir/both_file3",
- "$test_master_datadir/tst_standby_dir",
- "$test_master_datadir/tst_standby_dir/standby_file1",
- "$test_master_datadir/tst_standby_dir/standby_file2",
- "$test_master_datadir/tst_standby_dir/standby_subdir",
-"$test_master_datadir/tst_standby_dir/standby_subdir/standby_file3" ],
- "file lists match");
+ "$test_master_datadir/tst_master_dir",
+ "$test_master_datadir/tst_master_dir/master_file1",
+ "$test_master_datadir/tst_master_dir/master_file2",
+ "$test_master_datadir/tst_master_dir/master_subdir",
+ "$test_master_datadir/tst_master_dir/master_subdir/master_file3", ],
+ "file lists match") or (diag("Files found:"), diag(explain(\@paths)));
RewindTest::clean_rewind_test();
}
diff -r 699924885324 -r 8c43292e0012 src/test/perl/TestLib.pm
--- a/src/test/perl/TestLib.pm Sun Sep 03 11:12:29 2017 -0400
+++ b/src/test/perl/TestLib.pm Fri Oct 27 18:47:42 2017 +0200
@@ -10,6 +10,7 @@
use strict;
use warnings;
+use Carp::Always;
use Config;
use Exporter 'import';
use File::Basename;