From b70fbdccb84e98c128728b79bc85e94b802645a7 Mon Sep 17 00:00:00 2001 From: Sergey Levin Date: Thu, 5 Mar 2026 08:33:07 +0500 Subject: [PATCH v10] Migration of the pg_commit_ts directory --- doc/src/sgml/logical-replication.sgml | 8 +-- src/bin/pg_upgrade/check.c | 24 +++++++ src/bin/pg_upgrade/controldata.c | 20 ++++++ src/bin/pg_upgrade/meson.build | 1 + src/bin/pg_upgrade/pg_upgrade.c | 40 ++++++++--- src/bin/pg_upgrade/pg_upgrade.h | 2 + .../pg_upgrade/t/008_transfer_commit_ts.pl | 67 +++++++++++++++++++ 7 files changed, 148 insertions(+), 14 deletions(-) create mode 100644 src/bin/pg_upgrade/t/008_transfer_commit_ts.pl diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index bcb473c078b..0468ac36eef 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -2818,11 +2818,11 @@ CONTEXT: processing remote data for replication origin "pg_16395" during "INSER - Commit timestamps and origin data are not preserved during the upgrade. - As a result, even if + Parameters for physical replication slots are not preserved during the + upgrade. As a result, even if retain_dead_tuples - is enabled, the upgraded subscriber may be unable to detect conflicts or - log relevant commit timestamps and origins when applying changes from the + is enabled, the upgraded subscriber may be unable to detect + update_deleted conflicts when applying changes from the publisher occurred before the upgrade. Additionally, immediately after the upgrade, the vacuum may remove the deleted rows that are required for conflict detection. This can affect the changes that were not replicated diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index eb35c68d450..8e35057492f 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -34,6 +34,7 @@ static void check_new_cluster_replication_slots(void); static void check_new_cluster_subscription_configuration(void); static void check_old_cluster_for_valid_slots(void); static void check_old_cluster_subscription_state(void); +static void check_new_cluster_pg_commit_ts(void); static void check_old_cluster_global_names(ClusterInfo *cluster); /* @@ -791,9 +792,32 @@ check_new_cluster(void) check_new_cluster_replication_slots(); check_new_cluster_subscription_configuration(); + + check_new_cluster_pg_commit_ts(); + } +void +check_new_cluster_pg_commit_ts(void) +{ + PGconn *conn; + PGresult *res; + bool commit_ts_is_enabled; + + prep_status("Checking for new cluster configuration for commit timestamp"); + conn = connectToServer(&new_cluster, "template1"); + res = executeQueryOrDie(conn, "SELECT setting FROM pg_settings " + "WHERE name = 'track_commit_timestamp'"); + commit_ts_is_enabled = strcmp(PQgetvalue(res, 0, 0), "on") == 0; + PQclear(res); + PQfinish(conn); + if (!commit_ts_is_enabled && + old_cluster.controldata.chkpnt_newstCommitTsxid > 0) + pg_fatal("\"track_commit_timestamp\" must be \"on\" but is set to \"off\""); + + check_ok(); +} void report_clusters_compatible(void) { diff --git a/src/bin/pg_upgrade/controldata.c b/src/bin/pg_upgrade/controldata.c index aa6e8b4de5d..fa8b28adf43 100644 --- a/src/bin/pg_upgrade/controldata.c +++ b/src/bin/pg_upgrade/controldata.c @@ -321,6 +321,26 @@ get_control_data(ClusterInfo *cluster) cluster->controldata.chkpnt_nxtmulti = str2uint(p); got_multi = true; } + else if ((p = strstr(bufin, "Latest checkpoint's oldestCommitTsXid:")) != NULL) + { + p = strchr(p, ':'); + + if (p == NULL || strlen(p) <= 1) + pg_fatal("%d: controldata retrieval problem", __LINE__); + + p++; /* remove ':' char */ + cluster->controldata.chkpnt_oldstCommitTsxid = str2uint(p); + } + else if ((p = strstr(bufin, "Latest checkpoint's newestCommitTsXid:")) != NULL) + { + p = strchr(p, ':'); + + if (p == NULL || strlen(p) <= 1) + pg_fatal("%d: controldata retrieval problem", __LINE__); + + p++; /* remove ':' char */ + cluster->controldata.chkpnt_newstCommitTsxid = str2uint(p); + } else if ((p = strstr(bufin, "Latest checkpoint's oldestXID:")) != NULL) { p = strchr(p, ':'); diff --git a/src/bin/pg_upgrade/meson.build b/src/bin/pg_upgrade/meson.build index 49b1b624f25..b27477c3f8a 100644 --- a/src/bin/pg_upgrade/meson.build +++ b/src/bin/pg_upgrade/meson.build @@ -51,6 +51,7 @@ tests += { 't/005_char_signedness.pl', 't/006_transfer_modes.pl', 't/007_multixact_conversion.pl', + 't/008_transfer_commit_ts.pl', ], 'test_kwargs': {'priority': 40}, # pg_upgrade tests are slow }, diff --git a/src/bin/pg_upgrade/pg_upgrade.c b/src/bin/pg_upgrade/pg_upgrade.c index 2127d297bfe..456719383dc 100644 --- a/src/bin/pg_upgrade/pg_upgrade.c +++ b/src/bin/pg_upgrade/pg_upgrade.c @@ -216,13 +216,11 @@ main(int argc, char **argv) * as it only retains the dead tuples. It is created here for consistency. * Note that the new conflict detection slot uses the latest transaction * ID as xmin, so it cannot protect dead tuples that existed before the - * upgrade. Additionally, commit timestamps and origin data are not - * preserved during the upgrade. So, even after creating the slot, the - * upgraded subscriber may be unable to detect conflicts or log relevant - * commit timestamps and origins when applying changes from the publisher - * occurred before the upgrade especially if those changes were not - * replicated. It can only protect tuples that might be deleted after the - * new cluster starts. + * upgrade. It means even after creating the slot, the upgraded subscriber + * may be unable to detect update_deleted conflicts when applying changes + * from the publisher occurred before the upgrade especially if those + * changes were not replicated. It can only protect tuples that might be + * deleted after the new cluster starts. */ if (migrate_logical_slots || old_cluster.sub_retain_dead_tuples) { @@ -773,6 +771,9 @@ copy_subdir_files(const char *old_subdir, const char *new_subdir) static void copy_xact_xlog_xid(void) { + bool is_copy_commit_ts; + uint32 oldest_xid, newest_xid; + /* * Copy old commit logs to new data dir. pg_clog has been renamed to * pg_xact in post-10 clusters. @@ -782,6 +783,22 @@ copy_xact_xlog_xid(void) GET_MAJOR_VERSION(new_cluster.major_version) <= 906 ? "pg_clog" : "pg_xact"); + /* + * Copy old commit_timestamp data to new, if available. + */ + is_copy_commit_ts = + (old_cluster.controldata.chkpnt_oldstCommitTsxid > 0 && + old_cluster.controldata.chkpnt_newstCommitTsxid > 0); + + if (is_copy_commit_ts) + { + copy_subdir_files("pg_commit_ts", "pg_commit_ts"); + oldest_xid = old_cluster.controldata.chkpnt_oldstCommitTsxid; + newest_xid = old_cluster.controldata.chkpnt_newstCommitTsxid; + } + else + oldest_xid = newest_xid = old_cluster.controldata.chkpnt_nxtxid; + prep_status("Setting oldest XID for new cluster"); exec_prog(UTILITY_LOG_FILE, NULL, true, true, "\"%s/pg_resetwal\" -f -u %u \"%s\"", @@ -799,12 +816,15 @@ copy_xact_xlog_xid(void) "\"%s/pg_resetwal\" -f -e %u \"%s\"", new_cluster.bindir, old_cluster.controldata.chkpnt_nxtepoch, new_cluster.pgdata); - /* must reset commit timestamp limits also */ + + /* + * must reset commit timestamp limits also or copy from the old cluster + */ exec_prog(UTILITY_LOG_FILE, NULL, true, true, "\"%s/pg_resetwal\" -f -c %u,%u \"%s\"", new_cluster.bindir, - old_cluster.controldata.chkpnt_nxtxid, - old_cluster.controldata.chkpnt_nxtxid, + oldest_xid, + newest_xid, new_cluster.pgdata); check_ok(); diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index 1d767bbda2d..1c19a22f844 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -245,6 +245,8 @@ typedef struct uint64 chkpnt_nxtmxoff; uint32 chkpnt_oldstMulti; uint32 chkpnt_oldstxid; + uint32 chkpnt_oldstCommitTsxid; + uint32 chkpnt_newstCommitTsxid; uint32 align; uint32 blocksz; uint32 largesz; diff --git a/src/bin/pg_upgrade/t/008_transfer_commit_ts.pl b/src/bin/pg_upgrade/t/008_transfer_commit_ts.pl new file mode 100644 index 00000000000..b5e104ef23c --- /dev/null +++ b/src/bin/pg_upgrade/t/008_transfer_commit_ts.pl @@ -0,0 +1,67 @@ +# Copyright (c) 2025-2026, PostgreSQL Global Development Group + +# Tests for transfer pg_commit_ts directory. + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Can be changed to test the other modes +my $mode = $ENV{PG_TEST_PG_UPGRADE_MODE} || '--copy'; + +# Initialize old cluster +my $old = PostgreSQL::Test::Cluster->new('old'); +$old->init; +$old->append_conf('postgresql.conf', 'track_commit_timestamp = on'); +$old->start; +my $resold = $old->safe_psql( + 'postgres', qq{ + create table a(a int); + select xid,timestamp from pg_last_committed_xact(); +}); + +my ($xid) = $resold =~ /\s*(\d+)\s*\|.*/; +$old->stop; + +# Initialize new cluster +my $new = PostgreSQL::Test::Cluster->new('new'); +$new->init; + +# Setup a common pg_upgrade command to be used by all the test cases +my @pg_upgrade_cmd = ( + 'pg_upgrade', '--no-sync', + '--old-datadir' => $old->data_dir, + '--new-datadir' => $new->data_dir, + '--old-bindir' => $old->config_data('--bindir'), + '--new-bindir' => $new->config_data('--bindir'), + '--socketdir' => $new->host, + '--old-port' => $old->port, + '--new-port' => $new->port, + $mode); + +# In a VPATH build, we'll be started in the source directory, but we want +# to run pg_upgrade in the build directory so that any files generated finish +# in it, like delete_old_cluster.{sh,bat}. +chdir ${PostgreSQL::Test::Utils::tmp_check}; + +command_checks_all( + [@pg_upgrade_cmd], 1, + [qr{"track_commit_timestamp" must be "on" but is set to "off"}], [], + 'run of pg_upgrade for mismatch parameter track_commit_timestamp'); + +$new->append_conf('postgresql.conf', 'track_commit_timestamp = on'); + +command_ok([@pg_upgrade_cmd], 'run of pg_upgrade ok'); + +$new->start; +my $resnew = $new->safe_psql( + 'postgres', qq{ + select $xid,pg_xact_commit_timestamp(${xid}::text::xid); +}); +$new->stop; +ok($resold eq $resnew, "timestamp transferred successfully"); + +done_testing(); -- 2.50.1