From d713a32499802395639645412a7c605870280f3a Mon Sep 17 00:00:00 2001 From: Koval Dmitry Date: Mon, 14 Nov 2022 21:39:14 +0300 Subject: [PATCH v4] Operation log --- src/backend/access/transam/xlog.c | 10 + src/backend/backup/basebackup.c | 1 + src/backend/storage/lmgr/lwlocknames.txt | 1 + src/backend/utils/misc/Makefile | 1 + src/backend/utils/misc/meson.build | 1 + src/backend/utils/misc/pg_controllog.c | 142 ++++ src/bin/pg_checksums/pg_checksums.c | 1 + src/bin/pg_resetwal/pg_resetwal.c | 4 + src/bin/pg_rewind/file_ops.c | 28 + src/bin/pg_rewind/file_ops.h | 2 + src/bin/pg_rewind/pg_rewind.c | 59 ++ src/bin/pg_upgrade/controldata.c | 52 ++ src/bin/pg_upgrade/exec.c | 9 +- src/bin/pg_upgrade/pg_upgrade.c | 2 + src/bin/pg_upgrade/pg_upgrade.h | 2 + src/common/Makefile | 1 + src/common/controllog_utils.c | 667 ++++++++++++++++++ src/common/meson.build | 1 + src/include/catalog/pg_controllog.h | 142 ++++ src/include/catalog/pg_proc.dat | 9 + src/include/common/controllog_utils.h | 27 + src/test/modules/test_misc/meson.build | 1 + .../modules/test_misc/t/004_operation_log.pl | 136 ++++ src/tools/msvc/Mkvcbuild.pm | 4 +- 24 files changed, 1298 insertions(+), 5 deletions(-) create mode 100644 src/backend/utils/misc/pg_controllog.c create mode 100644 src/common/controllog_utils.c create mode 100644 src/include/catalog/pg_controllog.h create mode 100644 src/include/common/controllog_utils.h create mode 100644 src/test/modules/test_misc/t/004_operation_log.pl diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8f47fb7570..dd3c4c7ac4 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -68,6 +68,7 @@ #include "catalog/pg_control.h" #include "catalog/pg_database.h" #include "common/controldata_utils.h" +#include "common/controllog_utils.h" #include "common/file_utils.h" #include "executor/instrument.h" #include "miscadmin.h" @@ -4775,6 +4776,9 @@ BootStrapXLOG(void) /* some additional ControlFile fields are set in WriteControlFile() */ WriteControlFile(); + /* Put information into operation log. */ + put_operation_log_element(DataDir, OLT_BOOTSTRAP); + /* Bootstrap the commit log, too */ BootStrapCLOG(); BootStrapCommitTs(); @@ -5743,8 +5747,14 @@ StartupXLOG(void) SpinLockRelease(&XLogCtl->info_lck); UpdateControlFile(); + LWLockRelease(ControlFileLock); + /* Put information into operation log. */ + if (promoted) + put_operation_log_element(DataDir, OLT_PROMOTED); + put_operation_log_element(DataDir, OLT_STARTUP); + /* * Shutdown the recovery environment. This must occur after * RecoverPreparedTransactions() (see notes in lock_twophase_recover()) diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c index 3fb9451643..0ca709b5b2 100644 --- a/src/backend/backup/basebackup.c +++ b/src/backend/backup/basebackup.c @@ -211,6 +211,7 @@ static const struct exclude_list_item excludeFiles[] = */ static const struct exclude_list_item noChecksumFiles[] = { {"pg_control", false}, + {"pg_control_log", false}, {"pg_filenode.map", false}, {"pg_internal.init", true}, {"PG_VERSION", false}, diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt index 6c7cf6c295..5673de1669 100644 --- a/src/backend/storage/lmgr/lwlocknames.txt +++ b/src/backend/storage/lmgr/lwlocknames.txt @@ -53,3 +53,4 @@ XactTruncationLock 44 # 45 was XactTruncationLock until removal of BackendRandomLock WrapLimitsVacuumLock 46 NotifyQueueTailLock 47 +ControlLogFileLock 48 diff --git a/src/backend/utils/misc/Makefile b/src/backend/utils/misc/Makefile index b9ee4eb48a..3fa20e0368 100644 --- a/src/backend/utils/misc/Makefile +++ b/src/backend/utils/misc/Makefile @@ -23,6 +23,7 @@ OBJS = \ help_config.o \ pg_config.o \ pg_controldata.o \ + pg_controllog.o \ pg_rusage.o \ ps_status.o \ queryenvironment.o \ diff --git a/src/backend/utils/misc/meson.build b/src/backend/utils/misc/meson.build index e3e99ec5cb..9932aa637d 100644 --- a/src/backend/utils/misc/meson.build +++ b/src/backend/utils/misc/meson.build @@ -8,6 +8,7 @@ backend_sources += files( 'help_config.c', 'pg_config.c', 'pg_controldata.c', + 'pg_controllog.c', 'pg_rusage.c', 'ps_status.c', 'queryenvironment.c', diff --git a/src/backend/utils/misc/pg_controllog.c b/src/backend/utils/misc/pg_controllog.c new file mode 100644 index 0000000000..c47c3bf37f --- /dev/null +++ b/src/backend/utils/misc/pg_controllog.c @@ -0,0 +1,142 @@ +/*------------------------------------------------------------------------- + * + * pg_controllog.c + * + * Routines to expose the contents of the control log file via a set of SQL + * functions. + * + * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/utils/misc/pg_controllog.c + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/htup_details.h" +#include "access/transam.h" +#include "access/xlog.h" +#include "access/xlog_internal.h" +#include "catalog/pg_type.h" +#include "common/controllog_utils.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "utils/builtins.h" +#include "utils/pg_lsn.h" +#include "utils/timestamp.h" + +/* + * pg_operation_log + * + * Returns list of operation log data. + * NOTE: this is a set-returning-function (SRF). + */ +Datum +pg_operation_log(PG_FUNCTION_ARGS) +{ +#define PG_OPERATION_LOG_COLS 6 + FuncCallContext *funcctx; + OperationLogBuffer *log_buffer; + + /* + * Initialize tuple descriptor & function call context. + */ + if (SRF_IS_FIRSTCALL()) + { + MemoryContext oldcxt; + TupleDesc tupdesc; + bool crc_ok; + + /* create a function context for cross-call persistence */ + funcctx = SRF_FIRSTCALL_INIT(); + + /* switch to memory context appropriate for multiple function calls */ + oldcxt = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + + /* read the control file */ + log_buffer = get_operation_log(DataDir, &crc_ok); + if (!crc_ok) + ereport(ERROR, + (errmsg("calculated CRC checksum does not match value stored in file"))); + + tupdesc = CreateTemplateTupleDesc(PG_OPERATION_LOG_COLS); + + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "event", + TEXTOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 2, "edition", + TEXTOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "version", + TEXTOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "lsn", + PG_LSNOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "last", + TIMESTAMPTZOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "count", + INT4OID, -1, 0); + + funcctx->tuple_desc = BlessTupleDesc(tupdesc); + + /* The only state we need is the operation log buffer. */ + funcctx->user_fctx = (void *) log_buffer; + + MemoryContextSwitchTo(oldcxt); + } + + /* stuff done on every call of the function */ + funcctx = SRF_PERCALL_SETUP(); + log_buffer = (OperationLogBuffer *) funcctx->user_fctx; + + if (funcctx->call_cntr < get_operation_log_count(log_buffer)) + { + Datum result; + Datum values[PG_OPERATION_LOG_COLS]; + bool nulls[PG_OPERATION_LOG_COLS]; + HeapTuple tuple; + OperationLogData *data = get_operation_log_element(log_buffer, (uint16) funcctx->call_cntr); + int major_version, + minor_version, + patch_version; + + /* + * Form tuple with appropriate data. + */ + MemSet(nulls, 0, sizeof(nulls)); + MemSet(values, 0, sizeof(values)); + + /* event */ + values[0] = CStringGetTextDatum(get_operation_log_type_name(data->ol_type)); + + /* edition */ + values[1] = CStringGetTextDatum(get_str_edition(data->ol_edition)); + + /* version */ + patch_version = data->ol_version % 100; + minor_version = (data->ol_version / 100) % 100; + major_version = data->ol_version / 10000; + if (major_version < 1000) + values[2] = CStringGetTextDatum(psprintf("%u.%u.%u.%u", major_version / 100, + major_version % 100, + minor_version, patch_version)); + else + values[2] = CStringGetTextDatum(psprintf("%u.%u.%u", major_version / 100, + minor_version, patch_version)); + + /* lsn */ + values[3] = LSNGetDatum(data->ol_lsn); + + /* last */ + values[4] = TimestampTzGetDatum(time_t_to_timestamptz(data->ol_timestamp)); + + /* count */ + values[5] = Int32GetDatum(data->ol_count); + + tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls); + result = HeapTupleGetDatum(tuple); + SRF_RETURN_NEXT(funcctx, result); + } + + /* done when there are no more elements left */ + SRF_RETURN_DONE(funcctx); +} diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c index aa21007497..32122db023 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -115,6 +115,7 @@ struct exclude_list_item */ static const struct exclude_list_item skip[] = { {"pg_control", false}, + {"pg_control_log", false}, {"pg_filenode.map", false}, {"pg_internal.init", true}, {"PG_VERSION", false}, diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c index e7ef2b8bd0..5cd3fc29fb 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c @@ -50,6 +50,7 @@ #include "access/xlog.h" #include "access/xlog_internal.h" #include "common/controldata_utils.h" +#include "common/controllog_utils.h" #include "common/fe_memutils.h" #include "common/file_perm.h" #include "common/logging.h" @@ -885,6 +886,9 @@ RewriteControlFile(void) /* The control file gets flushed here. */ update_controlfile(".", &ControlFile, true); + + /* Put information into operation log. */ + put_operation_log_element(".", OLT_RESETWAL); } diff --git a/src/bin/pg_rewind/file_ops.c b/src/bin/pg_rewind/file_ops.c index 25996b4da4..fb6dc73309 100644 --- a/src/bin/pg_rewind/file_ops.c +++ b/src/bin/pg_rewind/file_ops.c @@ -354,6 +354,34 @@ slurpFile(const char *datadir, const char *path, size_t *filesize) return buffer; } +/* + * Try to open file. + * Returns true if file exists. + */ +bool +check_file_exists(const char *datadir, const char *path) +{ + char fullpath[MAXPGPATH]; + int fd; + + snprintf(fullpath, sizeof(fullpath), "%s/%s", datadir, path); + + errno = 0; + if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) == -1) + { + /* File doesn't exist */ + if (errno == ENOENT) + return false; + + pg_fatal("could not open file \"%s\" for reading: %m", + fullpath); + } + + close(fd); + + return true; +} + /* * Traverse through all files in a data directory, calling 'callback' * for each file. diff --git a/src/bin/pg_rewind/file_ops.h b/src/bin/pg_rewind/file_ops.h index 427cf8e0b5..904c697a9c 100644 --- a/src/bin/pg_rewind/file_ops.h +++ b/src/bin/pg_rewind/file_ops.h @@ -26,4 +26,6 @@ extern char *slurpFile(const char *datadir, const char *path, size_t *filesize); typedef void (*process_file_callback_t) (const char *path, file_type_t type, size_t size, const char *link_target); extern void traverse_datadir(const char *datadir, process_file_callback_t callback); +extern bool check_file_exists(const char *datadir, const char *path); + #endif /* FILE_OPS_H */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 858d8d9f2f..eb11531de1 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -19,6 +19,7 @@ #include "catalog/catversion.h" #include "catalog/pg_control.h" #include "common/controldata_utils.h" +#include "common/controllog_utils.h" #include "common/file_perm.h" #include "common/restricted_token.h" #include "common/string.h" @@ -43,6 +44,8 @@ static void createBackupLabel(XLogRecPtr startpoint, TimeLineID starttli, static void digestControlFile(ControlFileData *ControlFile, const char *content, size_t size); +static void digestOperationLog(OperationLogBuffer * LogBuffer, + const char *content, size_t size); static void getRestoreCommand(const char *argv0); static void sanityChecks(void); static void findCommonAncestorTimeline(XLogRecPtr *recptr, int *tliIndex); @@ -53,6 +56,8 @@ static ControlFileData ControlFile_target; static ControlFileData ControlFile_source; static ControlFileData ControlFile_source_after; +static OperationLogBuffer OperationLog_target = {0}; + const char *progname; int WalSegSz; @@ -330,6 +335,15 @@ main(int argc, char **argv) digestControlFile(&ControlFile_source, buffer, size); pg_free(buffer); + /* Read target operation log for prevent rewriting it. */ + if (check_file_exists(datadir_target, PG_OPERATION_LOG_FILE)) + { + buffer = slurpFile(datadir_target, PG_OPERATION_LOG_FILE, &size); + digestOperationLog(&OperationLog_target, buffer, size); + pg_free(buffer); + } + /* Otherwise we have OperationLog_target with zeros. */ + sanityChecks(); /* @@ -672,7 +686,14 @@ perform_rewind(filemap_t *filemap, rewind_source *source, ControlFile_new.minRecoveryPointTLI = endtli; ControlFile_new.state = DB_IN_ARCHIVE_RECOVERY; if (!dry_run) + { update_controlfile(datadir_target, &ControlFile_new, do_sync); + + /* Restore saved operation log. */ + update_operation_log(datadir_target, &OperationLog_target); + /* Put information about "pg_rewind" into operation log. */ + put_operation_log_element(datadir_target, OLT_REWIND); + } } static void @@ -1009,6 +1030,44 @@ digestControlFile(ControlFileData *ControlFile, const char *content, checkControlFile(ControlFile); } +/* + * Check CRC of operation log buffer + */ +static void +checkOperationLogBuffer(OperationLogBuffer * LogBuffer) +{ + pg_crc32c crc; + + /* Calculate CRC */ + INIT_CRC32C(crc); + COMP_CRC32C(crc, + (char *) LogBuffer + sizeof(pg_crc32c), + PG_OPERATION_LOG_FULL_SIZE - sizeof(pg_crc32c)); + FIN_CRC32C(crc); + + /* And simply compare it */ + if (!EQ_CRC32C(crc, LogBuffer->header.ol_crc)) + pg_fatal("unexpected operation log CRC"); +} + +/* + * Verify operation log contents in the buffer 'content', and copy it to + * *LogBuffer. + */ +static void +digestOperationLog(OperationLogBuffer * LogBuffer, const char *content, + size_t size) +{ + if (size != PG_OPERATION_LOG_FULL_SIZE) + pg_fatal("unexpected operation log size %d, expected %d", + (int) size, PG_OPERATION_LOG_FULL_SIZE); + + memcpy(LogBuffer, content, size); + + /* Additional checks on operation log */ + checkOperationLogBuffer(LogBuffer); +} + /* * Get value of GUC parameter restore_command from the target cluster. * diff --git a/src/bin/pg_upgrade/controldata.c b/src/bin/pg_upgrade/controldata.c index 9071a6fd45..f991ef5133 100644 --- a/src/bin/pg_upgrade/controldata.c +++ b/src/bin/pg_upgrade/controldata.c @@ -14,6 +14,7 @@ #include "pg_upgrade.h" #include "common/string.h" +#include "common/controllog_utils.h" /* * get_control_data() @@ -731,3 +732,54 @@ disable_old_cluster(void) "started once the new cluster has been started.", old_cluster.pgdata); } + + +/* + * copy_operation_log() + * + * Copy operation log from the old cluster to the new cluster and put info + * about upgrade. If operation log not exists in the old cluster then put + * startup message with version info of old cluster. + */ +void +copy_operation_log(void) +{ + OperationLogBuffer *log_buffer; + bool log_is_empty; + ClusterInfo *cluster; + bool crc_ok; + + /* Read operation log from the old cluster. */ + log_buffer = get_operation_log(old_cluster.pgdata, &crc_ok); + if (!crc_ok) + pg_fatal("pg_control operation log CRC value is incorrect"); + + /* + * Check operation log records in the old cluster. Need to put information + * about old version in case operation log is empty. + */ + log_is_empty = (get_operation_log_count(log_buffer) == 0); + + if (user_opts.transfer_mode == TRANSFER_MODE_LINK) + cluster = &old_cluster; + else + { + cluster = &new_cluster; + + /* Place operation log in the new cluster. */ + update_operation_log(cluster->pgdata, log_buffer); + } + + pfree(log_buffer); + + /* Put information about the old cluster if needed. */ + if (log_is_empty) + put_operation_log_element_version(cluster->pgdata, OLT_STARTUP, + ED_PG_ORIGINAL, + old_cluster.bin_version_num); + + /* + * Put information about upgrade in the operation log of the old cluster. + */ + put_operation_log_element(cluster->pgdata, OLT_UPGRADE); +} diff --git a/src/bin/pg_upgrade/exec.c b/src/bin/pg_upgrade/exec.c index 5b2edebe41..6728a617de 100644 --- a/src/bin/pg_upgrade/exec.c +++ b/src/bin/pg_upgrade/exec.c @@ -37,7 +37,8 @@ get_bin_version(ClusterInfo *cluster) FILE *output; int rc; int v1 = 0, - v2 = 0; + v2 = 0, + v3 = 0; snprintf(cmd, sizeof(cmd), "\"%s/pg_ctl\" --version", cluster->bindir); fflush(NULL); @@ -52,18 +53,20 @@ get_bin_version(ClusterInfo *cluster) pg_fatal("could not get pg_ctl version data using %s: %s", cmd, wait_result_to_str(rc)); - if (sscanf(cmd_output, "%*s %*s %d.%d", &v1, &v2) < 1) - pg_fatal("could not get pg_ctl version output from %s", cmd); + if (sscanf(cmd_output, "%*s %*s %d.%d.%d", &v1, &v2, &v3) < 1) + pg_fatal("could not get pg_ctl version output from %s\n", cmd); if (v1 < 10) { /* old style, e.g. 9.6.1 */ cluster->bin_version = v1 * 10000 + v2 * 100; + cluster->bin_version_num = (cluster->bin_version + v3) * 100; } else { /* new style, e.g. 10.1 */ cluster->bin_version = v1 * 10000; + cluster->bin_version_num = (cluster->bin_version + v2) * 100; } } diff --git a/src/bin/pg_upgrade/pg_upgrade.c b/src/bin/pg_upgrade/pg_upgrade.c index e5597d3105..df6258ab03 100644 --- a/src/bin/pg_upgrade/pg_upgrade.c +++ b/src/bin/pg_upgrade/pg_upgrade.c @@ -194,6 +194,8 @@ main(int argc, char **argv) check_ok(); } + copy_operation_log(); + create_script_for_old_cluster_deletion(&deletion_script_file_name); issue_warnings_and_set_wal_level(); diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index 5f2a116f23..a6dda7b0c3 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -264,6 +264,7 @@ typedef struct uint32 major_version; /* PG_VERSION of cluster */ char major_version_str[64]; /* string PG_VERSION of cluster */ uint32 bin_version; /* version returned from pg_ctl */ + uint32 bin_version_num; /* full version (incl. minor part) returned from pg_ctl */ const char *tablespace_suffix; /* directory specification */ } ClusterInfo; @@ -348,6 +349,7 @@ void create_script_for_old_cluster_deletion(char **deletion_script_file_name); void get_control_data(ClusterInfo *cluster, bool live_check); void check_control_data(ControlData *oldctrl, ControlData *newctrl); void disable_old_cluster(void); +void copy_operation_log(void); /* dump.c */ diff --git a/src/common/Makefile b/src/common/Makefile index 2f424a5735..aeac856045 100644 --- a/src/common/Makefile +++ b/src/common/Makefile @@ -51,6 +51,7 @@ OBJS_COMMON = \ compression.o \ config_info.o \ controldata_utils.o \ + controllog_utils.o \ d2s.o \ encnames.o \ exec.o \ diff --git a/src/common/controllog_utils.c b/src/common/controllog_utils.c new file mode 100644 index 0000000000..d8876d0914 --- /dev/null +++ b/src/common/controllog_utils.c @@ -0,0 +1,667 @@ +/*------------------------------------------------------------------------- + * + * controllog_utils.c + * Common code for operation log file output. + * + * + * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/common/controllog_utils.c + * + *------------------------------------------------------------------------- + */ + +#ifndef FRONTEND +#include "postgres.h" +#else +#include "postgres_fe.h" +#endif + +#include +#include +#include +#include + +#include "access/xlog_internal.h" +#include "catalog/pg_control.h" +#include "catalog/pg_controllog.h" +#include "common/controldata_utils.h" +#include "common/controllog_utils.h" +#include "common/file_perm.h" +#ifdef FRONTEND +#include "common/file_utils.h" +#include "common/logging.h" +#endif +#include "port/pg_crc32c.h" + +#ifndef FRONTEND +#include "pgstat.h" +#include "storage/fd.h" +#include "storage/lwlock.h" +#endif + +/* + * Descriptions of supported operations of operation log. + */ +OperationLogTypeDesc OperationLogTypesDescs[] = { + {OLT_BOOTSTRAP, OLM_INSERT, "bootstrap"}, + {OLT_STARTUP, OLM_MERGE, "startup"}, + {OLT_RESETWAL, OLM_MERGE, "pg_resetwal"}, + {OLT_REWIND, OLM_MERGE, "pg_rewind"}, + {OLT_UPGRADE, OLM_INSERT, "pg_upgrade"}, + {OLT_PROMOTED, OLM_INSERT, "promoted"} +}; + + +/* + * calculate_operation_log_crc() + * + * Calculate CRC of operation log. + */ +static pg_crc32c +calculate_operation_log_crc(OperationLogBuffer * log_buffer) +{ + pg_crc32c crc; + + INIT_CRC32C(crc); + COMP_CRC32C(crc, + (char *) log_buffer + sizeof(pg_crc32c), + PG_OPERATION_LOG_FULL_SIZE - sizeof(pg_crc32c)); + FIN_CRC32C(crc); + + return crc; +} + +/* + * get_empty_operation_log() + * + * Function returns empty operation log buffer. + */ +OperationLogBuffer * +get_empty_operation_log_buffer(void) +{ + OperationLogBuffer *log_buffer; + + /* Initialize operation log file with zeros. */ + log_buffer = palloc0(PG_OPERATION_LOG_FULL_SIZE); + + /* Calculate CRC. */ + log_buffer->header.ol_crc = calculate_operation_log_crc(log_buffer); + + return log_buffer; +} + +/* + * create_operation_log_file() + * + * Create file for operation log and initialize it with zeros. + * Function returns descriptor of created file or -1 in error case. + * Function cannot generate report with ERROR and FATAL for correct lock + * releasing on top level. + */ +static int +create_operation_log_file(char *ControlLogFilePath) +{ + int fd; + OperationLogBuffer *log_buffer; + +#ifndef FRONTEND + fd = OpenTransientFile(ControlLogFilePath, O_RDWR | O_CREAT | PG_BINARY); + + if (fd < 0) + { + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not create file \"%s\": %m", + PG_OPERATION_LOG_FILE))); + return -1; + } +#else + fd = open(ControlLogFilePath, O_RDWR | O_CREAT | PG_BINARY, pg_file_create_mode); + + if (fd < 0) + pg_fatal("could not create file \"%s\": %m", + ControlLogFilePath); +#endif + + /* Initialize operation log file with zeros. */ + log_buffer = get_empty_operation_log_buffer(); + + errno = 0; + if (write(fd, log_buffer, PG_OPERATION_LOG_FULL_SIZE) != PG_OPERATION_LOG_FULL_SIZE) + { + /* If write didn't set errno, assume problem is no disk space. */ + if (errno == 0) + errno = ENOSPC; + +#ifndef FRONTEND + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not write operation log in the file \"%s\": %m", + ControlLogFilePath))); +#else + pg_fatal("could not write operation log in the file \"%s\": %m", + ControlLogFilePath); +#endif + pfree(log_buffer); + return -1; + } + + pfree(log_buffer); + +#ifndef FRONTEND + if (pg_fsync(fd) != 0) + { + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", + ControlLogFilePath))); + return -1; + } +#else + if (fsync(fd) != 0) + pg_fatal("could not fsync file \"%s\": %m", ControlLogFilePath); +#endif + + if (lseek(fd, 0, SEEK_SET) != 0) + { +#ifndef FRONTEND + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not seek to position 0 of file \"%s\": %m", + ControlLogFilePath))); +#else + pg_fatal("could not seek to position 0 of file \"%s\": %m", + ControlLogFilePath); +#endif + return -1; + } + + return fd; +} + +#define LWLockReleaseSaveErrno(lock) \ + save_errno = errno; \ + LWLockRelease(lock); \ + errno = save_errno; \ + +/* + * get_operation_log() + * + * Get the operation log ring buffer. The result is returned as a palloc'd copy + * of operation log buffer. + * + * crc_ok_p can be used by the caller to see whether the CRC of the operation + * log is correct. + */ +OperationLogBuffer * +get_operation_log(const char *DataDir, bool *crc_ok_p) +{ + OperationLogBuffer *log_buffer = NULL; + int fd; + char ControlLogFilePath[MAXPGPATH]; + pg_crc32c crc; + int r; +#ifndef FRONTEND + int save_errno; +#endif + + Assert(crc_ok_p); + + snprintf(ControlLogFilePath, MAXPGPATH, "%s/%s", DataDir, PG_OPERATION_LOG_FILE); + +#ifndef FRONTEND + LWLockAcquire(ControlLogFileLock, LW_EXCLUSIVE); + fd = OpenTransientFile(ControlLogFilePath, O_RDONLY | PG_BINARY); +#else + fd = open(ControlLogFilePath, O_RDONLY | PG_BINARY, 0); +#endif + if (fd < 0) + { +#ifndef FRONTEND + LWLockReleaseSaveErrno(ControlLogFileLock); +#endif + /* File doesn't exist - return empty operation log buffer. */ + if (errno == ENOENT) + { + *crc_ok_p = true; /* CRC will be calculated below. */ + return get_empty_operation_log_buffer(); + } + else + { +#ifndef FRONTEND + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\" for reading: %m", + ControlLogFilePath))); +#else + pg_fatal("could not open file \"%s\" for reading: %m", + ControlLogFilePath); +#endif + } + } + + log_buffer = palloc(PG_OPERATION_LOG_FULL_SIZE); + + r = read(fd, log_buffer, PG_OPERATION_LOG_FULL_SIZE); + if (r != PG_OPERATION_LOG_FULL_SIZE) + { +#ifndef FRONTEND + LWLockReleaseSaveErrno(ControlLogFileLock); + + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("could not read operation log from the file \"%s\": read %d of %d", + ControlLogFilePath, r, PG_OPERATION_LOG_FULL_SIZE))); +#else + pg_fatal("could not read operation log from the file \"%s\": read %d of %d", + ControlLogFilePath, r, PG_OPERATION_LOG_FULL_SIZE); +#endif + } + +#ifndef FRONTEND + if (CloseTransientFile(fd) != 0) + { + LWLockReleaseSaveErrno(ControlLogFileLock); + + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", + ControlLogFilePath))); + } + LWLockRelease(ControlLogFileLock); +#else + if (close(fd) != 0) + pg_fatal("could not close file \"%s\": %m", ControlLogFilePath); +#endif + + /* Check the CRC. */ + crc = calculate_operation_log_crc(log_buffer); + + *crc_ok_p = EQ_CRC32C(crc, log_buffer->header.ol_crc); + + return log_buffer; +} + +/* + * update_operation_log() + * + * Update the operation log ring buffer. + * Note. To protect against failures a operation log file is written in two + * stages: first a temporary file is created, then the temporary file is + * renamed to the operation log file. + */ +void +update_operation_log(const char *DataDir, OperationLogBuffer * log_buffer) +{ + int fd; + char ControlLogFilePath[MAXPGPATH]; + char ControlLogFilePathTmp[MAXPGPATH]; +#ifndef FRONTEND + int save_errno; +#endif + + snprintf(ControlLogFilePath, sizeof(ControlLogFilePath), "%s/%s", DataDir, + PG_OPERATION_LOG_FILE); + snprintf(ControlLogFilePathTmp, sizeof(ControlLogFilePathTmp), "%s.tmp", + ControlLogFilePath); + + /* Recalculate CRC of operation log. */ + log_buffer->header.ol_crc = calculate_operation_log_crc(log_buffer); + +#ifndef FRONTEND + LWLockAcquire(ControlLogFileLock, LW_EXCLUSIVE); +#endif + + /* Create a temporary file. */ + fd = create_operation_log_file(ControlLogFilePathTmp); + if (fd < 0) + { +#ifndef FRONTEND + LWLockReleaseSaveErrno(ControlLogFileLock); + + ereport(PANIC, + (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", + ControlLogFilePathTmp))); +#else + pg_fatal("could not open file \"%s\": %m", ControlLogFilePathTmp); +#endif + } + + /* Place operation log buffer into temporary file. */ + errno = 0; + if (write(fd, log_buffer, PG_OPERATION_LOG_FULL_SIZE) != PG_OPERATION_LOG_FULL_SIZE) + { + /* if write didn't set errno, assume problem is no disk space */ + if (errno == 0) + errno = ENOSPC; + +#ifndef FRONTEND + LWLockReleaseSaveErrno(ControlLogFileLock); + + ereport(PANIC, + (errcode_for_file_access(), + errmsg("could not write operation log in the file \"%s\": %m", + ControlLogFilePathTmp))); +#else + pg_fatal("could not write operation log in the file \"%s\": %m", + ControlLogFilePathTmp); +#endif + } + + /* Close the temporary file. */ +#ifndef FRONTEND + if (CloseTransientFile(fd) != 0) + { + LWLockReleaseSaveErrno(ControlLogFileLock); + + ereport(PANIC, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", + ControlLogFilePathTmp))); + } +#else + if (close(fd) != 0) + pg_fatal("could not close file \"%s\": %m", ControlLogFilePathTmp); +#endif + + /* Rename temporary file to required name. */ +#ifndef FRONTEND + if (durable_rename(ControlLogFilePathTmp, ControlLogFilePath, LOG) != 0) + { + LWLockReleaseSaveErrno(ControlLogFileLock); + + ereport(PANIC, + (errcode_for_file_access(), + errmsg("could not rename file \"%s\" to \"%s\": %m", + ControlLogFilePathTmp, ControlLogFilePath))); + } + LWLockRelease(ControlLogFileLock); +#else + if (durable_rename(ControlLogFilePathTmp, ControlLogFilePath) != 0) + pg_fatal("could not rename file \"%s\" to \"%s\": %m", + ControlLogFilePathTmp, ControlLogFilePath); +#endif +} + +/* + * is_enum_value_correct() + * + * Function returns true in case value is correct value of enum. + * + * val - test value; + * minval - first enum value (correct value); + * maxval - last enum value (incorrect value). + */ +static bool +is_enum_value_correct(int16 val, int16 minval, int16 maxval) +{ + Assert(val >= minval || val < maxval); + + if (val < minval || val >= maxval) + return false; + return true; +} + +/* + * get_operation_log_type_desc() + * + * Function returns pointer to OperationLogTypeDesc struct for given type of + * operation ol_type. + */ +static OperationLogTypeDesc * +get_operation_log_type_desc(ol_type_enum ol_type) +{ + return &OperationLogTypesDescs[ol_type - 1]; +} + +/* + * fill_operation_log_element() + * + * Fill new operation log element. Value of ol_lsn is last checkpoint record + * pointer. + */ +static void +fill_operation_log_element(ControlFileData *ControlFile, + OperationLogTypeDesc * desc, + PgNumEdition edition, uint32 version_num, + OperationLogData * data) +{ + data->ol_type = desc->ol_type; + data->ol_edition = edition; + data->ol_count = 1; + data->ol_version = version_num; + data->ol_timestamp = (pg_time_t) time(NULL); + data->ol_lsn = ControlFile->checkPoint; +} + +/* + * find_operation_log_element_for_merge() + * + * Find element into operation log ring buffer by ol_type and version. + * Returns NULL in case element is not found. + */ +static OperationLogData * +find_operation_log_element_for_merge(ol_type_enum ol_type, + OperationLogBuffer * log_buffer, + PgNumEdition edition, uint32 version_num) +{ + uint32 first = log_buffer->header.ol_first; + uint32 count = get_operation_log_count(log_buffer); + OperationLogData *data; + uint32 i; + + Assert(first < PG_OPERATION_LOG_COUNT && count <= PG_OPERATION_LOG_COUNT); + + for (i = 0; i < count; i++) + { + data = &log_buffer->data[(first + i) % PG_OPERATION_LOG_COUNT]; + if (data->ol_type == ol_type && + data->ol_edition == edition && + data->ol_version == version_num) + return data; + } + + return NULL; +} + +/* + * put_operation_log_element(), put_operation_log_element_version() + * + * Put element into operation log ring buffer. + * + * DataDir is the path to the top level of the PGDATA directory tree; + * ol_type is type of operation; + * edition is edition of current PostgreSQL version; + * version_num is number of version (for example 13000802 for v13.8.2). + * + * Note that it is up to the caller to properly lock ControlFileLogLock when + * calling this routine in the backend. + */ +void +put_operation_log_element_version(const char *DataDir, ol_type_enum ol_type, + PgNumEdition edition, uint32 version_num) +{ + OperationLogBuffer *log_buffer; + ControlFileData *ControlFile; + bool crc_ok; + OperationLogTypeDesc *desc; + + if (!is_enum_value_correct(ol_type, OLT_BOOTSTRAP, OLT_NumberOfTypes)) + { +#ifndef FRONTEND + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("invalid type of operation (%u) for operation log", ol_type))); +#else + pg_fatal("invalid type of operation (%u) for operation log", ol_type); +#endif + } + + desc = get_operation_log_type_desc(ol_type); + + if (!is_enum_value_correct(desc->ol_mode, OLM_MERGE, OLM_NumberOfModes)) + { +#ifndef FRONTEND + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("invalid mode of operation (%u) for operation log", ol_type))); +#else + pg_fatal("invalid mode of operation (%u) for operation log", ol_type); +#endif + } + + /* get a copy of the control file */ + ControlFile = get_controlfile(DataDir, &crc_ok); + if (!crc_ok) +#ifndef FRONTEND + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("pg_control CRC value is incorrect"))); +#else + pg_fatal("pg_control CRC value is incorrect"); +#endif + + /* get a copy of the operation log */ + log_buffer = get_operation_log(DataDir, &crc_ok); + if (!crc_ok) +#ifndef FRONTEND + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("pg_control_log CRC value is incorrect"))); +#else + pg_fatal("pg_control_log CRC value is incorrect"); +#endif + + switch (desc->ol_mode) + { + case OLM_MERGE: + { + OperationLogData *data; + + data = find_operation_log_element_for_merge(ol_type, log_buffer, + edition, version_num); + if (data) + { + /* + * We just found the element with the same type and the + * same version. Update it. + */ + if (data->ol_count < PG_UINT16_MAX) /* prevent overflow */ + data->ol_count++; + data->ol_timestamp = (pg_time_t) time(NULL); + data->ol_lsn = ControlFile->checkPoint; + break; + } + } + /* FALLTHROUGH */ + + case OLM_INSERT: + { + uint16 first = log_buffer->header.ol_first; + uint16 count = log_buffer->header.ol_count; + uint16 current; + + Assert(first < PG_OPERATION_LOG_COUNT && count <= PG_OPERATION_LOG_COUNT); + + if (count == PG_OPERATION_LOG_COUNT) + { + current = first; + /* Owerflow, shift the first element */ + log_buffer->header.ol_first = (first + 1) % PG_OPERATION_LOG_COUNT; + } + else + { + current = first + count; + /* Increase number of elements: */ + log_buffer->header.ol_count++; + } + + /* Fill operation log element. */ + fill_operation_log_element(ControlFile, desc, edition, version_num, + &log_buffer->data[current]); + break; + } + + default: +#ifndef FRONTEND + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("unexpected operation log mode %d", + desc->ol_mode))); +#else + pg_fatal("unexpected operation log mode %d", desc->ol_mode); +#endif + } + + update_operation_log(DataDir, log_buffer); + + pfree(log_buffer); + + pfree(ControlFile); +} + +/* + * Helper constant for determine current edition. + * Here can be custom editions. + */ +static const uint8 current_edition = ED_PG_ORIGINAL; + +/* + * Helper constant for determine current version. + * Multiplier 100 used as reserve of last two digits for patch number. + */ +static const uint32 current_version_num = PG_VERSION_NUM * 100; + +void +put_operation_log_element(const char *DataDir, ol_type_enum ol_type) +{ + put_operation_log_element_version(DataDir, ol_type, current_edition, current_version_num); +} + +/* + * get_operation_log_element() + * + * Returns operation log buffer element with number num. + */ +OperationLogData * +get_operation_log_element(OperationLogBuffer * log_buffer, uint16 num) +{ + uint32 first = log_buffer->header.ol_first; +#ifdef USE_ASSERT_CHECKING + uint32 count = get_operation_log_count(log_buffer); + + Assert(num < count); +#endif + + return &log_buffer->data[(first + num) % PG_OPERATION_LOG_COUNT]; +} + +/* + * get_operation_log_count() + * + * Returns number of elements in given operation log buffer. + */ +uint16 +get_operation_log_count(OperationLogBuffer * log_buffer) +{ + return log_buffer->header.ol_count; +} + +/* + * get_operation_log_type_name() + * + * Returns name of given type. + */ +const char * +get_operation_log_type_name(ol_type_enum ol_type) +{ + if (is_enum_value_correct(ol_type, OLT_BOOTSTRAP, OLT_NumberOfTypes)) + return OperationLogTypesDescs[ol_type - 1].ol_name; + else + return psprintf("unknown name %u", ol_type); +} diff --git a/src/common/meson.build b/src/common/meson.build index 1caa1fed04..586c74ae41 100644 --- a/src/common/meson.build +++ b/src/common/meson.build @@ -5,6 +5,7 @@ common_sources = files( 'checksum_helper.c', 'compression.c', 'controldata_utils.c', + 'controllog_utils.c', 'encnames.c', 'exec.c', 'file_perm.c', diff --git a/src/include/catalog/pg_controllog.h b/src/include/catalog/pg_controllog.h new file mode 100644 index 0000000000..fddac25fea --- /dev/null +++ b/src/include/catalog/pg_controllog.h @@ -0,0 +1,142 @@ +/*------------------------------------------------------------------------- + * + * pg_controllog.h + * The system operation log file "pg_control_log" is not a heap + * relation. + * However, we define it here so that the format is documented. + * + * + * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/catalog/pg_controllog.h + * + *------------------------------------------------------------------------- + */ +#ifndef PG_CONTROLLOG_H +#define PG_CONTROLLOG_H + +#include "access/transam.h" +#include "access/xlogdefs.h" +#include "pgtime.h" /* for pg_time_t */ +#include "port/pg_crc32c.h" + +#define PG_OPERATION_LOG_FILE "global/pg_control_log" + +/* + * Type of operation for operation log. + */ +typedef enum +{ + OLT_BOOTSTRAP = 1, /* bootstrap */ + OLT_STARTUP, /* server startup */ + OLT_RESETWAL, /* pg_resetwal */ + OLT_REWIND, /* pg_rewind */ + OLT_UPGRADE, /* pg_upgrade */ + OLT_PROMOTED, /* promoted */ + OLT_NumberOfTypes /* should be last */ +} ol_type_enum; + +/* + * Mode of operation processing. + */ +typedef enum +{ + OLM_MERGE = 1, /* insert element only if not exists element + * with the same ol_type and ol_version; + * otherwise update existing element */ + OLM_INSERT, /* insert element into ring buffer 'as is' */ + OLM_NumberOfModes /* should be last */ +} ol_mode_enum; + +/* + * Helper struct for describing supported operations. + */ +typedef struct OperationLogTypeDesc +{ + ol_type_enum ol_type; /* element type */ + ol_mode_enum ol_mode; /* element mode */ + const char *ol_name; /* display name of element */ +} OperationLogTypeDesc; + +/* + * Element of operation log ring buffer (24 bytes). + */ +typedef struct OperationLogData +{ + uint8 ol_type; /* operation type */ + uint8 ol_edition; /* postgres edition */ + uint16 ol_count; /* number of operations */ + uint32 ol_version; /* postgres version */ + pg_time_t ol_timestamp; /* = int64, operation date/time */ + XLogRecPtr ol_lsn; /* = uint64, last check point record ptr */ +} OperationLogData; + +/* + * Header of operation log ring buffer (8 bytes). + */ +typedef struct OperationLogHeader +{ + pg_crc32c ol_crc; /* CRC of operation log ... MUST BE FIRST! */ + uint16 ol_first; /* position of first ring buffer element */ + uint16 ol_count; /* number of elements in ring buffer */ +} OperationLogHeader; + +/* + * Whole size of the operation log ring buffer (with header). + */ +#define PG_OPERATION_LOG_FULL_SIZE 8192 + +/* + * Size of elements of the operation log ring buffer. + * Value must be a multiple of sizeof(OperationLogData). + */ +#define PG_OPERATION_LOG_SIZE (PG_OPERATION_LOG_FULL_SIZE - sizeof(OperationLogHeader)) + +/* + * Number of elements in the operation log. + */ +#define PG_OPERATION_LOG_COUNT (PG_OPERATION_LOG_SIZE / sizeof(OperationLogData)) + +/* + * Operation log ring buffer. + */ +typedef struct OperationLogBuffer +{ + OperationLogHeader header; + OperationLogData data[PG_OPERATION_LOG_COUNT]; + +} OperationLogBuffer; + +StaticAssertDecl(sizeof(OperationLogBuffer) == PG_OPERATION_LOG_FULL_SIZE, + "structure OperationLogBuffer must have size PG_OPERATION_LOG_FULL_SIZE"); + +/* Enum for postgres edition. */ +typedef enum +{ + ED_PG_ORIGINAL = 0 + /* Here can be custom editions */ +} PgNumEdition; + +#define ED_PG_ORIGINAL_STR "vanilla" +#define ED_UNKNOWN_STR "unknown" + +/* + * get_str_edition() + * + * Returns edition string by edition number. + */ +static inline const char * +get_str_edition(PgNumEdition edition) +{ + switch (edition) + { + case ED_PG_ORIGINAL: + return ED_PG_ORIGINAL_STR; + + /* Here can be custom editions */ + } + return ED_UNKNOWN_STR; +} + +#endif /* PG_CONTROLLOG_H */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 86eb8e8c58..7b55e75d6b 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11891,4 +11891,13 @@ prorettype => 'bytea', proargtypes => 'pg_brin_minmax_multi_summary', prosrc => 'brin_minmax_multi_summary_send' }, +# operation log function +{ oid => '8110', descr => 'show operation log', + proname => 'pg_operation_log', prorows => '170', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => '', + proallargtypes => '{text,text,text,pg_lsn,timestamptz,int4}', + proargmodes => '{o,o,o,o,o,o}', + proargnames => '{event,edition,version,lsn,last,count}', + prosrc => 'pg_operation_log' }, + ] diff --git a/src/include/common/controllog_utils.h b/src/include/common/controllog_utils.h new file mode 100644 index 0000000000..dc5c01e87f --- /dev/null +++ b/src/include/common/controllog_utils.h @@ -0,0 +1,27 @@ +/* + * controllog_utils.h + * Common code for pg_control_log output + * + * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/common/controllog_utils.h + */ +#ifndef COMMON_CONTROLLOG_UTILS_H +#define COMMON_CONTROLLOG_UTILS_H + +#include "catalog/pg_controllog.h" + +extern OperationLogBuffer * get_operation_log(const char *DataDir, bool *crc_ok_p); +extern OperationLogBuffer * get_empty_operation_log_buffer(void); +extern void update_operation_log(const char *DataDir, OperationLogBuffer * log_buffer); + +extern void put_operation_log_element(const char *DataDir, ol_type_enum ol_type); +extern void put_operation_log_element_version(const char *DataDir, ol_type_enum ol_type, + PgNumEdition edition, uint32 version_num); +extern uint16 get_operation_log_count(OperationLogBuffer * log_buffer); +extern OperationLogData * get_operation_log_element(OperationLogBuffer * log_buffer, + uint16 num); +extern const char *get_operation_log_type_name(ol_type_enum ol_type); + +#endif /* COMMON_CONTROLLOG_UTILS_H */ diff --git a/src/test/modules/test_misc/meson.build b/src/test/modules/test_misc/meson.build index 21bde427b4..ec698284af 100644 --- a/src/test/modules/test_misc/meson.build +++ b/src/test/modules/test_misc/meson.build @@ -9,6 +9,7 @@ tests += { 't/001_constraint_validation.pl', 't/002_tablespace.pl', 't/003_check_guc.pl', + 't/004_operation_log.pl', ], }, } diff --git a/src/test/modules/test_misc/t/004_operation_log.pl b/src/test/modules/test_misc/t/004_operation_log.pl new file mode 100644 index 0000000000..3a07f78ffa --- /dev/null +++ b/src/test/modules/test_misc/t/004_operation_log.pl @@ -0,0 +1,136 @@ + +# Copyright (c) 2022, PostgreSQL Global Development Group + +# Test for operation log. +# +# Some events like +# "bootstrap", "startup", "pg_rewind", "pg_resetwal", "promoted", "pg_upgrade" +# should be registered in operation log. + +use strict; +use warnings; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; +use File::Copy; + +# Create and start primary node +my $node_primary = PostgreSQL::Test::Cluster->new('primary'); +$node_primary->init(allows_streaming => 1); +$node_primary->append_conf('postgresql.conf', qq(wal_keep_size = 100MB)); +$node_primary->start; + +# Get server version +my $server_version = $node_primary->safe_psql("postgres", "SELECT current_setting('server_version_num');") + 0; +my $major_version = $server_version / 10000; +my $minor_version = $server_version % 100; + +# Get primary node backup +$node_primary->backup('primary_backup'); + +# Initialize standby node from backup +my $node_standby = PostgreSQL::Test::Cluster->new('standby'); +$node_standby->init_from_backup($node_primary, 'primary_backup', has_streaming => 1); +$node_standby->start; + +# Promote the standby +$node_standby->promote; + +# Stop standby node +$node_standby->stop; + +my $node_standby_pgdata = $node_standby->data_dir; +my $node_primary_connstr = $node_primary->connstr; + +# Keep a temporary postgresql.conf or it would be overwritten during the rewind. +my $tmp_folder = PostgreSQL::Test::Utils::tempdir; +copy("$node_standby_pgdata/postgresql.conf", "$tmp_folder/node_standby-postgresql.conf.tmp"); + +# Get "pg_rewind" event +command_ok( + [ + 'pg_rewind', + "--source-server=$node_primary_connstr", + "--target-pgdata=$node_standby_pgdata", + "--debug" + ], + 'run pg_rewind'); + +# Move back postgresql.conf with old settings +move("$tmp_folder/node_standby-postgresql.conf.tmp", "$node_standby_pgdata/postgresql.conf"); + +# Start and stop standby before resetwal and upgrade +$node_standby->start; +$node_standby->stop; + +# Get first "pg_resetwal" event +system_or_bail('pg_resetwal', '-f', $node_standby->data_dir); + +# Get second "pg_resetwal" event +system_or_bail('pg_resetwal', '-f', $node_standby->data_dir); + +# Initialize a new node for the upgrade +my $node_new = PostgreSQL::Test::Cluster->new('new'); +$node_new->init; + +my $bindir_new = $node_new->config_data('--bindir'); +my $bindir_standby = $node_standby->config_data('--bindir'); + +# We want to run pg_upgrade in the build directory so that any files generated +# finish in it, like delete_old_cluster.{sh,bat}. +chdir ${PostgreSQL::Test::Utils::tmp_check}; + +# Run pg_upgrade +command_ok( + [ + 'pg_upgrade', '--no-sync', '-d', $node_standby->data_dir, + '-D', $node_new->data_dir, '-b', $bindir_standby, + '-B', $bindir_new, '-s', $node_new->host, + '-p', $node_standby->port, '-P', $node_new->port + ], + 'run pg_upgrade'); +# +# Need to check operation log +# +sub check_event +{ + my $event_name = shift; + my $result = shift; + my $func_args = shift ? "sum(count), count(*)" : "count(*)"; + + my $psql_stdout = $node_new->safe_psql('postgres', qq( + SELECT + $func_args, + min(split_part(version,'.','1')), + min(split_part(version,'.','2')) + FROM + pg_operation_log() + WHERE + event='$event_name')); + + is($psql_stdout, $result, 'check number of event ' . $event_name); + return; +} +#Start new node +$node_new->start; + +# Check number of event "bootstrap" +check_event('bootstrap', qq(1|1|$major_version|$minor_version), 1); + +# Check number of event "startup" +check_event('startup', qq(1|$major_version|$minor_version), 0); + +# Check number of event "promoted" +check_event('promoted', qq(1|1|$major_version|$minor_version), 1); + +# Check number of event "pg_upgrade" +check_event('pg_upgrade', qq(1|1|$major_version|$minor_version), 1); + +# Check number of event "pg_resetwal" +check_event('pg_resetwal', qq(2|1|$major_version|$minor_version), 1); + +# Check number of event "pg_rewind" +check_event('pg_rewind', qq(1|1|$major_version|$minor_version), 1); + +done_testing(); diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index ee49424d6f..08e150b040 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -133,8 +133,8 @@ sub mkvcbuild } our @pgcommonallfiles = qw( - base64.c checksum_helper.c compression.c - config_info.c controldata_utils.c d2s.c encnames.c exec.c + base64.c checksum_helper.c compression.c config_info.c + controldata_utils.c controllog_utils.c d2s.c encnames.c exec.c f2s.c file_perm.c file_utils.c hashfn.c ip.c jsonapi.c keywords.c kwlookup.c link-canary.c md5_common.c percentrepl.c pg_get_line.c pg_lzcompress.c pg_prng.c pgfnames.c psprintf.c relpath.c -- 2.31.0.windows.1