From cebcf65da3804654c1f42b18d9126fb5fdf9e924 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 14 Sep 2026 13:03:40 +0000 Subject: [PATCH v10 2/3] Add transactional filesystem operations (FILEOPS) backed by UNDO Make the structural filesystem mutations the server performs outside the buffer manager crash-safe and rollback-safe via a WAL resource manager (RM_FILEOPS_ID). Forward operations are WAL-logged and, inside a transaction, register a pending-op / UNDO record so an abort or crash recovery recursively reverts them. FILEOPS records its UNDO in the common WAL stream, so crash recovery reverts a loser transaction's structural changes through the cluster-wide UNDO engine's recovery pass; the AtPrepare path is handled so a prepared transaction's pending file operations survive 2PC. This commit adds the FILEOPS access method and its UNDO plumbing but does not yet route any in-tree DDL/storage caller through it (that is the next commit): the FileOps* API, fileops_redo, the fileops UNDO resource manager (FileopsUndoRmgrInit, registered via undormgrlist.h), the pg_xattr portability wrapper, rmgrdesc + pg_waldump support, the test_fileops module, SQL regression coverage, and recovery TAP tests for crash and 2PC. Author: Greg Burd Changes since v2: - Report FILEOPS UNDO apply failures as failures [f3bd4a3cc7] Tests: - Do not set logical_revert_naptime in 059 and 061 [3072609035] - Show that a crash right after commit loses deferred ops [b34b2463bb] Changes since v7: - recovery/060_fileops_standby_divergence is now 064: upstream added t/060_multixact_crashed_updater.pl - test_fileops gains 023 and 024, the tests for the two UNDO engine fixes above (in the previous commit) - recovery/072_perbackend_crash_undo is now 065_fileops_crash_undo; it never involved per-backend UNDO - Comments and tests no longer describe ROLLBACK PREPARED as handing the chain to the ATM and an asynchronous revert worker; in this series FinishPreparedTransaction() applies it synchronously - recovery/062 (Test 12) and 065 skip their file-mode checks on Windows, which has no POSIX modes; v8 carried these guards in 0005 - recovery/065 crashes the server before it closes the client. It closed the client first, so the backend could roll the transaction back online, and when that abort reached disk before the crash, recovery redid the changes Changes since v9 (the v8 fixes that the cut to three commits had lost): - An ordinary filesystem error is an ERROR again, not a PANIC. Each immediate operation makes its system call first and WAL-logs it only once it has succeeded; v9 ran the call inside the critical section after the WAL flush, so CREATE TABLESPACE on a directory the server cannot chmod took the server down, as in Shihao's v2 review (test_fileops/004). The window this leaves between the call and its WAL record is documented in fileops.c and the README - The abort record carries the transaction's FILEOPS inverses (XACT_XINFO_HAS_FILEOPS), and abort redo applies them. An online ROLLBACK reverted the files in the backend only, so a physical standby kept a change the primary rolled back, and so did the primary after a crash once the abort record was on disk (test_fileops/008). ROLLBACK PREPARED carries them too, read from the 2PC state, and keeps upstream's DELAY_CHKPT_START set until they are durable - FileOpsRename() renames at once and is undone by renaming back, instead of being deferred to commit (test_fileops/018) - An inverse reverts only an object that still shows the operation's effect, so a name reused since is left alone; a rollback's inverses are made durable once per path; the abort data a transaction may accumulate is capped (FILEOPS_MAX_ABORT_BYTES), so the abort record cannot fail - FileOpsMkdirTree() creates a directory that its UNDO removes with everything in it (used by the next commit for CREATE DATABASE) --- meson.build | 11 + src/Makefile.global.in | 5 + src/backend/access/rmgrdesc/Makefile | 1 + src/backend/access/rmgrdesc/fileopsdesc.c | 220 ++ src/backend/access/rmgrdesc/meson.build | 6 + src/backend/access/rmgrdesc/xactdesc.c | 8 + src/backend/access/transam/rmgr.c | 1 + src/backend/access/transam/twophase.c | 88 +- src/backend/access/transam/twophase_rmgr.c | 20 +- src/backend/access/transam/xact.c | 68 +- src/backend/storage/file/Makefile | 2 + src/backend/storage/file/README | 272 ++ src/backend/storage/file/fileops.c | 2575 +++++++++++++++++ src/backend/storage/file/fileops_undo.c | 1054 +++++++ src/backend/storage/file/meson.build | 2 + src/bin/pg_waldump/rmgrdesc.c | 1 + src/bin/pg_waldump/t/001_basic.pl | 3 +- src/include/access/fileops_xlog.h | 40 + src/include/access/rmgrlist.h | 3 + src/include/access/twophase_rmgr.h | 9 +- src/include/access/undormgrlist.h | 3 + src/include/access/xact.h | 8 +- src/include/port/pg_xattr.h | 40 + src/include/storage/fileops.h | 527 ++++ src/port/Makefile | 1 + src/port/meson.build | 1 + src/port/pg_xattr.c | 217 ++ src/test/modules/Makefile | 1 + src/test/modules/meson.build | 1 + src/test/modules/test_fileops/Makefile | 19 + .../test_fileops/expected/test_fileops.out | 413 +++ .../test_fileops/expected/test_fileops_1.out | 332 +++ .../test_fileops/expected/test_fileops_2.out | 18 + src/test/modules/test_fileops/meson.build | 48 + .../modules/test_fileops/sql/test_fileops.sql | 229 ++ .../t/001_fileops_apply_failure.pl | 38 + .../test_fileops/t/004_fileops_fs_errors.pl | 107 + .../t/005_fileops_checkpoint_loser.pl | 38 + .../t/007_fileops_commit_crash.pl | 77 + .../t/008_fileops_standby_abort.pl | 89 + .../test_fileops/t/018_fileops_rename.pl | 347 +++ .../t/023_fileops_inline_abort_ctx.pl | 60 + .../t/024_fileops_recovery_apply_failure.pl | 59 + .../test_fileops/test_fileops--1.0.sql | 99 + src/test/modules/test_fileops/test_fileops.c | 471 +++ .../modules/test_fileops/test_fileops.control | 4 + src/test/recovery/Makefile | 1 + src/test/recovery/meson.build | 5 + .../recovery/t/061_fileops_2pc_deferred.pl | 172 ++ src/test/recovery/t/062_fileops_recovery.pl | 423 +++ src/test/recovery/t/063_fileops_2pc.pl | 149 + .../t/064_fileops_standby_divergence.pl | 124 + src/test/recovery/t/065_fileops_crash_undo.pl | 137 + src/tools/pgindent/typedefs.list | 26 + 54 files changed, 8650 insertions(+), 23 deletions(-) create mode 100644 src/backend/access/rmgrdesc/fileopsdesc.c create mode 100644 src/backend/storage/file/README create mode 100644 src/backend/storage/file/fileops.c create mode 100644 src/backend/storage/file/fileops_undo.c create mode 100644 src/include/access/fileops_xlog.h create mode 100644 src/include/port/pg_xattr.h create mode 100644 src/include/storage/fileops.h create mode 100644 src/port/pg_xattr.c create mode 100644 src/test/modules/test_fileops/Makefile create mode 100644 src/test/modules/test_fileops/expected/test_fileops.out create mode 100644 src/test/modules/test_fileops/expected/test_fileops_1.out create mode 100644 src/test/modules/test_fileops/expected/test_fileops_2.out create mode 100644 src/test/modules/test_fileops/meson.build create mode 100644 src/test/modules/test_fileops/sql/test_fileops.sql create mode 100644 src/test/modules/test_fileops/t/001_fileops_apply_failure.pl create mode 100644 src/test/modules/test_fileops/t/004_fileops_fs_errors.pl create mode 100644 src/test/modules/test_fileops/t/005_fileops_checkpoint_loser.pl create mode 100644 src/test/modules/test_fileops/t/007_fileops_commit_crash.pl create mode 100644 src/test/modules/test_fileops/t/008_fileops_standby_abort.pl create mode 100644 src/test/modules/test_fileops/t/018_fileops_rename.pl create mode 100644 src/test/modules/test_fileops/t/023_fileops_inline_abort_ctx.pl create mode 100644 src/test/modules/test_fileops/t/024_fileops_recovery_apply_failure.pl create mode 100644 src/test/modules/test_fileops/test_fileops--1.0.sql create mode 100644 src/test/modules/test_fileops/test_fileops.c create mode 100644 src/test/modules/test_fileops/test_fileops.control create mode 100644 src/test/recovery/t/061_fileops_2pc_deferred.pl create mode 100644 src/test/recovery/t/062_fileops_recovery.pl create mode 100644 src/test/recovery/t/063_fileops_2pc.pl create mode 100644 src/test/recovery/t/064_fileops_standby_divergence.pl create mode 100644 src/test/recovery/t/065_fileops_crash_undo.pl diff --git a/meson.build b/meson.build index 3238c849e1b..935de4e5fc3 100644 --- a/meson.build +++ b/meson.build @@ -1586,6 +1586,17 @@ endif +############################################################### +# Library: FILEOPS +############################################################### + +# FILEOPS transactional filesystem operations (WAL rmgr RM_FILEOPS_ID) is +# always built. Mirrors USE_FILEOPS in src/Makefile.global.in: without it the +# rmgrlist.h entry is compiled out and every RM_FILEOPS_ID reference fails to +# build. +cdata.set('USE_FILEOPS', 1) + + ############################################################### # Library: selinux ############################################################### diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 38914773b83..b20d0065ce9 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -256,6 +256,11 @@ PG_SYSROOT = @PG_SYSROOT@ override CPPFLAGS += $(ICU_CFLAGS) $(LIBNUMA_CFLAGS) $(LIBURING_CFLAGS) +# FILEOPS transactional filesystem operations (WAL rmgr RM_FILEOPS_ID) is +# always built. +USE_FILEOPS = 1 +override CPPFLAGS += -DUSE_FILEOPS + ifdef PGXS override CPPFLAGS := -I$(includedir_server) -I$(includedir_internal) $(CPPFLAGS) else # not PGXS diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile index bf6709e738d..5b0d8922ea2 100644 --- a/src/backend/access/rmgrdesc/Makefile +++ b/src/backend/access/rmgrdesc/Makefile @@ -13,6 +13,7 @@ OBJS = \ clogdesc.o \ committsdesc.o \ dbasedesc.o \ + fileopsdesc.o \ genericdesc.o \ gindesc.o \ gistdesc.o \ diff --git a/src/backend/access/rmgrdesc/fileopsdesc.c b/src/backend/access/rmgrdesc/fileopsdesc.c new file mode 100644 index 00000000000..ba4cb111c37 --- /dev/null +++ b/src/backend/access/rmgrdesc/fileopsdesc.c @@ -0,0 +1,220 @@ +/*------------------------------------------------------------------------- + * + * fileopsdesc.c + * rmgr descriptor routines for storage/file/fileops.c + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/rmgrdesc/fileopsdesc.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "storage/fileops.h" + +void +fileops_desc(StringInfo buf, XLogReaderState *record) +{ + char *data = XLogRecGetData(record); + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + + switch (info) + { + case XLOG_FILEOPS_CREATE: + { + xl_fileops_create *xlrec = (xl_fileops_create *) data; + const char *path = data + SizeOfFileOpsCreate; + + appendStringInfo(buf, "create \"%s\" flags 0x%x mode 0%o", + path, xlrec->flags, xlrec->mode); + } + break; + + case XLOG_FILEOPS_WRITE: + { + xl_fileops_write *xlrec = (xl_fileops_write *) data; + const char *path = data + SizeOfFileOpsWrite; + + appendStringInfo(buf, "write \"%s\" offset %lld len %u", + path, (long long) xlrec->offset, xlrec->len); + } + break; + + case XLOG_FILEOPS_RENAME: + { + xl_fileops_rename *xlrec = (xl_fileops_rename *) data; + const char *oldpath = data + SizeOfFileOpsRename; + const char *newpath = oldpath + xlrec->oldpath_len; + + appendStringInfo(buf, "rename \"%s\" to \"%s\"", + oldpath, newpath); + } + break; + + case XLOG_FILEOPS_DELETE: + { + xl_fileops_delete *xlrec = (xl_fileops_delete *) data; + const char *path = data + SizeOfFileOpsDelete; + + appendStringInfo(buf, "delete \"%s\" at_%s", + path, + xlrec->at_commit ? "commit" : "abort"); + } + break; + + case XLOG_FILEOPS_SYMLINK: + { + xl_fileops_symlink *xlrec = (xl_fileops_symlink *) data; + const char *target = data + SizeOfFileOpsSymlink; + const char *linkpath = target + xlrec->target_len; + + appendStringInfo(buf, "symlink \"%s\" -> \"%s\"", + linkpath, target); + } + break; + + case XLOG_FILEOPS_LINK: + { + xl_fileops_link *xlrec = (xl_fileops_link *) data; + const char *oldpath = data + SizeOfFileOpsLink; + const char *newpath = oldpath + xlrec->oldpath_len; + + appendStringInfo(buf, "link \"%s\" -> \"%s\"", + newpath, oldpath); + } + break; + + case XLOG_FILEOPS_MKDIR: + { + xl_fileops_mkdir *xlrec = (xl_fileops_mkdir *) data; + const char *path = data + SizeOfFileOpsMkdir; + + appendStringInfo(buf, "mkdir \"%s\" mode 0%o%s", + path, (unsigned int) xlrec->mode, + (xlrec->flags & FILEOPS_MKDIR_TREE) ? " tree" : ""); + } + break; + + case XLOG_FILEOPS_RMDIR: + { + xl_fileops_rmdir *xlrec = (xl_fileops_rmdir *) data; + const char *path = data + SizeOfFileOpsRmdir; + + appendStringInfo(buf, "rmdir \"%s\" at_%s", + path, + xlrec->at_commit ? "commit" : "abort"); + } + break; + + case XLOG_FILEOPS_CHMOD: + { + xl_fileops_chmod *xlrec = (xl_fileops_chmod *) data; + const char *path = data + SizeOfFileOpsChmod; + + appendStringInfo(buf, "chmod \"%s\" mode 0%o", + path, (unsigned int) xlrec->mode); + } + break; + + case XLOG_FILEOPS_CHOWN: + { + xl_fileops_chown *xlrec = (xl_fileops_chown *) data; + const char *path = data + SizeOfFileOpsChown; + + appendStringInfo(buf, "chown \"%s\" uid %d gid %d", + path, (int) xlrec->uid, (int) xlrec->gid); + } + break; + + case XLOG_FILEOPS_TRUNCATE: + { + xl_fileops_truncate *xlrec = (xl_fileops_truncate *) data; + const char *path = data + SizeOfFileOpsTruncate; + + appendStringInfo(buf, "truncate \"%s\" to %lld bytes", + path, (long long) xlrec->length); + } + break; + + case XLOG_FILEOPS_SETXATTR: + { + xl_fileops_setxattr *xlrec = (xl_fileops_setxattr *) data; + const char *path = data + SizeOfFileOpsSetxattr; + const char *name = path + xlrec->path_len; + + appendStringInfo(buf, "setxattr \"%s\" name \"%s\" len %u", + path, name, xlrec->value_len); + } + break; + + case XLOG_FILEOPS_REMOVEXATTR: + { + xl_fileops_removexattr *xlrec = + (xl_fileops_removexattr *) data; + const char *path = data + SizeOfFileOpsRemovexattr; + const char *name = path + xlrec->path_len; + + appendStringInfo(buf, "removexattr \"%s\" name \"%s\"", + path, name); + } + break; + + default: + appendStringInfo(buf, "unknown fileops op code %u", info); + break; + } +} + +const char * +fileops_identify(uint8 info) +{ + const char *id = NULL; + + switch (info & ~XLR_INFO_MASK) + { + case XLOG_FILEOPS_CREATE: + id = "CREATE"; + break; + case XLOG_FILEOPS_DELETE: + id = "DELETE"; + break; + case XLOG_FILEOPS_RENAME: + id = "RENAME"; + break; + case XLOG_FILEOPS_WRITE: + id = "WRITE"; + break; + case XLOG_FILEOPS_TRUNCATE: + id = "TRUNCATE"; + break; + case XLOG_FILEOPS_CHMOD: + id = "CHMOD"; + break; + case XLOG_FILEOPS_CHOWN: + id = "CHOWN"; + break; + case XLOG_FILEOPS_MKDIR: + id = "MKDIR"; + break; + case XLOG_FILEOPS_RMDIR: + id = "RMDIR"; + break; + case XLOG_FILEOPS_SYMLINK: + id = "SYMLINK"; + break; + case XLOG_FILEOPS_LINK: + id = "LINK"; + break; + case XLOG_FILEOPS_SETXATTR: + id = "SETXATTR"; + break; + case XLOG_FILEOPS_REMOVEXATTR: + id = "REMOVEXATTR"; + break; + } + + return id; +} diff --git a/src/backend/access/rmgrdesc/meson.build b/src/backend/access/rmgrdesc/meson.build index d0dc4cb229a..c8af1ce82dd 100644 --- a/src/backend/access/rmgrdesc/meson.build +++ b/src/backend/access/rmgrdesc/meson.build @@ -27,4 +27,10 @@ rmgr_desc_sources = files( 'xlogdesc.c', ) +# FILEOPS is always built (USE_FILEOPS); rmgrlist.h registers +# fileops_desc/fileops_identify, so fileopsdesc.c must always be compiled or +# pg_waldump fails to link. The autoconf build picks this up via its +# *desc*.c wildcard; meson needs it listed explicitly. +rmgr_desc_sources += files('fileopsdesc.c') + backend_sources += rmgr_desc_sources diff --git a/src/backend/access/rmgrdesc/xactdesc.c b/src/backend/access/rmgrdesc/xactdesc.c index 96aa6e17bd5..f416e2d09f1 100644 --- a/src/backend/access/rmgrdesc/xactdesc.c +++ b/src/backend/access/rmgrdesc/xactdesc.c @@ -230,6 +230,12 @@ ParseAbortRecord(uint8 info, xl_xact_abort *xlrec, xl_xact_parsed_abort *parsed) data += sizeof(xl_xact_origin); } + + if (parsed->xinfo & XACT_XINFO_HAS_FILEOPS) + { + memcpy(&parsed->fileops_len, data, sizeof(uint32)); + parsed->fileops = data + sizeof(uint32); + } } /* @@ -391,6 +397,8 @@ xact_desc_abort(StringInfo buf, uint8 info, xl_xact_abort *xlrec, ReplOriginId o } xact_desc_stats(buf, "", parsed.nstats, parsed.stats); + if (parsed.fileops_len) + appendStringInfo(buf, "; fileops undo: %u bytes", parsed.fileops_len); } static void diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c index 130eb06bee3..7441cf2a716 100644 --- a/src/backend/access/transam/rmgr.c +++ b/src/backend/access/transam/rmgr.c @@ -6,6 +6,7 @@ * src/backend/access/transam/rmgr.c */ #include "postgres.h" +#include "storage/fileops.h" #include "access/rmgr.h" #include "access/xlog_internal.h" diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 9463c065e1f..45877b5b5e5 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -100,6 +100,7 @@ #include "replication/origin.h" #include "replication/syncrep.h" #include "storage/fd.h" +#include "storage/fileops.h" #include "storage/ipc.h" #include "storage/md.h" #include "storage/predicate.h" @@ -150,6 +151,15 @@ int max_prepared_xacts = 0; * twophase.h */ +/* + * FILEOPS 2PC prepare hook (fileops.c). Declared locally by prototype to + * avoid pulling storage/fileops.h into this file. Serializes this + * transaction's pending file operations into the 2PC state file so + * COMMIT/ROLLBACK PREPARED -- possibly from another backend after a crash -- + * can perform the deferred commit-time ops or the abort-time filesystem undo. + */ +extern void FileOpsAtPrepare(void); + typedef struct GlobalTransactionData { GlobalTransaction next; /* list link for free list */ @@ -241,7 +251,9 @@ static void RecordTransactionAbortPrepared(TransactionId xid, RelFileLocator *rels, int nstats, xl_xact_stats_item *stats, - const char *gid); + const char *gid, + const char *fileops, + uint32 fileops_len); static void ProcessRecords(char *bufptr, FullTransactionId fxid, const TwoPhaseCallback callbacks[]); static void RemoveGXact(GlobalTransaction gxact); @@ -996,11 +1008,17 @@ TwoPhaseFilePath(char *path, FullTransactionId fxid) * * TWOPHASE_MAGIC must be bumped whenever xl_xact_prepare changes layout. * The struct gained last_batch_lsn[NUndoPersistenceLevels] (24 bytes) for - * UNDO chain tracking across 2PC boundaries, requiring this bump from + * UNDO chain tracking across 2PC boundaries, requiring a bump from * 0x57F94534 to 0x57F94535 to prevent old servers from silently misreading * the variable-length arrays that follow the fixed header at the wrong offsets. + * + * Bumped again to 0x57F94536: FILEOPS now registers TWOPHASE_RM_FILEOPS_ID + * records so deferred commit-time file operations (RENAME/DELETE/RMDIR/RMTREE) + * and abort-time filesystem undo survive PREPARE. This adds a new 2PC record + * type (not a header field), but a bump still keeps a mixed-version cluster + * from replaying a state file whose record stream it does not understand. */ -#define TWOPHASE_MAGIC 0x57F94535 /* format identifier */ +#define TWOPHASE_MAGIC 0x57F94536 /* format identifier */ typedef xl_xact_prepare TwoPhaseFileHeader; @@ -1179,6 +1197,13 @@ EndPrepare(GlobalTransaction gxact) StateFileChunk *record; bool replorigin; + /* + * FILEOPS: persist this transaction's pending file operations into the + * 2PC state file, so COMMIT PREPARED can run them after a crash. Must + * run before the END sentinel while the record chain is still open. A no-op if this transaction registered no file operations. + */ + FileOpsAtPrepare(); + /* Add the end sentinel to the list of 2PC records */ RegisterTwoPhaseRecord(TWOPHASE_RM_END_ID, 0, NULL, 0); @@ -1652,12 +1677,38 @@ FinishPreparedTransaction(const char *gid, bool isCommit) hdr->initfileinval, gid); else { + char *fileops = NULL; + uint32 fileops_len = 0; + char *scan = bufptr; + + /* + * The abort decision and the FILEOPS inverses must be in the same WAL + * record, so a standby and crash recovery apply the inverses whenever + * they replay the abort. PREPARE saved them as a 2PC record. + */ + for (;;) + { + TwoPhaseRecordOnDisk *record = (TwoPhaseRecordOnDisk *) scan; + + if (record->rmid == TWOPHASE_RM_END_ID) + break; + scan += MAXALIGN(sizeof(TwoPhaseRecordOnDisk)); + if (record->rmid == TWOPHASE_RM_FILEOPS_ID && + record->info == FILEOPS_TWOPHASE_INVERSES) + { + fileops = scan; + fileops_len = record->len; + break; + } + scan += MAXALIGN(record->len); + } + RecordTransactionAbortPrepared(xid, hdr->nsubxacts, children, hdr->nabortrels, abortrels, hdr->nabortstats, abortstats, - gid); + gid, fileops, fileops_len); /* * ROLLBACK PREPARED: apply any cluster-wide UNDO chain synchronously, @@ -2574,7 +2625,9 @@ RecordTransactionAbortPrepared(TransactionId xid, RelFileLocator *rels, int nstats, xl_xact_stats_item *stats, - const char *gid) + const char *gid, + const char *fileops, + uint32 fileops_len) { XLogRecPtr recptr; bool replorigin; @@ -2619,7 +2672,7 @@ RecordTransactionAbortPrepared(TransactionId xid, nrels, rels, nstats, stats, MyXactFlags | XACT_FLAGS_ACQUIREDACCESSEXCLUSIVELOCK, - xid, gid); + xid, gid, fileops, fileops_len); if (replorigin) /* Move LSNs forward for this replication origin */ @@ -2632,11 +2685,30 @@ RecordTransactionAbortPrepared(TransactionId xid, /* Mark the transaction aborted in clog */ TransactionIdAbortTree(xid, nchildren, children); - /* Checkpoint can proceed now */ - MyProc->delayChkptFlags &= ~DELAY_CHKPT_START; + /* + * Checkpoint can proceed now, unless FILEOPS inverses still have to be + * applied: a checkpoint must not move the redo pointer past this abort + * record until they are durable, so keep it delayed until then. + */ + if (!fileops_len) + MyProc->delayChkptFlags &= ~DELAY_CHKPT_START; END_CRIT_SECTION(); + if (fileops_len) + { + PG_TRY(); + { + INJECTION_POINT("fileops-abort-after-flush", NULL); + FileOpsApplyAbort(fileops, fileops_len); + } + PG_FINALLY(); + { + MyProc->delayChkptFlags &= ~DELAY_CHKPT_START; + } + PG_END_TRY(); + } + /* * Wait for synchronous replication, if required. * diff --git a/src/backend/access/transam/twophase_rmgr.c b/src/backend/access/transam/twophase_rmgr.c index fae254c6e23..687e77f7e67 100644 --- a/src/backend/access/transam/twophase_rmgr.c +++ b/src/backend/access/transam/twophase_rmgr.c @@ -19,6 +19,14 @@ #include "pgstat.h" #include "storage/lock.h" #include "storage/predicate.h" +#include "storage/fileops.h" + +/* + * FILEOPS 2PC callbacks (fileops.c). Declared locally by prototype to avoid + * pulling storage/fileops.h in here; the signature is the generic + * TwoPhaseCallback. Postcommit performs the deferred commit-time file ops; + * postabort performs the abort-time filesystem undo. + */ const TwoPhaseCallback twophase_recover_callbacks[TWOPHASE_RM_MAX_ID + 1] = @@ -27,7 +35,8 @@ const TwoPhaseCallback twophase_recover_callbacks[TWOPHASE_RM_MAX_ID + 1] = lock_twophase_recover, /* Lock */ NULL, /* pgstat */ multixact_twophase_recover, /* MultiXact */ - predicatelock_twophase_recover /* PredicateLock */ + predicatelock_twophase_recover, /* PredicateLock */ + NULL /* FILEOPS (post-commit/abort only) */ }; const TwoPhaseCallback twophase_postcommit_callbacks[TWOPHASE_RM_MAX_ID + 1] = @@ -36,7 +45,8 @@ const TwoPhaseCallback twophase_postcommit_callbacks[TWOPHASE_RM_MAX_ID + 1] = lock_twophase_postcommit, /* Lock */ pgstat_twophase_postcommit, /* pgstat */ multixact_twophase_postcommit, /* MultiXact */ - NULL /* PredicateLock */ + NULL, /* PredicateLock */ + fileops_twophase_postcommit /* FILEOPS */ }; const TwoPhaseCallback twophase_postabort_callbacks[TWOPHASE_RM_MAX_ID + 1] = @@ -45,7 +55,8 @@ const TwoPhaseCallback twophase_postabort_callbacks[TWOPHASE_RM_MAX_ID + 1] = lock_twophase_postabort, /* Lock */ pgstat_twophase_postabort, /* pgstat */ multixact_twophase_postabort, /* MultiXact */ - NULL /* PredicateLock */ + NULL, /* PredicateLock */ + fileops_twophase_postabort /* FILEOPS */ }; const TwoPhaseCallback twophase_standby_recover_callbacks[TWOPHASE_RM_MAX_ID + 1] = @@ -54,5 +65,6 @@ const TwoPhaseCallback twophase_standby_recover_callbacks[TWOPHASE_RM_MAX_ID + 1 lock_twophase_standby_recover, /* Lock */ NULL, /* pgstat */ NULL, /* MultiXact */ - NULL /* PredicateLock */ + NULL, /* PredicateLock */ + NULL /* FILEOPS */ }; diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c index 7b0ef34508b..b53efdd5b55 100644 --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@ -59,6 +59,7 @@ #include "storage/aio_subsys.h" #include "storage/condition_variable.h" #include "storage/fd.h" +#include "storage/fileops.h" #include "storage/lmgr.h" #include "storage/md.h" #include "storage/predicate.h" @@ -1890,6 +1891,8 @@ RecordTransactionAbort(bool isSubXact) TransactionId *children; TimestampTz xact_time; bool replorigin; + char *fileops; + uint32 fileops_len; /* * If we haven't been assigned an XID, nobody will care whether we aborted @@ -1908,9 +1911,9 @@ RecordTransactionAbort(bool isSubXact) /* * We have a valid XID, so we should write an ABORT record for it. * - * We do not flush XLOG to disk here, since the default assumption after a - * crash would be that we aborted, anyway. For the same reason, we don't - * need to worry about interlocking against checkpoint start. + * An ordinary abort need not flush WAL: a crash treats it as a loser. + * FILEOPS inverses, however, must follow a durable abort decision, with + * checkpoint start delayed until their filesystem effects are synced. */ /* @@ -1932,6 +1935,10 @@ RecordTransactionAbort(bool isSubXact) nchildren = xactGetCommittedChildren(&children); ndroppedstats = pgstat_get_transactional_drops(false, &droppedstats); + FileOpsGetAbortData(&fileops, &fileops_len); + if (fileops_len) + FileOpsBeginAbort(); + /* XXX do we really need a critical section here? */ START_CRIT_SECTION(); @@ -1948,7 +1955,7 @@ RecordTransactionAbort(bool isSubXact) nrels, rels, ndroppedstats, droppedstats, MyXactFlags, InvalidTransactionId, - NULL); + NULL, fileops, fileops_len); if (replorigin) /* Move LSNs forward for this replication origin */ @@ -1977,8 +1984,26 @@ RecordTransactionAbort(bool isSubXact) */ TransactionIdAbortTree(xid, nchildren, children); + if (fileops_len) + XLogFlush(XactLastRecEnd); END_CRIT_SECTION(); + if (fileops_len) + { + PG_TRY(); + { + INJECTION_POINT("fileops-abort-after-flush", NULL); + FileOpsApplyAbort(fileops, fileops_len); + } + PG_FINALLY(); + { + FileOpsEndAbort(); + } + PG_END_TRY(); + } + if (fileops) + pfree(fileops); + /* Compute latestXid while we have the child XIDs handy */ latestXid = TransactionIdLatest(xid, nchildren, children); @@ -3112,8 +3137,21 @@ AbortTransaction(void) s->parallelModeLevel = 0; s->parallelChildXact = false; /* should be false already */ - /* Clean up transaction undo state (free per-persistence record sets) */ - AtAbort_XactUndo(); + /* + * Apply the transaction's UNDO, except FILEOPS: its inverses ride in the + * abort record (RecordTransactionAbort), so that a standby and crash + * recovery apply them too. The generic chain is for crash losers. + */ + FileOpsSkipUndo = true; + PG_TRY(); + { + AtAbort_XactUndo(); + } + PG_FINALLY(); + { + FileOpsSkipUndo = false; + } + PG_END_TRY(); /* * do abort processing @@ -6218,7 +6256,8 @@ XactLogAbortRecord(TimestampTz abort_time, int nrels, RelFileLocator *rels, int ndroppedstats, xl_xact_stats_item *droppedstats, int xactflags, TransactionId twophase_xid, - const char *twophase_gid) + const char *twophase_gid, + const char *fileops, uint32 fileops_len) { xl_xact_abort xlrec; xl_xact_xinfo xl_xinfo; @@ -6233,7 +6272,7 @@ XactLogAbortRecord(TimestampTz abort_time, Assert(CritSectionCount > 0); - xl_xinfo.xinfo = 0; + xl_xinfo.xinfo = fileops_len ? XACT_XINFO_HAS_FILEOPS : 0; /* decide between a plain and 2pc abort */ if (!TransactionIdIsValid(twophase_xid)) @@ -6346,6 +6385,12 @@ XactLogAbortRecord(TimestampTz abort_time, if (xl_xinfo.xinfo & XACT_XINFO_HAS_ORIGIN) XLogRegisterData(&xl_origin, sizeof(xl_xact_origin)); + if (fileops_len) + { + XLogRegisterData(&fileops_len, sizeof(fileops_len)); + XLogRegisterData(fileops, fileops_len); + } + /* Include the replication origin */ XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN); @@ -6568,6 +6613,13 @@ xact_redo_abort(xl_xact_parsed_abort *parsed, TransactionId xid, false /* backward */ , false /* WAL */ ); } + if (parsed->fileops_len) + { + XLogFlush(lsn); + INJECTION_POINT("fileops-abort-after-flush", NULL); + FileOpsApplyAbort(parsed->fileops, parsed->fileops_len); + } + /* Make sure files supposed to be dropped are dropped */ if (parsed->nrels > 0) { diff --git a/src/backend/storage/file/Makefile b/src/backend/storage/file/Makefile index 660ac51807e..8bcfafec7cf 100644 --- a/src/backend/storage/file/Makefile +++ b/src/backend/storage/file/Makefile @@ -16,6 +16,8 @@ OBJS = \ buffile.o \ copydir.o \ fd.o \ + fileops.o \ + fileops_undo.o \ fileset.o \ reinit.o \ sharedfileset.o diff --git a/src/backend/storage/file/README b/src/backend/storage/file/README new file mode 100644 index 00000000000..26c9a5b5120 --- /dev/null +++ b/src/backend/storage/file/README @@ -0,0 +1,272 @@ +src/backend/storage/file/README + +Transactional File Operations (FILEOPS) +======================================= + +This directory's fileops.c implements FILEOPS: a set of filesystem +operations that participate in PostgreSQL transactions. An ordinary +unlink() or rename() issued in the middle of a transaction is not covered by +WAL and is not reversed if the transaction aborts or the server crashes. +FILEOPS closes that gap for the filesystem mutations the server performs +outside the buffer manager. + +FILEOPS is NOT used for relation files. Creating and dropping tables, +indexes, TRUNCATE, and CREATE/DROP DATABASE|TABLESPACE at the catalog/smgr +level go through smgr and catalog/storage.c, which have zero FileOps calls. +The only in-core callers of the FileOps* API are: + + - dbcommands.c (CREATE/ALTER/DROP DATABASE: directory + PG_VERSION, + ALTER DATABASE ... SET TABLESPACE) + - tablespace.c (CREATE/DROP TABLESPACE: the per-tablespace directory + and symlink; the LOCATION paths are absolute by design) + - copydir.c (the destination tree of a directory copy) + + +When each operation takes effect, and what undoes it +---------------------------------------------------- + +The rule: everything that CREATES or CHANGES an object takes effect at once +and is undone if the transaction aborts; DELETING an object is DEFERRED to +transaction end, so nothing is ever destroyed that a rollback would need. + + operation takes effect undone on abort by (*) + --------------------- ---------------- ------------------------------ + FileOpsCreate now unlink the file + FileOpsMkdir now rmdir the directory + FileOpsMkdirTree now remove the whole tree + FileOpsChmod / Chown now restore the old mode / owner + FileOpsTruncate now restore length and removed bytes + FileOpsWrite now restore the overwritten bytes + FileOpsSymlink / Link now unlink the link + FileOpsRename now rename back + FileOpsSetXattr / now restore the old value + FileOpsRemoveXattr + FileOpsDelete at commit (**) nothing to undo: not done yet + FileOpsRmdir / at commit (**) nothing to undo: not done yet + FileOpsRmdirRecursive + + (*) at wal_level >= replica, by the operation's UNDO record: on ROLLBACK, + ROLLBACK TO SAVEPOINT and ROLLBACK PREPARED (applied with the abort + record, which carries the inverses), on a standby replaying that + abort record, and by the startup undo phase for a transaction a crash + left in flight. At wal_level = minimal, by an abort-time pending op + on ROLLBACK and ROLLBACK TO SAVEPOINT only (see below). + (**) or at abort, with at_commit = false. + +Each undo reverts only an object that still shows the operation's effect +(fileops_undo_apply() in fileops_undo.c), so a name reused in the meantime is +left alone. Undo runs newest first, so an operation on a renamed file's new +name is undone before the rename. + + +Immediate operations +-------------------- + +These run when the API is called, so the file or directory can be used +within the same transaction, and each registers an abort-time undo action +that reverses it if the transaction rolls back: + + FileOpsCreate -- create() now; UNDO/pending-delete on abort + FileOpsMkdir -- mkdir() now; UNDO rmdir on abort + FileOpsMkdirTree -- mkdir() now; UNDO removes the whole tree on abort + FileOpsChmod -- chmod() now; UNDO restore old mode on abort + FileOpsChown -- chown() now; UNDO restore old uid/gid on abort + FileOpsTruncate -- ftruncate() now; UNDO restore old length on abort + FileOpsSymlink -- symlink() now; UNDO unlink on abort + FileOpsLink -- link() now; UNDO unlink on abort + FileOpsRename -- rename() now; UNDO rename back on abort + FileOpsWrite -- pwrite() now (WAL-logged, bounded before-image UNDO) + FileOpsSetXattr / FileOpsRemoveXattr -- now; UNDO restore old value on abort + +FileOpsRename(oldpath, newpath) renames a file or directory. It never +replaces newpath: an existing newpath is an ERROR (ERRCODE_DUPLICATE_FILE), +as are a missing oldpath and a newpath on another filesystem, and each +changes nothing. Undoing a replacement would need the replaced object kept +somewhere until the transaction ends, and no caller needs one. The rename +is reversed only while newpath exists and oldpath is free. Like the undo of +FileOpsCreate and FileOpsMkdir, that test looks at the paths alone: the +device and inode of the object are not stable across a restore, a promoted +standby, or redo that makes the object again, so ROLLBACK PREPARED on a +promoted standby could not use them. +Unlike the other immediate operations it writes and flushes its WAL and UNDO +records BEFORE the rename() call, so a crash cannot lose the old name with +nothing on record: a crash before the call is harmless, since redo makes the +rename and the loser's UNDO reverses it. + + +Deferred operations +------------------- + +The deleting operations only record a PendingFileOp, and the filesystem is +touched at transaction end: + + FileOpsDelete -- deferred; at_commit chooses commit vs abort + FileOpsRmdir / FileOpsRmdirRecursive -- deferred; at_commit + +FileOpsDoPendingOps(isCommit) runs the pending list at transaction end: the +at_commit=true ops fire on commit, the at_commit=false ops fire on abort. + + +When FILEOPS is durable: only at wal_level >= replica +----------------------------------------------------- + +FILEOPS writes WAL and UNDO records ONLY when XLogIsNeeded() is true, i.e. +wal_level >= replica. At wal_level = minimal it emits no WAL and no UNDO: + + - There is NO crash safety at minimal. The immediate constructive ops + mutate the filesystem with no WAL record to redo, and the only reversal + for a crash-mid-transaction is the in-memory pendingFileOps list, which + does not survive a crash. + + - There is NO 2PC safety at minimal. With nothing serialized, a PREPARE'd + transaction's file operations cannot be reproduced after a crash. + +Every FILEOPS + crash-recovery + 2PC guarantee below assumes wal_level >= +replica, which is what any deployment relying on those guarantees must set. + + +Revert and recovery model (NO fileops_cleanup) +---------------------------------------------- + +There is NO fileops_cleanup() function. The RM_FILEOPS_ID WAL resource +manager registers only rm_redo/rm_desc/rm_identify; its rm_startup and +rm_cleanup slots are NULL (see access/rmgrlist.h). Reversal rides two +mechanisms, both on the COMMON WAL stream: + + 1. Live-transaction abort (in-memory). ROLLBACK runs + FileOpsDoPendingOps(false), executing each constructive op's registered + abort-time undo (delete the created file, restore the old + mode/length/xattr, rmdir the created directory, ...). No WAL needed; + the in-process list has the state. + + 2. Crash / async revert (common-WAL UNDO records). Each constructive op + also prepares an UNDO record (PrepareXactUndoData(UNDO_RMID_FILEOPS,...)) + and inserts it (InsertXactUndoData) in the same critical section as its + forward WAL record. On crash, a loser transaction's UNDO chain is + applied by the cluster-wide UNDO engine's recovery pass + (PerformUndoRecovery, at end of redo); the apply side is + fileops_undo_apply() in fileops_undo.c, registered through the UNDO rmgr + registry (FileopsUndoRmgrInit), NOT through a WAL rm_cleanup callback. + +Deferred ops emit their forward WAL record at commit-EXECUTION time, inside +FileOpsDoPendingOps() when the op actually happens on the primary, so a +physical standby reproduces it by redo. Their redo handlers are idempotent +(delete of an already-absent file, rmdir of an already-absent directory both +count as success). Only the commit path is logged; an at_commit=false +delete/rmdir run during abort emits no WAL, because a standby must never +apply an aborted transaction's effect. + +Redo replays each record onto the filesystem as it is now, which after a +crash may already include later operations, and so it cannot tell a name +reused later from the original. This is a known limit of FILEOPS redo: +making FILEOPS fully crash-atomic is future work (a resource manager that +logs enough to replay against the state the record was made in). Rename +redo converges on the primary's state as far as the paths allow: + + - oldpath absent: the rename was made already; nothing to do. + - newpath absent: rename oldpath to newpath. + - both present: oldpath was made again by the replay of an earlier record + (the CREATE, MKDIR or LINK that made it since the checkpoint), or by a + later operation, and newpath is the later state. Remove oldpath (a + directory only if it is empty) and keep newpath. If oldpath is a + non-empty directory, leave both, with a WARNING. + +Renaming over newpath instead would lose data. After a -> b, then b -> c, +then a new b, with a older than the checkpoint, redo of b -> c finds the new +b and c both present, and would put the new b in place of c. + +One deliberate exception: + + SYMLINK redo is an intentional no-op. FileOpsSymlink is used only for + tablespace symlinks; standby/crash-recovery replay of those is owned by + RM_TBLSPC, which applies the recovery-time layout remap. + + +Two-phase commit +---------------- + +PREPARE TRANSACTION serializes the transaction's pending FileOps into the +2PC state file so they survive to COMMIT/ROLLBACK PREPARED, including from a +different backend after a crash between PREPARE and the final decision: + + - FileOpsAtPrepare(), called from EndPrepare() between StartPrepare() and + the END sentinel, writes one TWOPHASE_RM_FILEOPS_ID record per pending + at_commit=true op into the durable 2PC state. + + - The FILEOPS 2PC post-commit callback performs those deferred commit-time + ops (delete/rmdir), reusing the same per-op executor as + FileOpsDoPendingOps(). COMMIT PREPARED thus applies them; a crash before + it re-drives them from the durable 2PC state. + + - The immediate ops' inverses are serialized separately, as one + FILEOPS_TWOPHASE_INVERSES record. ROLLBACK PREPARED carries them in its + abort record and applies them before it releases the prepared + transaction's locks, as an ordinary abort does; COMMIT PREPARED ignores + them. The FILEOPS post-abort callback is therefore effectively a no-op. + + - PostPrepare_FileOps() then discards this backend's in-memory list; the + durable 2PC state (for commit-time ops) and the UNDO chain (for abort-time + reversal), not this backend, drive the eventual work. + +Before this, PostPrepare_FileOps() freed the pending list without recording +it, so deferred commit-time ops were lost across PREPARE. Adding the +TWOPHASE_RM_FILEOPS_ID record type bumped TWOPHASE_MAGIC (twophase.c). + + +How to use the API +------------------ + +The public API is declared in src/include/storage/fileops.h. Call these in +place of the raw libc/syscall equivalents whenever the operation must be +transactional. + + register_delete (FileOpsCreate). When true, the newly created file is + registered for deletion if the transaction aborts. + + at_commit (FileOpsDelete, FileOpsRmdir, FileOpsRmdirRecursive). + at_commit = true removes on commit (dropping state the transaction is + logically deleting); at_commit = false removes only on abort. + +FileOpsMkdirTree creates a directory that the transaction then fills by +other means (smgr for CREATE DATABASE, copydir() for FILE_COPY and ALTER +DATABASE SET TABLESPACE). Its UNDO record and its abort-time pending op +remove the whole tree, whatever ended up in it, where FileOpsMkdir's only +rmdir() an empty directory. For a database directory the UNDO first drops +that database's shared buffers and pending fsync requests. + +The API hides platform differences: PG_O_DIRECT for unbuffered I/O, +pg_fsync() for the correct per-platform flush, and fsync_parent_path() for +directory durability on Unix. + + +Transaction lifecycle integration +---------------------------------- + +The deferred operations are driven by AM-neutral hooks that xact.c calls +(wired to the FileOps* functions in fileops_undo.c): + + FileOpsDoPendingOps(isCommit) -- executes the pending list at end of xact. + AtSubCommit_FileOps() -- reparents a subxact's pending ops. + AtSubAbort_FileOps() -- runs a subxact's abort-time ops. + FileOpsAtPrepare() -- serializes pending ops into 2PC state. + PostPrepare_FileOps() -- discards the in-memory list after PREPARE. + FileOpsCancelPendingDelete() -- removes a registered delete entry. + FileOpsSync() -- forces durability of a path immediately. + + +WAL replay +---------- + +fileops_redo() dispatches each XLOG_FILEOPS_* record to the matching action +during recovery; fileops_desc()/fileops_identify() render records for +pg_waldump. Because the redo handler is itself the recovery action, code +paths that run under WAL replay (the "redo" arms in dbcommands.c, +tablespace.c, and copydir.c) call the raw syscalls directly rather than +re-entering the FileOps* API. + +Tests: src/test/recovery/t/062_fileops_recovery.pl (crash recovery), +063_fileops_2pc.pl (constructive-op 2PC via chmod), +064_fileops_standby_divergence.pl (physical-standby convergence), +061_fileops_2pc_deferred.pl (deferred commit-time ops through PREPARE + +crash), and the test_fileops module's regression test and TAP tests +(018_fileops_rename.pl covers FileOpsRename). diff --git a/src/backend/storage/file/fileops.c b/src/backend/storage/file/fileops.c new file mode 100644 index 00000000000..386eb618910 --- /dev/null +++ b/src/backend/storage/file/fileops.c @@ -0,0 +1,2575 @@ +/*------------------------------------------------------------------------- + * + * fileops.c + * Transactional file operations with WAL logging + * + * This module provides transactional filesystem operations that integrate + * with PostgreSQL's WAL and transaction management. It is NOT used for + * relation files (smgr/catalog handle those); the only in-core callers are + * dbcommands.c, tablespace.c, and copydir.c. + * + * Two op classes: + * - IMMEDIATE ops (create, mkdir, chmod, chown, truncate, symlink, link, + * rename, write, setxattr) take effect at once, and each registers an + * abort-time undo action (a common-WAL UNDO record plus, where relevant, + * a pending op) that reverses it on rollback. + * - DESTRUCTIVE ops (delete, rmdir, rmtree) are DEFERRED: a PendingFileOp + * is recorded now and the filesystem is touched at transaction end by + * FileOpsDoPendingOps(). + * + * WAL and UNDO are written only when XLogIsNeeded() (wal_level >= replica); + * at wal_level = minimal there is no crash or 2PC safety. + * + * An immediate op performs its system call FIRST, outside any critical + * section, and WAL-logs itself only once the call has succeeded: the forward + * WAL record and its UNDO record are inserted and flushed together in a short + * critical section (fileops_log()). So an ordinary filesystem error + * (permission denied, no space, ...) is a plain ERROR, as it is for the same + * call without FILEOPS, and a failed op leaves nothing in the WAL for redo or + * a standby to replay. + * + * This is NOT atomic. Between the system call and the WAL flush there is a + * window in which the change is on disk with no record of it: a crash there + * leaves the change in place, unknown to redo, to a standby and to rollback, + * just as the bare system call would. Nothing reverts it. The + * opposite order (log, then act inside a critical section) closes that window + * only by turning every filesystem error into a PANIC. + * + * FileOpsRename() is the exception: there the window would lose the old name + * for good, so it logs first and renames after, outside the critical section. + * A failed rename() is still a plain ERROR, because its inverse renames back + * only while the new name exists and the old one does not (see there). + * + * Reverting checks the object at the recorded path before touching it (see + * fileops_undo.c), but a path, its type and, where recorded, its inode are + * evidence that the object is the one this transaction made, not proof: + * something else can have replaced it in between. + * + * The deferred-operations pattern works as follows: + * 1. A destructive API call records a PendingFileOp entry in a linked list + * 2. At commit/abort time, FileOpsDoPendingOps() executes or discards + * the pending operations based on transaction outcome (a commit-time + * destructive op emits its WAL record at execution time) + * 3. At PREPARE, FileOpsAtPrepare() serializes the pending ops into the + * 2PC state file so COMMIT/ROLLBACK PREPARED can perform them + * + * Subtransaction support: + * - At subtransaction commit, entries are reassigned to the parent level + * - At subtransaction abort, abort-time actions execute immediately + * + * Platform-specific handling: + * - O_DIRECT: Uses PG_O_DIRECT abstraction (Linux native O_DIRECT, + * macOS F_NOCACHE via fcntl, Windows FILE_FLAG_NO_BUFFERING) + * - fsync: Uses pg_fsync() which selects the appropriate mechanism + * (Linux fdatasync, macOS F_FULLFSYNC, Windows FlushFileBuffers, + * BSD fsync) + * - Directory sync: Uses fsync_fname()/fsync_parent_path() which + * handle directory fsync on Unix platforms (not needed on Windows) + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/storage/file/fileops.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include +#include +#ifdef HAVE_SYS_FCNTL_H +#include +#endif + +/* + * ENODATA is Linux-specific; FreeBSD and other BSDs don't define it. + * When removing extended attributes, ENODATA means "attribute does not + * exist" -- equivalent to ENOATTR on BSDs. + */ +#ifndef ENODATA +#ifdef ENOATTR +#define ENODATA ENOATTR +#else +#define ENODATA ENOENT +#endif +#endif + +#include "access/fileops_xlog.h" +#include "access/rmgr.h" +#include "access/twophase_rmgr.h" +#include "access/undolog.h" +#include "access/undorecord.h" +#include "access/undormgr.h" +#include "access/xact.h" +#include "access/xactundo.h" +#include "access/xlog.h" +#include "access/xloginsert.h" +#include "catalog/pg_class.h" +#include "miscadmin.h" +#include "port/pg_xattr.h" +#include "storage/fd.h" +#include "storage/fileops.h" +#include "storage/proc.h" +#include "utils/injection_point.h" +#include "utils/memutils.h" + + +/* Head of the pending file operations linked list */ +static PendingFileOp *pendingFileOps = NULL; +static Size inverseBytes = 0; + +static void FileOpsExecOne(PendingFileOp *pending, bool isCommit, + XLogRecPtr *max_lsn); + +/* + * fileops_fsync_parent -- fsync the parent directory of a file path + * + * This ensures that directory entry changes (create, delete, rename) + * are durable. On Windows, directory fsync is not needed because NTFS + * journals directory entries; fsync_fname_ext() handles this by being + * a no-op for directories on Windows. + */ +static void +fileops_fsync_parent(const char *fname, int elevel) +{ + char parentpath[MAXPGPATH]; + char *sep; + + strlcpy(parentpath, fname, MAXPGPATH); + + sep = strrchr(parentpath, '/'); + if (sep != NULL) + { + /* Got a path component, fsync the directory portion */ + if (sep == parentpath) + parentpath[1] = '\0'; /* root directory */ + else + *sep = '\0'; + + fsync_fname_ext(parentpath, true, true, elevel); + } +} + +/* + * fileops_pathlen - Length of a path including its terminating NUL + * + * WAL and UNDO records store path lengths in a uint16 and the UNDO payload + * buffers are sized for MAXPGPATH, so refuse a longer path rather than + * truncate it (or overrun a buffer). Called before the system call, so a + * refused path changes nothing. + */ +static int +fileops_pathlen(const char *path) +{ + size_t len = strlen(path) + 1; + + /* Reserve room before any mutation; abort itself must never exceed WAL. */ + if (inverseBytes + 4 * MAXPGPATH + 2 * FILEOPS_MAX_DATA_BYTES > + FILEOPS_MAX_ABORT_BYTES) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("too much transactional file operation undo data"))); + + if (len > MAXPGPATH) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("file operation path length %zu exceeds maximum %d", + len, MAXPGPATH))); + return (int) len; +} + +/* + * fileops_log - WAL-log an immediate operation that has already succeeded + * + * The caller has made the filesystem change, prepared its UNDO record (if + * any, undo_ctx != NULL) with PrepareXactUndoData(), and registered the + * forward record's data after XLogBeginInsert(). Insert both and flush them + * through the end of the UNDO batch, so that a crash leaves either both or + * neither. Only this is a critical section; see the file header comment. + */ +static void +fileops_log(uint8 info, XactUndoContext *undo_ctx) +{ + START_CRIT_SECTION(); + (void) XLogInsert(RM_FILEOPS_ID, info); + if (undo_ctx != NULL) + { + InsertXactUndoData(undo_ctx); + CleanupXactUndoInsertion(undo_ctx); + } + XLogFlush(XactLastRecEnd); + END_CRIT_SECTION(); +} + +/* + * AddPendingFileOp - Add a new pending file operation to the list + * + * All fields are deep-copied into TopMemoryContext to survive + * until transaction end, following the PendingRelDelete pattern. + */ +static void +AddPendingFileOp(PendingFileOpType type, const char *path, + const char *newpath, off_t length, bool at_commit) +{ + PendingFileOp *pending; + MemoryContext oldcxt; + + oldcxt = MemoryContextSwitchTo(TopMemoryContext); + + pending = (PendingFileOp *) palloc(sizeof(PendingFileOp)); + pending->type = type; + pending->path = pstrdup(path); + pending->newpath = newpath ? pstrdup(newpath) : NULL; + pending->length = length; + pending->data = NULL; + pending->data_len = 0; + pending->at_commit = at_commit; + pending->nestLevel = GetCurrentTransactionNestLevel(); + pending->next = pendingFileOps; + pendingFileOps = pending; + + MemoryContextSwitchTo(oldcxt); +} + +/* + * AddPendingFileOpWithData - Add a pending file operation with extra data + * + * Like AddPendingFileOp but also stores arbitrary data (e.g., original + * xattr value for restore on abort). + */ +static void +AddPendingFileOpWithData(PendingFileOpType type, const char *path, + const char *newpath, off_t length, + const void *data, size_t data_len, + bool at_commit) +{ + PendingFileOp *pending; + MemoryContext oldcxt; + + oldcxt = MemoryContextSwitchTo(TopMemoryContext); + + pending = (PendingFileOp *) palloc(sizeof(PendingFileOp)); + pending->type = type; + pending->path = pstrdup(path); + pending->newpath = newpath ? pstrdup(newpath) : NULL; + pending->length = length; + if (data && data_len > 0) + { + pending->data = palloc(data_len); + memcpy(pending->data, data, data_len); + pending->data_len = data_len; + } + else + { + pending->data = NULL; + pending->data_len = 0; + } + pending->at_commit = at_commit; + pending->nestLevel = GetCurrentTransactionNestLevel(); + pending->next = pendingFileOps; + pendingFileOps = pending; + + MemoryContextSwitchTo(oldcxt); +} + +/* Keep the same inverse for online abort, abort redo, and crash losers. */ +static void +fileops_prepare_undo(XactUndoContext *ctx, char persistence, uint8 rmid, + uint16 info, Oid reloid, const char *payload, Size len) +{ + AddPendingFileOpWithData(PENDING_FILEOP_INVERSE, "", NULL, + info, payload, len, false); + inverseBytes += len + 2 * sizeof(uint32); + FileOpsReserveInverseSyncs(); + PrepareXactUndoData(ctx, persistence, rmid, info, reloid, payload, len); +} + +/* Serialize in execution (newest-first) order, without alignment assumptions. */ +void +FileOpsGetAbortData(char **data, uint32 *len) +{ + PendingFileOp *p; + StringInfoData buf; + int level = GetCurrentTransactionNestLevel(); + + *data = NULL; + *len = 0; + if (inverseBytes == 0) + return; + initStringInfo(&buf); + for (p = pendingFileOps; p; p = p->next) + { + uint32 info = p->length; + uint32 size = p->data_len; + + if (p->type != PENDING_FILEOP_INVERSE || p->nestLevel < level) + continue; + appendBinaryStringInfo(&buf, &info, sizeof(info)); + appendBinaryStringInfo(&buf, &size, sizeof(size)); + appendBinaryStringInfo(&buf, p->data, size); + } + *data = buf.data; + *len = buf.len; +} + +/* + * CreateCheckPoint waits for this flag after choosing its redo pointer. Even + * though inverses ride in abort WAL, a checkpoint must not omit that record + * until the corresponding filesystem changes have been fsynced. A database + * directory tree can make this interval long; there is no retry loop here. + */ +void +FileOpsBeginAbort(void) +{ + MyProc->delayChkptFlags |= DELAY_CHKPT_START; +} + +void +FileOpsEndAbort(void) +{ + MyProc->delayChkptFlags &= ~DELAY_CHKPT_START; +} + +/* Apply the self-contained abort payload, also used during WAL replay. */ +void +FileOpsApplyAbort(const char *data, uint32 len) +{ + while (len > 0) + { + uint32 info; + uint32 size; + + if (len < 2 * sizeof(uint32)) + elog(PANIC, "invalid FILEOPS abort payload"); + memcpy(&info, data, sizeof(info)); + memcpy(&size, data + sizeof(info), sizeof(size)); + data += 2 * sizeof(uint32); + len -= 2 * sizeof(uint32); + if (size > len) + elog(PANIC, "invalid FILEOPS inverse length"); + FileOpsApplyInverse(info, data, size); + data += size; + len -= size; + } + FileOpsSyncInverses(PANIC); +} + +/* + * FreePendingFileOp - Free a pending file operation entry + */ +static void +FreePendingFileOp(PendingFileOp *pending) +{ + if (pending->type == PENDING_FILEOP_INVERSE) + inverseBytes -= pending->data_len + 2 * sizeof(uint32); + if (pending->path) + pfree(pending->path); + if (pending->newpath) + pfree(pending->newpath); + if (pending->data) + pfree(pending->data); + pfree(pending); +} + +/* + * FileOpsCancelPendingDelete - Cancel a pending file deletion + * + * This removes matching DELETE entries from the pendingFileOps list. + * It is called by RelationPreserveStorage() to ensure that when a + * relation's storage is preserved (e.g., during index reuse in ALTER TABLE), + * the corresponding FileOps DELETE entry is also cancelled, preventing + * FileOpsDoPendingOps from deleting the file at commit time. + */ +void +FileOpsCancelPendingDelete(const char *path, bool at_commit) +{ + PendingFileOp *pending; + PendingFileOp *prev; + PendingFileOp *next; + + prev = NULL; + for (pending = pendingFileOps; pending != NULL; pending = next) + { + next = pending->next; + if (pending->type == PENDING_FILEOP_DELETE && + pending->at_commit == at_commit && + strcmp(pending->path, path) == 0) + { + /* unlink and free list entry */ + if (prev) + prev->next = next; + else + pendingFileOps = next; + FreePendingFileOp(pending); + /* prev does not change */ + } + else + { + prev = pending; + } + } +} + +/* + * FileOpsSync - Ensure a file's data is durably written to disk + * + * This is a convenience wrapper around fsync_fname() that uses the + * platform-appropriate sync mechanism: + * - Linux: fdatasync() (only flushes data, not metadata unless needed) + * - macOS: fcntl(F_FULLFSYNC) (flushes disk write cache) + * - FreeBSD: fsync() + * - Windows: FlushFileBuffers() + * + * An ERROR is raised if the sync fails. + */ +/* + * FileOpsCreate - Create a file within a transaction + * + * Creates the file immediately (so it can be used within the transaction) + * and logs the creation to WAL. If register_delete is true, the file will + * be deleted if the transaction aborts. + * + * Platform handling: + * - Linux/FreeBSD: O_DIRECT passed directly to open() + * - macOS: F_NOCACHE fcntl applied after open() + * - Windows: FILE_FLAG_NO_BUFFERING (handled by port layer) + * + * Returns the file descriptor; failing to create the file is an ERROR. + */ +int +FileOpsCreate(const char *path, int flags, mode_t mode, bool register_delete) +{ + int fd; + int pathlen = fileops_pathlen(path); + + Assert(!IsInParallelMode()); + + fd = OpenTransientFilePerm(path, flags | O_CREAT, mode); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create file \"%s\": %m", path))); + + if (register_delete && !XLogIsNeeded()) + AddPendingFileOp(PENDING_FILEOP_DELETE, path, NULL, 0, false); + + if (enableFsync) + { + if (pg_fsync(fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", path))); + fileops_fsync_parent(path, data_sync_elevel(ERROR)); + } + + if (XLogIsNeeded()) + { + xl_fileops_create xlrec; + char undo_payload[sizeof(FileopsUndoCreate) + MAXPGPATH]; + FileopsUndoCreate *hdr = (FileopsUndoCreate *) undo_payload; + XactUndoContext undo_ctx; + + xlrec.flags = flags; + xlrec.mode = mode; + xlrec.register_delete = register_delete; + + hdr->path_len = (uint16) pathlen; + memcpy(undo_payload + sizeof(FileopsUndoCreate), path, pathlen); + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_CREATE, + InvalidOid, undo_payload, + sizeof(FileopsUndoCreate) + pathlen); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsCreate); + XLogRegisterData(path, pathlen); + fileops_log(XLOG_FILEOPS_CREATE, &undo_ctx); + + INJECTION_POINT("fileops-create-after-wal-flush", NULL); + } + + return fd; +} + +/* + * FileOpsDelete - Schedule a file deletion within a transaction + * + * The file is not deleted immediately. Instead, the deletion is deferred + * to transaction commit (if at_commit is true) or abort (if false). + * On Unix: unlink() with parent dir fsync. + * On Windows: pgunlink() with retry on EACCES. + */ +void +FileOpsDelete(const char *path, bool at_commit) +{ + Assert(!IsInParallelMode()); + + /* + * no WAL is emitted at registration time. The deferred delete is + * WAL-logged in FileOpsDoPendingOps() at commit-execution time, when it + * actually happens on the primary, so a physical standby reproduces it + * via redo. Registration-time logging would be a no-op on redo (the + * standby never runs FileOpsDoPendingOps()) and, for at_commit=false + * delete-on-abort registrations, would wrongly apply an aborted + * transaction's effect on the standby. + */ + AddPendingFileOp(PENDING_FILEOP_DELETE, path, NULL, 0, at_commit); +} + +/* + * FileOpsRename - Rename a file or directory within a transaction + * + * Immediate execution: when this returns, newpath exists and oldpath does + * not, and the rename is reversed if the transaction or subtransaction + * aborts (rename back, while newpath exists and oldpath is free). Unlike + * rename(2), it never replaces newpath: rolling back a + * replacement would need the replaced object kept somewhere, and no caller + * needs one. An existing newpath, a missing oldpath and a newpath on another + * filesystem are each an ERROR that changes nothing. The check that newpath + * is free is not atomic with the rename; callers serialize the use of a path, + * as they must for any FILEOPS operation. + * + * Unlike the other immediate operations, the WAL and UNDO records are + * written and flushed BEFORE the system call. The other order would let a + * crash between rename() and the flush lose the old name for good, with + * nothing on record to undo it. In this order a crash before rename() is + * harmless: redo performs the rename and the loser's UNDO reverses it. If + * rename() fails after the WAL, it is an ERROR: the abort's inverse finds + * oldpath still in place and leaves it, and a standby that replayed the + * forward record reverses it when it replays the abort record. + * + * At wal_level = minimal nothing is logged, and an abort-time pending op + * renames the object back. + * + * On Windows, pgrename() retries MoveFileEx() while the file is in use. + * + * Returns 0; any failure is an ERROR. + */ +int +FileOpsRename(const char *oldpath, const char *newpath) +{ + int oldpathlen = fileops_pathlen(oldpath); + int newpathlen = fileops_pathlen(newpath); + struct stat st; + struct stat dst; + char parent[MAXPGPATH]; + + Assert(!IsInParallelMode()); + + if (lstat(oldpath, &st) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", oldpath))); + if (lstat(newpath, &dst) == 0) + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_FILE), + errmsg("could not rename file \"%s\" to \"%s\": file already exists", + oldpath, newpath))); + if (errno != ENOENT) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", newpath))); + strlcpy(parent, newpath, sizeof(parent)); + get_parent_directory(parent); + if (parent[0] == '\0') + strlcpy(parent, ".", sizeof(parent)); + if (stat(parent, &dst) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat directory \"%s\": %m", parent))); + if (dst.st_dev != st.st_dev) + { + errno = EXDEV; + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("could not rename file \"%s\" to \"%s\": %m", + oldpath, newpath))); + } + + if (XLogIsNeeded()) + { + xl_fileops_rename xlrec; + XactUndoContext undo_ctx; + char payload[sizeof(FileopsUndoRename) + 2 * MAXPGPATH]; + FileopsUndoRename *hdr = (FileopsUndoRename *) payload; + + xlrec.oldpath_len = (uint16) oldpathlen; + + hdr->oldpath_len = (uint16) oldpathlen; + hdr->newpath_len = (uint16) newpathlen; + memcpy(payload + sizeof(FileopsUndoRename), oldpath, oldpathlen); + memcpy(payload + sizeof(FileopsUndoRename) + oldpathlen, newpath, + newpathlen); + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_RENAME, + InvalidOid, payload, + sizeof(FileopsUndoRename) + oldpathlen + + newpathlen); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsRename); + XLogRegisterData(oldpath, oldpathlen); + XLogRegisterData(newpath, newpathlen); + fileops_log(XLOG_FILEOPS_RENAME, &undo_ctx); + + INJECTION_POINT("fileops-rename-after-wal-flush", NULL); + } + else + { + /* Registered first: it does nothing if the rename does not happen. */ + AddPendingFileOp(PENDING_FILEOP_RENAME, newpath, oldpath, 0, false); + } + + if (rename(oldpath, newpath) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not rename file \"%s\" to \"%s\": %m", + oldpath, newpath))); + + if (enableFsync) + { + fileops_fsync_parent(newpath, data_sync_elevel(ERROR)); + fileops_fsync_parent(oldpath, data_sync_elevel(ERROR)); + } + + return 0; +} + +/* Capture a bounded range before changing it, including shrink tails. */ +static char * +fileops_before_image(int fd, const char *path, off_t offset, Size len) +{ + char *data; + + if (len > FILEOPS_MAX_DATA_BYTES) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("file operation before-image exceeds %d bytes", + FILEOPS_MAX_DATA_BYTES))); + data = palloc(len ? len : 1); + if (len && pg_pread(fd, data, len, offset) != len) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read before-image of file \"%s\": %m", path))); + return data; +} + +/* + * FileOpsWrite - Write data to a file at a specific offset + * + * Immediate execution, WAL-logged for crash recovery replay. + * Uses pwrite() on POSIX. On Windows: SetFilePointerEx + WriteFile. + * + * Bounded before-images restore overwritten bytes and the original length. + * + * Returns 0; a failed write is an ERROR. + */ +int +FileOpsWrite(const char *path, off_t offset, const void *data, uint32 len) +{ + int fd; + ssize_t nbytes; + int pathlen = fileops_pathlen(path); + struct stat st; + char *before; + uint32 before_len; + char *payload; + Size payload_len; + FileopsUndoWrite *hdr; + XactUndoContext undo_ctx; + + Assert(!IsInParallelMode()); + + fd = OpenTransientFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\" for writing: %m", path))); + + if (offset < 0 || len > FILEOPS_MAX_DATA_BYTES || + offset > PG_INT64_MAX - len) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("invalid or excessive transactional file write range"))); + if (fstat(fd, &st) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", path))); + before_len = offset < st.st_size ? Min((off_t) len, st.st_size - offset) : 0; + before = fileops_before_image(fd, path, offset, before_len); + payload_len = sizeof(FileopsUndoWrite) + pathlen + before_len + len; + payload = palloc0(payload_len); + hdr = (FileopsUndoWrite *) payload; + hdr->orig_length = st.st_size; + hdr->new_length = Max(st.st_size, offset + len); + hdr->offset = offset; + hdr->before_len = before_len; + hdr->after_len = len; + hdr->path_len = pathlen; + memcpy(payload + sizeof(*hdr), path, pathlen); + memcpy(payload + sizeof(*hdr) + pathlen, before, before_len); + memcpy(payload + sizeof(*hdr) + pathlen + before_len, data, len); + + /* Keep an inverse even if pwrite changes only part of the range. */ + if (XLogIsNeeded()) + { + (void) GetCurrentTransactionId(); + AddPendingFileOpWithData(PENDING_FILEOP_INVERSE, "", NULL, + FILEOPS_UNDO_WRITE, payload, payload_len, false); + inverseBytes += payload_len + 2 * sizeof(uint32); + } + + nbytes = pg_pwrite(fd, data, + IS_INJECTION_POINT_ATTACHED("fileops-write-short") ? len / 2 : len, + offset); + if (nbytes != (ssize_t) len) + { + int save_errno = errno; + + /* Match the actual after-image, not the requested complete write. */ + if (XLogIsNeeded()) + { + FileopsUndoWrite *inverse = (FileopsUndoWrite *) pendingFileOps->data; + + inverse->after_len = nbytes > 0 ? nbytes : 0; + inverse->new_length = nbytes > 0 ? + Max(st.st_size, offset + nbytes) : st.st_size; + pendingFileOps->data_len -= len - inverse->after_len; + } + CloseTransientFile(fd); + errno = save_errno; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write %u bytes to file \"%s\" at offset %lld: %m", + len, path, (long long) offset))); + } + + if (enableFsync && pg_fsync(fd) != 0) + { + int save_errno = errno; + + CloseTransientFile(fd); + errno = save_errno; + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\" after write: %m", path))); + } + + if (CloseTransientFile(fd) != 0) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", path))); + + if (XLogIsNeeded()) + { + xl_fileops_write xlrec; + + PrepareXactUndoData(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_WRITE, + InvalidOid, payload, payload_len); + xlrec.offset = offset; + xlrec.len = len; + xlrec.path_len = (uint16) pathlen; + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsWrite); + XLogRegisterData(path, pathlen); + XLogRegisterData(data, len); + fileops_log(XLOG_FILEOPS_WRITE, &undo_ctx); + } + + pfree(before); + pfree(payload); + return 0; +} + +/* + * FileOpsTruncate - Truncate a file within a transaction + * + * Executed immediately and WAL-logged. Uses ftruncate() on POSIX, + * SetEndOfFile() on Windows. File is fsynced after truncation. The + * original size is restored if the transaction aborts. + */ +void +FileOpsTruncate(const char *path, off_t length) +{ + int fd; + struct stat st; + off_t orig_length = 0; + int pathlen = fileops_pathlen(path); + char *before; + uint32 before_len; + + Assert(!IsInParallelMode()); + + /* Capture original length for UNDO before truncation */ + if (stat(path, &st) == 0) + orig_length = st.st_size; + + fd = OpenTransientFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\" for truncation: %m", path))); + + if (length < 0) + ereport(ERROR, (errmsg("negative file truncation length"))); + before_len = orig_length > length ? + Min(orig_length - length, (off_t) FILEOPS_MAX_DATA_BYTES + 1) : 0; + before = fileops_before_image(fd, path, length, before_len); + + if (ftruncate(fd, length) < 0) + { + int save_errno = errno; + + CloseTransientFile(fd); + errno = save_errno; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not truncate file \"%s\" to %lld bytes: %m", + path, (long long) length))); + } + + /* + * Register abort-time pending op to restore original size on rollback. + * This provides immediate rollback without waiting for the background + * UNDO worker. + */ + if (!XLogIsNeeded()) + AddPendingFileOp(PENDING_FILEOP_TRUNCATE, path, NULL, orig_length, false); + + if (enableFsync && pg_fsync(fd) != 0) + { + int save_errno = errno; + + CloseTransientFile(fd); + errno = save_errno; + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\" after truncation: %m", + path))); + } + + if (CloseTransientFile(fd) != 0) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", path))); + + if (XLogIsNeeded()) + { + xl_fileops_truncate xlrec; + XactUndoContext undo_ctx; + char *payload = palloc0(sizeof(FileopsUndoTruncate) + pathlen + before_len); + FileopsUndoTruncate *hdr = (FileopsUndoTruncate *) payload; + + xlrec.length = length; + + hdr->before_len = before_len; + hdr->orig_length = orig_length; + hdr->new_length = length; + hdr->path_len = (uint16) pathlen; + memcpy(payload + sizeof(FileopsUndoTruncate), path, pathlen); + memcpy(payload + sizeof(FileopsUndoTruncate) + pathlen, before, before_len); + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_TRUNCATE, + InvalidOid, payload, + sizeof(FileopsUndoTruncate) + pathlen + before_len); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsTruncate); + XLogRegisterData(path, pathlen); + fileops_log(XLOG_FILEOPS_TRUNCATE, &undo_ctx); + pfree(payload); + } + pfree(before); +} + +/* + * FileOpsChmod - Change file permissions within a transaction + * + * Immediate execution, WAL-logged. + * On POSIX: chmod(). On Windows: _chmod() with limited mode bits + * (only _S_IREAD/_S_IWRITE; no group/other). Logs WARNING for + * unsupported mode bits on Windows. + * + * Returns 0 on success, -1 on failure with errno left set, so the caller + * can tell ENOENT from other errors and report it as it would a plain + * chmod() failure (tablespace.c does). Nothing is logged or registered + * when it fails, because nothing changed. + */ +int +FileOpsChmod(const char *path, mode_t mode) +{ + struct stat st; + mode_t orig_mode; + int pathlen = fileops_pathlen(path); + + Assert(!IsInParallelMode()); + + /* Capture original mode for UNDO before chmod */ + if (stat(path, &st) != 0) + return -1; /* leave errno set by stat() for the caller */ + orig_mode = st.st_mode & 07777; + + if (chmod(path, mode) < 0) + return -1; /* leave errno set by chmod() for the caller */ + + /* + * Register abort-time pending op to restore original mode on rollback. + * Store mode in the length field (mode_t fits in off_t). + */ + if (!XLogIsNeeded()) + AddPendingFileOp(PENDING_FILEOP_CHMOD, path, NULL, (off_t) orig_mode, false); + + if (XLogIsNeeded()) + { + xl_fileops_chmod xlrec; + XactUndoContext undo_ctx; + char payload[sizeof(FileopsUndoChmod) + MAXPGPATH]; + FileopsUndoChmod *hdr = (FileopsUndoChmod *) payload; + + xlrec.mode = mode; + xlrec.orig_mode = orig_mode; + + hdr->orig_mode = orig_mode; + hdr->new_mode = mode & 07777; + hdr->path_len = (uint16) pathlen; + memcpy(payload + sizeof(FileopsUndoChmod), path, pathlen); + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_CHMOD, + InvalidOid, payload, + sizeof(FileopsUndoChmod) + pathlen); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsChmod); + XLogRegisterData(path, pathlen); + fileops_log(XLOG_FILEOPS_CHMOD, &undo_ctx); + + INJECTION_POINT("fileops-chmod-after-wal-flush", NULL); + } + + return 0; +} + +/* + * FileOpsChown - Change file ownership within a transaction + * + * Immediate execution, WAL-logged. + * On POSIX: chown(). On Windows: no-op with WARNING (Windows uses + * ACLs for ownership, not uid/gid). + * + * Returns 0; a failed chown() is an ERROR. + */ +int +FileOpsChown(const char *path, uid_t uid, gid_t gid) +{ + struct stat st; + uid_t orig_uid = (uid_t) -1; + gid_t orig_gid = (gid_t) -1; + int pathlen = fileops_pathlen(path); + + Assert(!IsInParallelMode()); + + /* Capture original ownership for UNDO before chown */ + if (stat(path, &st) == 0) + { + orig_uid = st.st_uid; + orig_gid = st.st_gid; + } + +#ifndef WIN32 + if (chown(path, uid, gid) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not chown file \"%s\" to %d:%d: %m", + path, (int) uid, (int) gid))); +#else + ereport(WARNING, + (errmsg("chown is not supported on Windows, skipping for \"%s\"", + path))); + return 0; /* nothing changed, so nothing to log */ +#endif + + if (XLogIsNeeded()) + { + xl_fileops_chown xlrec; + XactUndoContext undo_ctx; + char payload[sizeof(FileopsUndoChown) + MAXPGPATH]; + FileopsUndoChown *hdr = (FileopsUndoChown *) payload; + + xlrec.uid = uid; + xlrec.gid = gid; + xlrec.orig_uid = orig_uid; + xlrec.orig_gid = orig_gid; + + hdr->orig_uid = orig_uid; + hdr->orig_gid = orig_gid; + /* chown() leaves an id of -1 unchanged */ + hdr->new_uid = (uid == (uid_t) -1) ? orig_uid : uid; + hdr->new_gid = (gid == (gid_t) -1) ? orig_gid : gid; + hdr->path_len = (uint16) pathlen; + memcpy(payload + sizeof(FileopsUndoChown), path, pathlen); + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_CHOWN, + InvalidOid, payload, + sizeof(FileopsUndoChown) + pathlen); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsChown); + XLogRegisterData(path, pathlen); + fileops_log(XLOG_FILEOPS_CHOWN, &undo_ctx); + } + + return 0; +} + +void +FileOpsSync(const char *path) +{ + fsync_fname(path, false); +} + +/* + * fileops_mkdir - Create a directory within a transaction + * + * Shared by FileOpsMkdir() and FileOpsMkdirTree(); see there. + */ +static int +fileops_mkdir(const char *path, mode_t mode, uint16 flags, Oid dboid) +{ + int pathlen = fileops_pathlen(path); + + Assert(!IsInParallelMode()); + + if (MakePGDirectory(path) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\": %m", path))); + + /* Remove the directory (or tree) if the transaction aborts */ + if (!XLogIsNeeded()) + AddPendingFileOp((flags & FILEOPS_MKDIR_TREE) ? + PENDING_FILEOP_RMTREE : PENDING_FILEOP_RMDIR, + path, NULL, 0, false); + + if (enableFsync) + fileops_fsync_parent(path, data_sync_elevel(ERROR)); + + if (XLogIsNeeded()) + { + xl_fileops_mkdir xlrec; + XactUndoContext undo_ctx; + char payload[sizeof(FileopsUndoMkdir) + MAXPGPATH]; + FileopsUndoMkdir *hdr = (FileopsUndoMkdir *) payload; + struct stat st; + + xlrec.mode = mode; + xlrec.flags = flags; + + if (stat(path, &st) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat directory \"%s\": %m", path))); + hdr->dev = st.st_dev; + hdr->ino = st.st_ino; + hdr->dboid = dboid; + hdr->flags = flags; + hdr->path_len = (uint16) pathlen; + memcpy(payload + sizeof(FileopsUndoMkdir), path, pathlen); + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_MKDIR, + InvalidOid, payload, + sizeof(FileopsUndoMkdir) + pathlen); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsMkdir); + XLogRegisterData(path, pathlen); + fileops_log(XLOG_FILEOPS_MKDIR, &undo_ctx); + } + + return 0; +} + +/* + * FileOpsMkdir - Create a directory within a transaction + * + * Immediate execution; the directory is removed (rmdir) if the transaction + * aborts, so it must then be empty. + * Uses MakePGDirectory() pattern (mkdir with pg_dir_create_mode). + * On Windows: _mkdir() (no mode parameter, permissions from parent). + * + * Returns 0; a failed mkdir() is an ERROR. + */ +int +FileOpsMkdir(const char *path, mode_t mode) +{ + return fileops_mkdir(path, mode, 0, InvalidOid); +} + +/* + * FileOpsMkdirTree - Create a directory the transaction will fill + * + * Like FileOpsMkdir(), but if the transaction aborts the whole tree is + * removed, including files created in it by other means (smgr, copydir()). + * For a database directory pass its OID as dboid: before the tree is removed + * by the UNDO record, that database's shared buffers and pending fsync + * requests are dropped, as they would be by DROP DATABASE. + */ +int +FileOpsMkdirTree(const char *path, mode_t mode, Oid dboid) +{ + return fileops_mkdir(path, mode, FILEOPS_MKDIR_TREE, dboid); +} + +/* + * FileOpsRmdir - Remove a directory within a transaction + * + * Deferred to commit time (like DELETE). Uses rmdir() on all platforms. + * On Windows: _rmdir(). + */ +void +FileOpsRmdir(const char *path, bool at_commit) +{ + Assert(!IsInParallelMode()); + + /* + * WAL is emitted at commit-execution time in FileOpsDoPendingOps(), not + * at registration, so the standby reproduces the rmdir via redo. + */ + AddPendingFileOp(PENDING_FILEOP_RMDIR, path, NULL, 0, at_commit); +} + +/* + * FileOpsRmdirRecursive - Schedule recursive directory removal. + * + * Enumerates directory tree and schedules: + * - FileOpsDelete() for each file (at_commit) + * - FileOpsRmdir() for each subdirectory (at_commit, leaf-first order) + * + * Permissive: ENOENT is logged but not an error. + */ +void +FileOpsRmdirRecursive(const char *path, bool at_commit) +{ + DIR *dir; + struct dirent *de; + + dir = AllocateDir(path); + if (dir == NULL) + { + if (errno == ENOENT) + { + elog(LOG, "FileOpsRmdirRecursive: directory \"%s\" does not exist", + path); + return; + } + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open directory \"%s\": %m", path))); + } + + while ((de = ReadDirExtended(dir, path, LOG)) != NULL) + { + char subpath[MAXPGPATH]; + struct stat st; + + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) + continue; + + snprintf(subpath, MAXPGPATH, "%s/%s", path, de->d_name); + if (lstat(subpath, &st) == 0) + { + if (S_ISDIR(st.st_mode)) + FileOpsRmdirRecursive(subpath, at_commit); + else + FileOpsDelete(subpath, at_commit); + } + } + FreeDir(dir); + FileOpsRmdir(path, at_commit); +} + +/* + * FileOpsSymlink - Create a symbolic link within a transaction + * + * Immediate execution. Registers delete-on-abort for cleanup. + * On POSIX: symlink(). On Windows: pgsymlink() which creates NTFS + * junction points via DeviceIoControl(). Note: junction points only + * work for directories on Windows. + * + * Returns 0; a failed symlink() is an ERROR. + */ +int +FileOpsSymlink(const char *target, const char *linkpath) +{ + int targetlen = fileops_pathlen(target); + int linkpathlen = fileops_pathlen(linkpath); + + Assert(!IsInParallelMode()); + + if (symlink(target, linkpath) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create symbolic link \"%s\" -> \"%s\": %m", + linkpath, target))); + + /* Register delete-on-abort to clean up the symlink on rollback */ + if (!XLogIsNeeded()) + AddPendingFileOp(PENDING_FILEOP_DELETE, linkpath, NULL, 0, false); + + if (enableFsync) + fileops_fsync_parent(linkpath, data_sync_elevel(ERROR)); + + if (XLogIsNeeded()) + { + xl_fileops_symlink xlrec; + XactUndoContext undo_ctx; + char payload[sizeof(FileopsUndoSymlink) + 2 * MAXPGPATH]; + FileopsUndoSymlink *hdr = (FileopsUndoSymlink *) payload; + + xlrec.target_len = (uint16) targetlen; + + /* the target lets the UNDO recognize the link it would remove */ + hdr->linkpath_len = (uint16) linkpathlen; + hdr->target_len = (uint16) targetlen; + memcpy(payload + sizeof(FileopsUndoSymlink), linkpath, linkpathlen); + memcpy(payload + sizeof(FileopsUndoSymlink) + linkpathlen, target, + targetlen); + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_SYMLINK, + InvalidOid, payload, + sizeof(FileopsUndoSymlink) + linkpathlen + + targetlen); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsSymlink); + XLogRegisterData(target, targetlen); + XLogRegisterData(linkpath, linkpathlen); + fileops_log(XLOG_FILEOPS_SYMLINK, &undo_ctx); + } + + return 0; +} + +/* + * FileOpsLink - Create a hard link within a transaction + * + * Immediate execution. Registers delete-on-abort for cleanup. + * On POSIX: link(). On Windows: CreateHardLinkA() (NTFS only). + * + * Returns 0; a failed link() is an ERROR. + */ +int +FileOpsLink(const char *oldpath, const char *newpath) +{ + int oldpathlen = fileops_pathlen(oldpath); + int newpathlen = fileops_pathlen(newpath); + struct stat st; + + Assert(!IsInParallelMode()); + + /* + * The new link will be this inode; the UNDO removes newpath only while it + * still is. + */ + if (stat(oldpath, &st) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", oldpath))); + + if (link(oldpath, newpath) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create hard link \"%s\" -> \"%s\": %m", + newpath, oldpath))); + + /* Register delete-on-abort to clean up the link on rollback */ + if (!XLogIsNeeded()) + AddPendingFileOp(PENDING_FILEOP_DELETE, newpath, NULL, 0, false); + + if (enableFsync) + fileops_fsync_parent(newpath, data_sync_elevel(ERROR)); + + if (XLogIsNeeded()) + { + xl_fileops_link xlrec; + XactUndoContext undo_ctx; + char payload[sizeof(FileopsUndoLink) + MAXPGPATH]; + FileopsUndoLink *hdr = (FileopsUndoLink *) payload; + + xlrec.oldpath_len = (uint16) oldpathlen; + + hdr->dev = st.st_dev; + hdr->ino = st.st_ino; + hdr->newpath_len = (uint16) newpathlen; + memcpy(payload + sizeof(FileopsUndoLink), newpath, newpathlen); + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_LINK, + InvalidOid, payload, + sizeof(FileopsUndoLink) + newpathlen); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsLink); + XLogRegisterData(oldpath, oldpathlen); + XLogRegisterData(newpath, newpathlen); + fileops_log(XLOG_FILEOPS_LINK, &undo_ctx); + } + + return 0; +} + +/* + * FileOpsSetXattr - Set an extended attribute on a file + * + * Immediate execution, WAL-logged. + * Uses pg_setxattr() portability layer which handles: + * - Linux: setxattr() + * - macOS: setxattr() with extra options parameter + * - FreeBSD: extattr_set_file() + * - Windows: NTFS Alternate Data Streams + * - Other: returns ENOTSUP + * + * Returns 0 on success. Returns -1 with errno set if the filesystem does + * not support extended attributes (ENOTSUP/EOPNOTSUPP) or refuses this one + * (EPERM/EACCES); any other failure is an ERROR. Nothing is logged or + * registered when it fails, because nothing changed. + */ +int +FileOpsSetXattr(const char *path, const char *name, + const void *value, size_t len) +{ + bool had_value = false; + char *orig_value = NULL; + ssize_t orig_value_len = 0; + int pathlen = fileops_pathlen(path); + ssize_t vlen; + + Assert(!IsInParallelMode()); + + /* + * Capture existing xattr value before overwriting, for the abort-time + * pending op and the UNDO record. If the attribute doesn't exist, + * had_value stays false. + */ + vlen = pg_getxattr(path, name, NULL, 0); + if (vlen > 0) + { + orig_value = (char *) palloc(vlen); + orig_value_len = pg_getxattr(path, name, orig_value, vlen); + if (orig_value_len >= 0) + had_value = true; + else + { + pfree(orig_value); + orig_value = NULL; + orig_value_len = 0; + } + } + else if (vlen == 0) + { + /* Attribute exists but has zero-length value */ + had_value = true; + } + + if (pg_setxattr(path, name, value, len) < 0) + { + int save_errno = errno; + + if (orig_value) + pfree(orig_value); + errno = save_errno; + if (errno == ENOTSUP || errno == EOPNOTSUPP || + errno == EPERM || errno == EACCES) + return -1; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not set extended attribute \"%s\" on \"%s\": %m", + name, path))); + } + + /* + * Register abort-time pending op to restore/remove xattr on rollback. If + * had_value: restore original value. If !had_value: remove the xattr. + */ + if (!XLogIsNeeded() && had_value) + AddPendingFileOpWithData(PENDING_FILEOP_SETXATTR, path, name, + 0, orig_value, (size_t) orig_value_len, false); + else if (!XLogIsNeeded()) + AddPendingFileOp(PENDING_FILEOP_REMOVEXATTR, path, name, 0, false); + + if (XLogIsNeeded()) + { + xl_fileops_setxattr xlrec; + int namelen = (int) strlen(name) + 1; + XactUndoContext undo_ctx; + Size payload_len; + char *payload; + FileopsUndoSetxattr *hdr; + + xlrec.path_len = (uint16) pathlen; + xlrec.name_len = (uint16) namelen; + xlrec.value_len = (uint32) len; + xlrec.had_value = had_value; + xlrec.orig_value_len = (uint32) (had_value ? orig_value_len : 0); + + payload_len = sizeof(FileopsUndoSetxattr) + pathlen + namelen + + (had_value ? orig_value_len : 0) + len; + payload = (char *) palloc(payload_len); + hdr = (FileopsUndoSetxattr *) payload; + + hdr->path_len = (uint16) pathlen; + hdr->name_len = (uint16) namelen; + hdr->orig_value_len = (uint32) (had_value ? orig_value_len : 0); + hdr->new_value_len = (uint32) len; + hdr->had_value = had_value; + + memcpy(payload + sizeof(FileopsUndoSetxattr), path, pathlen); + memcpy(payload + sizeof(FileopsUndoSetxattr) + pathlen, name, namelen); + if (had_value && orig_value_len > 0) + memcpy(payload + sizeof(FileopsUndoSetxattr) + pathlen + namelen, + orig_value, orig_value_len); + if (len > 0) + memcpy(payload + sizeof(FileopsUndoSetxattr) + pathlen + namelen + + hdr->orig_value_len, value, len); + + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_SETXATTR, + InvalidOid, payload, payload_len); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsSetxattr); + XLogRegisterData(path, pathlen); + XLogRegisterData(name, namelen); + XLogRegisterData(value, (uint32) len); + if (had_value && orig_value_len > 0) + XLogRegisterData(orig_value, (uint32) orig_value_len); + fileops_log(XLOG_FILEOPS_SETXATTR, &undo_ctx); + pfree(payload); + } + + if (orig_value) + pfree(orig_value); + + return 0; +} + +/* + * FileOpsRemoveXattr - Remove an extended attribute from a file + * + * Immediate execution, WAL-logged. + * Uses pg_removexattr() portability layer. Removing an attribute that does + * not exist (PG_ENOATTR) succeeds. + * + * Returns 0 on success, -1 if extended attributes are not supported on + * this filesystem/platform (errno left as ENOTSUP/EOPNOTSUPP); any other + * failure is an ERROR. Nothing is logged or registered when it fails. + */ +int +FileOpsRemoveXattr(const char *path, const char *name) +{ + char *saved_value = NULL; + ssize_t saved_value_len = 0; + int pathlen = fileops_pathlen(path); + ssize_t vlen; + + Assert(!IsInParallelMode()); + + /* + * Capture existing xattr value before removal, to restore it on rollback. + * A NULL saved_value means there was none. + */ + vlen = pg_getxattr(path, name, NULL, 0); + if (vlen > 0) + { + saved_value = (char *) palloc(vlen); + saved_value_len = pg_getxattr(path, name, saved_value, vlen); + if (saved_value_len < 0) + { + pfree(saved_value); + saved_value = NULL; + saved_value_len = 0; + } + } + else if (vlen == 0) + { + /* Attribute exists but has zero-length value */ + saved_value_len = 0; + saved_value = (char *) palloc(1); /* non-NULL marker */ + } + + if (pg_removexattr(path, name) < 0 && errno != PG_ENOATTR) + { + int save_errno = errno; + + if (saved_value) + pfree(saved_value); + errno = save_errno; + if (errno == ENOTSUP || errno == EOPNOTSUPP) + return -1; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not remove extended attribute \"%s\" from \"%s\": %m", + name, path))); + } + + /* + * Register abort-time pending op to restore the removed xattr on + * rollback. Use SETXATTR type: on abort, set the xattr back to its saved + * value. + */ + if (!XLogIsNeeded() && saved_value != NULL) + AddPendingFileOpWithData(PENDING_FILEOP_SETXATTR, path, name, + 0, saved_value, (size_t) saved_value_len, false); + + if (XLogIsNeeded()) + { + xl_fileops_removexattr xlrec; + int namelen = (int) strlen(name) + 1; + XactUndoContext undo_ctx; + char *payload = NULL; + bool had_value = (saved_value != NULL); + + xlrec.path_len = (uint16) pathlen; + xlrec.name_len = (uint16) namelen; + xlrec.had_value = had_value; + xlrec.orig_value_len = (uint32) (had_value ? saved_value_len : 0); + + if (had_value) + { + Size payload_len; + FileopsUndoRemovexattr *hdr; + + payload_len = sizeof(FileopsUndoRemovexattr) + pathlen + namelen + + saved_value_len; + payload = (char *) palloc(payload_len); + hdr = (FileopsUndoRemovexattr *) payload; + + hdr->path_len = (uint16) pathlen; + hdr->name_len = (uint16) namelen; + hdr->value_len = (uint32) saved_value_len; + + memcpy(payload + sizeof(FileopsUndoRemovexattr), path, pathlen); + memcpy(payload + sizeof(FileopsUndoRemovexattr) + pathlen, + name, namelen); + if (saved_value_len > 0) + memcpy(payload + sizeof(FileopsUndoRemovexattr) + pathlen + namelen, + saved_value, saved_value_len); + + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_REMOVEXATTR, + InvalidOid, payload, payload_len); + } + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsRemovexattr); + XLogRegisterData(path, pathlen); + XLogRegisterData(name, namelen); + if (had_value && saved_value_len > 0) + XLogRegisterData(saved_value, (uint32) saved_value_len); + fileops_log(XLOG_FILEOPS_REMOVEXATTR, had_value ? &undo_ctx : NULL); + if (payload) + pfree(payload); + } + + if (saved_value) + pfree(saved_value); + + return 0; +} + +/* + * FileOpsDoPendingOps - Execute pending file operations at transaction end + * + * At commit, operations with at_commit=true are executed. + * At abort, operations with at_commit=false are executed. + * + * This is called from xact.c at transaction commit/abort, analogous + * to smgrDoPendingDeletes(). + */ +void +FileOpsDoPendingOps(bool isCommit) +{ + int nestLevel = GetCurrentTransactionNestLevel(); + PendingFileOp *pending; + PendingFileOp *prev; + PendingFileOp *next; + PendingFileOp *exec_head = NULL; + PendingFileOp *exec_tail = NULL; + XLogRecPtr max_lsn = InvalidXLogRecPtr; + + /* + * Pass 1: detach the entries belonging to this (or an inner) nesting + * level from the pending list. Those matching the transaction outcome + * are queued for execution; the rest are discarded. Outer-level entries + * stay on the list for a later call. The queue is built in pending-list + * order, which is newest-first because AddPendingFileOp prepends. + */ + prev = NULL; + for (pending = pendingFileOps; pending != NULL; pending = next) + { + next = pending->next; + + if (pending->nestLevel < nestLevel) + { + /* outer-level entries should not be processed yet */ + prev = pending; + continue; + } + + /* unlink from list first, so we don't retry on failure */ + if (prev) + prev->next = next; + else + pendingFileOps = next; + + if (pending->at_commit == isCommit) + { + /* + * queue for execution, preserving pending-list (newest-first) + * order + */ + pending->next = NULL; + if (exec_tail != NULL) + exec_tail->next = pending; + else + exec_head = pending; + exec_tail = pending; + } + else + FreePendingFileOp(pending); + } + + /* + * Commit-time ops must run in registration order (oldest-first) so that + * nested directory removals proceed deepest-first. The pending list is + * LIFO, so the queue collected above is newest-first; reverse it for + * commit. Abort-time ops must keep LIFO order, which the collected queue + * already preserves, so they are left as-is. + */ + if (isCommit) + { + PendingFileOp *rev = NULL; + + while (exec_head != NULL) + { + next = exec_head->next; + exec_head->next = rev; + rev = exec_head; + exec_head = next; + } + exec_head = rev; + } + + /* Commit record written, nothing executed yet (for tests) */ + if (isCommit && exec_head != NULL) + INJECTION_POINT("fileops-commit-before-pending-ops", NULL); + + /* Pass 2: execute the queued ops in order, freeing each when done. */ + for (pending = exec_head; pending != NULL; pending = next) + { + next = pending->next; + + FileOpsExecOne(pending, isCommit, &max_lsn); + + FreePendingFileOp(pending); + } + + /* + * Each destructive commit op above XLogFlush()es its own record BEFORE + * performing the filesystem mutation (WAL-before-mutation), so no primary + * FS change can outrun its WAL record. This trailing flush is therefore + * redundant, kept only as a no-op safety backstop (max_lsn is already + * durable). Only the commit path ever sets max_lsn. + */ + if (!XLogRecPtrIsInvalid(max_lsn)) + XLogFlush(max_lsn); +} + +/* + * FileOpsExecOne - Execute one pending file operation + * + * Performs a single deferred op (commit- or abort-time) exactly as it would + * run in an ordinary transaction end. Shared by FileOpsDoPendingOps() and by + * the 2PC post-commit/post-abort callbacks, so an op prepared with PREPARE + * TRANSACTION runs identically to one that never left this backend. + * + * *max_lsn accumulates the highest WAL LSN written by a commit-time op so the + * caller can flush it. Each destructive commit op also flushes its own record + * before mutating the filesystem (WAL-before-mutation), so *max_lsn is only a + * backstop. + */ +static void +FileOpsExecOne(PendingFileOp *pending, bool isCommit, XLogRecPtr *max_lsn) +{ + { + switch (pending->type) + { + case PENDING_FILEOP_INVERSE: + break; + case PENDING_FILEOP_CREATE: + /* Creates are executed immediately, nothing to do */ + break; + + case PENDING_FILEOP_RENAME: + + /* + * Abort-time rename back of a FileOpsRename() at wal_level = + * minimal: path is the new name, newpath the old one. Only + * while the new name exists and the old one is free, as the + * UNDO does at higher wal_level. + */ + Assert(!pending->at_commit); + { + struct stat st; + + if (lstat(pending->path, &st) != 0 || + lstat(pending->newpath, &st) == 0) + break; + } + if (rename(pending->path, pending->newpath) < 0) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not rename file \"%s\" back to \"%s\": %m", + pending->path, pending->newpath))); + else if (enableFsync) + { + fileops_fsync_parent(pending->newpath, WARNING); + fileops_fsync_parent(pending->path, WARNING); + } + break; + + case PENDING_FILEOP_DELETE: + + /* + * log-at-commit-execution, but ONLY for the commit path. A + * delete-on-abort registration (at_commit=false) executes + * here during abort; that must NOT be WAL-logged, because a + * standby must never apply an aborted transaction's + * filesystem effect. + */ + if (isCommit && XLogIsNeeded()) + { + xl_fileops_delete xlrec; + int pathlen = (int) strlen(pending->path) + 1; + + xlrec.at_commit = true; + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsDelete); + XLogRegisterData(pending->path, pathlen); + *max_lsn = XLogInsert(RM_FILEOPS_ID, XLOG_FILEOPS_DELETE); + + /* + * WAL-before-mutation durability: flush this record + * BEFORE touching the filesystem. A crash after the + * unlink but before a batched flush would mutate the + * primary's FS without the record ever reaching the + * standby, diverging the two. + */ + XLogFlush(*max_lsn); + } + if (unlink(pending->path) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove file \"%s\": %m", + pending->path))); + } + else + { + if (enableFsync) + fileops_fsync_parent(pending->path, WARNING); + } + break; + + case PENDING_FILEOP_RMDIR: + + /* + * log-at-commit-execution, commit path only (see the DELETE + * case above for the abort-path rationale). + */ + if (isCommit && XLogIsNeeded()) + { + xl_fileops_rmdir xlrec; + int pathlen = (int) strlen(pending->path) + 1; + + xlrec.at_commit = true; + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsRmdir); + XLogRegisterData(pending->path, pathlen); + *max_lsn = XLogInsert(RM_FILEOPS_ID, XLOG_FILEOPS_RMDIR); + + /* WAL-before-mutation durability (see DELETE above). */ + XLogFlush(*max_lsn); + } + if (rmdir(pending->path) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove directory \"%s\": %m", + pending->path))); + } + else + { + if (enableFsync) + fileops_fsync_parent(pending->path, WARNING); + } + break; + + case PENDING_FILEOP_RMTREE: + + /* + * Abort-time removal of a FileOpsMkdirTree() directory and + * whatever ended up in it. The UNDO record, applied earlier + * in the abort, or the caller's own error cleanup has usually + * removed it already. + */ + { + struct stat st; + + if (lstat(pending->path, &st) != 0 || !S_ISDIR(st.st_mode)) + break; + } + if (!rmtree(pending->path, true)) + ereport(WARNING, + (errmsg("could not remove directory tree \"%s\"", + pending->path))); + else if (enableFsync) + fileops_fsync_parent(pending->path, WARNING); + break; + + case PENDING_FILEOP_TRUNCATE: + { + /* + * Restore original file size on abort. The original + * length is stored in pending->length. + */ + int trunc_fd; + + trunc_fd = OpenTransientFile(pending->path, + O_RDWR | PG_BINARY); + if (trunc_fd >= 0) + { + if (ftruncate(trunc_fd, pending->length) < 0) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not restore file \"%s\" to original size %lld: %m", + pending->path, + (long long) pending->length))); + else if (enableFsync && pg_fsync(trunc_fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\" after restoring size: %m", + pending->path))); + CloseTransientFile(trunc_fd); + } + else + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not open file \"%s\" to restore size: %m", + pending->path))); + } + } + break; + + case PENDING_FILEOP_CHMOD: + { + /* + * Restore original file mode on abort. The original mode + * is stored in pending->length (cast from mode_t). + */ + mode_t restore_mode = (mode_t) pending->length; + + if (chmod(pending->path, restore_mode) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not restore file \"%s\" to original mode 0%o: %m", + pending->path, + (unsigned int) restore_mode))); + } + } + break; + + case PENDING_FILEOP_SETXATTR: + { + /* + * Restore an xattr to its original value on abort. path = + * file, newpath = attr name, data = orig value. + */ + if (pending->data && pending->data_len > 0) + { + if (pg_setxattr(pending->path, pending->newpath, + pending->data, pending->data_len) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not restore extended attribute \"%s\" on \"%s\": %m", + pending->newpath, + pending->path))); + } + } + else + { + /* Zero-length original value */ + if (pg_setxattr(pending->path, pending->newpath, + "", 0) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not restore extended attribute \"%s\" on \"%s\": %m", + pending->newpath, + pending->path))); + } + } + } + break; + + case PENDING_FILEOP_REMOVEXATTR: + { + /* + * Remove an xattr on abort (attribute was newly created + * in the aborted transaction). + */ + if (pg_removexattr(pending->path, pending->newpath) < 0) + { + /* + * "Attribute already absent" is success for a + * remove-on-abort (idempotent, like our unlink/rmdir + * redo). Use PG_ENOATTR, the portable + * absent-attribute errno: on Linux that is ENODATA, + * on macOS/BSD ENOATTR -- the raw ENODATA/ENOENT + * check missed the macOS case and warned spuriously + * (sec3 portability bug). + */ + if (errno != ENOENT && errno != PG_ENOATTR) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove extended attribute \"%s\" from \"%s\": %m", + pending->newpath, + pending->path))); + } + } + break; + + default: + break; + } + } +} + +/* + * AtSubCommit_FileOps - Handle subtransaction commit + * + * Reassign all pending ops from the current nesting level to the parent. + */ +void +AtSubCommit_FileOps(void) +{ + int nestLevel = GetCurrentTransactionNestLevel(); + PendingFileOp *pending; + + for (pending = pendingFileOps; pending != NULL; pending = pending->next) + { + if (pending->nestLevel >= nestLevel) + pending->nestLevel = nestLevel - 1; + } +} + +/* + * AtSubAbort_FileOps - Handle subtransaction abort + * + * Execute abort-time actions for the current nesting level immediately. + */ +void +AtSubAbort_FileOps(void) +{ + FileOpsDoPendingOps(false); +} + +/* + * PostPrepare_FileOps - Clean up after PREPARE TRANSACTION + * + * FileOpsAtPrepare() (called from EndPrepare, before the 2PC END sentinel) + * has already serialized every pending file operation into the two-phase + * state file, so this backend can now discard its in-memory list. The ops + * are performed later by the FILEOPS 2PC post-commit/post-abort callbacks + * (fileops_twophase_postcommit/postabort), possibly from a different backend + * after a crash, driven from the durable 2PC state. + */ +void +PostPrepare_FileOps(void) +{ + PendingFileOp *pending; + PendingFileOp *next; + + for (pending = pendingFileOps; pending != NULL; pending = next) + { + next = pending->next; + pendingFileOps = next; + FreePendingFileOp(pending); + } +} + +/* + * Serialized form of a PendingFileOp for the 2PC state file. + * + * A fixed header followed by two variable-length blobs (path, data), in that + * order. Self-describing so a crash-recovered COMMIT/ROLLBACK + * PREPARED -- run by any backend -- can rebuild an equivalent PendingFileOp + * and hand it to FileOpsExecOne(). Only the fields FileOpsExecOne() reads + * are carried; nestLevel is irrelevant once prepared. + */ +typedef struct FileOpsTwoPhaseRecord +{ + int type; /* PendingFileOpType */ + bool at_commit; /* run at COMMIT PREPARED vs ROLLBACK PREPARED */ + off_t length; /* truncate length / write offset / mode */ + uint32 path_len; /* bytes, including NUL, 0 if none */ + uint32 data_len; /* bytes, 0 if none */ + /* followed by path, data */ +} FileOpsTwoPhaseRecord; + +/* + * FileOpsAtPrepare - Serialize deferred commit-time file ops into 2PC state + * + * Called from EndPrepare() between StartPrepare() and the END sentinel. Each + * pending at_commit=true op (DELETE/RMDIR) is written as one + * TWOPHASE_RM_FILEOPS_ID record. This is what makes those deferred + * commit-time ops survive PREPARE for COMMIT PREPARED across a crash between + * PREPARE and the final decision. + * + * The immediate-operation inverse list is serialized separately. ROLLBACK + * PREPARED includes it in its abort record and applies it before releasing + * the prepared transaction's locks. + */ +void +FileOpsAtPrepare(void) +{ + PendingFileOp *pending; + char *data; + uint32 len; + + FileOpsGetAbortData(&data, &len); + if (len) + RegisterTwoPhaseRecord(TWOPHASE_RM_FILEOPS_ID, + FILEOPS_TWOPHASE_INVERSES, data, len); + if (data) + pfree(data); + + + for (pending = pendingFileOps; pending != NULL; pending = pending->next) + { + FileOpsTwoPhaseRecord rec; + char *buf; + uint32 path_len; + uint32 data_len; + uint32 total; + char *p; + + /* Immediate inverses were serialized as one blob above. */ + if (!pending->at_commit) + continue; + + path_len = pending->path ? (uint32) strlen(pending->path) + 1 : 0; + /* No commit-time op has a newpath. */ + Assert(pending->newpath == NULL); + data_len = (pending->data && pending->data_len > 0) ? + (uint32) pending->data_len : 0; + + rec.type = (int) pending->type; + rec.at_commit = pending->at_commit; + rec.length = pending->length; + rec.path_len = path_len; + rec.data_len = data_len; + + total = sizeof(FileOpsTwoPhaseRecord) + path_len + data_len; + buf = palloc(total); + memcpy(buf, &rec, sizeof(FileOpsTwoPhaseRecord)); + p = buf + sizeof(FileOpsTwoPhaseRecord); + if (path_len > 0) + { + memcpy(p, pending->path, path_len); + p += path_len; + } + if (data_len > 0) + memcpy(p, pending->data, data_len); + + RegisterTwoPhaseRecord(TWOPHASE_RM_FILEOPS_ID, 0, buf, total); + pfree(buf); + } +} + +/* + * fileops_twophase_do - Shared body for the FILEOPS 2PC callbacks + * + * Rebuild a PendingFileOp from the serialized 2PC record and, if it matches + * the transaction outcome, perform it exactly as FileOpsDoPendingOps() would. + * A commit-time op emits its own WAL (idempotent redo on a standby); the + * trailing flush makes any accumulated LSN durable. + */ +static void +fileops_twophase_do(bool isCommit, void *recdata, uint32 len) +{ + FileOpsTwoPhaseRecord rec; + PendingFileOp pending; + const char *p; + XLogRecPtr max_lsn = InvalidXLogRecPtr; + + if (len < sizeof(FileOpsTwoPhaseRecord)) + { + elog(WARNING, "FILEOPS 2PC record too short (%u bytes)", len); + return; + } + memcpy(&rec, recdata, sizeof(FileOpsTwoPhaseRecord)); + + /* Only ops matching this outcome run; the rest are simply discarded. */ + if (rec.at_commit != isCommit) + return; + + memset(&pending, 0, sizeof(pending)); + pending.type = (PendingFileOpType) rec.type; + pending.at_commit = rec.at_commit; + pending.length = rec.length; + + p = (const char *) recdata + sizeof(FileOpsTwoPhaseRecord); + if (rec.path_len > 0) + { + pending.path = (char *) p; + p += rec.path_len; + } + if (rec.data_len > 0) + { + pending.data = (void *) p; + pending.data_len = rec.data_len; + } + + FileOpsExecOne(&pending, isCommit, &max_lsn); + + if (!XLogRecPtrIsInvalid(max_lsn)) + XLogFlush(max_lsn); +} + +/* + * fileops_twophase_postcommit - COMMIT PREPARED callback (perform at_commit ops) + */ +void +fileops_twophase_postcommit(FullTransactionId fxid, uint16 info, + void *recdata, uint32 len) +{ + if (info != FILEOPS_TWOPHASE_INVERSES) + fileops_twophase_do(true, recdata, len); +} + +/* + * fileops_twophase_postabort - ROLLBACK PREPARED callback + * + * Immediate inverses are applied with the abort decision, not here. Deferred + * commit-time operations are discarded. + */ +void +fileops_twophase_postabort(FullTransactionId fxid, uint16 info, + void *recdata, uint32 len) +{ + /* Inverses were applied with the durable abort decision, before unlock. */ + if (info != FILEOPS_TWOPHASE_INVERSES) + fileops_twophase_do(false, recdata, len); +} + +/* + * fileops_redo - WAL redo function for FILEOPS records + * + * Replay file operations during crash recovery or standby apply. + * Each operation type has its own redo handler added in separate commits. + */ +void +fileops_redo(XLogReaderState *record) +{ + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + char *data = XLogRecGetData(record); + + switch (info) + { + case XLOG_FILEOPS_CREATE: + { + xl_fileops_create *xlrec = (xl_fileops_create *) data; + const char *path = data + SizeOfFileOpsCreate; + int fd; + + fd = BasicOpenFilePerm(path, + (xlrec->flags & ~PG_O_DIRECT) | O_CREAT, + xlrec->mode); + if (fd < 0) + { + if (errno == ENOENT) + { + char parentpath[MAXPGPATH]; + char *sep; + + strlcpy(parentpath, path, MAXPGPATH); + sep = strrchr(parentpath, '/'); + if (sep != NULL) + { + *sep = '\0'; + if (MakePGDirectory(parentpath) < 0 && + errno != EEXIST) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\" during WAL replay: %m", + parentpath))); + } + + fd = BasicOpenFilePerm(path, + (xlrec->flags & ~PG_O_DIRECT) | O_CREAT, + xlrec->mode); + } + + if (fd < 0 && errno != EEXIST) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not create file \"%s\" during WAL replay: %m", + path))); + } + + if (fd >= 0) + { + if (enableFsync && pg_fsync(fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\" during WAL replay: %m", + path))); + close(fd); + if (enableFsync) + fileops_fsync_parent(path, data_sync_elevel(ERROR)); + } + } + break; + + case XLOG_FILEOPS_WRITE: + { + xl_fileops_write *xlrec = (xl_fileops_write *) data; + const char *path = data + SizeOfFileOpsWrite; + const char *wdata = path + xlrec->path_len; + int fd; + + fd = BasicOpenFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not open file \"%s\" for write during WAL replay: %m", + path))); + } + else + { + if (pg_pwrite(fd, wdata, xlrec->len, xlrec->offset) != + (ssize_t) xlrec->len) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not write to file \"%s\" during WAL replay: %m", + path))); + else if (enableFsync && pg_fsync(fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\" during WAL replay: %m", + path))); + close(fd); + } + } + break; + + case XLOG_FILEOPS_RENAME: + + /* + * Redo of FileOpsRename(), logged before the rename was made. + * Converge on the state the primary reached, whatever part of it + * is on disk already: + * + * - oldpath absent: the rename was made already. + * + * - newpath absent: make it. + * + * - both present: the replay of earlier records (the CREATE, + * MKDIR or LINK of oldpath since the checkpoint) has made oldpath + * again, or a later operation has; newpath, which the primary + * never let the rename replace, is the later state. Remove + * oldpath, a directory only if empty, and keep newpath. Renaming + * over newpath would lose data: after a -> b, b -> c and a new b, + * with a older than the checkpoint, redo of b -> c finds the new + * b and would put it in place of c. A non-empty oldpath + * directory is left, with a WARNING. + * + * Redo replays onto the current, possibly later, filesystem state + * and cannot tell an object made again by redo from one made at + * the same path later (see README). + */ + { + xl_fileops_rename *xlrec = (xl_fileops_rename *) data; + const char *oldpath = data + SizeOfFileOpsRename; + const char *newpath = oldpath + xlrec->oldpath_len; + struct stat st; + struct stat nst; + + if (lstat(oldpath, &st) != 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\" during WAL replay: %m", + oldpath))); + } + else if (lstat(newpath, &nst) != 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\" during WAL replay: %m", + newpath))); + else if (rename(oldpath, newpath) < 0) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not rename file \"%s\" to \"%s\" during WAL replay: %m", + oldpath, newpath))); + else if (enableFsync) + { + fileops_fsync_parent(newpath, data_sync_elevel(ERROR)); + fileops_fsync_parent(oldpath, data_sync_elevel(ERROR)); + } + } + else if ((S_ISDIR(st.st_mode) ? rmdir(oldpath) : + unlink(oldpath)) < 0) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove \"%s\", renamed to \"%s\", during WAL replay: %m", + oldpath, newpath))); + else if (enableFsync) + fileops_fsync_parent(oldpath, data_sync_elevel(ERROR)); + } + break; + + case XLOG_FILEOPS_DELETE: + + /* + * idempotent redo of a deferred delete executed at commit on the + * primary. An already-absent file (ENOENT) means the delete + * already took effect -- treat as success. Only commit-path + * deletes are ever logged (see FileOpsDoPendingOps), so redo of + * an aborted transaction's delete-on-abort can never occur here. + */ + { + xl_fileops_delete *xlrec pg_attribute_unused() = + (xl_fileops_delete *) data; + const char *path = data + SizeOfFileOpsDelete; + + if (unlink(path) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove file \"%s\" during WAL replay: %m", + path))); + } + else if (enableFsync) + fileops_fsync_parent(path, data_sync_elevel(ERROR)); + } + break; + + case XLOG_FILEOPS_SYMLINK: + + /* + * sec2: intentional no-op during redo. + * + * FileOpsSymlink() has exactly one caller, + * create_tablespace_directories() in tablespace.c, and it is used + * ONLY for tablespace symlinks under $PGDATA/pg_tblspc. Standby + * (and primary crash-recovery) replay of those symlinks is OWNED + * by RM_TBLSPC: tblspc_redo() -> create_tablespace_directories() + * runs with InRecovery set, first removing any stale symlink + * (remove_tablespace_symlink) and then re-creating it with the + * recovery-time layout logic that honors the standby's own + * tablespace layout / allow_in_place_tablespaces / tablespace_map + * remap. + * + * Replaying the FILEOPS record here would be redundant (RM_TBLSPC + * already creates the link) and, worse, actively wrong on a + * remapped standby: it would unlink the correctly-remapped link + * and re-point it at the primary's LITERAL absolute target, which + * may not exist or may belong to a different location on the + * standby. So FILEOPS defers tablespace-symlink replay to + * RM_TBLSPC and does nothing here. + * + * If FileOpsSymlink ever gains a non-tablespace caller, this must + * be revisited to replay those (and only those) symlinks; that + * would require distinguishing the two cases in the record. + */ + break; + + case XLOG_FILEOPS_LINK: + { + xl_fileops_link *xlrec = (xl_fileops_link *) data; + const char *oldpath = data + SizeOfFileOpsLink; + const char *newpath = oldpath + xlrec->oldpath_len; + + /* Remove existing link first for idempotent redo */ + unlink(newpath); + if (link(oldpath, newpath) < 0 && errno != EEXIST) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not create hard link \"%s\" during WAL replay: %m", + newpath))); + else if (enableFsync) + fileops_fsync_parent(newpath, data_sync_elevel(ERROR)); + } + break; + + case XLOG_FILEOPS_MKDIR: + { + xl_fileops_mkdir *xlrec pg_attribute_unused() = + (xl_fileops_mkdir *) data; + const char *path = data + SizeOfFileOpsMkdir; + + if (MakePGDirectory(path) < 0 && errno != EEXIST) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\" during WAL replay: %m", + path))); + else if (enableFsync) + fileops_fsync_parent(path, data_sync_elevel(ERROR)); + } + break; + + case XLOG_FILEOPS_RMDIR: + + /* + * idempotent redo of a deferred rmdir executed at commit on the + * primary. An already-absent directory (ENOENT) means the rmdir + * already took effect -- treat as success. + */ + { + xl_fileops_rmdir *xlrec pg_attribute_unused() = + (xl_fileops_rmdir *) data; + const char *path = data + SizeOfFileOpsRmdir; + + if (rmdir(path) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove directory \"%s\" during WAL replay: %m", + path))); + } + else if (enableFsync) + fileops_fsync_parent(path, data_sync_elevel(ERROR)); + } + break; + + case XLOG_FILEOPS_CHMOD: + { + xl_fileops_chmod *xlrec = (xl_fileops_chmod *) data; + const char *path = data + SizeOfFileOpsChmod; + + if (chmod(path, xlrec->mode) < 0 && errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not chmod file \"%s\" during WAL replay: %m", + path))); + } + break; + + case XLOG_FILEOPS_CHOWN: + { +#ifndef WIN32 + xl_fileops_chown *xlrec = (xl_fileops_chown *) data; + const char *path = data + SizeOfFileOpsChown; + + if (chown(path, xlrec->uid, xlrec->gid) < 0 && + errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not chown file \"%s\" during WAL replay: %m", + path))); +#endif + } + break; + + case XLOG_FILEOPS_TRUNCATE: + { + xl_fileops_truncate *xlrec = (xl_fileops_truncate *) data; + const char *path = data + SizeOfFileOpsTruncate; + int fd; + + XLogFlush(record->EndRecPtr); + + fd = BasicOpenFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not open file \"%s\" for truncation during WAL replay: %m", + path))); + } + else + { + if (ftruncate(fd, xlrec->length) < 0) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not truncate file \"%s\" to %lld bytes during WAL replay: %m", + path, (long long) xlrec->length))); + else if (enableFsync && pg_fsync(fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\" during WAL replay: %m", + path))); + close(fd); + } + } + break; + + case XLOG_FILEOPS_SETXATTR: + { + xl_fileops_setxattr *xlrec = (xl_fileops_setxattr *) data; + const char *path = data + SizeOfFileOpsSetxattr; + const char *name = path + xlrec->path_len; + const void *value = name + xlrec->name_len; + + if (pg_setxattr(path, name, value, xlrec->value_len) < 0 && + errno != ENOTSUP && errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not set extended attribute \"%s\" on \"%s\" during WAL replay: %m", + name, path))); + } + break; + + case XLOG_FILEOPS_REMOVEXATTR: + { + xl_fileops_removexattr *xlrec = + (xl_fileops_removexattr *) data; + const char *path = data + SizeOfFileOpsRemovexattr; + const char *name = path + xlrec->path_len; + + if (pg_removexattr(path, name) < 0 && + errno != ENOTSUP && errno != ENOENT && + errno != PG_ENOATTR) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove extended attribute \"%s\" from \"%s\" during WAL replay: %m", + name, path))); + } + break; + + default: + elog(PANIC, "fileops_redo: unknown op code %u", info); + break; + } +} diff --git a/src/backend/storage/file/fileops_undo.c b/src/backend/storage/file/fileops_undo.c new file mode 100644 index 00000000000..bde79d929b5 --- /dev/null +++ b/src/backend/storage/file/fileops_undo.c @@ -0,0 +1,1054 @@ +/*------------------------------------------------------------------------- + * + * fileops_undo.c + * FILEOPS UNDO resource manager + * + * This module implements the UNDO apply callbacks for transactional file + * operations. The inverse payloads cover structural changes and bounded + * before-images for overwritten or truncated file contents. + * + * On transaction abort, each FILEOPS UNDO record reverses the structural + * change made by the corresponding FileOps* function, if the object still + * shows that change (see fileops_undo_apply): + * - CREATE: unlink the created file + * - WRITE: restore overwritten bytes and original length + * - TRUNCATE: restore removed bytes and original length + * - CHMOD: chmod to original mode + * - CHOWN: chown to original uid/gid + * - MKDIR: rmdir the created directory, or remove its whole tree + * - SYMLINK: unlink the symlink + * - LINK: unlink the hard link + * - RENAME: rename the new name back to the old one + * - SETXATTR: restore/remove the original xattr + * - REMOVEXATTR: restore the original xattr value + * + * DELETE and RMDIR are handled by PendingFileOps (deferred execution) and do + * not need UNDO records since they execute only at commit time. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/storage/file/fileops_undo.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include + +#include "access/undormgr.h" +#include "access/xact.h" +#include "access/transam.h" +#include "access/xlog.h" +#include "access/xlogutils.h" +#include "port/pg_xattr.h" +#include "miscadmin.h" +#include "storage/bufmgr.h" +#include "storage/fd.h" +#include "storage/fileops.h" +#include "storage/md.h" +#include "utils/hsearch.h" +#include "utils/injection_point.h" +#include "utils/memutils.h" + +/* + * UNDO subtype constants and payload structures are defined in fileops.h, + * shared between this file (apply side) and fileops.c (insert side). + */ + +static bool applyingAbort = false; +bool FileOpsSkipUndo = false; + +/* Forward declarations */ +static void fileops_note_inverse(uint16 info, const char *data, uint32 len, + int elevel); +static UndoApplyResult fileops_undo_apply(uint8 rmid, uint16 info, + TransactionId xid, Oid reloid, + const char *payload, Size payload_len, + UndoRecPtr urec_ptr); +static void fileops_undo_desc(StringInfo buf, uint8 rmid, uint16 info, + const char *payload, Size payload_len); + +/* + * Report an xattr restore that failed, and return UNDO_APPLY_ERROR so the + * caller counts it as a record that could not be applied. Like the other + * subtypes, a file that no longer exists is not a failure. + */ +static UndoApplyResult +fileops_undo_xattr_failed(const char *op, const char *path, const char *name) +{ + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO %s: could not restore attribute \"%s\" of \"%s\": %m", + op, name, path))); + return UNDO_APPLY_ERROR; +} + +/* The FILEOPS UNDO RM registration entry */ +static const UndoRmgrData fileops_undo_rmgr = { + .rm_name = "fileops", + .rm_undo = fileops_undo_apply, + .rm_desc = fileops_undo_desc, +}; + +/* + * FileopsUndoRmgrInit - Register the FILEOPS UNDO resource manager + */ +void +FileopsUndoRmgrInit(void) +{ + RegisterUndoRmgr(UNDO_RMID_FILEOPS, &fileops_undo_rmgr); + + /* + * Wire FILEOPS's pending-structural-file-operation callbacks into the + * AM-neutral hooks in xact.c (see access/xact.h for why xact.c cannot + * call these by name directly). + */ + PendingPhysOpsDo_hook = FileOpsDoPendingOps; + PendingPhysOpsPostPrepare_hook = PostPrepare_FileOps; + PendingPhysOpsAtSubCommit_hook = AtSubCommit_FileOps; + PendingPhysOpsAtSubAbort_hook = AtSubAbort_FileOps; +} + +/* + * fileops_undo_lstat - lstat() for an UNDO action + * + * Returns 1 if path exists, 0 if it does not (nothing left to undo), and -1 + * after a WARNING for any other failure. + */ +static int +fileops_undo_lstat(const char *op, const char *path, struct stat *st) +{ + if (lstat(path, st) == 0) + return 1; + if (errno == ENOENT) + return 0; + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO %s: could not stat \"%s\": %m", op, path))); + return -1; +} + +/* + * fileops_undo_superseded - Report an object that no longer shows the effect + * being undone: it was already reverted by an earlier walk of this chain, or + * changed since. Either way it is not ours to touch. + */ +static UndoApplyResult +fileops_undo_superseded(const char *op, const char *path) +{ + ereport(DEBUG2, + (errmsg("FILEOPS UNDO %s: \"%s\" was changed since, leaving it", + op, path))); + return UNDO_APPLY_SUCCESS; +} + +/* + * fileops_xattr_is - Does attribute name of path hold exactly value? + * + * Returns 1 if so, 0 if not (or the attribute is absent), and -1 with errno + * set if it cannot be read. + */ +static int +fileops_xattr_is(const char *path, const char *name, const char *value, + uint32 len) +{ + ssize_t cur_len; + char *cur; + int result; + + cur_len = pg_getxattr(path, name, NULL, 0); + if (cur_len < 0) + return (errno == PG_ENOATTR) ? 0 : -1; + if ((size_t) cur_len != len) + return 0; + if (len == 0) + return 1; + + cur = palloc(len); + cur_len = pg_getxattr(path, name, cur, len); + if (cur_len < 0) + result = (errno == PG_ENOATTR || errno == ERANGE) ? 0 : -1; + else + result = ((size_t) cur_len == len && memcmp(cur, value, len) == 0); + pfree(cur); + return result; +} + +/* + * fileops_undo_apply - Apply a single FILEOPS UNDO record + * + * Reverses the structural filesystem operation, but only if the object still + * shows its effect, as recorded in the payload. The same chain can be walked + * again later (see storage/fileops.h), when the object may already have been + * reverted, or changed by someone else; then the record is a no-op. A CREATE + * or MKDIR cannot tell its own file or directory from one created again at + * the same path, since the inode is not known when the UNDO is written, and + * a REMOVEXATTR cannot tell an attribute it removed from one removed again; + * nor can a RENAME tell the object it renamed from one put at the new name + * since. + */ +static UndoApplyResult +fileops_undo_apply(uint8 rmid, uint16 info, TransactionId xid, Oid reloid, + const char *payload, Size payload_len, UndoRecPtr urec_ptr) +{ + UndoApplyResult result = UNDO_APPLY_SUCCESS; + struct stat st; + + Assert(rmid == UNDO_RMID_FILEOPS); + + /* Abort WAL owns completed aborts; generic UNDO is for crash losers. */ + if (!applyingAbort && FileOpsSkipUndo) + return UNDO_APPLY_SUCCESS; + + INJECTION_POINT("fileops-undo-apply-begin", NULL); + + switch (info) + { + case FILEOPS_UNDO_CREATE: + { + FileopsUndoCreate hdr; + const char *path; + + if (payload_len < sizeof(FileopsUndoCreate)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoCreate)); + path = payload + sizeof(FileopsUndoCreate); + + switch (fileops_undo_lstat("CREATE", path, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + if (!S_ISREG(st.st_mode)) + result = fileops_undo_superseded("CREATE", path); + else if (unlink(path) < 0 && errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO CREATE: could not unlink \"%s\": %m", + path))); + result = UNDO_APPLY_ERROR; + } + break; + } + } + break; + + case FILEOPS_UNDO_WRITE: + { + FileopsUndoWrite hdr; + const char *path; + const char *before; + const char *after; + char *current; + int fd; + int rc; + + if (payload_len < sizeof(hdr)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(hdr)); + if (payload_len != sizeof(hdr) + hdr.path_len + + hdr.before_len + hdr.after_len) + return UNDO_APPLY_ERROR; + path = payload + sizeof(hdr); + before = path + hdr.path_len; + after = before + hdr.before_len; + + /* + * 0 (gone) is nothing to undo, but -1 (a stat error) must fail + * like every other subtype, so a crash loser's overwritten + * bytes are reported as not restored rather than silently + * dropped. + */ + rc = fileops_undo_lstat("WRITE", path, &st); +#ifdef USE_INJECTION_POINTS + /* Lets a test reach the stat-error (-1) arm. */ + if (rc == 1 && + IS_INJECTION_POINT_ATTACHED("fileops-undo-write-lstat-error")) + rc = -1; +#endif + if (rc == -1) + result = UNDO_APPLY_ERROR; + if (rc != 1) + break; + if (!S_ISREG(st.st_mode) || st.st_size != hdr.new_length) + break; + fd = BasicOpenFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO WRITE: could not open \"%s\": %m", path))); + return UNDO_APPLY_ERROR; + } + current = palloc(hdr.after_len ? hdr.after_len : 1); + if (pg_pread(fd, current, hdr.after_len, hdr.offset) == hdr.after_len && + memcmp(current, after, hdr.after_len) == 0) + { + if ((hdr.before_len && + pg_pwrite(fd, before, hdr.before_len, hdr.offset) != hdr.before_len) || + ftruncate(fd, hdr.orig_length) != 0) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO WRITE: could not restore \"%s\": %m", path))); + result = UNDO_APPLY_ERROR; + } + } + pfree(current); + close(fd); + } + break; + + case FILEOPS_UNDO_TRUNCATE: + { + FileopsUndoTruncate hdr; + const char *path; + int fd; + + if (payload_len < sizeof(FileopsUndoTruncate)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoTruncate)); + path = payload + sizeof(FileopsUndoTruncate); + + switch (fileops_undo_lstat("TRUNCATE", path, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + if (!S_ISREG(st.st_mode) || st.st_size != hdr.new_length) + { + result = fileops_undo_superseded("TRUNCATE", path); + break; + } + fd = BasicOpenFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + { + if (errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO TRUNCATE: could not open \"%s\": %m", + path))); + result = UNDO_APPLY_ERROR; + } + } + else + { + if (hdr.before_len && + pg_pwrite(fd, path + hdr.path_len, hdr.before_len, + hdr.new_length) != hdr.before_len) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not restore bytes of file \"%s\": %m", path))); + result = UNDO_APPLY_ERROR; + } + if (ftruncate(fd, hdr.orig_length) < 0) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO TRUNCATE: could not restore length of \"%s\": %m", + path))); + result = UNDO_APPLY_ERROR; + } + close(fd); + } + break; + } + } + break; + + case FILEOPS_UNDO_CHMOD: + { + FileopsUndoChmod hdr; + const char *path; + + if (payload_len < sizeof(FileopsUndoChmod)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoChmod)); + path = payload + sizeof(FileopsUndoChmod); + + switch (fileops_undo_lstat("CHMOD", path, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + if ((st.st_mode & 07777) != hdr.new_mode) + result = fileops_undo_superseded("CHMOD", path); + else if (chmod(path, hdr.orig_mode) < 0 && + errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO CHMOD: could not restore mode of \"%s\": %m", + path))); + result = UNDO_APPLY_ERROR; + } + break; + } + } + break; + + case FILEOPS_UNDO_CHOWN: + { +#ifndef WIN32 + FileopsUndoChown hdr; + const char *path; + + if (payload_len < sizeof(FileopsUndoChown)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoChown)); + path = payload + sizeof(FileopsUndoChown); + + switch (fileops_undo_lstat("CHOWN", path, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + if (st.st_uid != hdr.new_uid || st.st_gid != hdr.new_gid) + result = fileops_undo_superseded("CHOWN", path); + else if (chown(path, hdr.orig_uid, hdr.orig_gid) < 0 && + errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO CHOWN: could not restore ownership of \"%s\": %m", + path))); + result = UNDO_APPLY_ERROR; + } + break; + } +#endif + } + break; + + case FILEOPS_UNDO_MKDIR: + { + FileopsUndoMkdir hdr; + const char *path; + + if (payload_len < sizeof(FileopsUndoMkdir)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoMkdir)); + path = payload + sizeof(FileopsUndoMkdir); + + switch (fileops_undo_lstat("MKDIR", path, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + if (!S_ISDIR(st.st_mode)) + result = fileops_undo_superseded("MKDIR", path); + else if ((hdr.flags & FILEOPS_MKDIR_TREE) && + !InRecovery && + (st.st_dev != hdr.dev || st.st_ino != hdr.ino)) + result = fileops_undo_superseded("MKDIR", path); + else if (hdr.flags & FILEOPS_MKDIR_TREE) + { + if (OidIsValid(hdr.dboid)) + { + DropDatabaseBuffers(hdr.dboid); + ForgetDatabaseSyncRequests(hdr.dboid); + } + if (!rmtree(path, true)) + { + ereport(WARNING, + (errmsg("FILEOPS UNDO MKDIR: could not remove directory tree \"%s\"", + path))); + result = UNDO_APPLY_ERROR; + } + } + else if (rmdir(path) < 0 && errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO MKDIR: could not rmdir \"%s\": %m", + path))); + result = UNDO_APPLY_ERROR; + } + break; + } + } + break; + + case FILEOPS_UNDO_SYMLINK: + { + FileopsUndoSymlink hdr; + const char *linkpath; + const char *target; + char cur[MAXPGPATH]; + ssize_t cur_len; + + if (payload_len < sizeof(FileopsUndoSymlink)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoSymlink)); + if (payload_len < sizeof(FileopsUndoSymlink) + + hdr.linkpath_len + hdr.target_len) + return UNDO_APPLY_ERROR; + linkpath = payload + sizeof(FileopsUndoSymlink); + target = linkpath + hdr.linkpath_len; + + cur_len = readlink(linkpath, cur, sizeof(cur)); + if (cur_len < 0) + { + if (errno == ENOENT) + break; + if (errno == EINVAL) /* not a symlink any more */ + { + result = fileops_undo_superseded("SYMLINK", linkpath); + break; + } + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO SYMLINK: could not read symbolic link \"%s\": %m", + linkpath))); + result = UNDO_APPLY_ERROR; + } + else if ((size_t) cur_len + 1 != hdr.target_len || + memcmp(cur, target, cur_len) != 0) + result = fileops_undo_superseded("SYMLINK", linkpath); + else if (unlink(linkpath) < 0 && errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO SYMLINK: could not unlink \"%s\": %m", + linkpath))); + result = UNDO_APPLY_ERROR; + } + } + break; + + case FILEOPS_UNDO_LINK: + { + FileopsUndoLink hdr; + const char *newpath; + + if (payload_len < sizeof(FileopsUndoLink)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoLink)); + newpath = payload + sizeof(FileopsUndoLink); + + switch (fileops_undo_lstat("LINK", newpath, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + if (!InRecovery && (st.st_dev != hdr.dev || st.st_ino != hdr.ino)) + result = fileops_undo_superseded("LINK", newpath); + else if (unlink(newpath) < 0 && errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO LINK: could not unlink \"%s\": %m", + newpath))); + result = UNDO_APPLY_ERROR; + } + break; + } + } + break; + + case FILEOPS_UNDO_RENAME: + { + FileopsUndoRename hdr; + const char *oldpath; + const char *newpath; + + if (payload_len < sizeof(FileopsUndoRename)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoRename)); + if (payload_len < sizeof(FileopsUndoRename) + + hdr.oldpath_len + hdr.newpath_len) + return UNDO_APPLY_ERROR; + oldpath = payload + sizeof(FileopsUndoRename); + newpath = oldpath + hdr.oldpath_len; + + /* + * Rename back while newpath exists and oldpath is free (see + * FileopsUndoRename). An old name reused since is not ours + * to replace. + */ + switch (fileops_undo_lstat("RENAME", newpath, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + switch (fileops_undo_lstat("RENAME", oldpath, &st)) + { + case 0: + if (rename(newpath, oldpath) < 0) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO RENAME: could not rename \"%s\" back to \"%s\": %m", + newpath, oldpath))); + result = UNDO_APPLY_ERROR; + } + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + result = fileops_undo_superseded("RENAME", + oldpath); + break; + } + break; + } + } + break; + + case FILEOPS_UNDO_SETXATTR: + { + FileopsUndoSetxattr hdr; + const char *path; + const char *name; + const char *orig_value; + const char *new_value; + + if (payload_len < sizeof(FileopsUndoSetxattr)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoSetxattr)); + if (payload_len < sizeof(FileopsUndoSetxattr) + hdr.path_len + + hdr.name_len + hdr.orig_value_len + hdr.new_value_len) + return UNDO_APPLY_ERROR; + path = payload + sizeof(FileopsUndoSetxattr); + name = path + hdr.path_len; + orig_value = name + hdr.name_len; + new_value = orig_value + hdr.orig_value_len; + + switch (fileops_undo_lstat("SETXATTR", path, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + switch (fileops_xattr_is(path, name, new_value, + hdr.new_value_len)) + { + case 0: + result = fileops_undo_superseded("SETXATTR", + path); + break; + case -1: + result = fileops_undo_xattr_failed("SETXATTR", + path, name); + break; + default: + if (hdr.had_value) + { + if (pg_setxattr(path, name, orig_value, + hdr.orig_value_len) < 0 && + errno != ENOENT) + result = fileops_undo_xattr_failed("SETXATTR", + path, name); + } + else if (pg_removexattr(path, name) < 0 && + errno != ENOENT && + errno != PG_ENOATTR) + result = fileops_undo_xattr_failed("SETXATTR", + path, name); + break; + } + break; + } + } + break; + + case FILEOPS_UNDO_REMOVEXATTR: + { + FileopsUndoRemovexattr hdr; + const char *path; + const char *name; + const void *value; + + if (payload_len < sizeof(FileopsUndoRemovexattr)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoRemovexattr)); + path = payload + sizeof(FileopsUndoRemovexattr); + name = path + hdr.path_len; + value = name + hdr.name_len; + + switch (fileops_undo_lstat("REMOVEXATTR", path, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + /* Restore the removed xattr, unless it is back */ + if (pg_getxattr(path, name, NULL, 0) >= 0) + result = fileops_undo_superseded("REMOVEXATTR", + path); + else if (errno != PG_ENOATTR) + result = fileops_undo_xattr_failed("REMOVEXATTR", + path, name); + else if (pg_setxattr(path, name, value, + hdr.value_len) < 0 && + errno != ENOENT) + result = fileops_undo_xattr_failed("REMOVEXATTR", + path, name); + break; + } + } + break; + + default: + ereport(WARNING, + (errmsg("FILEOPS UNDO: unknown subtype %u", info))); + return UNDO_APPLY_ERROR; + } + + INJECTION_POINT("fileops-undo-apply-end", NULL); + + /* + * The startup undo phase records the walk's completion after it + * returns, so the inverse must be durable by then; the walk makes it so + * (FileOpsSyncInverses). + */ + if (!applyingAbort && result == UNDO_APPLY_SUCCESS) + fileops_note_inverse(info, payload, payload_len, + data_sync_elevel(ERROR)); + + return result; +} + +/* + * Apply an abort-WAL inverse with the same identity tests as loser recovery. + * Ordinary syscall failures remain warnings. A failed inverse can therefore + * leave an online effect until recovery replays the abort record; callers must + * not infer successful filesystem rollback merely from transaction abort. + * + * The caller makes the inverses durable with FileOpsSyncInverses(PANIC) once + * it has applied them all. Failing to is a PANIC regardless of + * data_sync_retry. Online, we run after the abort record is flushed and the + * transaction is marked aborted in clog, so an ERROR would re-enter + * AbortTransaction and log a second abort record. The caller holds off + * checkpoints until the inverses are durable, so crash recovery replays the + * abort record and applies the inverses again. + */ +void +FileOpsApplyInverse(uint16 info, const char *data, uint32 len) +{ + applyingAbort = true; + PG_TRY(); + { + (void) fileops_undo_apply(UNDO_RMID_FILEOPS, info, InvalidTransactionId, + InvalidOid, data, len, InvalidUndoRecPtr); + fileops_note_inverse(info, data, len, PANIC); + } + PG_FINALLY(); + { + applyingAbort = false; + } + PG_END_TRY(); +} + +/* + * Paths that applied inverses changed and that are not durable yet, each with + * its parent directory. N inverses on one file need one fsync of the file + * and one of its directory, not N of each: every caller makes a whole + * rollback durable before it records the rollback as complete, by calling + * FileOpsSyncInverses() once. + * + * Online, the inverses are applied inside AbortTransaction, where an ERROR + * would re-enter it. So the table is created when the transaction writes its + * first FILEOPS UNDO (FileOpsReserveInverseSyncs), and an entry that cannot be + * allocated there is not an error: that path is synced at once instead. + */ +typedef struct FileOpsSyncEntry +{ + char path[MAXPGPATH]; /* hash key */ + bool target; /* an inverse's own path, not just a parent */ +} FileOpsSyncEntry; + +static HTAB *fileopsPendingSyncs = NULL; + +/* Create the table now, where an out-of-memory ERROR is still harmless. */ +void +FileOpsReserveInverseSyncs(void) +{ + HASHCTL ctl; + + if (fileopsPendingSyncs != NULL) + return; + ctl.keysize = MAXPGPATH; + ctl.entrysize = sizeof(FileOpsSyncEntry); + ctl.hcxt = TopMemoryContext; + fileopsPendingSyncs = hash_create("FILEOPS inverses to sync", 16, &ctl, + HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); +} + +/* fsync one path an inverse changed, as it is now; see FileOpsSyncInverses. */ +static void +fileops_sync_path(const char *path, bool target, int elevel) +{ + struct stat st; + +#ifdef USE_INJECTION_POINTS + if (target && IS_INJECTION_POINT_ATTACHED("fileops-inverse-fsync-fail")) + { + errno = EIO; + ereport(elevel, + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", path))); + } +#endif + if (lstat(path, &st) == 0 && !S_ISLNK(st.st_mode)) + { + INJECTION_POINT("fileops-inverse-fsync", unconstify(char *, path)); + fsync_fname_ext(path, S_ISDIR(st.st_mode), true, elevel); + } +} + +static void +fileops_note_sync(const char *path, bool target, int elevel) +{ + FileOpsSyncEntry *entry = NULL; + bool found; + + if (!enableFsync) + return; + + /* + * Inside AbortTransaction, an out-of-memory ERROR would re-enter it, so + * the table was created with the transaction's first FILEOPS UNDO. + * Elsewhere (abort redo, the startup undo phase) the ERROR is harmless. + */ + if (fileopsPendingSyncs == NULL && !(applyingAbort && !InRecovery)) + FileOpsReserveInverseSyncs(); + if (fileopsPendingSyncs != NULL) + entry = hash_search(fileopsPendingSyncs, path, HASH_ENTER_NULL, + &found); + if (entry == NULL) + { + /* No memory to defer it: make this path durable now. */ + fileops_sync_path(path, target, elevel); + return; + } + if (!found) + entry->target = false; + entry->target |= target; +} + +/* Remember the directory of a path an inverse changed, for syncing. */ +static void +fileops_note_parent(const char *path, int elevel) +{ + char parent[MAXPGPATH]; + char *sep; + + strlcpy(parent, path, sizeof(parent)); + sep = strrchr(parent, '/'); + if (sep == NULL) + strlcpy(parent, ".", sizeof(parent)); + else if (sep == parent) + parent[1] = '\0'; + else + *sep = '\0'; + + fileops_note_sync(parent, false, elevel); +} + +/* Remember the paths an inverse changed, and their directories. */ +static void +fileops_note_inverse(uint16 info, const char *data, uint32 len, int elevel) +{ + Size hdrlen = 0; + + switch (info) + { + case FILEOPS_UNDO_CREATE: + hdrlen = sizeof(FileopsUndoCreate); + break; + case FILEOPS_UNDO_WRITE: + hdrlen = sizeof(FileopsUndoWrite); + break; + case FILEOPS_UNDO_TRUNCATE: + hdrlen = sizeof(FileopsUndoTruncate); + break; + case FILEOPS_UNDO_CHMOD: + hdrlen = sizeof(FileopsUndoChmod); + break; + case FILEOPS_UNDO_CHOWN: + hdrlen = sizeof(FileopsUndoChown); + break; + case FILEOPS_UNDO_MKDIR: + hdrlen = sizeof(FileopsUndoMkdir); + break; + case FILEOPS_UNDO_SYMLINK: + hdrlen = sizeof(FileopsUndoSymlink); + break; + case FILEOPS_UNDO_LINK: + hdrlen = sizeof(FileopsUndoLink); + break; + case FILEOPS_UNDO_RENAME: + { + FileopsUndoRename hdr; + + /* + * The old name is back, in its directory, and the new name is + * gone from its own. + */ + if (len < sizeof(hdr)) + break; + memcpy(&hdr, data, sizeof(hdr)); + if (hdr.oldpath_len == 0 || hdr.newpath_len == 0 || + len < sizeof(hdr) + hdr.oldpath_len + hdr.newpath_len) + break; + fileops_note_sync(data + sizeof(hdr), true, elevel); + fileops_note_parent(data + sizeof(hdr), elevel); + fileops_note_parent(data + sizeof(hdr) + hdr.oldpath_len, + elevel); + return; + } + case FILEOPS_UNDO_SETXATTR: + hdrlen = sizeof(FileopsUndoSetxattr); + break; + case FILEOPS_UNDO_REMOVEXATTR: + hdrlen = sizeof(FileopsUndoRemovexattr); + break; + } + if (hdrlen == 0 || len <= hdrlen) + elog(PANIC, "invalid FILEOPS inverse payload"); + fileops_note_sync(data + hdrlen, true, elevel); + fileops_note_parent(data + hdrlen, elevel); +} + +/* + * FileOpsSyncInverses - Make the inverses applied so far durable + * + * fsyncs each path noted since the last call once, as it is now: a file or + * directory that still exists (a symlink is covered by its directory), and + * the directory of each. A path a later inverse removed is skipped; its + * removal is in its directory, which an inverse that removed the directory + * itself has noted in turn. Failures are reported at elevel. After an + * ERROR the paths not yet synced stay noted, and the next call syncs them. + */ +void +FileOpsSyncInverses(int elevel) +{ + HASH_SEQ_STATUS seq; + FileOpsSyncEntry *entry; + FileOpsSyncEntry **entries; + long n; + long i = 0; + + if (fileopsPendingSyncs == NULL || + hash_get_num_entries(fileopsPendingSyncs) == 0) + return; + + /* + * Sync from a copy of the entries: an ERROR must not leave a hash scan + * open, and every entry must stay noted until it is synced. Without + * memory for the copy (we may be inside AbortTransaction), take the + * entries one at a time instead, each with a scan of its own. + */ + n = hash_get_num_entries(fileopsPendingSyncs); + entries = palloc_extended(n * sizeof(FileOpsSyncEntry *), + MCXT_ALLOC_NO_OOM); + if (entries != NULL) + { + hash_seq_init(&seq, fileopsPendingSyncs); + while ((entry = hash_seq_search(&seq)) != NULL) + entries[i++] = entry; + for (i = 0; i < n; i++) + { + fileops_sync_path(entries[i]->path, entries[i]->target, elevel); + (void) hash_search(fileopsPendingSyncs, entries[i]->path, + HASH_REMOVE, NULL); + } + pfree(entries); + } + else + { + for (;;) + { + hash_seq_init(&seq, fileopsPendingSyncs); + entry = hash_seq_search(&seq); + if (entry == NULL) + break; + hash_seq_term(&seq); + fileops_sync_path(entry->path, entry->target, elevel); + (void) hash_search(fileopsPendingSyncs, entry->path, HASH_REMOVE, + NULL); + } + } +} + +/* + * fileops_undo_desc - Describe a FILEOPS UNDO record for debugging + */ +static void +fileops_undo_desc(StringInfo buf, uint8 rmid, uint16 info, + const char *payload, Size payload_len) +{ + const char *opname; + + switch (info) + { + case FILEOPS_UNDO_CREATE: + opname = "CREATE"; + break; + case FILEOPS_UNDO_TRUNCATE: + opname = "TRUNCATE"; + break; + case FILEOPS_UNDO_CHMOD: + opname = "CHMOD"; + break; + case FILEOPS_UNDO_CHOWN: + opname = "CHOWN"; + break; + case FILEOPS_UNDO_MKDIR: + opname = "MKDIR"; + break; + case FILEOPS_UNDO_SYMLINK: + opname = "SYMLINK"; + break; + case FILEOPS_UNDO_LINK: + opname = "LINK"; + break; + case FILEOPS_UNDO_RENAME: + opname = "RENAME"; + break; + case FILEOPS_UNDO_SETXATTR: + opname = "SETXATTR"; + break; + case FILEOPS_UNDO_REMOVEXATTR: + opname = "REMOVEXATTR"; + break; + default: + opname = "UNKNOWN"; + break; + } + + appendStringInfo(buf, "fileops %s", opname); +} diff --git a/src/backend/storage/file/meson.build b/src/backend/storage/file/meson.build index 795402589b0..7e487324c5f 100644 --- a/src/backend/storage/file/meson.build +++ b/src/backend/storage/file/meson.build @@ -4,6 +4,8 @@ backend_sources += files( 'buffile.c', 'copydir.c', 'fd.c', + 'fileops.c', + 'fileops_undo.c', 'fileset.c', 'reinit.c', 'sharedfileset.c', diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c index 8570f17916f..ed27a243feb 100644 --- a/src/bin/pg_waldump/rmgrdesc.c +++ b/src/bin/pg_waldump/rmgrdesc.c @@ -17,6 +17,7 @@ #include "access/hash_xlog.h" #include "access/heapam_xlog.h" #include "access/multixact.h" +#include "access/fileops_xlog.h" #include "access/nbtxlog.h" #include "access/rmgr.h" #include "access/spgxlog.h" diff --git a/src/bin/pg_waldump/t/001_basic.pl b/src/bin/pg_waldump/t/001_basic.pl index fbeb19ee4a4..ded5e37feda 100644 --- a/src/bin/pg_waldump/t/001_basic.pl +++ b/src/bin/pg_waldump/t/001_basic.pl @@ -97,7 +97,8 @@ ReplicationOrigin Generic LogicalMessage XLOG2 -Undo$/, +Undo +FileOps$/, 'rmgr list'); diff --git a/src/include/access/fileops_xlog.h b/src/include/access/fileops_xlog.h new file mode 100644 index 00000000000..b3ece9a2686 --- /dev/null +++ b/src/include/access/fileops_xlog.h @@ -0,0 +1,40 @@ +/*------------------------------------------------------------------------- + * + * fileops_xlog.h + * Transactional file operations XLOG resource manager definitions + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/include/access/fileops_xlog.h + * + *------------------------------------------------------------------------- + */ +#ifndef FILEOPS_XLOG_H +#define FILEOPS_XLOG_H + +#include "access/xlogreader.h" +#include "lib/stringinfo.h" + +/* XLOG stuff - all record types defined upfront for WAL compatibility */ +#define XLOG_FILEOPS_CREATE 0x00 +#define XLOG_FILEOPS_DELETE 0x10 +#define XLOG_FILEOPS_RENAME 0x20 +#define XLOG_FILEOPS_WRITE 0x30 +#define XLOG_FILEOPS_TRUNCATE 0x40 +#define XLOG_FILEOPS_CHMOD 0x50 +#define XLOG_FILEOPS_CHOWN 0x60 +#define XLOG_FILEOPS_MKDIR 0x70 +#define XLOG_FILEOPS_RMDIR 0x80 +#define XLOG_FILEOPS_SYMLINK 0x90 +#define XLOG_FILEOPS_LINK 0xA0 +#define XLOG_FILEOPS_SETXATTR 0xB0 +#define XLOG_FILEOPS_REMOVEXATTR 0xC0 + +/* Resource manager functions */ +extern void fileops_redo(XLogReaderState *record); +extern void fileops_desc(StringInfo buf, XLogReaderState *record); +extern const char *fileops_identify(uint8 info); + +#endif /* FILEOPS_XLOG_H */ diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h index 046ba45cb25..a34394a1e23 100644 --- a/src/include/access/rmgrlist.h +++ b/src/include/access/rmgrlist.h @@ -49,3 +49,6 @@ PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL, logicalmsg_decode) PG_RMGR(RM_XLOG2_ID, "XLOG2", xlog2_redo, xlog2_desc, xlog2_identify, NULL, NULL, NULL, xlog2_decode) PG_RMGR(RM_UNDO_ID, "Undo", undo_redo, undo_desc, undo_identify, NULL, NULL, NULL, NULL) +#ifdef USE_FILEOPS +PG_RMGR(RM_FILEOPS_ID, "FileOps", fileops_redo, fileops_desc, fileops_identify, NULL, NULL, NULL, NULL) +#endif diff --git a/src/include/access/twophase_rmgr.h b/src/include/access/twophase_rmgr.h index 8927f369c39..37be3c7b290 100644 --- a/src/include/access/twophase_rmgr.h +++ b/src/include/access/twophase_rmgr.h @@ -28,7 +28,8 @@ typedef uint8 TwoPhaseRmgrId; #define TWOPHASE_RM_PGSTAT_ID 2 #define TWOPHASE_RM_MULTIXACT_ID 3 #define TWOPHASE_RM_PREDICATELOCK_ID 4 -#define TWOPHASE_RM_MAX_ID TWOPHASE_RM_PREDICATELOCK_ID +#define TWOPHASE_RM_FILEOPS_ID 5 +#define TWOPHASE_RM_MAX_ID TWOPHASE_RM_FILEOPS_ID extern PGDLLIMPORT const TwoPhaseCallback twophase_recover_callbacks[]; extern PGDLLIMPORT const TwoPhaseCallback twophase_postcommit_callbacks[]; @@ -39,4 +40,10 @@ extern PGDLLIMPORT const TwoPhaseCallback twophase_standby_recover_callbacks[]; extern void RegisterTwoPhaseRecord(TwoPhaseRmgrId rmid, uint16 info, const void *data, uint32 len); +/* FILEOPS two-phase commit callbacks (defined in storage/file/fileops.c) */ +extern void fileops_twophase_postcommit(FullTransactionId fxid, uint16 info, + void *recdata, uint32 len); +extern void fileops_twophase_postabort(FullTransactionId fxid, uint16 info, + void *recdata, uint32 len); + #endif /* TWOPHASE_RMGR_H */ diff --git a/src/include/access/undormgrlist.h b/src/include/access/undormgrlist.h index 4ab60e21418..a9f8ec5afd7 100644 --- a/src/include/access/undormgrlist.h +++ b/src/include/access/undormgrlist.h @@ -31,3 +31,6 @@ */ /* there is deliberately not an #ifndef UNDORMGRLIST_H here */ + +/* FILEOPS: transactional filesystem operations UNDO rmgr (UNDO_RMID_FILEOPS=2) */ +UNDO_RMGR_INIT(FileopsUndoRmgrInit) diff --git a/src/include/access/xact.h b/src/include/access/xact.h index 8341d2015d3..32a1c1f394f 100644 --- a/src/include/access/xact.h +++ b/src/include/access/xact.h @@ -213,6 +213,7 @@ typedef struct SavedTransactionCharacteristics #define XACT_XINFO_HAS_AE_LOCKS (1U << 6) #define XACT_XINFO_HAS_GID (1U << 7) #define XACT_XINFO_HAS_DROPPED_STATS (1U << 8) +#define XACT_XINFO_HAS_FILEOPS (1U << 9) /* * Also stored in xinfo, these indicating a variety of additional actions that @@ -468,6 +469,10 @@ typedef struct xl_xact_parsed_abort XLogRecPtr origin_lsn; TimestampTz origin_timestamp; + + /* FILEOPS inverses to apply on redo (XACT_XINFO_HAS_FILEOPS) */ + uint32 fileops_len; + const char *fileops; } xl_xact_parsed_abort; @@ -559,7 +564,8 @@ extern XLogRecPtr XactLogAbortRecord(TimestampTz abort_time, int ndroppedstats, xl_xact_stats_item *droppedstats, int xactflags, TransactionId twophase_xid, - const char *twophase_gid); + const char *twophase_gid, + const char *fileops, uint32 fileops_len); extern void xact_redo(XLogReaderState *record); /* xactdesc.c */ diff --git a/src/include/port/pg_xattr.h b/src/include/port/pg_xattr.h new file mode 100644 index 00000000000..7159fc27a26 --- /dev/null +++ b/src/include/port/pg_xattr.h @@ -0,0 +1,40 @@ +/*------------------------------------------------------------------------- + * + * pg_xattr.h + * Cross-platform extended attribute abstraction + * + * Provides pg_setxattr() and pg_removexattr() that work across: + * - Linux: setxattr/removexattr + * - macOS: setxattr/removexattr (extra options param) + * - FreeBSD: extattr_set_file/extattr_delete_file + * - Windows: NTFS Alternate Data Streams + * - Fallback: returns ENOTSUP with WARNING + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * + * src/include/port/pg_xattr.h + * + *------------------------------------------------------------------------- + */ +#ifndef PG_XATTR_H +#define PG_XATTR_H + +/* + * ENODATA is Linux-specific. FreeBSD/macOS use ENOATTR for "attribute not + * found". Provide a portable PG_ENOATTR so callers don't need #ifdefs. + */ +#if defined(ENOATTR) +#define PG_ENOATTR ENOATTR +#elif defined(ENODATA) +#define PG_ENOATTR ENODATA +#else +#define PG_ENOATTR ENOENT /* last-resort fallback */ +#endif + +extern int pg_setxattr(const char *path, const char *name, + const void *value, size_t size); +extern ssize_t pg_getxattr(const char *path, const char *name, + void *value, size_t size); +extern int pg_removexattr(const char *path, const char *name); + +#endif /* PG_XATTR_H */ diff --git a/src/include/storage/fileops.h b/src/include/storage/fileops.h new file mode 100644 index 00000000000..7215ec207ec --- /dev/null +++ b/src/include/storage/fileops.h @@ -0,0 +1,527 @@ +/*------------------------------------------------------------------------- + * + * fileops.h + * Transactional file operations API + * + * This module provides transactional filesystem operations that are + * WAL-logged and integrated with PostgreSQL's transaction management. + * Constructive operations and renames take effect at once and are reversed + * if the transaction aborts; deletions are deferred to transaction end (see + * src/backend/storage/file/README). + * + * The RM_FILEOPS_ID resource manager handles WAL replay for these + * operations, ensuring correct behavior during crash recovery and + * standby replay. + * + * The operation set follows the Berkeley DB fileops.src model: each + * filesystem operation is a composable unit with its own WAL record + * type, redo handler, and descriptor. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/storage/fileops.h + * + *------------------------------------------------------------------------- + */ +#ifndef FILEOPS_H +#define FILEOPS_H + +#include "access/xlogreader.h" +#include "lib/stringinfo.h" + +/* + * WAL record types for FILEOPS operations. + * + * The high 4 bits of the info byte are used for record type, + * leaving the low bits for flags (following PostgreSQL convention). + * + * Following the Berkeley DB fileops.src model, each filesystem + * operation has its own WAL record type for independent redo. + */ +#define XLOG_FILEOPS_CREATE 0x00 +#define XLOG_FILEOPS_DELETE 0x10 +#define XLOG_FILEOPS_RENAME 0x20 +#define XLOG_FILEOPS_WRITE 0x30 +#define XLOG_FILEOPS_TRUNCATE 0x40 +#define XLOG_FILEOPS_CHMOD 0x50 +#define XLOG_FILEOPS_CHOWN 0x60 +#define XLOG_FILEOPS_MKDIR 0x70 +#define XLOG_FILEOPS_RMDIR 0x80 +#define XLOG_FILEOPS_SYMLINK 0x90 +#define XLOG_FILEOPS_LINK 0xA0 +#define XLOG_FILEOPS_SETXATTR 0xB0 +#define XLOG_FILEOPS_REMOVEXATTR 0xC0 + +/* + * PendingFileOp - Deferred file operation entry + * + * File operations are collected in a linked list during a transaction + * and executed at commit or abort time. This follows the same pattern + * used by PendingRelDelete in catalog/storage.c. + */ +typedef enum PendingFileOpType +{ + PENDING_FILEOP_CREATE, + PENDING_FILEOP_DELETE, + PENDING_FILEOP_RENAME, + PENDING_FILEOP_WRITE, + PENDING_FILEOP_TRUNCATE, + PENDING_FILEOP_CHMOD, + PENDING_FILEOP_CHOWN, + PENDING_FILEOP_MKDIR, + PENDING_FILEOP_RMDIR, + PENDING_FILEOP_RMTREE, /* abort: remove a FileOpsMkdirTree() tree */ + PENDING_FILEOP_SYMLINK, + PENDING_FILEOP_LINK, + PENDING_FILEOP_SETXATTR, + PENDING_FILEOP_REMOVEXATTR, + PENDING_FILEOP_INVERSE +} PendingFileOpType; + +typedef struct PendingFileOp +{ + PendingFileOpType type; /* operation type */ + char *path; /* primary file path */ + char *newpath; /* old path to restore (RENAME), or xattr name */ + off_t length; /* truncation length, write offset, or mode */ + void *data; /* generic data (e.g., original xattr value) */ + size_t data_len; /* length of data */ + bool at_commit; /* execute at commit (true) or abort (false) */ + int nestLevel; /* transaction nesting level */ + struct PendingFileOp *next; /* linked list link */ +} PendingFileOp; + +/* + * Public API for transactional file operations + * + * These functions handle platform-specific differences automatically: + * - O_DIRECT: PG_O_DIRECT (Linux/FreeBSD native, macOS F_NOCACHE, + * Windows FILE_FLAG_NO_BUFFERING) + * - fsync: pg_fsync() (Linux fdatasync, macOS F_FULLFSYNC, + * BSD fsync, Windows FlushFileBuffers) + * - Directory sync: fsync_parent_path() (Unix only, no-op on Windows) + * + * Operation-specific API functions are declared below their WAL + * record structures in subsequent sections. + */ + +/* Utility functions */ +extern void FileOpsCancelPendingDelete(const char *path, bool at_commit); +extern void FileOpsSync(const char *path); + +/* Abort WAL carries a length-prefixed sequence of inverse payloads. */ +#define FILEOPS_MAX_ABORT_BYTES (16 * 1024 * 1024) +#define FILEOPS_MAX_DATA_BYTES (1024 * 1024) +#define FILEOPS_TWOPHASE_INVERSES 1 +extern void FileOpsGetAbortData(char **data, uint32 *len); +extern void FileOpsApplyAbort(const char *data, uint32 len); + +/* Scoped to a generic walk whose FILEOPS inverses belong to abort WAL. */ +extern PGDLLIMPORT bool FileOpsSkipUndo; +extern void FileOpsApplyInverse(uint16 info, const char *data, uint32 len); +extern void FileOpsReserveInverseSyncs(void); +extern void FileOpsSyncInverses(int elevel); +extern void FileOpsBeginAbort(void); +extern void FileOpsEndAbort(void); + +/* Transaction lifecycle hooks */ +extern void FileOpsDoPendingOps(bool isCommit); +extern void AtSubCommit_FileOps(void); +extern void AtSubAbort_FileOps(void); +extern void PostPrepare_FileOps(void); +extern void FileOpsAtPrepare(void); + +/* + * xl_fileops_create - WAL record for file creation + * + * Records that a file was created within a transaction. If the transaction + * aborts, the file will be deleted. The path is stored as variable-length + * data following the fixed header. + */ +typedef struct xl_fileops_create +{ + int flags; /* open flags used for creation */ + mode_t mode; /* file permission mode */ + bool register_delete; /* register for delete-on-abort */ + /* variable-length path follows */ +} xl_fileops_create; + +#define SizeOfFileOpsCreate (offsetof(xl_fileops_create, register_delete) + sizeof(bool)) + +/* File creation API */ +extern int FileOpsCreate(const char *path, int flags, mode_t mode, + bool register_delete); + +/* + * xl_fileops_delete - WAL record for file deletion + * + * Records that a file deletion was requested. The at_commit flag indicates + * whether the deletion should happen at commit (true) or was registered + * as a delete-on-abort from a prior create (false). + */ +typedef struct xl_fileops_delete +{ + bool at_commit; /* true = delete at commit, false = at abort */ + /* variable-length path follows */ +} xl_fileops_delete; + +#define SizeOfFileOpsDelete (offsetof(xl_fileops_delete, at_commit) + sizeof(bool)) + +/* File deletion API */ +extern void FileOpsDelete(const char *path, bool at_commit); + +/* + * xl_fileops_rename - WAL record for file rename + * + * Records that a file or directory is being renamed; it is written before + * the rename is made. Both old and new paths are stored as variable-length + * data: oldpath_len bytes of old path, then the new path follows. + */ +typedef struct xl_fileops_rename +{ + uint16 oldpath_len; /* length of old path (including NUL) */ + /* variable-length old path follows, then new path */ +} xl_fileops_rename; + +#define SizeOfFileOpsRename (offsetof(xl_fileops_rename, oldpath_len) + sizeof(uint16)) + +/* + * File rename API: immediate, refuses an existing newpath, renamed back if + * the transaction aborts. + */ +extern int FileOpsRename(const char *oldpath, const char *newpath); + +/* + * xl_fileops_write - WAL record for file write at offset + * + * Records that data was written to a file at a specific offset. + * The path and data are stored as variable-length data following + * the fixed header. + */ +typedef struct xl_fileops_write +{ + off_t offset; /* write offset in file */ + uint32 len; /* data length */ + uint16 path_len; /* length of path (including NUL) */ + /* variable-length path follows, then data */ +} xl_fileops_write; + +#define SizeOfFileOpsWrite (offsetof(xl_fileops_write, path_len) + sizeof(uint16)) + +/* File write API */ +extern int FileOpsWrite(const char *path, off_t offset, + const void *data, uint32 len); + +/* + * xl_fileops_truncate - WAL record for file truncation + */ +typedef struct xl_fileops_truncate +{ + off_t length; /* new file length */ + /* variable-length path follows */ +} xl_fileops_truncate; + +#define SizeOfFileOpsTruncate (offsetof(xl_fileops_truncate, length) + sizeof(off_t)) + +/* File truncation API */ +extern void FileOpsTruncate(const char *path, off_t length); + +/* + * xl_fileops_chmod - WAL record for file permission change + */ +typedef struct xl_fileops_chmod +{ + mode_t mode; /* new permission mode */ + mode_t orig_mode; /* pre-chmod mode (crash-revert before-state) */ + /* variable-length path follows */ +} xl_fileops_chmod; + +#define SizeOfFileOpsChmod (offsetof(xl_fileops_chmod, orig_mode) + sizeof(mode_t)) + +/* + * xl_fileops_chown - WAL record for file ownership change + */ +typedef struct xl_fileops_chown +{ + uid_t uid; /* new owner user id */ + gid_t gid; /* new owner group id */ + uid_t orig_uid; /* pre-chown uid (crash-revert before-state) */ + gid_t orig_gid; /* pre-chown gid (crash-revert before-state) */ + /* variable-length path follows */ +} xl_fileops_chown; + +#define SizeOfFileOpsChown (offsetof(xl_fileops_chown, orig_gid) + sizeof(gid_t)) + +/* File metadata API */ +extern int FileOpsChmod(const char *path, mode_t mode); +extern int FileOpsChown(const char *path, uid_t uid, gid_t gid); + +/* + * xl_fileops_mkdir - WAL record for directory creation + */ +typedef struct xl_fileops_mkdir +{ + mode_t mode; /* directory permission mode */ + uint16 flags; /* FILEOPS_MKDIR_* */ + /* variable-length path follows */ +} xl_fileops_mkdir; + +#define SizeOfFileOpsMkdir (offsetof(xl_fileops_mkdir, flags) + sizeof(uint16)) + +/* + * The transaction fills the directory itself (with files FILEOPS does not + * track, e.g. through smgr), so undoing the mkdir removes the whole tree. + */ +#define FILEOPS_MKDIR_TREE 0x0001 + +/* + * xl_fileops_rmdir - WAL record for directory removal + */ +typedef struct xl_fileops_rmdir +{ + bool at_commit; /* true = rmdir at commit, false = at abort */ + /* variable-length path follows */ +} xl_fileops_rmdir; + +#define SizeOfFileOpsRmdir (offsetof(xl_fileops_rmdir, at_commit) + sizeof(bool)) + +/* Directory lifecycle API */ +extern int FileOpsMkdir(const char *path, mode_t mode); +extern int FileOpsMkdirTree(const char *path, mode_t mode, Oid dboid); +extern void FileOpsRmdir(const char *path, bool at_commit); +extern void FileOpsRmdirRecursive(const char *path, bool at_commit); + +/* + * xl_fileops_symlink - WAL record for symbolic link creation + */ +typedef struct xl_fileops_symlink +{ + uint16 target_len; /* length of target (including NUL) */ + /* variable-length target follows, then linkpath */ +} xl_fileops_symlink; + +#define SizeOfFileOpsSymlink (offsetof(xl_fileops_symlink, target_len) + sizeof(uint16)) + +/* + * xl_fileops_link - WAL record for hard link creation + */ +typedef struct xl_fileops_link +{ + uint16 oldpath_len; /* length of old path (including NUL) */ + /* variable-length old path follows, then new path */ +} xl_fileops_link; + +#define SizeOfFileOpsLink (offsetof(xl_fileops_link, oldpath_len) + sizeof(uint16)) + +/* Link operations API */ +extern int FileOpsSymlink(const char *target, const char *linkpath); +extern int FileOpsLink(const char *oldpath, const char *newpath); + +/* + * xl_fileops_setxattr - WAL record for setting an extended attribute + */ +typedef struct xl_fileops_setxattr +{ + uint16 name_len; /* attribute name length (including NUL) */ + uint32 value_len; /* attribute value length */ + uint16 path_len; /* file path length (including NUL) */ + uint32 orig_value_len; /* pre-set value length (crash-revert) */ + bool had_value; /* true if xattr existed before this set */ + /* variable-length: path, name, value, orig_value (if had_value) */ +} xl_fileops_setxattr; + +#define SizeOfFileOpsSetxattr (offsetof(xl_fileops_setxattr, had_value) + sizeof(bool)) + +/* + * xl_fileops_removexattr - WAL record for removing an extended attribute + */ +typedef struct xl_fileops_removexattr +{ + uint16 name_len; /* attribute name length (including NUL) */ + uint16 path_len; /* file path length (including NUL) */ + uint32 orig_value_len; /* removed value length (crash-revert); 0 = + * none */ + bool had_value; /* true if xattr existed before this remove */ + /* variable-length: path, name, orig_value (if had_value) */ +} xl_fileops_removexattr; + +#define SizeOfFileOpsRemovexattr (offsetof(xl_fileops_removexattr, had_value) + sizeof(bool)) + +/* Extended attribute API */ +extern int FileOpsSetXattr(const char *path, const char *name, + const void *value, size_t len); +extern int FileOpsRemoveXattr(const char *path, const char *name); + +/* WAL redo and descriptor functions */ +extern void fileops_redo(XLogReaderState *record); +extern void fileops_desc(StringInfo buf, XLogReaderState *record); +extern const char *fileops_identify(uint8 info); + +/* FILEOPS UNDO RM (fileops_undo.c) */ +extern void FileopsUndoRmgrInit(void); + +/* + * UNDO resource-manager ID for FILEOPS (stored in urec_rmid). Defined here, + * with FILEOPS, rather than in the generic access/undormgr.h, so the UNDO + * core names no specific consumer. See access/undormgr.h for the shared, + * WAL-durable ID number space. + */ +#define UNDO_RMID_FILEOPS 2 + +/* FILEOPS UNDO subtypes (stored in urec_info) */ +#define FILEOPS_UNDO_WRITE 0x0002 +#define FILEOPS_UNDO_CREATE 0x0001 +#define FILEOPS_UNDO_TRUNCATE 0x0003 +#define FILEOPS_UNDO_CHMOD 0x0004 +#define FILEOPS_UNDO_CHOWN 0x0005 +#define FILEOPS_UNDO_MKDIR 0x0006 +#define FILEOPS_UNDO_SYMLINK 0x0007 +#define FILEOPS_UNDO_LINK 0x0008 +#define FILEOPS_UNDO_SETXATTR 0x0009 +#define FILEOPS_UNDO_REMOVEXATTR 0x000A +#define FILEOPS_UNDO_RENAME 0x000B + +/* + * FILEOPS UNDO payload structures + * + * These structs define the on-disk layout of UNDO record payloads for each + * FILEOPS operation. Variable-length paths and data follow the fixed header. + * + * An inverse can be applied more than once (online at ROLLBACK, and again by + * redo of the abort record after a crash), and the object may have changed in + * between. + * So each payload also records what the operation left behind (its post-state), + * and fileops_undo_apply() reverts only an object still in that state. + */ + +/* CREATE undo payload: the path to unlink, if it is still a regular file */ +typedef struct FileopsUndoCreate +{ + uint16 path_len; /* including NUL */ + /* followed by path */ +} FileopsUndoCreate; + +/* WRITE undo: path, before-image, then after-image. */ +typedef struct FileopsUndoWrite +{ + off_t orig_length; + off_t new_length; + off_t offset; + uint32 before_len; + uint32 after_len; + uint16 path_len; +} FileopsUndoWrite; + +/* TRUNCATE undo payload: path, followed by the bytes removed by a shrink. */ +typedef struct FileopsUndoTruncate +{ + off_t orig_length; + off_t new_length; + uint32 before_len; + uint16 path_len; + /* followed by path */ +} FileopsUndoTruncate; + +/* CHMOD undo payload: path + original and new mode */ +typedef struct FileopsUndoChmod +{ + mode_t orig_mode; + mode_t new_mode; + uint16 path_len; + /* followed by path */ +} FileopsUndoChmod; + +/* CHOWN undo payload: path + original and new uid/gid */ +typedef struct FileopsUndoChown +{ + uid_t orig_uid; + gid_t orig_gid; + uid_t new_uid; + gid_t new_gid; + uint16 path_len; + /* followed by path */ +} FileopsUndoChown; + +/* + * MKDIR undo payload: the path to rmdir, if it is still a directory. + * + * With FILEOPS_MKDIR_TREE the whole tree is removed instead, after dropping + * the shared buffers and pending fsync requests of database dboid (if valid), + * as a database directory's files are written through shared buffers. Since + * that destroys whatever is in the tree, outside recovery it is done only + * while the path is still the directory created (dev, ino): a later walk of + * the chain must not remove a directory created again at the same path, e.g. + * by a retried ALTER DATABASE SET TABLESPACE. Crash recovery cannot rely on + * the inode (redo on a standby or a restored backup creates the directory + * anew), and needs not: until the transaction's locks are gone no other + * transaction can have made that path. Even so, the inode is evidence that + * the directory is ours, not proof. + */ +typedef struct FileopsUndoMkdir +{ + dev_t dev; /* the directory created */ + ino_t ino; + Oid dboid; /* database filling the tree, or InvalidOid */ + uint16 flags; /* FILEOPS_MKDIR_* */ + uint16 path_len; + /* followed by path */ +} FileopsUndoMkdir; + +/* SYMLINK undo payload: the linkpath to unlink, if it still points to target */ +typedef struct FileopsUndoSymlink +{ + uint16 linkpath_len; + uint16 target_len; + /* followed by linkpath, then target */ +} FileopsUndoSymlink; + +/* LINK undo payload: the newpath to unlink, if it is still the linked inode */ +typedef struct FileopsUndoLink +{ + dev_t dev; /* device and inode of the link source */ + ino_t ino; + uint16 newpath_len; + /* followed by newpath */ +} FileopsUndoLink; + +/* + * RENAME undo payload: rename newpath back to oldpath, while newpath exists + * and oldpath does not. Like CREATE and MKDIR, the test is the state of the + * two paths alone, not the device and inode of the object renamed: those are + * not stable across a restore, a promoted standby or redo that makes the + * object again, so ROLLBACK PREPARED on a promoted standby, for one, would + * take its own copy of the object for a different one and leave it renamed. + * The paths are evidence that the object is the one renamed, not proof. + */ +typedef struct FileopsUndoRename +{ + uint16 oldpath_len; /* including NUL */ + uint16 newpath_len; /* including NUL */ + /* followed by oldpath, then newpath */ +} FileopsUndoRename; + +/* SETXATTR undo payload: path + name + original value (if any) + new value */ +typedef struct FileopsUndoSetxattr +{ + uint16 path_len; + uint16 name_len; + uint32 orig_value_len; /* 0 if xattr didn't exist before */ + uint32 new_value_len; + bool had_value; /* true if xattr existed before setxattr */ + /* followed by path, name, original value (if had_value), new value */ +} FileopsUndoSetxattr; + +/* REMOVEXATTR undo payload: path + name + removed value */ +typedef struct FileopsUndoRemovexattr +{ + uint16 path_len; + uint16 name_len; + uint32 value_len; + /* followed by path, name, value */ +} FileopsUndoRemovexattr; + +/* Recursive directory removal */ +extern void FileOpsRmdirRecursive(const char *path, bool at_commit); + +#endif /* FILEOPS_H */ diff --git a/src/port/Makefile b/src/port/Makefile index 7e9b5877652..39e1388227a 100644 --- a/src/port/Makefile +++ b/src/port/Makefile @@ -51,6 +51,7 @@ OBJS = \ pg_popcount_aarch64.o \ pg_popcount_x86.o \ pg_strong_random.o \ + pg_xattr.o \ pgcheckdir.o \ pgmkdirp.o \ pgsleep.o \ diff --git a/src/port/meson.build b/src/port/meson.build index 922b3f64676..20e1ca00431 100644 --- a/src/port/meson.build +++ b/src/port/meson.build @@ -14,6 +14,7 @@ pgport_sources = [ 'pg_popcount_aarch64.c', 'pg_popcount_x86.c', 'pg_strong_random.c', + 'pg_xattr.c', 'pgcheckdir.c', 'pgmkdirp.c', 'pgsleep.c', diff --git a/src/port/pg_xattr.c b/src/port/pg_xattr.c new file mode 100644 index 00000000000..58a3d422817 --- /dev/null +++ b/src/port/pg_xattr.c @@ -0,0 +1,217 @@ +/*------------------------------------------------------------------------- + * + * pg_xattr.c + * Cross-platform extended attribute abstraction + * + * Platform detection uses compiler-defined macros rather than + * configure-time checks, avoiding meson.build/configure.ac changes. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/port/pg_xattr.c + * + *------------------------------------------------------------------------- + */ +#include "c.h" + +#include +#include "port/pg_xattr.h" + +/* + * Platform detection via compiler macros. + * Linux and macOS both provide but with different APIs. + * FreeBSD uses . Windows uses NTFS Alternate Data Streams. + */ +#if defined(__linux__) || defined(__APPLE__) +#include +#define PG_HAVE_XATTR 1 +#elif defined(__FreeBSD__) +#include +#include +#define PG_HAVE_EXTATTR 1 +#elif defined(WIN32) +#define PG_HAVE_ADS 1 +#endif + +#if defined(PG_HAVE_ADS) +/* + * Set errno for a failed operation on the stream "path:name". A missing + * stream fails with ERROR_FILE_NOT_FOUND, like a missing file; report it as + * PG_ENOATTR when the file itself exists, as the other platforms do. + */ +static void +pg_ads_set_errno(const char *path, DWORD err) +{ + _dosmaperr(err); + if (errno == ENOENT && GetFileAttributesA(path) != INVALID_FILE_ATTRIBUTES) + errno = PG_ENOATTR; +} +#endif + +/* + * pg_setxattr - Set an extended attribute on a file + * + * Returns 0 on success, -1 on failure (errno set). + */ +int +pg_setxattr(const char *path, const char *name, + const void *value, size_t size) +{ +#if defined(PG_HAVE_XATTR) +#if defined(__APPLE__) + return setxattr(path, name, value, size, 0, 0); +#else + return setxattr(path, name, value, size, 0); +#endif + +#elif defined(PG_HAVE_EXTATTR) + ssize_t ret; + + ret = extattr_set_file(path, EXTATTR_NAMESPACE_USER, + name, value, size); + return (ret >= 0) ? 0 : -1; + +#elif defined(PG_HAVE_ADS) + char ads_path[MAXPGPATH]; + HANDLE hFile; + DWORD written; + + snprintf(ads_path, sizeof(ads_path), "%s:%s", path, name); + + hFile = CreateFileA(ads_path, GENERIC_WRITE, 0, NULL, + CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (hFile == INVALID_HANDLE_VALUE) + { + _dosmaperr(GetLastError()); + return -1; + } + + if (!WriteFile(hFile, value, (DWORD) size, &written, NULL) || + written != (DWORD) size) + { + _dosmaperr(GetLastError()); + CloseHandle(hFile); + return -1; + } + + CloseHandle(hFile); + return 0; + +#else + /* Unsupported platform: succeed in WAL but no-op locally */ + (void) path; + (void) name; + (void) value; + (void) size; + errno = ENOTSUP; + return -1; +#endif +} + +/* + * pg_getxattr - Get an extended attribute value from a file + * + * Returns the number of bytes placed in value on success, + * or -1 on failure (errno set). If value is NULL or size is 0, + * returns the size of the attribute value without reading it. + */ +ssize_t +pg_getxattr(const char *path, const char *name, + void *value, size_t size) +{ +#if defined(PG_HAVE_XATTR) +#if defined(__APPLE__) + return getxattr(path, name, value, size, 0, 0); +#else + return getxattr(path, name, value, size); +#endif + +#elif defined(PG_HAVE_EXTATTR) + return extattr_get_file(path, EXTATTR_NAMESPACE_USER, + name, value, size); + +#elif defined(PG_HAVE_ADS) + char ads_path[MAXPGPATH]; + HANDLE hFile; + DWORD bytesRead; + LARGE_INTEGER fileSize; + + snprintf(ads_path, sizeof(ads_path), "%s:%s", path, name); + + hFile = CreateFileA(ads_path, GENERIC_READ, FILE_SHARE_READ, NULL, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (hFile == INVALID_HANDLE_VALUE) + { + pg_ads_set_errno(path, GetLastError()); + return -1; + } + + if (!GetFileSizeEx(hFile, &fileSize)) + { + _dosmaperr(GetLastError()); + CloseHandle(hFile); + return -1; + } + + if (value == NULL || size == 0) + { + CloseHandle(hFile); + return (ssize_t) fileSize.QuadPart; + } + + if (!ReadFile(hFile, value, (DWORD) size, &bytesRead, NULL)) + { + _dosmaperr(GetLastError()); + CloseHandle(hFile); + return -1; + } + + CloseHandle(hFile); + return (ssize_t) bytesRead; + +#else + (void) path; + (void) name; + (void) value; + (void) size; + errno = ENOTSUP; + return -1; +#endif +} + +/* + * pg_removexattr - Remove an extended attribute from a file + * + * Returns 0 on success, -1 on failure (errno set). + */ +int +pg_removexattr(const char *path, const char *name) +{ +#if defined(PG_HAVE_XATTR) +#if defined(__APPLE__) + return removexattr(path, name, 0); +#else + return removexattr(path, name); +#endif + +#elif defined(PG_HAVE_EXTATTR) + return extattr_delete_file(path, EXTATTR_NAMESPACE_USER, name); + +#elif defined(PG_HAVE_ADS) + char ads_path[MAXPGPATH]; + + snprintf(ads_path, sizeof(ads_path), "%s:%s", path, name); + if (DeleteFileA(ads_path)) + return 0; + + pg_ads_set_errno(path, GetLastError()); + return -1; + +#else + (void) path; + (void) name; + errno = ENOTSUP; + return -1; +#endif +} diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index 6c1504ac932..f16d803fa2c 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -32,6 +32,7 @@ SUBDIRS = \ test_escape \ test_extensible \ test_extensions \ + test_fileops \ test_ginpostinglist \ test_int128 \ test_integerset \ diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build index ef3cdb8faf9..49cc09d041b 100644 --- a/src/test/modules/meson.build +++ b/src/test/modules/meson.build @@ -34,6 +34,7 @@ subdir('test_dsm_registry') subdir('test_escape') subdir('test_extensible') subdir('test_extensions') +subdir('test_fileops') subdir('test_ginpostinglist') subdir('test_int128') subdir('test_integerset') diff --git a/src/test/modules/test_fileops/Makefile b/src/test/modules/test_fileops/Makefile new file mode 100644 index 00000000000..69100c75272 --- /dev/null +++ b/src/test/modules/test_fileops/Makefile @@ -0,0 +1,19 @@ +MODULE_big = test_fileops +OBJS = test_fileops.o +EXTENSION = test_fileops +DATA = test_fileops--1.0.sql +PGFILEDESC = "test_fileops - test transactional file operations" +REGRESS = test_fileops +TAP_TESTS = 1 +EXTRA_INSTALL = src/test/modules/injection_points +export enable_injection_points +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/test_fileops +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/test_fileops/expected/test_fileops.out b/src/test/modules/test_fileops/expected/test_fileops.out new file mode 100644 index 00000000000..b529cfe3bb2 --- /dev/null +++ b/src/test/modules/test_fileops/expected/test_fileops.out @@ -0,0 +1,413 @@ +-- +-- Tests for FILEOPS UNDO rollback of direct file operations. +-- +-- This test module provides SQL-callable wrappers around FileOps C functions +-- that have no DDL-level callers, enabling direct testing of UNDO rollback +-- for: FileOpsTruncate, FileOpsChmod, FileOpsLink, FileOpsSetXattr, +-- FileOpsRemoveXattr. +-- +-- REQUIRES: UNDO subsystem (always active) +-- +-- These are POSIX file semantics: modes, hard links that do not share the +-- target's attributes, extended attributes. Windows has none of them (chmod +-- only toggles the read-only attribute, which a hard link shares), so the test +-- does not run there; the TAP tests cover the operations FILEOPS supports on +-- Windows. +SELECT version() ~ '(Visual C\+\+|mingw32|msvc|windows)' AS skip_test \gset +\if :skip_test +\quit +\endif +CREATE EXTENSION test_fileops; +-- ================================================================ +-- Setup: create test files in the data directory +-- ================================================================ +SELECT test_fileops_create_tempfile('test_fileops_a.dat') AS filepath_a \gset +SELECT test_fileops_create_tempfile('test_fileops_b.dat') AS filepath_b \gset +-- Record initial state (umask may affect mode) +SELECT test_fileops_file_size(:'filepath_a') AS initial_size; + initial_size +-------------- + 1024 +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS initial_mode; + initial_mode +-------------- + 384 +(1 row) + +-- ================================================================ +-- Test 1: FileOpsTruncate UNDO - rollback restores original size +-- ================================================================ +BEGIN; +SELECT test_fileops_truncate(:'filepath_a', 256); + test_fileops_truncate +----------------------- + +(1 row) + +SELECT test_fileops_file_size(:'filepath_a') AS size_during_txn; + size_during_txn +----------------- + 256 +(1 row) + +ROLLBACK; +-- After rollback, UNDO should restore original size (1024) +SELECT test_fileops_file_size(:'filepath_a') AS size_after_rollback; + size_after_rollback +--------------------- + 1024 +(1 row) + +-- ================================================================ +-- Test 2: FileOpsTruncate commit - change persists +-- ================================================================ +BEGIN; +SELECT test_fileops_truncate(:'filepath_b', 512); + test_fileops_truncate +----------------------- + +(1 row) + +COMMIT; +SELECT test_fileops_file_size(:'filepath_b') AS size_after_commit; + size_after_commit +------------------- + 512 +(1 row) + +-- ================================================================ +-- Test 3: FileOpsChmod UNDO - rollback restores original permissions +-- ================================================================ +-- Change to a known starting mode first (committed) +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 = 420 decimal + test_fileops_chmod +-------------------- + +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_baseline; + mode_baseline +--------------- + 420 +(1 row) + +BEGIN; +-- Change to 0600 = 384 decimal +SELECT test_fileops_chmod(:'filepath_a', 384); + test_fileops_chmod +-------------------- + +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_during_txn; + mode_during_txn +----------------- + 384 +(1 row) + +ROLLBACK; +-- After rollback, UNDO should restore to 0644 = 420 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_rollback; + mode_after_rollback +--------------------- + 420 +(1 row) + +-- ================================================================ +-- Test 4: FileOpsChmod commit - change persists +-- ================================================================ +BEGIN; +SELECT test_fileops_chmod(:'filepath_a', 448); -- 0700 = 448 decimal + test_fileops_chmod +-------------------- + +(1 row) + +COMMIT; +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_commit; + mode_after_commit +------------------- + 448 +(1 row) + +-- Restore for subsequent tests +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 + test_fileops_chmod +-------------------- + +(1 row) + +-- ================================================================ +-- Test 5: FileOpsLink UNDO - rollback removes the hard link +-- ================================================================ +SELECT test_fileops_data_dir() || '/test_fileops_link.dat' AS linkpath \gset +BEGIN; +SELECT test_fileops_link(:'filepath_a', :'linkpath'); + test_fileops_link +------------------- + +(1 row) + +SELECT test_fileops_file_exists(:'linkpath') AS link_during_txn; + link_during_txn +----------------- + t +(1 row) + +ROLLBACK; +-- After rollback, UNDO should remove the hard link +SELECT test_fileops_file_exists(:'linkpath') AS link_after_rollback; + link_after_rollback +--------------------- + f +(1 row) + +-- ================================================================ +-- Test 6: FileOpsLink commit - link persists +-- ================================================================ +BEGIN; +SELECT test_fileops_link(:'filepath_a', :'linkpath'); + test_fileops_link +------------------- + +(1 row) + +COMMIT; +SELECT test_fileops_file_exists(:'linkpath') AS link_after_commit; + link_after_commit +------------------- + t +(1 row) + +-- ================================================================ +-- Test 7: FileOpsTruncate in subtransaction - ROLLBACK TO undoes it +-- ================================================================ +-- Ensure file is 1024 bytes +SELECT test_fileops_file_size(:'filepath_a') AS size_before_test7; + size_before_test7 +------------------- + 1024 +(1 row) + +BEGIN; +-- Outer truncate (should persist after commit) +SELECT test_fileops_truncate(:'filepath_a', 800); + test_fileops_truncate +----------------------- + +(1 row) + +SAVEPOINT sp1; +-- Inner truncate (should be rolled back) +SELECT test_fileops_truncate(:'filepath_a', 100); + test_fileops_truncate +----------------------- + +(1 row) + +SELECT test_fileops_file_size(:'filepath_a') AS size_in_savepoint; + size_in_savepoint +------------------- + 100 +(1 row) + +ROLLBACK TO sp1; +-- After ROLLBACK TO, should be back to 800 (the outer truncate value) +SELECT test_fileops_file_size(:'filepath_a') AS size_after_rollback_to; + size_after_rollback_to +------------------------ + 800 +(1 row) + +COMMIT; +-- After commit, the outer truncate to 800 should persist +SELECT test_fileops_file_size(:'filepath_a') AS size_after_commit; + size_after_commit +------------------- + 800 +(1 row) + +-- ================================================================ +-- Test 8: FileOpsChmod in subtransaction - ROLLBACK TO restores mode +-- ================================================================ +-- Set a known mode baseline +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 + test_fileops_chmod +-------------------- + +(1 row) + +BEGIN; +-- Outer chmod (persists) +SELECT test_fileops_chmod(:'filepath_a', 448); -- 0700 + test_fileops_chmod +-------------------- + +(1 row) + +SAVEPOINT sp1; +-- Inner chmod (rolled back) +SELECT test_fileops_chmod(:'filepath_a', 256); -- 0400 + test_fileops_chmod +-------------------- + +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_in_savepoint; + mode_in_savepoint +------------------- + 256 +(1 row) + +ROLLBACK TO sp1; +-- After ROLLBACK TO, should be back to 0700 = 448 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_rollback_to; + mode_after_rollback_to +------------------------ + 448 +(1 row) + +COMMIT; +-- After commit, outer chmod (0700 = 448) persists +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_commit; + mode_after_commit +------------------- + 448 +(1 row) + +-- ================================================================ +-- Test 9: FileOpsLink in subtransaction - ROLLBACK TO removes link +-- ================================================================ +SELECT test_fileops_data_dir() || '/test_fileops_link2.dat' AS linkpath2 \gset +BEGIN; +SAVEPOINT sp1; +SELECT test_fileops_link(:'filepath_a', :'linkpath2'); + test_fileops_link +------------------- + +(1 row) + +SELECT test_fileops_file_exists(:'linkpath2') AS link2_in_savepoint; + link2_in_savepoint +-------------------- + t +(1 row) + +ROLLBACK TO sp1; +-- Link should be removed by UNDO +SELECT test_fileops_file_exists(:'linkpath2') AS link2_after_rollback_to; + link2_after_rollback_to +------------------------- + f +(1 row) + +COMMIT; +SELECT test_fileops_file_exists(:'linkpath2') AS link2_after_commit; + link2_after_commit +-------------------- + f +(1 row) + +-- ================================================================ +-- Test 10: FileOpsSetXattr / FileOpsRemoveXattr UNDO +-- (skipped on platforms without xattr support) +-- ================================================================ +-- Try to set an xattr; returns false if unsupported (ENOTSUP/EPERM/EACCES) +SELECT test_fileops_setxattr(:'filepath_a', 'user.test_key', 'initial_value') + AS xattr_supported \gset +\if :xattr_supported +-- Test 10a: SetXattr UNDO - rollback removes newly-set xattr +BEGIN; +SELECT test_fileops_setxattr(:'filepath_a', 'user.rollback_test', 'will_vanish'); + test_fileops_setxattr +----------------------- + t +(1 row) + +SELECT test_fileops_getxattr(:'filepath_a', 'user.rollback_test') AS xattr_during_txn; + xattr_during_txn +------------------ + will_vanish +(1 row) + +ROLLBACK; +SELECT test_fileops_getxattr(:'filepath_a', 'user.rollback_test') AS xattr_after_rollback; + xattr_after_rollback +---------------------- + +(1 row) + +-- Test 10b: SetXattr overwrite UNDO - rollback restores old value +BEGIN; +SELECT test_fileops_setxattr(:'filepath_a', 'user.test_key', 'overwritten'); + test_fileops_setxattr +----------------------- + t +(1 row) + +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_overwritten; + xattr_overwritten +------------------- + overwritten +(1 row) + +ROLLBACK; +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_restored; + xattr_restored +---------------- + initial_value +(1 row) + +-- Test 10c: RemoveXattr UNDO - rollback restores removed xattr +BEGIN; +SELECT test_fileops_removexattr(:'filepath_a', 'user.test_key'); + test_fileops_removexattr +-------------------------- + t +(1 row) + +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_after_remove; + xattr_after_remove +-------------------- + +(1 row) + +ROLLBACK; +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_after_remove_rollback; + xattr_after_remove_rollback +----------------------------- + initial_value +(1 row) + +-- Test 10d: Xattr in subtransaction +BEGIN; +SAVEPOINT sp1; +SELECT test_fileops_setxattr(:'filepath_a', 'user.sub_key', 'sub_value'); + test_fileops_setxattr +----------------------- + t +(1 row) + +SELECT test_fileops_getxattr(:'filepath_a', 'user.sub_key') AS xattr_in_savepoint; + xattr_in_savepoint +-------------------- + sub_value +(1 row) + +ROLLBACK TO sp1; +SELECT test_fileops_getxattr(:'filepath_a', 'user.sub_key') AS xattr_after_sp_rollback; + xattr_after_sp_rollback +------------------------- + +(1 row) + +COMMIT; +\else +SELECT 'xattr tests skipped (not supported on this platform/filesystem)' AS notice; +\endif +-- ================================================================ +-- Cleanup +-- ================================================================ +DROP EXTENSION test_fileops; diff --git a/src/test/modules/test_fileops/expected/test_fileops_1.out b/src/test/modules/test_fileops/expected/test_fileops_1.out new file mode 100644 index 00000000000..41193304cd6 --- /dev/null +++ b/src/test/modules/test_fileops/expected/test_fileops_1.out @@ -0,0 +1,332 @@ +-- +-- Tests for FILEOPS UNDO rollback of direct file operations. +-- +-- This test module provides SQL-callable wrappers around FileOps C functions +-- that have no DDL-level callers, enabling direct testing of UNDO rollback +-- for: FileOpsTruncate, FileOpsChmod, FileOpsLink, FileOpsSetXattr, +-- FileOpsRemoveXattr. +-- +-- REQUIRES: UNDO subsystem (always active) +-- +-- These are POSIX file semantics: modes, hard links that do not share the +-- target's attributes, extended attributes. Windows has none of them (chmod +-- only toggles the read-only attribute, which a hard link shares), so the test +-- does not run there; the TAP tests cover the operations FILEOPS supports on +-- Windows. +SELECT version() ~ '(Visual C\+\+|mingw32|msvc|windows)' AS skip_test \gset +\if :skip_test +\quit +\endif +CREATE EXTENSION test_fileops; +-- ================================================================ +-- Setup: create test files in the data directory +-- ================================================================ +SELECT test_fileops_create_tempfile('test_fileops_a.dat') AS filepath_a \gset +SELECT test_fileops_create_tempfile('test_fileops_b.dat') AS filepath_b \gset +-- Record initial state (umask may affect mode) +SELECT test_fileops_file_size(:'filepath_a') AS initial_size; + initial_size +-------------- + 1024 +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS initial_mode; + initial_mode +-------------- + 384 +(1 row) + +-- ================================================================ +-- Test 1: FileOpsTruncate UNDO - rollback restores original size +-- ================================================================ +BEGIN; +SELECT test_fileops_truncate(:'filepath_a', 256); + test_fileops_truncate +----------------------- + +(1 row) + +SELECT test_fileops_file_size(:'filepath_a') AS size_during_txn; + size_during_txn +----------------- + 256 +(1 row) + +ROLLBACK; +-- After rollback, UNDO should restore original size (1024) +SELECT test_fileops_file_size(:'filepath_a') AS size_after_rollback; + size_after_rollback +--------------------- + 1024 +(1 row) + +-- ================================================================ +-- Test 2: FileOpsTruncate commit - change persists +-- ================================================================ +BEGIN; +SELECT test_fileops_truncate(:'filepath_b', 512); + test_fileops_truncate +----------------------- + +(1 row) + +COMMIT; +SELECT test_fileops_file_size(:'filepath_b') AS size_after_commit; + size_after_commit +------------------- + 512 +(1 row) + +-- ================================================================ +-- Test 3: FileOpsChmod UNDO - rollback restores original permissions +-- ================================================================ +-- Change to a known starting mode first (committed) +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 = 420 decimal + test_fileops_chmod +-------------------- + +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_baseline; + mode_baseline +--------------- + 420 +(1 row) + +BEGIN; +-- Change to 0600 = 384 decimal +SELECT test_fileops_chmod(:'filepath_a', 384); + test_fileops_chmod +-------------------- + +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_during_txn; + mode_during_txn +----------------- + 384 +(1 row) + +ROLLBACK; +-- After rollback, UNDO should restore to 0644 = 420 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_rollback; + mode_after_rollback +--------------------- + 420 +(1 row) + +-- ================================================================ +-- Test 4: FileOpsChmod commit - change persists +-- ================================================================ +BEGIN; +SELECT test_fileops_chmod(:'filepath_a', 448); -- 0700 = 448 decimal + test_fileops_chmod +-------------------- + +(1 row) + +COMMIT; +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_commit; + mode_after_commit +------------------- + 448 +(1 row) + +-- Restore for subsequent tests +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 + test_fileops_chmod +-------------------- + +(1 row) + +-- ================================================================ +-- Test 5: FileOpsLink UNDO - rollback removes the hard link +-- ================================================================ +SELECT test_fileops_data_dir() || '/test_fileops_link.dat' AS linkpath \gset +BEGIN; +SELECT test_fileops_link(:'filepath_a', :'linkpath'); + test_fileops_link +------------------- + +(1 row) + +SELECT test_fileops_file_exists(:'linkpath') AS link_during_txn; + link_during_txn +----------------- + t +(1 row) + +ROLLBACK; +-- After rollback, UNDO should remove the hard link +SELECT test_fileops_file_exists(:'linkpath') AS link_after_rollback; + link_after_rollback +--------------------- + f +(1 row) + +-- ================================================================ +-- Test 6: FileOpsLink commit - link persists +-- ================================================================ +BEGIN; +SELECT test_fileops_link(:'filepath_a', :'linkpath'); + test_fileops_link +------------------- + +(1 row) + +COMMIT; +SELECT test_fileops_file_exists(:'linkpath') AS link_after_commit; + link_after_commit +------------------- + t +(1 row) + +-- ================================================================ +-- Test 7: FileOpsTruncate in subtransaction - ROLLBACK TO undoes it +-- ================================================================ +-- Ensure file is 1024 bytes +SELECT test_fileops_file_size(:'filepath_a') AS size_before_test7; + size_before_test7 +------------------- + 1024 +(1 row) + +BEGIN; +-- Outer truncate (should persist after commit) +SELECT test_fileops_truncate(:'filepath_a', 800); + test_fileops_truncate +----------------------- + +(1 row) + +SAVEPOINT sp1; +-- Inner truncate (should be rolled back) +SELECT test_fileops_truncate(:'filepath_a', 100); + test_fileops_truncate +----------------------- + +(1 row) + +SELECT test_fileops_file_size(:'filepath_a') AS size_in_savepoint; + size_in_savepoint +------------------- + 100 +(1 row) + +ROLLBACK TO sp1; +-- After ROLLBACK TO, should be back to 800 (the outer truncate value) +SELECT test_fileops_file_size(:'filepath_a') AS size_after_rollback_to; + size_after_rollback_to +------------------------ + 800 +(1 row) + +COMMIT; +-- After commit, the outer truncate to 800 should persist +SELECT test_fileops_file_size(:'filepath_a') AS size_after_commit; + size_after_commit +------------------- + 800 +(1 row) + +-- ================================================================ +-- Test 8: FileOpsChmod in subtransaction - ROLLBACK TO restores mode +-- ================================================================ +-- Set a known mode baseline +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 + test_fileops_chmod +-------------------- + +(1 row) + +BEGIN; +-- Outer chmod (persists) +SELECT test_fileops_chmod(:'filepath_a', 448); -- 0700 + test_fileops_chmod +-------------------- + +(1 row) + +SAVEPOINT sp1; +-- Inner chmod (rolled back) +SELECT test_fileops_chmod(:'filepath_a', 256); -- 0400 + test_fileops_chmod +-------------------- + +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_in_savepoint; + mode_in_savepoint +------------------- + 256 +(1 row) + +ROLLBACK TO sp1; +-- After ROLLBACK TO, should be back to 0700 = 448 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_rollback_to; + mode_after_rollback_to +------------------------ + 448 +(1 row) + +COMMIT; +-- After commit, outer chmod (0700 = 448) persists +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_commit; + mode_after_commit +------------------- + 448 +(1 row) + +-- ================================================================ +-- Test 9: FileOpsLink in subtransaction - ROLLBACK TO removes link +-- ================================================================ +SELECT test_fileops_data_dir() || '/test_fileops_link2.dat' AS linkpath2 \gset +BEGIN; +SAVEPOINT sp1; +SELECT test_fileops_link(:'filepath_a', :'linkpath2'); + test_fileops_link +------------------- + +(1 row) + +SELECT test_fileops_file_exists(:'linkpath2') AS link2_in_savepoint; + link2_in_savepoint +-------------------- + t +(1 row) + +ROLLBACK TO sp1; +-- Link should be removed by UNDO +SELECT test_fileops_file_exists(:'linkpath2') AS link2_after_rollback_to; + link2_after_rollback_to +------------------------- + f +(1 row) + +COMMIT; +SELECT test_fileops_file_exists(:'linkpath2') AS link2_after_commit; + link2_after_commit +-------------------- + f +(1 row) + +-- ================================================================ +-- Test 10: FileOpsSetXattr / FileOpsRemoveXattr UNDO +-- (skipped on platforms without xattr support) +-- ================================================================ +-- Try to set an xattr; returns false if unsupported (ENOTSUP/EPERM/EACCES) +SELECT test_fileops_setxattr(:'filepath_a', 'user.test_key', 'initial_value') + AS xattr_supported \gset +\if :xattr_supported +\else +SELECT 'xattr tests skipped (not supported on this platform/filesystem)' AS notice; + notice +----------------------------------------------------------------- + xattr tests skipped (not supported on this platform/filesystem) +(1 row) + +\endif +-- ================================================================ +-- Cleanup +-- ================================================================ +DROP EXTENSION test_fileops; diff --git a/src/test/modules/test_fileops/expected/test_fileops_2.out b/src/test/modules/test_fileops/expected/test_fileops_2.out new file mode 100644 index 00000000000..d25aa81d54b --- /dev/null +++ b/src/test/modules/test_fileops/expected/test_fileops_2.out @@ -0,0 +1,18 @@ +-- +-- Tests for FILEOPS UNDO rollback of direct file operations. +-- +-- This test module provides SQL-callable wrappers around FileOps C functions +-- that have no DDL-level callers, enabling direct testing of UNDO rollback +-- for: FileOpsTruncate, FileOpsChmod, FileOpsLink, FileOpsSetXattr, +-- FileOpsRemoveXattr. +-- +-- REQUIRES: UNDO subsystem (always active) +-- +-- These are POSIX file semantics: modes, hard links that do not share the +-- target's attributes, extended attributes. Windows has none of them (chmod +-- only toggles the read-only attribute, which a hard link shares), so the test +-- does not run there; the TAP tests cover the operations FILEOPS supports on +-- Windows. +SELECT version() ~ '(Visual C\+\+|mingw32|msvc|windows)' AS skip_test \gset +\if :skip_test +\quit diff --git a/src/test/modules/test_fileops/meson.build b/src/test/modules/test_fileops/meson.build new file mode 100644 index 00000000000..5d886f13739 --- /dev/null +++ b/src/test/modules/test_fileops/meson.build @@ -0,0 +1,48 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group + +test_fileops_sources = files( + 'test_fileops.c', +) + +if host_system == 'windows' + test_fileops_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'test_fileops', + '--FILEDESC', 'test_fileops - regression testing of FILEOPS subsystem',]) +endif + +test_fileops = shared_module('test_fileops', + test_fileops_sources, + kwargs: pg_test_mod_args, +) +test_install_libs += test_fileops + +test_install_data += files( + 'test_fileops.control', + 'test_fileops--1.0.sql', +) + +tests += { + 'name': 'test_fileops', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'regress': { + 'sql': [ + 'test_fileops', + ], + }, + 'tap': { + 'env': { + 'enable_injection_points': get_option('injection_points') ? 'yes' : 'no', + }, + 'tests': [ + 't/001_fileops_apply_failure.pl', + 't/004_fileops_fs_errors.pl', + 't/005_fileops_checkpoint_loser.pl', + 't/007_fileops_commit_crash.pl', + 't/008_fileops_standby_abort.pl', + 't/018_fileops_rename.pl', + 't/023_fileops_inline_abort_ctx.pl', + 't/024_fileops_recovery_apply_failure.pl', + ], + }, +} diff --git a/src/test/modules/test_fileops/sql/test_fileops.sql b/src/test/modules/test_fileops/sql/test_fileops.sql new file mode 100644 index 00000000000..ef1a10523d1 --- /dev/null +++ b/src/test/modules/test_fileops/sql/test_fileops.sql @@ -0,0 +1,229 @@ +-- +-- Tests for FILEOPS UNDO rollback of direct file operations. +-- +-- This test module provides SQL-callable wrappers around FileOps C functions +-- that have no DDL-level callers, enabling direct testing of UNDO rollback +-- for: FileOpsTruncate, FileOpsChmod, FileOpsLink, FileOpsSetXattr, +-- FileOpsRemoveXattr. +-- +-- REQUIRES: UNDO subsystem (always active) +-- +-- These are POSIX file semantics: modes, hard links that do not share the +-- target's attributes, extended attributes. Windows has none of them (chmod +-- only toggles the read-only attribute, which a hard link shares), so the test +-- does not run there; the TAP tests cover the operations FILEOPS supports on +-- Windows. +SELECT version() ~ '(Visual C\+\+|mingw32|msvc|windows)' AS skip_test \gset +\if :skip_test +\quit +\endif + +CREATE EXTENSION test_fileops; + +-- ================================================================ +-- Setup: create test files in the data directory +-- ================================================================ + +SELECT test_fileops_create_tempfile('test_fileops_a.dat') AS filepath_a \gset +SELECT test_fileops_create_tempfile('test_fileops_b.dat') AS filepath_b \gset + +-- Record initial state (umask may affect mode) +SELECT test_fileops_file_size(:'filepath_a') AS initial_size; +SELECT test_fileops_get_mode(:'filepath_a') AS initial_mode; + +-- ================================================================ +-- Test 1: FileOpsTruncate UNDO - rollback restores original size +-- ================================================================ + +BEGIN; +SELECT test_fileops_truncate(:'filepath_a', 256); +SELECT test_fileops_file_size(:'filepath_a') AS size_during_txn; +ROLLBACK; + +-- After rollback, UNDO should restore original size (1024) +SELECT test_fileops_file_size(:'filepath_a') AS size_after_rollback; + +-- ================================================================ +-- Test 2: FileOpsTruncate commit - change persists +-- ================================================================ + +BEGIN; +SELECT test_fileops_truncate(:'filepath_b', 512); +COMMIT; + +SELECT test_fileops_file_size(:'filepath_b') AS size_after_commit; + +-- ================================================================ +-- Test 3: FileOpsChmod UNDO - rollback restores original permissions +-- ================================================================ + +-- Change to a known starting mode first (committed) +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 = 420 decimal +SELECT test_fileops_get_mode(:'filepath_a') AS mode_baseline; + +BEGIN; +-- Change to 0600 = 384 decimal +SELECT test_fileops_chmod(:'filepath_a', 384); +SELECT test_fileops_get_mode(:'filepath_a') AS mode_during_txn; +ROLLBACK; + +-- After rollback, UNDO should restore to 0644 = 420 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_rollback; + +-- ================================================================ +-- Test 4: FileOpsChmod commit - change persists +-- ================================================================ + +BEGIN; +SELECT test_fileops_chmod(:'filepath_a', 448); -- 0700 = 448 decimal +COMMIT; + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_commit; + +-- Restore for subsequent tests +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 + +-- ================================================================ +-- Test 5: FileOpsLink UNDO - rollback removes the hard link +-- ================================================================ + +SELECT test_fileops_data_dir() || '/test_fileops_link.dat' AS linkpath \gset + +BEGIN; +SELECT test_fileops_link(:'filepath_a', :'linkpath'); +SELECT test_fileops_file_exists(:'linkpath') AS link_during_txn; +ROLLBACK; + +-- After rollback, UNDO should remove the hard link +SELECT test_fileops_file_exists(:'linkpath') AS link_after_rollback; + +-- ================================================================ +-- Test 6: FileOpsLink commit - link persists +-- ================================================================ + +BEGIN; +SELECT test_fileops_link(:'filepath_a', :'linkpath'); +COMMIT; + +SELECT test_fileops_file_exists(:'linkpath') AS link_after_commit; + +-- ================================================================ +-- Test 7: FileOpsTruncate in subtransaction - ROLLBACK TO undoes it +-- ================================================================ + +-- Ensure file is 1024 bytes +SELECT test_fileops_file_size(:'filepath_a') AS size_before_test7; + +BEGIN; +-- Outer truncate (should persist after commit) +SELECT test_fileops_truncate(:'filepath_a', 800); + +SAVEPOINT sp1; +-- Inner truncate (should be rolled back) +SELECT test_fileops_truncate(:'filepath_a', 100); +SELECT test_fileops_file_size(:'filepath_a') AS size_in_savepoint; +ROLLBACK TO sp1; + +-- After ROLLBACK TO, should be back to 800 (the outer truncate value) +SELECT test_fileops_file_size(:'filepath_a') AS size_after_rollback_to; +COMMIT; + +-- After commit, the outer truncate to 800 should persist +SELECT test_fileops_file_size(:'filepath_a') AS size_after_commit; + +-- ================================================================ +-- Test 8: FileOpsChmod in subtransaction - ROLLBACK TO restores mode +-- ================================================================ + +-- Set a known mode baseline +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 + +BEGIN; +-- Outer chmod (persists) +SELECT test_fileops_chmod(:'filepath_a', 448); -- 0700 + +SAVEPOINT sp1; +-- Inner chmod (rolled back) +SELECT test_fileops_chmod(:'filepath_a', 256); -- 0400 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_in_savepoint; +ROLLBACK TO sp1; + +-- After ROLLBACK TO, should be back to 0700 = 448 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_rollback_to; +COMMIT; + +-- After commit, outer chmod (0700 = 448) persists +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_commit; + +-- ================================================================ +-- Test 9: FileOpsLink in subtransaction - ROLLBACK TO removes link +-- ================================================================ + +SELECT test_fileops_data_dir() || '/test_fileops_link2.dat' AS linkpath2 \gset + +BEGIN; +SAVEPOINT sp1; +SELECT test_fileops_link(:'filepath_a', :'linkpath2'); +SELECT test_fileops_file_exists(:'linkpath2') AS link2_in_savepoint; +ROLLBACK TO sp1; + +-- Link should be removed by UNDO +SELECT test_fileops_file_exists(:'linkpath2') AS link2_after_rollback_to; +COMMIT; + +SELECT test_fileops_file_exists(:'linkpath2') AS link2_after_commit; + +-- ================================================================ +-- Test 10: FileOpsSetXattr / FileOpsRemoveXattr UNDO +-- (skipped on platforms without xattr support) +-- ================================================================ + +-- Try to set an xattr; returns false if unsupported (ENOTSUP/EPERM/EACCES) +SELECT test_fileops_setxattr(:'filepath_a', 'user.test_key', 'initial_value') + AS xattr_supported \gset + +\if :xattr_supported + +-- Test 10a: SetXattr UNDO - rollback removes newly-set xattr +BEGIN; +SELECT test_fileops_setxattr(:'filepath_a', 'user.rollback_test', 'will_vanish'); +SELECT test_fileops_getxattr(:'filepath_a', 'user.rollback_test') AS xattr_during_txn; +ROLLBACK; + +SELECT test_fileops_getxattr(:'filepath_a', 'user.rollback_test') AS xattr_after_rollback; + +-- Test 10b: SetXattr overwrite UNDO - rollback restores old value +BEGIN; +SELECT test_fileops_setxattr(:'filepath_a', 'user.test_key', 'overwritten'); +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_overwritten; +ROLLBACK; + +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_restored; + +-- Test 10c: RemoveXattr UNDO - rollback restores removed xattr +BEGIN; +SELECT test_fileops_removexattr(:'filepath_a', 'user.test_key'); +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_after_remove; +ROLLBACK; + +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_after_remove_rollback; + +-- Test 10d: Xattr in subtransaction +BEGIN; +SAVEPOINT sp1; +SELECT test_fileops_setxattr(:'filepath_a', 'user.sub_key', 'sub_value'); +SELECT test_fileops_getxattr(:'filepath_a', 'user.sub_key') AS xattr_in_savepoint; +ROLLBACK TO sp1; + +SELECT test_fileops_getxattr(:'filepath_a', 'user.sub_key') AS xattr_after_sp_rollback; +COMMIT; + +\else +SELECT 'xattr tests skipped (not supported on this platform/filesystem)' AS notice; +\endif + +-- ================================================================ +-- Cleanup +-- ================================================================ + +DROP EXTENSION test_fileops; diff --git a/src/test/modules/test_fileops/t/001_fileops_apply_failure.pl b/src/test/modules/test_fileops/t/001_fileops_apply_failure.pl new file mode 100644 index 00000000000..77a13129264 --- /dev/null +++ b/src/test/modules/test_fileops/t/001_fileops_apply_failure.pl @@ -0,0 +1,38 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +# A FILEOPS inverse that cannot be applied at ROLLBACK is reported as a +# failure. The inverse rides in the abort record, so crash replay retries it. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('fileops_apply_failure'); +$node->init; +# FILEOPS writes UNDO only when WAL is needed. +$node->append_conf( + 'postgresql.conf', q{ +wal_level = replica +}); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $path = $node->safe_psql('postgres', + "SELECT test_fileops_create_tempfile('fileops_fail.dat')"); + +my $bg = $node->background_psql('postgres'); +$bg->query_safe('BEGIN'); +$bg->query_safe("SELECT test_fileops_truncate('$path', 10)"); +chmod(0444, $path) or die "chmod $path: $!"; +$bg->query('ROLLBACK'); +$bg->quit; + +like( + slurp_file($node->logfile), + qr/FILEOPS UNDO TRUNCATE: could not open/, + 'the failure was reported'); + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/004_fileops_fs_errors.pl b/src/test/modules/test_fileops/t/004_fileops_fs_errors.pl new file mode 100644 index 00000000000..7d947c62b9f --- /dev/null +++ b/src/test/modules/test_fileops/t/004_fileops_fs_errors.pl @@ -0,0 +1,107 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group + +# An ordinary filesystem error in an immediate FileOps operation (permission +# denied, no space, ...) must be a plain ERROR, exactly as without FILEOPS: +# the server must not PANIC, and the failed operation must leave nothing in +# the WAL for a standby to replay. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +if ($windows_os) +{ + plan skip_all => 'POSIX permission semantics required'; +} + +my $node = PostgreSQL::Test::Cluster->new('fileops_fs_errors'); +$node->init; +$node->append_conf('postgresql.conf', "wal_level = replica\n"); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $datadir = $node->data_dir; +my $ro = "$datadir/ro_dir"; +mkdir($ro) or die "mkdir $ro: $!"; +my $file = $node->safe_psql('postgres', + "SELECT test_fileops_create_tempfile('fserr.dat')"); +my $rofile = $node->safe_psql('postgres', + "SELECT test_fileops_create_tempfile('fserr_ro.dat')"); +my $has_xattr = $node->safe_psql('postgres', + "SELECT test_fileops_setxattr('$rofile', 'user.pgtest', 'v')"); +chmod(0500, $ro) or die "chmod $ro: $!"; +chmod(0444, $rofile) or die "chmod $rofile: $!"; + +my $start_time = + $node->safe_psql('postgres', 'SELECT pg_postmaster_start_time()'); +my $start_lsn = + $node->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn()'); + +my %cases = ( + create => "SELECT test_fileops_create('$ro/new.dat', 384)", + write => "SELECT test_fileops_write('/dev/full', 0, 'x')", + truncate => "SELECT test_fileops_truncate('$rofile', 10)", + chmod => "SELECT test_fileops_chmod('/', 448)", + chown => "SELECT test_fileops_chown('$file', 0, 0)", + mkdir => "SELECT test_fileops_mkdir('$ro/sub', 448)", + symlink => "SELECT test_fileops_symlink('$file', '$ro/lnk')", + link => "SELECT test_fileops_link('$file', '$ro/hard')",); + +if ($has_xattr eq 't') +{ + $cases{setxattr} = + "SELECT test_fileops_setxattr('$rofile', 'user.pgtest2', 'v')"; + $cases{removexattr} = + "SELECT test_fileops_removexattr('$rofile', 'user.pgtest')"; +} + +foreach my $op (sort keys %cases) +{ + my ($ret, $stdout, $stderr) = $node->psql('postgres', $cases{$op}); + + # The xattr wrappers report a refused operation as false. + ok( ($stderr =~ /ERROR:/ && $stderr !~ /PANIC|FATAL/) + || ($stdout eq 'f' && $stderr eq ''), + "$op: a filesystem error is an ERROR, not a PANIC" + ) or diag("stdout: $stdout\nstderr: $stderr"); + is($node->safe_psql('postgres', 'SELECT pg_postmaster_start_time()'), + $start_time, "$op: the server did not restart"); +} + +unlike( + slurp_file($node->logfile), + qr/PANIC|terminated by signal/, + 'no PANIC or backend crash in the server log'); + +# CREATE TABLESPACE on a directory we do not own fails as it does without +# FILEOPS. "/" is owned by root, and the server never runs as root. +my ($ret, $stdout, $stderr) = + $node->psql('postgres', "CREATE TABLESPACE t LOCATION '/'"); +like( + $stderr, + qr/ERROR: could not set permissions on directory "\/": (?:Operation not permitted|Read-only file system)/, + 'CREATE TABLESPACE on a root-owned directory gives the usual error'); +is($node->safe_psql('postgres', 'SELECT pg_postmaster_start_time()'), + $start_time, 'the server survived CREATE TABLESPACE'); + +# Nothing the failed operations attempted reached the WAL. +my $end_lsn = + $node->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn()'); +my ($walout, $walerr); +IPC::Run::run [ + 'pg_waldump', + '--path' => "$datadir/pg_wal", + '--start' => $start_lsn, + '--end' => $end_lsn, + '--rmgr' => 'FileOps' + ], + '>' => \$walout, + '2>' => \$walerr; +is($walout, '', 'no FILEOPS WAL record was written for a failed operation'); + +chmod(0700, $ro); +chmod(0644, $rofile); +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/005_fileops_checkpoint_loser.pl b/src/test/modules/test_fileops/t/005_fileops_checkpoint_loser.pl new file mode 100644 index 00000000000..034ac1ccc2a --- /dev/null +++ b/src/test/modules/test_fileops/t/005_fileops_checkpoint_loser.pl @@ -0,0 +1,38 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# A transaction that the crash leaves in progress must be rolled back even +# when all of its UNDO precedes the last checkpoint's redo pointer. Redo +# starts there, so it sees the chain only through the checkpoint's +# XLOG_UNDO_INFLIGHT record. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('checkpoint_loser'); +$node->init; +$node->append_conf('postgresql.conf', + "wal_level = replica\nautovacuum = off\ncheckpoint_timeout = '1h'\n"); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +my $pd = $node->data_dir; +foreach my $ckpt (0, 1) +{ + my $f = "loser_$ckpt.dat"; + my $bg = $node->background_psql('postgres'); + $bg->query_until(qr/ready/, + "BEGIN;\nSELECT test_fileops_create('$f', 384);\n\\echo ready\n"); + $node->safe_psql('postgres', 'CHECKPOINT') if $ckpt; + + # Crash before closing the client, so the backend cannot roll back first. + $node->stop('immediate'); + $bg->{run}->kill_kill; + $node->start; + ok(!-e "$pd/$f", + "checkpoint=$ckpt: the loser's file is removed after a crash"); +} + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/007_fileops_commit_crash.pl b/src/test/modules/test_fileops/t/007_fileops_commit_crash.pl new file mode 100644 index 00000000000..0ff3e80890f --- /dev/null +++ b/src/test/modules/test_fileops/t/007_fileops_commit_crash.pl @@ -0,0 +1,77 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group + +# A deferred commit-time file operation must survive a crash right after +# its transaction's commit record: the commit is durable, so the deletion, +# rename or rmdir it implies must happen too, on restart if not before. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $node = PostgreSQL::Test::Cluster->new('fileops_commit_crash'); +$node->init; +$node->append_conf( + 'postgresql.conf', q{ +shared_preload_libraries = 'injection_points' +wal_level = replica +}); +$node->start; +if (!$node->check_extension('injection_points')) +{ + plan skip_all => 'Extension injection_points not installed'; +} +$node->safe_psql('postgres', 'CREATE EXTENSION injection_points'); +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +my $datadir = $node->data_dir; + +# Commit $sql, crash right after the commit record, restart. +sub commit_and_crash +{ + my ($sql) = @_; + $node->safe_psql('postgres', + "SELECT injection_points_attach('fileops-commit-before-pending-ops', 'wait')" + ); + my $bg = $node->background_psql('postgres', on_error_stop => 0); + $bg->query_until(qr/start/, "\\echo start\n$sql\n"); + $node->wait_for_event('client backend', + 'fileops-commit-before-pending-ops'); + $node->stop('immediate'); + $bg->{run}->finish; + $node->start; +} + +$node->safe_psql('postgres', + "SELECT test_fileops_create_tempfile('del.dat')"); +$node->safe_psql('postgres', + "SELECT test_fileops_create_tempfile('ren.dat')"); +$node->safe_psql('postgres', "SELECT test_fileops_mkdir('rmd', 448)"); +commit_and_crash( + q{BEGIN; +SELECT test_fileops_delete('del.dat'); +SELECT test_fileops_rename('ren.dat', 'ren2.dat'); +SELECT test_fileops_rmdir('rmd'); +COMMIT;}); +TODO: +{ + # The deferred ops are WAL-logged only when executed, after the commit + # record, so this crash loses them. Their intent must be durable before + # the commit record and executed by commit redo. + local $TODO = + 'commit-time FILEOPS ops are logged after the commit record'; + ok(!-e "$datadir/del.dat", + 'committed FileOpsDelete happened despite the crash'); + ok( !-e "$datadir/ren.dat" && -e "$datadir/ren2.dat", + 'committed FileOpsRename happened despite the crash'); + ok(!-e "$datadir/rmd", + 'committed FileOpsRmdir happened despite the crash'); +} + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/008_fileops_standby_abort.pl b/src/test/modules/test_fileops/t/008_fileops_standby_abort.pl new file mode 100644 index 00000000000..aee36a32fc9 --- /dev/null +++ b/src/test/modules/test_fileops/t/008_fileops_standby_abort.pl @@ -0,0 +1,89 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1); +$primary->append_conf('postgresql.conf', "max_prepared_transactions = 5\n"); +$primary->start; +$primary->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +$primary->safe_psql('postgres', 'CREATE EXTENSION injection_points') + if $ENV{enable_injection_points} eq 'yes'; +$primary->backup('backup'); +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, 'backup', has_streaming => 1); +$standby->start; +$primary->safe_psql( + 'postgres', q{ +BEGIN; +SELECT test_fileops_create('base.dat', 384); +SELECT test_fileops_write('base.dat', 0, 'committed-base'); +COMMIT; +BEGIN; +SELECT test_fileops_create('abort.dat', 384); +SELECT test_fileops_write('abort.dat', 0, 'aborted'); +ROLLBACK; +BEGIN; SELECT test_fileops_mkdir('abortdir', 448); ROLLBACK; +BEGIN; SELECT test_fileops_chmod('base.dat', 448); ROLLBACK; +BEGIN; SELECT test_fileops_write('base.dat', 0, 'OVERWRITTEN!!!'); ROLLBACK; +BEGIN; SELECT test_fileops_truncate('base.dat', 3); ROLLBACK; +}); +$primary->safe_psql( + 'postgres', q{ +BEGIN; +SELECT test_fileops_write('base.dat', 0, 'prepared'); +SELECT test_fileops_truncate('base.dat', 2); +PREPARE TRANSACTION 'p'; +ROLLBACK PREPARED 'p'; +BEGIN; +SAVEPOINT a; +SELECT test_fileops_chmod('base.dat', 448); +ROLLBACK TO a; +COMMIT; +}); +$primary->wait_for_catchup($standby); +if ($ENV{enable_injection_points} eq 'yes') +{ + $standby->safe_psql('postgres', + "SELECT injection_points_attach('fileops-abort-after-flush', 'wait')" + ); + $primary->safe_psql( + 'postgres', q{ +BEGIN; +SELECT test_fileops_create('standby-window.dat', 384); +ROLLBACK; +}); + $standby->wait_for_event('startup', 'fileops-abort-after-flush'); + ok(-e $standby->data_dir . '/standby-window.dat', + 'standby paused before inverse'); + $standby->stop('immediate'); + $standby->start; + $primary->wait_for_catchup($standby); + ok(!-e $standby->data_dir . '/standby-window.dat', + 'standby crash replays inverse'); +} +for my $stage ('streaming', 'promoted') +{ + $standby->promote if $stage eq 'promoted'; + for my $node ($primary, $standby) + { + my $dir = $node->data_dir; + my $name = $node->name; + ok(!-e "$dir/abort.dat", "$stage $name: aborted create absent"); + ok(!-e "$dir/abortdir", "$stage $name: aborted mkdir absent"); + SKIP: + { + skip 'POSIX permission semantics required', 1 if $windows_os; + is((stat("$dir/base.dat"))[2] & 07777, + 0600, "$stage $name: original mode"); + } + is(slurp_file("$dir/base.dat"), + 'committed-base', "$stage $name: original bytes"); + } +} +$standby->stop; +$primary->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/018_fileops_rename.pl b/src/test/modules/test_fileops/t/018_fileops_rename.pl new file mode 100644 index 00000000000..1967f84427b --- /dev/null +++ b/src/test/modules/test_fileops/t/018_fileops_rename.pl @@ -0,0 +1,347 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# FileOpsRename() renames at once, so the new name is usable inside the +# transaction, and the rename is reversed like every other immediate FILEOPS +# operation: on ROLLBACK, ROLLBACK TO SAVEPOINT and ROLLBACK PREPARED, by the +# startup undo phase after a crash, and on a standby that replays the abort. +# At wal_level = minimal an abort-time action renames it back. A rename onto +# an existing name, or of a missing file, is refused and changes nothing. +# Crash redo of committed renames converges on the state before the crash. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $injection = $ENV{enable_injection_points} eq 'yes'; + +my $node = PostgreSQL::Test::Cluster->new('fileops_rename'); +$node->init(allows_streaming => 1); +$node->append_conf( + 'postgresql.conf', q{ +autovacuum = off +max_prepared_transactions = 5 +}); +$node->append_conf('postgresql.conf', + "shared_preload_libraries = 'injection_points'\n") + if $injection; +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +my $datadir = $node->data_dir; + +# Commit file $name holding $content on $n. FILEOPS paths are relative to +# the data directory, and these are WAL-logged, so a standby gets them. +sub mkfile +{ + my ($n, $name, $content) = @_; + $n->safe_psql( + 'postgres', qq{ +BEGIN; +SELECT test_fileops_create('$name', 384); +SELECT test_fileops_write('$name', 0, '$content'); +COMMIT; +}); +} + +# What file $name in $dir holds, or ''. +sub content +{ + my ($dir, $name) = @_; + return -f "$dir/$name" ? slurp_file("$dir/$name") : ''; +} + +# Rename $from to $to in $bg's open transaction; it must show at once. +sub rename_in +{ + my ($bg, $dir, $from, $to, $tag) = @_; + $bg->query_safe("SELECT test_fileops_rename('$from', '$to')"); + ok( !-e "$dir/$from" && -e "$dir/$to", + "$tag: renamed inside the transaction"); +} + +my $bg = $node->background_psql('postgres', on_error_stop => 0); + +# a. COMMIT keeps the rename +mkfile($node, 'a.dat', 'content-a'); +$bg->query_safe('BEGIN'); +rename_in($bg, $datadir, 'a.dat', 'a2.dat', 'a'); +is(content($datadir, 'a2.dat'), + 'content-a', 'a: the new name has the content inside the transaction'); +$bg->query_safe('COMMIT'); +is(content($datadir, 'a2.dat'), 'content-a', 'a: COMMIT keeps the new name'); +ok(!-e "$datadir/a.dat", 'a: COMMIT keeps the old name gone'); + +# b. ROLLBACK restores the old name +mkfile($node, 'b.dat', 'content-b'); +$bg->query_safe('BEGIN'); +rename_in($bg, $datadir, 'b.dat', 'b2.dat', 'b'); +$bg->query_safe('ROLLBACK'); +is(content($datadir, 'b.dat'), + 'content-b', 'b: ROLLBACK restores the old name with its content'); +ok(!-e "$datadir/b2.dat", 'b: ROLLBACK removes the new name'); + +# c. ROLLBACK TO SAVEPOINT restores it, and COMMIT does not redo it +mkfile($node, 'c.dat', 'content-c'); +$bg->query_safe('BEGIN'); +$bg->query_safe('SAVEPOINT s'); +rename_in($bg, $datadir, 'c.dat', 'c2.dat', 'c'); +$bg->query_safe('ROLLBACK TO SAVEPOINT s'); +is(content($datadir, 'c.dat'), + 'content-c', 'c: ROLLBACK TO SAVEPOINT restores the old name'); +ok(!-e "$datadir/c2.dat", 'c: ROLLBACK TO SAVEPOINT removes the new name'); +$bg->query_safe('COMMIT'); +is(content($datadir, 'c.dat'), 'content-c', 'c: still restored after COMMIT'); +ok(!-e "$datadir/c2.dat", 'c: new name still absent after COMMIT'); + +# d. a later operation on the new name is undone first (LIFO) +mkfile($node, 'd.dat', 'content-d'); +my ($ret, $stdout, $stderr) = $node->psql( + 'postgres', q{BEGIN; +SELECT test_fileops_rename('d.dat', 'd2.dat'); +SELECT test_fileops_write('d2.dat', 0, 'CHANGED-AND-LONGER'); +SELECT pg_read_file('d2.dat'); +ROLLBACK;}); +like($stdout, qr/^CHANGED-AND-LONGER$/m, 'd: the write went to the new name'); +is(content($datadir, 'd.dat'), + 'content-d', 'd: ROLLBACK undid the write, then the rename'); +ok(!-e "$datadir/d2.dat", 'd: new name absent after ROLLBACK'); + +# e. refused: an existing destination, a missing source +mkfile($node, 'e.dat', 'content-e'); +mkfile($node, 'e2.dat', 'content-e2'); +($ret, $stdout, $stderr) = + $node->psql('postgres', "SELECT test_fileops_rename('e.dat', 'e2.dat')"); +isnt($ret, 0, 'e: renaming onto an existing name fails'); +like($stderr, qr/already exists/, 'e: ... saying the name exists'); +is(content($datadir, 'e.dat'), 'content-e', 'e: source unchanged'); +is(content($datadir, 'e2.dat'), 'content-e2', 'e: destination unchanged'); +($ret, $stdout, $stderr) = $node->psql('postgres', + "SELECT test_fileops_rename('e-missing.dat', 'e3.dat')"); +isnt($ret, 0, 'e: renaming a missing file fails'); +like($stderr, qr/could not stat file/, 'e: ... saying it is missing'); +ok(!-e "$datadir/e3.dat", 'e: ... and creates nothing'); + +# f. a directory, with a file in it +$node->safe_psql('postgres', "SELECT test_fileops_mkdir('fdir', 448)"); +mkfile($node, 'fdir/inner.dat', 'content-f'); +$bg->query_safe('BEGIN'); +rename_in($bg, $datadir, 'fdir', 'fdir2', 'f'); +is(content($datadir, 'fdir2/inner.dat'), + 'content-f', 'f: the directory contents moved with it'); +$bg->query_safe('ROLLBACK'); +is(content($datadir, 'fdir/inner.dat'), + 'content-f', 'f: ROLLBACK restores the directory and its contents'); +ok(!-e "$datadir/fdir2", 'f: new directory name absent after ROLLBACK'); + +# l. the rollback leaves an old name that was reused in the meantime alone +mkfile($node, 'l.dat', 'content-l'); +$bg->query_safe('BEGIN'); +rename_in($bg, $datadir, 'l.dat', 'l2.dat', 'l: old name reused'); +append_to_file("$datadir/l.dat", 'reused'); +$bg->query_safe('ROLLBACK'); +is(content($datadir, 'l.dat'), 'reused', 'l: the reused old name is kept'); +is(content($datadir, 'l2.dat'), + 'content-l', 'l: the renamed file stays under the new name'); +$bg->quit; + +# g. crash recovery reverses the rename of an in-flight transaction +mkfile($node, 'g.dat', 'content-g'); +my $bgc = $node->background_psql('postgres', on_error_stop => 0); +$bgc->query_safe('BEGIN'); +rename_in($bgc, $datadir, 'g.dat', 'g2.dat', 'g'); +$node->stop('immediate'); +$bgc->{run}->finish; +$node->start; +is(content($datadir, 'g.dat'), + 'content-g', 'g: crash recovery restores the old name'); +ok(!-e "$datadir/g2.dat", 'g: crash recovery removes the new name'); + +# g. a committed rename of a file created since the last checkpoint: redo +# re-creates the old name from the earlier records, and the rename's redo must +# not leave it behind +mkfile($node, 'g3.dat', 'content-g3'); +$node->safe_psql('postgres', + "SELECT test_fileops_rename('g3.dat', 'g4.dat')"); +$node->stop('immediate'); +$node->start; +is(content($datadir, 'g4.dat'), + 'content-g3', 'g: a committed rename survives a crash'); +ok(!-e "$datadir/g3.dat", 'g: ... and redo leaves no old name behind'); + +# g. committed history a -> b, b -> c, then b made again, where a predates the +# checkpoint. Redo of b -> c finds the later b and c both present; renaming +# over c would replace the renamed file with the later b. +mkfile($node, 'ga.dat', 'content-ga'); +$node->safe_psql('postgres', 'CHECKPOINT'); +$node->safe_psql('postgres', + "SELECT test_fileops_rename('ga.dat', 'gb.dat')"); +$node->safe_psql('postgres', + "SELECT test_fileops_rename('gb.dat', 'gc.dat')"); +mkfile($node, 'gb.dat', 'new-gb'); +$node->stop('immediate'); +$node->start; +ok(!-e "$datadir/ga.dat", 'g: a -> b -> c, b again: no a after the crash'); +is(content($datadir, 'gb.dat'), 'new-gb', 'g: ... b is the later file'); +is(content($datadir, 'gc.dat'), + 'content-ga', 'g: ... c is still the renamed file'); + +# g. a renamed directory that redo re-creates and fills under its old name is +# not removed: both names are left, with a WARNING, and nothing is lost +$node->safe_psql('postgres', "SELECT test_fileops_mkdir('gdir', 448)"); +mkfile($node, 'gdir/x.dat', 'content-gx'); +$node->safe_psql('postgres', "SELECT test_fileops_rename('gdir', 'gdir2')"); +my $offset = -s $node->logfile; +$node->stop('immediate'); +$node->start; +is(content($datadir, 'gdir2/x.dat'), + 'content-gx', 'g: a renamed directory keeps its contents after a crash'); +like( + slurp_file($node->logfile, $offset), + qr/WARNING: .*"gdir"/, + 'g: redo warns about the re-created old directory it leaves'); + +# h. two-phase commit +mkfile($node, 'h1.dat', 'content-h1'); +$node->safe_psql('postgres', + "BEGIN; SELECT test_fileops_rename('h1.dat', 'h1b.dat'); PREPARE TRANSACTION 'h1';" +); +ok(!-e "$datadir/h1.dat" && -e "$datadir/h1b.dat", + 'h: renamed while prepared'); +$node->safe_psql('postgres', "ROLLBACK PREPARED 'h1'"); +is(content($datadir, 'h1.dat'), + 'content-h1', 'h: ROLLBACK PREPARED restores the old name'); +ok(!-e "$datadir/h1b.dat", 'h: ROLLBACK PREPARED removes the new name'); + +mkfile($node, 'h2.dat', 'content-h2'); +$node->safe_psql('postgres', + "BEGIN; SELECT test_fileops_rename('h2.dat', 'h2b.dat'); PREPARE TRANSACTION 'h2';" +); +$node->stop('immediate'); +$node->start; +ok( !-e "$datadir/h2.dat" && -e "$datadir/h2b.dat", + 'h: still renamed while prepared after a crash'); +$node->safe_psql('postgres', "ROLLBACK PREPARED 'h2'"); +is(content($datadir, 'h2.dat'), + 'content-h2', 'h: ROLLBACK PREPARED after a crash restores the old name'); +ok(!-e "$datadir/h2b.dat", + 'h: ROLLBACK PREPARED after a crash removes the new name'); + +mkfile($node, 'h3.dat', 'content-h3'); +$node->safe_psql('postgres', + "BEGIN; SELECT test_fileops_rename('h3.dat', 'h3b.dat'); PREPARE TRANSACTION 'h3';" +); +$node->safe_psql('postgres', "COMMIT PREPARED 'h3'"); +is(content($datadir, 'h3b.dat'), + 'content-h3', 'h: COMMIT PREPARED keeps the new name'); +ok(!-e "$datadir/h3.dat", 'h: COMMIT PREPARED keeps the old name gone'); + +# i. a streaming standby converges, for COMMIT and for ROLLBACK +$node->backup('bk'); +my $standby = PostgreSQL::Test::Cluster->new('fileops_rename_standby'); +$standby->init_from_backup($node, 'bk', has_streaming => 1); +$standby->start; +my $sdir = $standby->data_dir; +mkfile($node, 'i1.dat', 'content-i1'); +mkfile($node, 'i2.dat', 'content-i2'); +$node->safe_psql('postgres', + "SELECT test_fileops_rename('i1.dat', 'i1b.dat')"); +my $bgi = $node->background_psql('postgres', on_error_stop => 0); +$bgi->query_safe('BEGIN'); +$bgi->query_safe("SELECT test_fileops_rename('i2.dat', 'i2b.dat')"); +$node->wait_for_catchup($standby); +is(content($sdir, 'i1b.dat'), + 'content-i1', 'i: standby has the committed new name'); +ok(!-e "$sdir/i1.dat", 'i: standby lost the committed old name'); +ok( !-e "$sdir/i2.dat" && -e "$sdir/i2b.dat", + 'i: standby shows a rename in flight'); +$bgi->query_safe('ROLLBACK'); +$bgi->quit; +$node->wait_for_catchup($standby); +is(content($sdir, 'i2.dat'), + 'content-i2', 'i: standby restores the old name of a rolled-back rename'); +ok(!-e "$sdir/i2b.dat", 'i: standby has no new name of a rolled-back rename'); + +# i. ROLLBACK PREPARED on a promoted standby renames back, though the +# standby's file is not the primary's inode +mkfile($node, 'i3.dat', 'content-i3'); +$node->safe_psql('postgres', + "BEGIN; SELECT test_fileops_rename('i3.dat', 'i3b.dat'); PREPARE TRANSACTION 'i3';" +); +$node->wait_for_catchup($standby); +ok( !-e "$sdir/i3.dat" && -e "$sdir/i3b.dat", + 'i: standby shows the prepared rename'); +$standby->promote; +$standby->safe_psql('postgres', "ROLLBACK PREPARED 'i3'"); +is(content($sdir, 'i3.dat'), + 'content-i3', + 'i: ROLLBACK PREPARED on the promoted standby restores the old name'); +ok(!-e "$sdir/i3b.dat", + 'i: ROLLBACK PREPARED on the promoted standby removes the new name'); +$node->safe_psql('postgres', "ROLLBACK PREPARED 'i3'"); +$standby->stop; + +# j. wal_level = minimal: no WAL, an abort-time action renames back +my $min = PostgreSQL::Test::Cluster->new('fileops_rename_minimal'); +$min->init; +$min->append_conf('postgresql.conf', + "wal_level = minimal\nmax_wal_senders = 0\n"); +$min->start; +$min->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +is($min->safe_psql('postgres', 'SHOW wal_level'), + 'minimal', 'j: wal_level is minimal'); +my $mdir = $min->data_dir; +mkfile($min, 'j.dat', 'content-j'); +my $bgm = $min->background_psql('postgres', on_error_stop => 0); +$bgm->query_safe('BEGIN'); +rename_in($bgm, $mdir, 'j.dat', 'j2.dat', 'j'); +$bgm->query_safe('ROLLBACK'); +is(content($mdir, 'j.dat'), 'content-j', 'j: ROLLBACK restores the old name'); +ok(!-e "$mdir/j2.dat", 'j: ROLLBACK removes the new name'); +$bgm->query_safe('BEGIN'); +$bgm->query_safe('SAVEPOINT s'); +rename_in($bgm, $mdir, 'j.dat', 'j2.dat', 'j: subtransaction'); +$bgm->query_safe('ROLLBACK TO SAVEPOINT s'); +is(content($mdir, 'j.dat'), + 'content-j', 'j: ROLLBACK TO SAVEPOINT restores the old name'); +rename_in($bgm, $mdir, 'j.dat', 'j3.dat', 'j: then'); +$bgm->query_safe('COMMIT'); +is(content($mdir, 'j3.dat'), 'content-j', 'j: COMMIT keeps the new name'); +ok(!-e "$mdir/j.dat" && !-e "$mdir/j2.dat", 'j: and no other name'); +$bgm->quit; +$min->stop; + +# k. a crash after the WAL is flushed but before rename(): redo performs the +# rename, and the loser's UNDO reverses it +SKIP: +{ + skip 'Injection points not supported by this build', 4 unless $injection; + $node->safe_psql('postgres', 'CREATE EXTENSION injection_points'); + mkfile($node, 'k.dat', 'content-k'); + $node->safe_psql('postgres', + "SELECT injection_points_attach('fileops-rename-after-wal-flush', 'wait')" + ); + my $bgk = $node->background_psql('postgres', on_error_stop => 0); + $bgk->query_until( + qr/start/, q(\echo start +BEGIN; +SELECT test_fileops_rename('k.dat', 'k2.dat'); +)); + $node->wait_for_event('client backend', 'fileops-rename-after-wal-flush'); + ok(-e "$datadir/k.dat" && !-e "$datadir/k2.dat", + 'k: logged, not yet renamed'); + $offset = -s $node->logfile; + $node->stop('immediate'); + $bgk->{run}->finish; + $node->start; + like( + slurp_file($node->logfile, $offset), + qr/UNDO recovery: rolling back transaction/, + 'k: the transaction was rolled back as a loser'); + is(content($datadir, 'k.dat'), 'content-k', 'k: the old name is kept'); + ok(!-e "$datadir/k2.dat", 'k: the new name is absent'); +} + +$node->stop; + +done_testing(); diff --git a/src/test/modules/test_fileops/t/023_fileops_inline_abort_ctx.pl b/src/test/modules/test_fileops/t/023_fileops_inline_abort_ctx.pl new file mode 100644 index 00000000000..edce8683773 --- /dev/null +++ b/src/test/modules/test_fileops/t/023_fileops_inline_abort_ctx.pl @@ -0,0 +1,60 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# AtAbort_XactUndo()'s inline-abort path must restore the memory context and +# free its working context on every exit, including the early exit taken when +# the batch LSN does not point at a valid UNDO batch. +# +# That exit jumped over the MemoryContextSwitchTo(old_ctx) / +# MemoryContextDelete(undo_ctx) pair, so the "Inline UNDO Apply" context (a +# child of TopMemoryContext) leaked on every such abort, and the rest of the +# abort ran in, and returned into, that context. One leaked context is +# enough to show it: it survives into the next statement of the same +# backend, where pg_backend_memory_contexts lists it. +# +# The invalid-batch exit is forced with the "inline-undo-invalid-batch" +# injection point. A FILEOPS chmod puts one UNDO record in the +# transaction's chain so the inline path runs at all. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $node = PostgreSQL::Test::Cluster->new('fileops_inline_abort_ctx'); +$node->init; +# FILEOPS writes UNDO only when WAL is needed. +$node->append_conf('postgresql.conf', 'wal_level = replica'); +$node->start; +$node->safe_psql('postgres', + 'CREATE EXTENSION test_fileops; CREATE EXTENSION injection_points;'); + +my $path = $node->safe_psql('postgres', + "SELECT test_fileops_create_tempfile('inline_abort_ctx.dat')"); + +# One backend: the leaked context has to be looked for in the backend that +# aborted, after its transaction has ended. +my $bg = $node->background_psql('postgres'); +$bg->query_safe('SELECT injection_points_set_local()'); +$bg->query_safe( + "SELECT injection_points_attach('inline-undo-invalid-batch', 'error')"); + +$bg->query_safe('BEGIN'); +$bg->query_safe("SELECT test_fileops_chmod('$path', 384)"); +$bg->query('ROLLBACK'); + +# The ROLLBACK warns "inline UNDO failed"; query() leaves that on stderr. +my $leaked = $bg->query( + "SELECT count(*) FROM pg_backend_memory_contexts " + . "WHERE name = 'Inline UNDO Apply'"); +chomp $leaked; +is($leaked, '0', + 'the inline-abort working context is freed on the invalid-batch exit'); + +$bg->quit; +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/024_fileops_recovery_apply_failure.pl b/src/test/modules/test_fileops/t/024_fileops_recovery_apply_failure.pl new file mode 100644 index 00000000000..1dd3b70f34c --- /dev/null +++ b/src/test/modules/test_fileops/t/024_fileops_recovery_apply_failure.pl @@ -0,0 +1,59 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# The startup undo phase must not report a crash loser as rolled back when one +# of its UNDO records could not be applied. +# +# A crash loser changed a file's mode through FILEOPS. Before the restart the +# file's directory loses its search permission, so the inverse chmod() fails +# with a real EACCES. Nothing retries it, so the mode stays changed; the +# summary at the end of the undo phase has to say so. +# +# Must not run as root (root ignores the directory permission). +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +plan skip_all => 'running as root: EACCES cannot be produced' if $> == 0; +plan skip_all => 'directory search permission is POSIX-only' + if $PostgreSQL::Test::Utils::windows_os; + +my $node = PostgreSQL::Test::Cluster->new('fileops_recovery_apply_failure'); +$node->init; +# FILEOPS writes UNDO only when WAL is needed. +$node->append_conf('postgresql.conf', "wal_level = replica\nautovacuum = off\n"); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $dir = PostgreSQL::Test::Utils::tempdir('eacces'); +my $path = "$dir/m.dat"; +open(my $fh, '>', $path) or die "open $path: $!"; +close($fh); +chmod(0644, $path) or die "chmod $path: $!"; + +# A chmod in flight when the server crashes: a crash loser. +my $bg = $node->background_psql('postgres', on_error_stop => 0); +$bg->query_safe('BEGIN'); +$bg->query_safe("SELECT test_fileops_chmod('$path', 384)"); # 0600 +$node->safe_psql('postgres', 'SELECT pg_switch_wal()'); +my $offset = -s $node->logfile; +$node->stop('immediate'); +$bg->{run}->finish; + +chmod(0600, $dir) or die "chmod $dir: $!"; +$node->start; +chmod(0700, $dir) or die "chmod $dir: $!"; + +my $log = slurp_file($node->logfile, $offset); +like($log, qr/FILEOPS UNDO CHMOD: could not (?:stat|restore mode of) .*: Permission denied/, + 'the inverse failed with a real EACCES'); +like($log, qr/UNDO recovery complete: 1 transactions processed, 0 records applied, 1 records could not be applied/, + 'the undo phase reports the record it could not apply'); +unlike($log, qr/UNDO recovery complete: \d+ transactions rolled back/, + 'the loser is not reported as rolled back'); +is(sprintf('%04o', (stat($path))[2] & 07777), '0600', + 'the mode change was not rolled back'); + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/test_fileops--1.0.sql b/src/test/modules/test_fileops/test_fileops--1.0.sql new file mode 100644 index 00000000000..80545574727 --- /dev/null +++ b/src/test/modules/test_fileops/test_fileops--1.0.sql @@ -0,0 +1,99 @@ +/* src/test/modules/test_fileops/test_fileops--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION test_fileops" to load this file. \quit + +-- Create a temporary file and return its path +CREATE FUNCTION test_fileops_create_tempfile(filename text) + RETURNS text + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Create a file via FileOpsCreate (exercises the WAL-before-syscall path) +CREATE FUNCTION test_fileops_create(filepath text, mode int) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Truncate a file to the given length (uses FileOpsTruncate) +CREATE FUNCTION test_fileops_truncate(filepath text, length bigint) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Get file size +CREATE FUNCTION test_fileops_file_size(filepath text) + RETURNS bigint + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Chmod a file (uses FileOpsChmod) +CREATE FUNCTION test_fileops_chmod(filepath text, mode int) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Get file permissions (mode) +CREATE FUNCTION test_fileops_get_mode(filepath text) + RETURNS int + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Create a hard link (uses FileOpsLink) +CREATE FUNCTION test_fileops_link(oldpath text, newpath text) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Check if a file exists +CREATE FUNCTION test_fileops_file_exists(filepath text) + RETURNS boolean + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Set extended attribute (uses FileOpsSetXattr) +CREATE FUNCTION test_fileops_setxattr(filepath text, attrname text, attrvalue text) + RETURNS boolean + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Get extended attribute value +CREATE FUNCTION test_fileops_getxattr(filepath text, attrname text) + RETURNS text + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Remove extended attribute (uses FileOpsRemoveXattr) +CREATE FUNCTION test_fileops_removexattr(filepath text, attrname text) + RETURNS boolean + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Get the data directory path (for constructing absolute paths) +CREATE FUNCTION test_fileops_data_dir() + RETURNS text + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Deferred delete at commit (uses FileOpsDelete, at_commit=true) +CREATE FUNCTION test_fileops_delete(filepath text) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Deferred rename at commit (uses FileOpsRename) +CREATE FUNCTION test_fileops_rename(oldpath text, newpath text) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Create a directory (uses FileOpsMkdir) +CREATE FUNCTION test_fileops_mkdir(path text, mode int) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Deferred rmdir at commit (uses FileOpsRmdir, at_commit=true) +CREATE FUNCTION test_fileops_rmdir(path text) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Write a string at an offset (uses FileOpsWrite) +CREATE FUNCTION test_fileops_write(filepath text, "offset" bigint, data text) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Change ownership (uses FileOpsChown) +CREATE FUNCTION test_fileops_chown(filepath text, uid int, gid int) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Create a symbolic link (uses FileOpsSymlink) +CREATE FUNCTION test_fileops_symlink(target text, linkpath text) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; diff --git a/src/test/modules/test_fileops/test_fileops.c b/src/test/modules/test_fileops/test_fileops.c new file mode 100644 index 00000000000..c871d974427 --- /dev/null +++ b/src/test/modules/test_fileops/test_fileops.c @@ -0,0 +1,471 @@ +/*------------------------------------------------------------------------- + * + * test_fileops.c + * Test module exposing FileOps C API to SQL for regression testing. + * + * This module provides SQL-callable wrappers around the FileOps functions + * that have no DDL-level callers, enabling direct testing of: + * - FileOpsTruncate (with UNDO rollback of file size) + * - FileOpsChmod (with UNDO rollback of permissions) + * - FileOpsLink (with UNDO rollback of hard links) + * - FileOpsSetXattr / FileOpsRemoveXattr (with UNDO rollback) + * + * Copyright (c) 2024-2026, PostgreSQL Global Development Group + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include +#include + +#include "fmgr.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "port/pg_xattr.h" +#include "access/xlogdefs.h" +#include "storage/fd.h" +#include "storage/fileops.h" +#include "utils/builtins.h" +#include "utils/pg_lsn.h" + +PG_MODULE_MAGIC; + +PG_FUNCTION_INFO_V1(test_fileops_create_tempfile); +PG_FUNCTION_INFO_V1(test_fileops_create); +PG_FUNCTION_INFO_V1(test_fileops_truncate); +PG_FUNCTION_INFO_V1(test_fileops_file_size); +PG_FUNCTION_INFO_V1(test_fileops_chmod); +PG_FUNCTION_INFO_V1(test_fileops_get_mode); +PG_FUNCTION_INFO_V1(test_fileops_link); +PG_FUNCTION_INFO_V1(test_fileops_file_exists); +PG_FUNCTION_INFO_V1(test_fileops_setxattr); +PG_FUNCTION_INFO_V1(test_fileops_getxattr); +PG_FUNCTION_INFO_V1(test_fileops_removexattr); +PG_FUNCTION_INFO_V1(test_fileops_data_dir); +PG_FUNCTION_INFO_V1(test_fileops_write); +PG_FUNCTION_INFO_V1(test_fileops_chown); +PG_FUNCTION_INFO_V1(test_fileops_symlink); + +PG_FUNCTION_INFO_V1(test_fileops_delete); +PG_FUNCTION_INFO_V1(test_fileops_rename); +PG_FUNCTION_INFO_V1(test_fileops_mkdir); +PG_FUNCTION_INFO_V1(test_fileops_rmdir); + +/* + * test_fileops_create_tempfile - Create a test file in data directory. + * + * Creates a file with some content using FileOpsCreate + write, returning + * the absolute path. The file is created with 1024 bytes of content. + */ +Datum +test_fileops_create_tempfile(PG_FUNCTION_ARGS) +{ + text *filename = PG_GETARG_TEXT_PP(0); + char *fname = text_to_cstring(filename); + char filepath[MAXPGPATH]; + int fd; + char buf[1024]; + + snprintf(filepath, MAXPGPATH, "%s/%s", DataDir, fname); + + /* Use raw create + write so we have a file to manipulate */ + fd = open(filepath, O_CREAT | O_WRONLY | O_TRUNC | PG_BINARY, 0644); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create file \"%s\": %m", filepath))); + + /* Fill with 1024 bytes */ + memset(buf, 'X', sizeof(buf)); + if (write(fd, buf, sizeof(buf)) != sizeof(buf)) + { + int save_errno = errno; + + close(fd); + errno = save_errno; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write to file \"%s\": %m", filepath))); + } + close(fd); + + PG_RETURN_TEXT_P(cstring_to_text(filepath)); +} + +/* + * test_fileops_create - Create a file using FileOpsCreate. + * + * Unlike test_fileops_create_tempfile (which uses raw open()/write() to set + * up fixture files outside any transactional path), this calls FileOpsCreate + * directly so tests can exercise its WAL-before-syscall crash-recovery + * behavior. + */ +Datum +test_fileops_create(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + int mode = PG_GETARG_INT32(1); + char *filepath = text_to_cstring(filepath_text); + int fd; + + fd = FileOpsCreate(filepath, O_WRONLY | PG_BINARY, (mode_t) mode, true); + if (fd >= 0) + CloseTransientFile(fd); + + pfree(filepath); + PG_RETURN_VOID(); +} + +/* + * test_fileops_truncate - Truncate a file using FileOpsTruncate. + * + * This exercises the UNDO path: on rollback, the original file size + * should be restored. + */ +Datum +test_fileops_truncate(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + int64 length = PG_GETARG_INT64(1); + char *filepath = text_to_cstring(filepath_text); + + FileOpsTruncate(filepath, (off_t) length); + + pfree(filepath); + PG_RETURN_VOID(); +} + +/* + * test_fileops_file_size - Get the current size of a file. + */ +Datum +test_fileops_file_size(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + char *filepath = text_to_cstring(filepath_text); + struct stat st; + + if (stat(filepath, &st) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", filepath))); + + pfree(filepath); + PG_RETURN_INT64((int64) st.st_size); +} + +/* + * test_fileops_chmod - Change file permissions using FileOpsChmod. + * + * This exercises the UNDO path: on rollback, the original permissions + * should be restored. + */ +Datum +test_fileops_chmod(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + int mode = PG_GETARG_INT32(1); + char *filepath = text_to_cstring(filepath_text); + + if (FileOpsChmod(filepath, (mode_t) mode) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not chmod file \"%s\": %m", filepath))); + + pfree(filepath); + PG_RETURN_VOID(); +} + +/* + * test_fileops_get_mode - Get file permission bits. + */ +Datum +test_fileops_get_mode(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + char *filepath = text_to_cstring(filepath_text); + struct stat st; + + if (stat(filepath, &st) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", filepath))); + + pfree(filepath); + PG_RETURN_INT32((int32) (st.st_mode & 0777)); +} + +/* + * test_fileops_link - Create a hard link using FileOpsLink. + * + * This exercises the UNDO path: on rollback, the link should be removed. + */ +Datum +test_fileops_link(PG_FUNCTION_ARGS) +{ + text *oldpath_text = PG_GETARG_TEXT_PP(0); + text *newpath_text = PG_GETARG_TEXT_PP(1); + char *oldpath = text_to_cstring(oldpath_text); + char *newpath = text_to_cstring(newpath_text); + + if (FileOpsLink(oldpath, newpath) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not link \"%s\" to \"%s\": %m", + oldpath, newpath))); + + pfree(oldpath); + pfree(newpath); + PG_RETURN_VOID(); +} + +/* + * test_fileops_file_exists - Check whether a file exists. + */ +Datum +test_fileops_file_exists(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + char *filepath = text_to_cstring(filepath_text); + struct stat st; + bool exists; + + exists = (stat(filepath, &st) == 0); + + pfree(filepath); + PG_RETURN_BOOL(exists); +} + +/* + * test_fileops_setxattr - Set an extended attribute using FileOpsSetXattr. + * + * Returns true if the platform supports xattrs, false if ENOTSUP. + * This exercises the UNDO path: on rollback, the original xattr value + * (or absence) should be restored. + */ +Datum +test_fileops_setxattr(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + text *name_text = PG_GETARG_TEXT_PP(1); + text *value_text = PG_GETARG_TEXT_PP(2); + char *filepath = text_to_cstring(filepath_text); + char *name = text_to_cstring(name_text); + char *value = text_to_cstring(value_text); + int ret; + + ret = FileOpsSetXattr(filepath, name, value, strlen(value)); + if (ret != 0) + { + if (errno == ENOTSUP || errno == EOPNOTSUPP || + errno == EPERM || errno == EACCES) + { + pfree(filepath); + pfree(name); + pfree(value); + PG_RETURN_BOOL(false); + } + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not set xattr \"%s\" on \"%s\": %m", + name, filepath))); + } + + pfree(filepath); + pfree(name); + pfree(value); + PG_RETURN_BOOL(true); +} + +/* + * test_fileops_getxattr - Get an extended attribute value. + * + * Returns NULL if the attribute doesn't exist or xattrs are unsupported. + */ +Datum +test_fileops_getxattr(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + text *name_text = PG_GETARG_TEXT_PP(1); + char *filepath = text_to_cstring(filepath_text); + char *name = text_to_cstring(name_text); + char buf[1024]; + ssize_t len; + + len = pg_getxattr(filepath, name, buf, sizeof(buf) - 1); + if (len < 0) + { + pfree(filepath); + pfree(name); + PG_RETURN_NULL(); + } + + buf[len] = '\0'; + + pfree(filepath); + pfree(name); + PG_RETURN_TEXT_P(cstring_to_text(buf)); +} + +/* + * test_fileops_removexattr - Remove an extended attribute using + * FileOpsRemoveXattr. + * + * Returns true if successful, false if xattrs are unsupported. + * This exercises the UNDO path: on rollback, the removed xattr should + * be restored. + */ +Datum +test_fileops_removexattr(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + text *name_text = PG_GETARG_TEXT_PP(1); + char *filepath = text_to_cstring(filepath_text); + char *name = text_to_cstring(name_text); + int ret; + + ret = FileOpsRemoveXattr(filepath, name); + if (ret != 0) + { + if (errno == ENOTSUP || errno == EOPNOTSUPP) + { + pfree(filepath); + pfree(name); + PG_RETURN_BOOL(false); + } + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not remove xattr \"%s\" from \"%s\": %m", + name, filepath))); + } + + pfree(filepath); + pfree(name); + PG_RETURN_BOOL(true); +} + +/* + * test_fileops_data_dir - Return the data directory path. + */ +Datum +test_fileops_data_dir(PG_FUNCTION_ARGS) +{ + PG_RETURN_TEXT_P(cstring_to_text(DataDir)); +} + +/* + * test_fileops_delete - Deferred delete via FileOpsDelete(at_commit=true). + * + * Exercises the log-at-commit-execution path: the delete happens (and is + * WAL-logged) in FileOpsDoPendingOps() at commit, so a physical standby + * reproduces it by replay. + */ +Datum +test_fileops_delete(PG_FUNCTION_ARGS) +{ + char *filepath = text_to_cstring(PG_GETARG_TEXT_PP(0)); + + FileOpsDelete(filepath, true); + pfree(filepath); + PG_RETURN_VOID(); +} + +/* + * test_fileops_rename - Deferred rename via FileOpsRename (always at_commit). + */ +Datum +test_fileops_rename(PG_FUNCTION_ARGS) +{ + char *oldpath = text_to_cstring(PG_GETARG_TEXT_PP(0)); + char *newpath = text_to_cstring(PG_GETARG_TEXT_PP(1)); + + if (FileOpsRename(oldpath, newpath) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not rename \"%s\" to \"%s\": %m", + oldpath, newpath))); + pfree(oldpath); + pfree(newpath); + PG_RETURN_VOID(); +} + +/* + * test_fileops_mkdir - Create a directory via FileOpsMkdir. + * + * FileOpsMkdir creates the directory immediately and registers an + * rmdir-on-abort; on commit the directory persists. Used to set up a + * directory the test can then remove with test_fileops_rmdir. + */ +Datum +test_fileops_mkdir(PG_FUNCTION_ARGS) +{ + char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + int mode = PG_GETARG_INT32(1); + + if (FileOpsMkdir(path, (mode_t) mode) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\": %m", path))); + pfree(path); + PG_RETURN_VOID(); +} + +/* + * test_fileops_rmdir - Deferred rmdir via FileOpsRmdir(at_commit=true). + */ +Datum +test_fileops_rmdir(PG_FUNCTION_ARGS) +{ + char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + + FileOpsRmdir(path, true); + pfree(path); + PG_RETURN_VOID(); +} + +/* + * test_fileops_write - Write a string at an offset via FileOpsWrite. + */ +Datum +test_fileops_write(PG_FUNCTION_ARGS) +{ + char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + int64 offset = PG_GETARG_INT64(1); + char *data = text_to_cstring(PG_GETARG_TEXT_PP(2)); + + (void) FileOpsWrite(path, (off_t) offset, data, (uint32) strlen(data)); + pfree(path); + pfree(data); + PG_RETURN_VOID(); +} + +/* + * test_fileops_chown - Change ownership via FileOpsChown. + */ +Datum +test_fileops_chown(PG_FUNCTION_ARGS) +{ + char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + + (void) FileOpsChown(path, (uid_t) PG_GETARG_INT32(1), + (gid_t) PG_GETARG_INT32(2)); + pfree(path); + PG_RETURN_VOID(); +} + +/* + * test_fileops_symlink - Create a symbolic link via FileOpsSymlink. + */ +Datum +test_fileops_symlink(PG_FUNCTION_ARGS) +{ + char *target = text_to_cstring(PG_GETARG_TEXT_PP(0)); + char *linkpath = text_to_cstring(PG_GETARG_TEXT_PP(1)); + + (void) FileOpsSymlink(target, linkpath); + pfree(target); + pfree(linkpath); + PG_RETURN_VOID(); +} diff --git a/src/test/modules/test_fileops/test_fileops.control b/src/test/modules/test_fileops/test_fileops.control new file mode 100644 index 00000000000..1d39ab2a916 --- /dev/null +++ b/src/test/modules/test_fileops/test_fileops.control @@ -0,0 +1,4 @@ +comment = 'Test module for transactional file operations (FILEOPS)' +default_version = '1.0' +module_pathname = '$libdir/test_fileops' +relocatable = true diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile index 9c4102b6b2c..cb169662fb6 100644 --- a/src/test/recovery/Makefile +++ b/src/test/recovery/Makefile @@ -13,6 +13,7 @@ EXTRA_INSTALL=contrib/pg_prewarm \ contrib/pg_stat_statements \ contrib/test_decoding \ src/test/modules/injection_points \ + src/test/modules/test_fileops \ src/test/modules/test_wait_lsn subdir = src/test/recovery diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build index b6ac9dd07a7..daa07c38560 100644 --- a/src/test/recovery/meson.build +++ b/src/test/recovery/meson.build @@ -69,6 +69,11 @@ tests += { 't/058_shutdown_crash_restart.pl', 't/059_remote_apply_status_interval.pl', 't/060_multixact_crashed_updater.pl', + 't/064_fileops_standby_divergence.pl', + 't/061_fileops_2pc_deferred.pl', + 't/062_fileops_recovery.pl', + 't/063_fileops_2pc.pl', + 't/065_fileops_crash_undo.pl', ], }, } diff --git a/src/test/recovery/t/061_fileops_2pc_deferred.pl b/src/test/recovery/t/061_fileops_2pc_deferred.pl new file mode 100644 index 00000000000..0b1690658c5 --- /dev/null +++ b/src/test/recovery/t/061_fileops_2pc_deferred.pl @@ -0,0 +1,172 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group +# +# Test that DEFERRED commit-time FILEOPS (delete) and abort-time filesystem +# undo (of a create and a rename) survive PREPARE TRANSACTION across a crash. +# +# The bug: PostPrepare_FileOps() used to free the pending-op list without +# recording it anywhere, so a transaction PREPARE'd with a deferred delete +# (at_commit=true) lost that op entirely -- COMMIT PREPARED performed +# nothing -- and an immediate create's abort-time undo was lost for ROLLBACK +# PREPARED. The fix serializes the pending ops into the 2PC state file at +# PREPARE (TWOPHASE_RM_FILEOPS_ID records) and performs them in the FILEOPS +# post-commit / post-abort callbacks, so they survive a crash between PREPARE +# and the final decision, driven from the durable 2PC state. +# +# Unlike 063_fileops_2pc.pl (which only exercises chmod, a constructive op +# reversed via the cluster-wide UNDO chain), this test drives the deferred +# commit-time delete path -- the one that was silently dropped -- next to an +# immediate rename, which ROLLBACK PREPARED must rename back. +# +# NB: FILEOPS writes WAL/UNDO only when XLogIsNeeded() (wal_level >= replica), +# so 2PC durability requires wal_level = replica; this test runs there. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('fileops_2pc_deferred'); +$node->init; +$node->append_conf( + "postgresql.conf", qq( +autovacuum = off +max_prepared_transactions = 10 +wal_level = replica +)); +$node->start; + +if (!$node->check_extension('test_fileops')) +{ + plan skip_all => 'Extension test_fileops not installed'; +} +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $datadir = $node->data_dir; + +sub file_exists +{ + my ($path) = @_; + return $node->safe_psql('postgres', + qq{SELECT test_fileops_file_exists('$path')}); +} + +# Poll until the file's existence matches $want ('t'/'f'). +sub wait_for_exists +{ + my ($path, $want, $desc) = @_; + my $ok = $node->poll_query_until('postgres', + qq{SELECT test_fileops_file_exists('$path') = '$want'}); + is($ok, '1', $desc); +} + +# ================================================================ +# Scenario A: COMMIT PREPARED performs the deferred commit-time ops +# - deferred delete of an existing file +# - immediate rename of an existing file +# - immediate create of a new file (register_delete) +# PREPARE, crash, restart, then COMMIT PREPARED. All three must take effect. +# ================================================================ +my $del_a = "$datadir/fo2pc_del_a.dat"; +my $ren_a = "$datadir/fo2pc_ren_a.dat"; +my $ren_a2 = "$datadir/fo2pc_ren_a.renamed"; +my $new_a = "$datadir/fo2pc_new_a.dat"; + +$node->safe_psql('postgres', + qq{SELECT test_fileops_create_tempfile('fo2pc_del_a.dat')}); +$node->safe_psql('postgres', + qq{SELECT test_fileops_create_tempfile('fo2pc_ren_a.dat')}); + +is(file_exists($del_a), 't', 'A: delete target exists before txn'); +is(file_exists($ren_a), 't', 'A: rename source exists before txn'); + +$node->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_delete('$del_a'); +SELECT test_fileops_rename('$ren_a', '$ren_a2'); +SELECT test_fileops_create('$new_a', 384); +PREPARE TRANSACTION 'fo2pc_commit'; +)); + +# While prepared, the deferred delete has NOT run yet; the immediate rename +# and create have. +is(file_exists($del_a), 't', + 'A: delete deferred -- file still present while prepared'); +is(file_exists($ren_a), 'f', + 'A: rename immediate -- source gone while prepared'); +is(file_exists($ren_a2), 't', + 'A: rename immediate -- dest present while prepared'); +is(file_exists($new_a), 't', 'A: immediate create present while prepared'); + +# Crash between PREPARE and COMMIT PREPARED. +$node->stop('immediate'); +$node->start; + +is($node->safe_psql('postgres', q{SELECT gid FROM pg_prepared_xacts}), + 'fo2pc_commit', 'A: prepared xact survives crash'); + +$node->safe_psql('postgres', q{COMMIT PREPARED 'fo2pc_commit'}); + +is(file_exists($del_a), 'f', + 'A: COMMIT PREPARED performed the deferred delete'); +is(file_exists($ren_a), 'f', + 'A: COMMIT PREPARED kept the rename (source gone)'); +is(file_exists($ren_a2), 't', + 'A: COMMIT PREPARED kept the rename (dest present)'); +is(file_exists($new_a), 't', + 'A: created file persists after COMMIT PREPARED'); + +# ================================================================ +# Scenario B: ROLLBACK PREPARED does NOT perform the deferred commit-time +# delete, and DOES undo the immediate rename and create. +# PREPARE, crash, restart, then ROLLBACK PREPARED. +# ================================================================ +my $del_b = "$datadir/fo2pc_del_b.dat"; +my $ren_b = "$datadir/fo2pc_ren_b.dat"; +my $ren_b2 = "$datadir/fo2pc_ren_b.renamed"; +my $new_b = "$datadir/fo2pc_new_b.dat"; + +$node->safe_psql('postgres', + qq{SELECT test_fileops_create_tempfile('fo2pc_del_b.dat')}); +$node->safe_psql('postgres', + qq{SELECT test_fileops_create_tempfile('fo2pc_ren_b.dat')}); + +$node->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_delete('$del_b'); +SELECT test_fileops_rename('$ren_b', '$ren_b2'); +SELECT test_fileops_create('$new_b', 384); +PREPARE TRANSACTION 'fo2pc_rollback'; +)); + +is(file_exists($ren_b2), 't', 'B: immediate rename made while prepared'); +is(file_exists($new_b), 't', 'B: immediate create present while prepared'); + +# Crash between PREPARE and ROLLBACK PREPARED. +$node->stop('immediate'); +$node->start; + +is($node->safe_psql('postgres', q{SELECT gid FROM pg_prepared_xacts}), + 'fo2pc_rollback', 'B: prepared xact survives crash'); + +$node->safe_psql('postgres', q{ROLLBACK PREPARED 'fo2pc_rollback'}); + +is(file_exists($del_b), 't', + 'B: ROLLBACK PREPARED did NOT delete (file still present)'); +is(file_exists($ren_b), 't', + 'B: ROLLBACK PREPARED renamed back (source present)'); +is(file_exists($ren_b2), 'f', + 'B: ROLLBACK PREPARED renamed back (dest absent)'); +# The immediate create is undone by the constructive op's cluster-wide UNDO +# record on ROLLBACK PREPARED. +wait_for_exists($new_b, 'f', + 'B: ROLLBACK PREPARED undid the immediate create'); + +# Server healthy at the end. +is($node->safe_psql('postgres', 'SELECT 1'), + '1', 'server operational after all deferred-2PC scenarios'); + +$node->stop; +done_testing(); diff --git a/src/test/recovery/t/062_fileops_recovery.pl b/src/test/recovery/t/062_fileops_recovery.pl new file mode 100644 index 00000000000..fb8f23b63ae --- /dev/null +++ b/src/test/recovery/t/062_fileops_recovery.pl @@ -0,0 +1,423 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group +# +# Test crash recovery for transactional file operations (FILEOPS). +# +# These tests verify that FILEOPS WAL replay correctly handles: +# - Crash during file creation (with delete-on-abort) +# - Crash during deferred file deletion +# - Crash during file operations on standby +# - Multiple sequential crashes + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('fileops_recovery'); +$node->init; +$node->append_conf( + "postgresql.conf", qq( +autovacuum = off +log_min_messages = debug2 +)); +$node->start; + +# ================================================================ +# Test 1: CREATE TABLE survives crash +# ================================================================ + +$node->safe_psql( + "postgres", qq( +CREATE TABLE fileops_test (id int, data text); +INSERT INTO fileops_test VALUES (1, 'created_table'); +)); + +$node->stop('immediate'); +$node->start; + +my $result = + $node->safe_psql("postgres", "SELECT data FROM fileops_test WHERE id = 1"); +is($result, 'created_table', 'CREATE TABLE survives crash'); + +# ================================================================ +# Test 2: DROP TABLE is properly handled after crash +# ================================================================ + +$node->safe_psql( + "postgres", qq( +CREATE TABLE drop_me (id int); +INSERT INTO drop_me VALUES (1); +)); + +# Get the relfilenode before dropping +my $relpath = + $node->safe_psql("postgres", "SELECT pg_relation_filepath('drop_me')"); + +$node->safe_psql("postgres", "DROP TABLE drop_me"); + +$node->stop('immediate'); +$node->start; + +# Table should be gone +my ($ret, $stdout, $stderr) = + $node->psql("postgres", "SELECT * FROM drop_me"); +isnt($ret, 0, 'dropped table is gone after crash recovery'); + +# ================================================================ +# Test 3: Crash during transaction with CREATE TABLE (uncommitted) +# ================================================================ + +# This table is committed +$node->safe_psql( + "postgres", qq( +CREATE TABLE committed_table (id int); +INSERT INTO committed_table VALUES (42); +)); + +# Crash the server +$node->stop('immediate'); +$node->start; + +# Committed table should exist +$result = $node->safe_psql("postgres", "SELECT id FROM committed_table"); +is($result, '42', 'committed CREATE TABLE survives crash'); + +# ================================================================ +# Test 4: Multiple CREATE and DROP operations then crash +# ================================================================ + +$node->safe_psql( + "postgres", qq( +CREATE TABLE t1 (id int); +CREATE TABLE t2 (id int); +CREATE TABLE t3 (id int); +INSERT INTO t1 VALUES (1); +INSERT INTO t2 VALUES (2); +INSERT INTO t3 VALUES (3); +DROP TABLE t2; +)); + +$node->stop('immediate'); +$node->start; + +$result = $node->safe_psql("postgres", "SELECT id FROM t1"); +is($result, '1', 't1 survives crash'); + +($ret, $stdout, $stderr) = $node->psql("postgres", "SELECT * FROM t2"); +isnt($ret, 0, 't2 (dropped) is gone after crash'); + +$result = $node->safe_psql("postgres", "SELECT id FROM t3"); +is($result, '3', 't3 survives crash'); + +# ================================================================ +# Test 5: Crash after checkpoint with file operations +# ================================================================ + +$node->safe_psql( + "postgres", qq( +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t3; +CREATE TABLE checkpoint_test (id int); +INSERT INTO checkpoint_test VALUES (1); +CHECKPOINT; +INSERT INTO checkpoint_test VALUES (2); +)); + +$node->stop('immediate'); +$node->start; + +$result = + $node->safe_psql("postgres", "SELECT count(*) FROM checkpoint_test"); +is($result, '2', 'data after checkpoint survives crash'); + +# ================================================================ +# Test 6: Multiple crashes in sequence with file operations +# ================================================================ + +$node->safe_psql( + "postgres", qq( +DROP TABLE IF EXISTS checkpoint_test; +CREATE TABLE multi_crash (id int); +INSERT INTO multi_crash VALUES (1); +)); + +$node->stop('immediate'); +$node->start; + +$node->safe_psql( + "postgres", qq( +INSERT INTO multi_crash VALUES (2); +CREATE TABLE multi_crash_2 (id int); +INSERT INTO multi_crash_2 VALUES (10); +)); + +$node->stop('immediate'); +$node->start; + +$result = $node->safe_psql("postgres", "SELECT count(*) FROM multi_crash"); +is($result, '2', 'multi_crash table correct after double crash'); + +$result = $node->safe_psql("postgres", "SELECT id FROM multi_crash_2"); +is($result, '10', 'multi_crash_2 table correct after double crash'); + +# ================================================================ +# Test 7: CREATE TABLESPACE survives crash +# ================================================================ + +$node->safe_psql( + "postgres", qq( +DROP TABLE IF EXISTS multi_crash; +DROP TABLE IF EXISTS multi_crash_2; +SET allow_in_place_tablespaces = on; +CREATE TABLESPACE fileops_crash_ts LOCATION ''; +CREATE TABLE ts_crash_table (id int) TABLESPACE fileops_crash_ts; +INSERT INTO ts_crash_table VALUES (99); +)); + +$node->stop('immediate'); +$node->start; + +$result = $node->safe_psql("postgres", "SELECT id FROM ts_crash_table"); +is($result, '99', 'table in tablespace survives crash'); + +$result = $node->safe_psql("postgres", + "SELECT spcname FROM pg_tablespace WHERE spcname = 'fileops_crash_ts'"); +is($result, 'fileops_crash_ts', 'tablespace survives crash'); + +# ================================================================ +# Test 8: DROP TABLESPACE completes after crash +# ================================================================ + +$node->safe_psql( + "postgres", qq( +DROP TABLE ts_crash_table; +DROP TABLESPACE fileops_crash_ts; +)); + +$node->stop('immediate'); +$node->start; + +$result = $node->safe_psql("postgres", + "SELECT count(*) FROM pg_tablespace WHERE spcname = 'fileops_crash_ts'"); +is($result, '0', 'dropped tablespace is gone after crash'); + +# ================================================================ +# Test 9: CREATE DATABASE survives crash +# ================================================================ + +$node->safe_psql( + "postgres", qq( +CREATE DATABASE fileops_crash_db; +)); + +$node->stop('immediate'); +$node->start; + +$result = $node->safe_psql("postgres", + "SELECT datname FROM pg_database WHERE datname = 'fileops_crash_db'"); +is($result, 'fileops_crash_db', 'CREATE DATABASE survives crash'); + +$node->safe_psql("postgres", "DROP DATABASE fileops_crash_db"); + +# ================================================================ +# Test 10: Standby crash during FILEOPS replay +# ================================================================ + +# Set up primary + standby +my $primary = PostgreSQL::Test::Cluster->new('fileops_primary'); +$primary->init(allows_streaming => 1); +$primary->append_conf( + "postgresql.conf", qq( +autovacuum = off +)); +$primary->start; +$primary->backup('backup'); + +my $standby = PostgreSQL::Test::Cluster->new('fileops_standby'); +$standby->init_from_backup($primary, 'backup', has_streaming => 1); +$standby->start; + +# Create table on primary and wait for standby to catch up +$primary->safe_psql( + "postgres", qq( +CREATE TABLE standby_test (id int); +INSERT INTO standby_test VALUES (1); +)); + +$primary->wait_for_catchup($standby); + +# Verify on standby +$result = $standby->safe_psql("postgres", "SELECT id FROM standby_test"); +is($result, '1', 'CREATE TABLE replicated to standby'); + +# Crash the standby +$standby->stop('immediate'); +$standby->start; + +# Add more data on primary +$primary->safe_psql( + "postgres", qq( +INSERT INTO standby_test VALUES (2); +)); + +$primary->wait_for_catchup($standby); + +$result = + $standby->safe_psql("postgres", "SELECT count(*) FROM standby_test"); +is($result, '2', 'standby recovers and catches up after crash'); + +# Clean up primary/standby +$standby->stop; +$primary->stop; + +# Clean up original node +$node->stop; + +# ================================================================ +# Test 11/12: Crash right after an immediate op's WAL is flushed, before +# its transaction commits (FileOpsCreate, FileOpsChmod). +# +# An immediate op makes its filesystem change first and then flushes its +# WAL and UNDO records. The injection points pause the backend just after +# that flush, inside the still-open implicit transaction, and the postmaster +# is then hard-killed. Redo replays the change, and crash recovery applies +# the UNDO of the never-committed transaction, so the end state must be as +# if the transaction never ran: no orphaned file, no leftover mode change. +# ================================================================ + +if ($ENV{enable_injection_points} eq 'yes') +{ + my $inode = PostgreSQL::Test::Cluster->new('fileops_wal_ordering'); + $inode->init; + $inode->append_conf( + 'postgresql.conf', qq( +shared_preload_libraries = 'injection_points' +autovacuum = off +log_min_messages = debug2 +wal_level = replica +)); + $inode->start; + + if ( $inode->check_extension('injection_points') + && $inode->check_extension('test_fileops')) + { + $inode->safe_psql('postgres', 'CREATE EXTENSION injection_points'); + $inode->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + + my $idatadir = $inode->data_dir; + + # --- Test 11: FileOpsCreate ----------------------------------- + $inode->safe_psql('postgres', + q{SELECT injection_points_attach('fileops-create-after-wal-flush', 'wait')} + ); + + my $create_path = "$idatadir/wal_ordering_create.dat"; + my $bgpsql = $inode->background_psql('postgres'); + $bgpsql->query_until( + qr/starting_create/, + qq(\\echo starting_create +SELECT test_fileops_create('$create_path', 384); +\\q +)); + + $inode->wait_for_event('client backend', + 'fileops-create-after-wal-flush'); + + # The file exists and its WAL is durable; the transaction is open. + ok(-e $create_path, + 'Test 11: file exists at the injection point (WAL durable)'); + + # Hard-kill the postmaster from exactly this window. The wrapping + # implicit transaction is still in progress (blocked on the + # injection point), so it never committed. + $inode->stop('immediate'); + $inode->start; + + # Recovery redoes the physical CREATE from the durable WAL record + # unconditionally, then determines the transaction never committed + # and applies its UNDO record to reverse the create. Net effect: + # the file must NOT exist -- exactly as if the transaction had + # never run, with no orphaned state left behind by the crash. + ok( !-e $create_path, + 'Test 11: FileOpsCreate of a not-committed transaction is ' + . 'reversed after a crash'); + + # Server is healthy after the reconciliation. + my $result11 = $inode->safe_psql('postgres', 'SELECT 1'); + is($result11, '1', 'Test 11: server operational after recovery'); + + $bgpsql->{run}->finish; + + # --- Test 12: FileOpsChmod ------------------------------------- + $inode->safe_psql('postgres', + qq{SELECT test_fileops_create_tempfile('wal_ordering_chmod.dat')} + ); + my $chmod_path = "$idatadir/wal_ordering_chmod.dat"; + $inode->safe_psql('postgres', + qq{SELECT test_fileops_chmod('$chmod_path', 420)}); # 0644 + my $mode_before = $inode->safe_psql('postgres', + qq{SELECT test_fileops_get_mode('$chmod_path')}); + # Windows has no POSIX file modes: chmod only toggles read-only. + my $posix_modes = !$windows_os; + is($mode_before, '420', 'Test 12: baseline mode is 0644') + if $posix_modes; + + $inode->safe_psql('postgres', + q{SELECT injection_points_attach('fileops-chmod-after-wal-flush', 'wait')} + ); + + $bgpsql = $inode->background_psql('postgres'); + $bgpsql->query_until( + qr/starting_chmod/, + qq(\\echo starting_chmod +SELECT test_fileops_chmod('$chmod_path', 448); -- 0700 +\\q +)); + + $inode->wait_for_event('client backend', + 'fileops-chmod-after-wal-flush'); + + # chmod() has run and its WAL is durable; the transaction is open. + my $mode_at_injection = $inode->safe_psql('postgres', + qq{SELECT test_fileops_get_mode('$chmod_path')}); + is($mode_at_injection, '448', + 'Test 12: mode changed on disk at the injection point (WAL durable)' + ) if $posix_modes; + + # Hard-kill, same as Test 11. The chmod's wrapping implicit + # transaction never committed. + $inode->stop('immediate'); + $inode->start; + + # Recovery redoes the physical CHMOD unconditionally, then reverses + # it via UNDO because the transaction never committed. Net effect: + # mode must be back to the pre-transaction baseline (0644), not the + # attempted 0700, and not some undefined intermediate state. + my $mode_after = $inode->safe_psql('postgres', + qq{SELECT test_fileops_get_mode('$chmod_path')}); + is($mode_after, '420', + 'Test 12: FileOpsChmod of a not-committed transaction is ' + . 'reversed after a crash') + if $posix_modes; + + # Server is healthy after the reconciliation. + my $result12 = $inode->safe_psql('postgres', 'SELECT 1'); + is($result12, '1', 'Test 12: server operational after recovery'); + + $bgpsql->{run}->finish; + } + else + { + note + 'skipping Test 11/12 (injection_points or test_fileops extension not installed)'; + } + + $inode->stop; +} +else +{ + note 'skipping Test 11/12 (injection points not supported by this build)'; +} + +done_testing(); diff --git a/src/test/recovery/t/063_fileops_2pc.pl b/src/test/recovery/t/063_fileops_2pc.pl new file mode 100644 index 00000000000..b88c8dd5910 --- /dev/null +++ b/src/test/recovery/t/063_fileops_2pc.pl @@ -0,0 +1,149 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group +# +# Test two-phase commit (PREPARE TRANSACTION / COMMIT PREPARED / ROLLBACK +# PREPARED) for transactions that generated cluster-wide FILEOPS UNDO. +# +# FILEOPS uses the undo-in-common-WAL mechanism. At PREPARE the permanent +# UNDO chain-head LSN is durably saved in the 2PC state (xl_xact_prepare / +# gxact->undo_batch_lsn), and its WAL is retained while the xact stays +# prepared. ROLLBACK PREPARED (even from a different backend, post-crash) +# applies that chain, which reverses the filesystem change. COMMIT PREPARED +# keeps it. +# +# Vehicle: the test_fileops extension's SQL wrappers for FileOpsChmod, which +# are legal inside a transaction block that can then be PREPARE'd. +# +# NB: FILEOPS only writes UNDO WAL when XLogIsNeeded() (wal_level >= replica). +# At wal_level = minimal it relies on the backend-private pending-op list, +# which is discarded at PREPARE and cannot survive 2PC. This test therefore +# runs at wal_level = replica, which is what any FILEOPS + 2PC deployment +# requires. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Every test here is a chmod reversal; Windows has no POSIX file modes. +plan skip_all => 'POSIX permission semantics required' if $windows_os; + +my $node = PostgreSQL::Test::Cluster->new('fileops_2pc'); +$node->init; +$node->append_conf( + "postgresql.conf", qq( +autovacuum = off +max_prepared_transactions = 10 +wal_level = replica +)); +$node->start; + +if (!$node->check_extension('test_fileops')) +{ + plan skip_all => 'Extension test_fileops not installed'; +} +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $datadir = $node->data_dir; +my $file = "$datadir/twopc.dat"; + +# Poll until the file mode equals $want, or fail. +sub wait_for_mode +{ + my ($want, $desc) = @_; + my $ok = $node->poll_query_until('postgres', + qq{SELECT test_fileops_get_mode('$file') = $want}); + is($ok, '1', $desc); +} + +# 0644 = 420, 0700 = 448, 0600 = 384 (decimal) +$node->safe_psql('postgres', + qq{SELECT test_fileops_create_tempfile('twopc.dat')}); +$node->safe_psql('postgres', qq{SELECT test_fileops_chmod('$file', 420)}); +my $baseline = + $node->safe_psql('postgres', qq{SELECT test_fileops_get_mode('$file')}); +is($baseline, '420', 'baseline mode is 0644'); + +# ================================================================ +# Test 1: ROLLBACK PREPARED reverses the chmod +# ================================================================ +$node->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_chmod('$file', 448); -- 0700 +PREPARE TRANSACTION 'fo_rb'; +)); +my $m = + $node->safe_psql('postgres', qq{SELECT test_fileops_get_mode('$file')}); +is($m, '448', 'Test 1: chmod applied and still visible while prepared'); + +$node->safe_psql('postgres', q{ROLLBACK PREPARED 'fo_rb'}); +wait_for_mode(420, 'Test 1: ROLLBACK PREPARED reverses chmod (0700 -> 0644)'); + +# ================================================================ +# Test 2: COMMIT PREPARED keeps the chmod +# ================================================================ +$node->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_chmod('$file', 448); -- 0700 +PREPARE TRANSACTION 'fo_cp'; +)); +$node->safe_psql('postgres', q{COMMIT PREPARED 'fo_cp'}); +$m = $node->safe_psql('postgres', qq{SELECT test_fileops_get_mode('$file')}); +is($m, '448', 'Test 2: COMMIT PREPARED keeps chmod (0700)'); + +# Reset to baseline for the crash test. +$node->safe_psql('postgres', qq{SELECT test_fileops_chmod('$file', 420)}); + +# ================================================================ +# Test 3: WAL retention across checkpoints while prepared +# ================================================================ +$node->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_chmod('$file', 384); -- 0600 +PREPARE TRANSACTION 'fo_ckpt'; +)); +# Churn WAL and checkpoint; the prepared xact's UNDO batch must be retained. +$node->safe_psql('postgres', 'CHECKPOINT'); +for my $i (1 .. 20) +{ + $node->safe_psql('postgres', + "CREATE TABLE junk_$i(x int); " + . "INSERT INTO junk_$i SELECT g FROM generate_series(1,5000) g; " + . "DROP TABLE junk_$i;"); +} +$node->safe_psql('postgres', 'CHECKPOINT'); +$node->safe_psql('postgres', q{ROLLBACK PREPARED 'fo_ckpt'}); +wait_for_mode(420, + 'Test 3: ROLLBACK PREPARED still reverses after WAL churn + checkpoints'); + +# ================================================================ +# Test 4: crash recovery -- prepared UNDO xact survives, then reverts +# ================================================================ +$node->safe_psql('postgres', qq{SELECT test_fileops_chmod('$file', 420)}); +$node->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_chmod('$file', 448); -- 0700 +PREPARE TRANSACTION 'fo_crash'; +)); +$node->stop('immediate'); +$node->start; + +my $prep = $node->safe_psql('postgres', q{SELECT gid FROM pg_prepared_xacts}); +is($prep, 'fo_crash', 'Test 4: prepared xact survives crash recovery'); +$m = $node->safe_psql('postgres', qq{SELECT test_fileops_get_mode('$file')}); +is($m, '448', 'Test 4: chmod still applied (prepared, not yet rolled back)'); + +$node->safe_psql('postgres', q{ROLLBACK PREPARED 'fo_crash'}); +wait_for_mode(420, + 'Test 4: ROLLBACK PREPARED after crash+restart reverses chmod'); + +# Server healthy at the end. +is($node->safe_psql('postgres', 'SELECT 1'), + '1', 'server operational after all 2PC UNDO scenarios'); + +$node->stop; +done_testing(); diff --git a/src/test/recovery/t/064_fileops_standby_divergence.pl b/src/test/recovery/t/064_fileops_standby_divergence.pl new file mode 100644 index 00000000000..65a07fb873e --- /dev/null +++ b/src/test/recovery/t/064_fileops_standby_divergence.pl @@ -0,0 +1,124 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group +# +# Regression test: physical-standby reproduction of FILEOPS deferred +# DESTRUCTIVE operations (DELETE, RENAME, RMDIR). +# +# These three ops execute on the primary at commit inside +# FileOpsDoPendingOps(). Each is logged at commit-EXECUTION time with a +# real, idempotent redo handler, so a physical standby (which only replays +# WAL, never running FileOpsDoPendingOps) reproduces them and its filesystem +# converges with the primary's. +# +# This test sets up a streaming physical standby, performs a committed +# deferred DELETE, RENAME, and RMDIR on the primary, waits for catchup, and +# asserts the standby's filesystem MATCHES the primary's. It MUST FAIL on the +# pre-fix (no-op redo) code -- see the header note in fileops.c and the +# "prove non-vacuity" comment below. +# +# All FILEOPS paths are RELATIVE to the data directory, matching every core +# FILEOPS caller (dbcommands.c, tablespace.c): the backend's cwd is PGDATA, so +# a relative path resolves to the right file on both the primary and a standby +# whose absolute data_dir differs. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $primary = PostgreSQL::Test::Cluster->new('fileops_div_primary'); +$primary->init(allows_streaming => 1); +$primary->append_conf( + "postgresql.conf", qq( +autovacuum = off +wal_level = replica +)); +$primary->start; + +if (!$primary->check_extension('test_fileops')) +{ + plan skip_all => 'Extension test_fileops not installed'; +} +$primary->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $primary_dir = $primary->data_dir; + +# Fixture, all committed and WAL-logged so the standby has them before we +# start mutating. Use FileOps CREATE (WAL-logged, standby-reproduced) and +# FileOps MKDIR (WAL-logged) rather than raw open(), so the fixture itself +# replicates. Paths are relative to PGDATA. +$primary->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_create('div_delete_me.dat', 384); -- 0600 +SELECT test_fileops_create('div_rename_src.dat', 384); +SELECT test_fileops_mkdir('div_rmdir_me', 448); -- 0700 +COMMIT; +)); + +# Take the base backup AFTER the fixture exists so init_from_backup gives the +# standby the fixture files, then stream the mutations. +$primary->backup('backup'); +my $standby = PostgreSQL::Test::Cluster->new('fileops_div_standby'); +$standby->init_from_backup($primary, 'backup', has_streaming => 1); +$standby->start; + +my $standby_dir = $standby->data_dir; + +$primary->wait_for_catchup($standby); + +# Sanity: the fixture replicated to the standby. +ok(-e "$standby_dir/div_delete_me.dat", 'fixture: delete target on standby'); +ok(-e "$standby_dir/div_rename_src.dat", 'fixture: rename source on standby'); +ok(-d "$standby_dir/div_rmdir_me", 'fixture: rmdir target on standby'); + +# Perform the three deferred destructive ops in a committed transaction on the +# primary. Each goes through the FileOps deferred path (at_commit=true) and, +# with the fix, is WAL-logged at commit-execution. +$primary->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_delete('div_delete_me.dat'); +SELECT test_fileops_rename('div_rename_src.dat', 'div_rename_dst.dat'); +SELECT test_fileops_rmdir('div_rmdir_me'); +COMMIT; +)); + +# Confirm the primary applied them (the op ran in FileOpsDoPendingOps). +ok(!-e "$primary_dir/div_delete_me.dat", 'primary: delete applied'); +ok(!-e "$primary_dir/div_rename_src.dat", 'primary: rename source gone'); +ok(-e "$primary_dir/div_rename_dst.dat", 'primary: rename dest present'); +ok(!-d "$primary_dir/div_rmdir_me", 'primary: rmdir applied'); + +$primary->wait_for_catchup($standby); + +# The diff the reviewer asked for: the standby's filesystem must MATCH the +# primary's. Pre-fix (no-op redo) the standby still has the original file / +# dir and lacks the renamed file -- these four assertions all fail, which is +# the non-vacuity proof. Post-fix they pass. +ok(!-e "$standby_dir/div_delete_me.dat", + 'deferred DELETE reproduced on standby (file gone)'); +ok(!-e "$standby_dir/div_rename_src.dat", + 'deferred RENAME reproduced on standby (old name gone)'); +ok(-e "$standby_dir/div_rename_dst.dat", + 'deferred RENAME reproduced on standby (new name present)'); +ok(!-d "$standby_dir/div_rmdir_me", + 'deferred RMDIR reproduced on standby (dir gone)'); + +# Idempotent redo across a standby restart: replaying the same records again +# after a crash must not error or diverge. +$standby->stop('immediate'); +$standby->start; +$primary->safe_psql('postgres', 'SELECT txid_current()'); +$primary->wait_for_catchup($standby); +ok(!-e "$standby_dir/div_delete_me.dat", + 'standby end state stable after restart (delete)'); +ok(-e "$standby_dir/div_rename_dst.dat", + 'standby end state stable after restart (rename)'); +ok(!-d "$standby_dir/div_rmdir_me", + 'standby end state stable after restart (rmdir)'); + +$standby->stop; +$primary->stop; + +done_testing(); diff --git a/src/test/recovery/t/065_fileops_crash_undo.pl b/src/test/recovery/t/065_fileops_crash_undo.pl new file mode 100644 index 00000000000..f598c7e7418 --- /dev/null +++ b/src/test/recovery/t/065_fileops_crash_undo.pl @@ -0,0 +1,137 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group +# +# FILEOPS crash-recovery revert via the cluster-wide UNDO engine (common WAL). +# +# FILEOPS is a consumer of the cluster-wide UNDO engine: its structural +# operations (file existence, permissions, directory structure) are +# WAL-durable via RM_FILEOPS_ID forward records (fileops_redo), and each +# reversible op also writes an UNDO record into the common WAL +# (PrepareXactUndoData(UNDO_RMID_FILEOPS,...) / InsertXactUndoData) in the +# same critical section as its forward record. Filesystem structural +# operations are cluster-wide changes whose durability/revert belong in the +# common WAL. This test proves that model is crash-safe. +# +# A transaction produces FILEOPS create/chmod (immediate, WAL-flushed), then the +# server is crashed (immediate stop) BEFORE the transaction commits. On +# recovery: +# - fileops_redo() REDOES the forward create/chmod unconditionally, and +# - PerformUndoRecovery() (end of redo) applies the never-committed +# transaction's FILEOPS UNDO chain via the RM_FILEOPS_ID rm_undo +# callbacks (fileops_undo.c), reversing every op: +# - the created file is unlinked (CREATE revert), and +# - the chmod'd file's mode is restored (CHMOD revert, before-state in WAL). +# +# FILEOPS only writes WAL at wal_level >= replica (XLogIsNeeded()), so the test +# runs at wal_level = replica, matching any FILEOPS deployment. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('fileops_crash'); +$node->init; +$node->append_conf( + 'postgresql.conf', qq( +autovacuum = off +wal_level = replica +)); +$node->start; + +if (!$node->check_extension('test_fileops')) +{ + plan skip_all => 'Extension test_fileops not installed'; +} +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $datadir = $node->data_dir; +my $create_path = "$datadir/fileops_crash_create.dat"; +my $chmod_path = "$datadir/fileops_crash_chmod.dat"; + +# --------------------------------------------------------------------------- +# Set up a pre-existing file to chmod (this part is committed). 0644 = 420. +# --------------------------------------------------------------------------- +$node->safe_psql('postgres', + q{SELECT test_fileops_create_tempfile('fileops_crash_chmod.dat')}); +$node->safe_psql('postgres', + qq{SELECT test_fileops_chmod('$chmod_path', 420)}); +SKIP: +{ + skip 'POSIX permission semantics required', 1 if $windows_os; + is( $node->safe_psql( + 'postgres', qq{SELECT test_fileops_get_mode('$chmod_path')}), + '420', + 'baseline mode of pre-existing file is 0644'); +} + +# --------------------------------------------------------------------------- +# Open a transaction that produces FILEOPS UNDO (create a new file and +# chmod the existing one) and DO NOT commit it. The FILEOPS forward operations +# execute and are WAL-flushed immediately (the file exists on disk, the mode is +# changed), but the wrapping transaction stays in progress. +# --------------------------------------------------------------------------- +my $bg = $node->background_psql('postgres'); +$bg->query_until( + qr/fileops_ready/, qq( +BEGIN; +SELECT test_fileops_create('$create_path', 384); -- 0600, new file +SELECT test_fileops_chmod('$chmod_path', 448); -- 0700, existing file +\\echo fileops_ready +)); + +# The forward ops are visible on disk while the xact is in progress. +ok(-e $create_path, + 'created file exists on disk before crash (in-progress xact)'); +SKIP: +{ + skip 'POSIX permission semantics required', 1 if $windows_os; + is( $node->safe_psql( + 'postgres', qq{SELECT test_fileops_get_mode('$chmod_path')}), + '448', + 'chmod applied on disk before crash (in-progress xact)'); +} + +# --------------------------------------------------------------------------- +# Crash immediately: the transaction never commits. Stop the server before +# closing the client: closing it first would let the backend roll the +# transaction back online, which is a different case. The background psql +# connection dies with the server; abandon it (do not try to drain/finish it, +# which would hang waiting on the dead connection). +# --------------------------------------------------------------------------- +$node->stop('immediate'); +$bg->{run}->kill_kill; +$node->start; + +# --------------------------------------------------------------------------- +# Recovery redoes the forward ops from WAL, then PerformUndoRecovery() reverses +# the never-committed transaction's ops from its common-WAL UNDO chain: +# - the created file is unlinked, and +# - the chmod is restored to the original 0644. +# --------------------------------------------------------------------------- +ok( !-e $create_path, + 'FILEOPS COMMON-WAL CRASH RECOVERY: created file was REVERTED (unlinked) after crash' +); +SKIP: +{ + skip 'POSIX permission semantics required', 1 if $windows_os; + is( $node->safe_psql( + 'postgres', qq{SELECT test_fileops_get_mode('$chmod_path')}), + '420', + 'FILEOPS COMMON-WAL CRASH RECOVERY: chmod was REVERTED (0700 -> 0644) after crash' + ); +} + +# Recovery must be clean. +my $rlog = slurp_file($node->logfile); +unlike( + $rlog, + qr/PANIC|inconsistent|corrupt/i, + 'clean crash recovery, no PANIC/corruption'); + +# Server healthy afterwards. +is($node->safe_psql('postgres', 'SELECT 1'), + '1', 'server operational after crash-recovery undo'); + +$node->stop; +done_testing(); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 7e745e36b22..818b6d3eddd 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -836,6 +836,17 @@ ExtensionVersionInfo FDWCollateState FD_SET FILE +FileOpsTwoPhaseRecord +FileopsUndoChmod +FileopsUndoChown +FileopsUndoCreate +FileopsUndoLink +FileopsUndoMkdir +FileopsUndoRemovexattr +FileopsUndoRename +FileopsUndoSetxattr +FileopsUndoSymlink +FileopsUndoTruncate FILETIME FPI FSMAddress @@ -1912,6 +1923,8 @@ PACE_HEADER PACL PATH PCtxtHandle +PendingFileOp +PendingFileOpType PERL_CONTEXT PERL_SI PFN @@ -3515,6 +3528,19 @@ X86FeatureId XactUndoContext XactUndoData XactUndoSubTransactionState +xl_fileops_chmod +xl_fileops_chown +xl_fileops_create +xl_fileops_delete +xl_fileops_link +xl_fileops_mkdir +xl_fileops_removexattr +xl_fileops_rename +xl_fileops_rmdir +xl_fileops_setxattr +xl_fileops_symlink +xl_fileops_truncate +xl_fileops_write xl_undo_allocate xl_undo_apply xl_undo_batch -- 2.54.0