From d62255e8ba9306d6660e09cb38cc09de68309312 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 14 Sep 2026 13:03:40 +0000 Subject: [PATCH v9 2/3] Add transactional filesystem operations (FILEOPS) backed by UNDO Make the structural filesystem mutations the server performs outside the buffer manager crash-safe and rollback-safe via a WAL resource manager (RM_FILEOPS_ID). Forward operations are WAL-logged and, inside a transaction, register a pending-op / UNDO record so an abort or crash recovery recursively reverts them. FILEOPS records its UNDO in the common WAL stream, so crash recovery reverts a loser transaction's structural changes through the cluster-wide UNDO engine's recovery pass; the AtPrepare path is handled so a prepared transaction's pending file operations survive 2PC. This commit adds the FILEOPS access method and its UNDO plumbing but does not yet route any in-tree DDL/storage caller through it (that is the next commit): the FileOps* API, fileops_redo, the fileops UNDO resource manager (FileopsUndoRmgrInit, registered via undormgrlist.h), the pg_xattr portability wrapper, rmgrdesc + pg_waldump support, the test_fileops module, SQL regression coverage, and recovery TAP tests for crash and 2PC. Author: Greg Burd Changes since v2: - Report FILEOPS UNDO apply failures as failures [f3bd4a3cc7] Tests: - Do not set logical_revert_naptime in 059 and 061 [3072609035] - Show that a crash right after commit loses deferred ops [b34b2463bb] Changes since v7: - recovery/060_fileops_standby_divergence is now 064: upstream added t/060_multixact_crashed_updater.pl - test_fileops gains 023 and 024, the tests for the two UNDO engine fixes above (in the previous commit) - recovery/072_perbackend_crash_undo is now 065_fileops_crash_undo; it never involved per-backend UNDO - Comments and tests no longer describe ROLLBACK PREPARED as handing the chain to the ATM and an asynchronous revert worker; in this series FinishPreparedTransaction() applies it synchronously - recovery/062 (Test 12) and 065 skip their file-mode checks on Windows, which has no POSIX modes; v8 carried these guards in 0005 - recovery/065 crashes the server before it closes the client. It closed the client first, so the backend could roll the transaction back online, and when that abort reached disk before the crash, recovery redid the changes (see the known gap in the cover letter) --- meson.build | 11 + src/Makefile.global.in | 5 + src/backend/access/rmgrdesc/Makefile | 1 + src/backend/access/rmgrdesc/fileopsdesc.c | 219 ++ src/backend/access/rmgrdesc/meson.build | 6 + src/backend/access/transam/rmgr.c | 1 + src/backend/access/transam/twophase.c | 27 +- src/backend/access/transam/twophase_rmgr.c | 20 +- src/backend/storage/file/Makefile | 2 + src/backend/storage/file/README | 201 ++ src/backend/storage/file/fileops.c | 2561 +++++++++++++++++ src/backend/storage/file/fileops_undo.c | 431 +++ src/backend/storage/file/meson.build | 2 + src/bin/pg_waldump/rmgrdesc.c | 1 + src/bin/pg_waldump/t/001_basic.pl | 3 +- src/include/access/fileops_xlog.h | 40 + src/include/access/rmgrlist.h | 3 + src/include/access/twophase_rmgr.h | 9 +- src/include/access/undormgrlist.h | 3 + src/include/port/pg_xattr.h | 40 + src/include/storage/fileops.h | 463 +++ src/port/Makefile | 1 + src/port/meson.build | 1 + src/port/pg_xattr.c | 217 ++ src/test/modules/Makefile | 1 + src/test/modules/meson.build | 1 + src/test/modules/test_fileops/Makefile | 19 + .../test_fileops/expected/test_fileops.out | 413 +++ .../test_fileops/expected/test_fileops_1.out | 332 +++ .../test_fileops/expected/test_fileops_2.out | 18 + src/test/modules/test_fileops/meson.build | 44 + .../modules/test_fileops/sql/test_fileops.sql | 229 ++ .../t/001_fileops_apply_failure.pl | 42 + .../t/007_fileops_commit_crash.pl | 77 + .../t/023_fileops_inline_abort_ctx.pl | 60 + .../t/024_fileops_recovery_apply_failure.pl | 59 + .../test_fileops/test_fileops--1.0.sql | 99 + src/test/modules/test_fileops/test_fileops.c | 471 +++ .../modules/test_fileops/test_fileops.control | 4 + src/test/recovery/Makefile | 1 + src/test/recovery/meson.build | 5 + .../recovery/t/061_fileops_2pc_deferred.pl | 167 ++ src/test/recovery/t/062_fileops_recovery.pl | 446 +++ src/test/recovery/t/063_fileops_2pc.pl | 149 + .../t/064_fileops_standby_divergence.pl | 124 + src/test/recovery/t/065_fileops_crash_undo.pl | 137 + src/tools/pgindent/typedefs.list | 26 + 47 files changed, 7184 insertions(+), 8 deletions(-) create mode 100644 src/backend/access/rmgrdesc/fileopsdesc.c create mode 100644 src/backend/storage/file/README create mode 100644 src/backend/storage/file/fileops.c create mode 100644 src/backend/storage/file/fileops_undo.c create mode 100644 src/include/access/fileops_xlog.h create mode 100644 src/include/port/pg_xattr.h create mode 100644 src/include/storage/fileops.h create mode 100644 src/port/pg_xattr.c create mode 100644 src/test/modules/test_fileops/Makefile create mode 100644 src/test/modules/test_fileops/expected/test_fileops.out create mode 100644 src/test/modules/test_fileops/expected/test_fileops_1.out create mode 100644 src/test/modules/test_fileops/expected/test_fileops_2.out create mode 100644 src/test/modules/test_fileops/meson.build create mode 100644 src/test/modules/test_fileops/sql/test_fileops.sql create mode 100644 src/test/modules/test_fileops/t/001_fileops_apply_failure.pl create mode 100644 src/test/modules/test_fileops/t/007_fileops_commit_crash.pl create mode 100644 src/test/modules/test_fileops/t/023_fileops_inline_abort_ctx.pl create mode 100644 src/test/modules/test_fileops/t/024_fileops_recovery_apply_failure.pl create mode 100644 src/test/modules/test_fileops/test_fileops--1.0.sql create mode 100644 src/test/modules/test_fileops/test_fileops.c create mode 100644 src/test/modules/test_fileops/test_fileops.control create mode 100644 src/test/recovery/t/061_fileops_2pc_deferred.pl create mode 100644 src/test/recovery/t/062_fileops_recovery.pl create mode 100644 src/test/recovery/t/063_fileops_2pc.pl create mode 100644 src/test/recovery/t/064_fileops_standby_divergence.pl create mode 100644 src/test/recovery/t/065_fileops_crash_undo.pl diff --git a/meson.build b/meson.build index 3238c849e1b..935de4e5fc3 100644 --- a/meson.build +++ b/meson.build @@ -1586,6 +1586,17 @@ endif +############################################################### +# Library: FILEOPS +############################################################### + +# FILEOPS transactional filesystem operations (WAL rmgr RM_FILEOPS_ID) is +# always built. Mirrors USE_FILEOPS in src/Makefile.global.in: without it the +# rmgrlist.h entry is compiled out and every RM_FILEOPS_ID reference fails to +# build. +cdata.set('USE_FILEOPS', 1) + + ############################################################### # Library: selinux ############################################################### diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 38914773b83..b20d0065ce9 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -256,6 +256,11 @@ PG_SYSROOT = @PG_SYSROOT@ override CPPFLAGS += $(ICU_CFLAGS) $(LIBNUMA_CFLAGS) $(LIBURING_CFLAGS) +# FILEOPS transactional filesystem operations (WAL rmgr RM_FILEOPS_ID) is +# always built. +USE_FILEOPS = 1 +override CPPFLAGS += -DUSE_FILEOPS + ifdef PGXS override CPPFLAGS := -I$(includedir_server) -I$(includedir_internal) $(CPPFLAGS) else # not PGXS diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile index bf6709e738d..5b0d8922ea2 100644 --- a/src/backend/access/rmgrdesc/Makefile +++ b/src/backend/access/rmgrdesc/Makefile @@ -13,6 +13,7 @@ OBJS = \ clogdesc.o \ committsdesc.o \ dbasedesc.o \ + fileopsdesc.o \ genericdesc.o \ gindesc.o \ gistdesc.o \ diff --git a/src/backend/access/rmgrdesc/fileopsdesc.c b/src/backend/access/rmgrdesc/fileopsdesc.c new file mode 100644 index 00000000000..d13891729ee --- /dev/null +++ b/src/backend/access/rmgrdesc/fileopsdesc.c @@ -0,0 +1,219 @@ +/*------------------------------------------------------------------------- + * + * fileopsdesc.c + * rmgr descriptor routines for storage/file/fileops.c + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/rmgrdesc/fileopsdesc.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "storage/fileops.h" + +void +fileops_desc(StringInfo buf, XLogReaderState *record) +{ + char *data = XLogRecGetData(record); + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + + switch (info) + { + case XLOG_FILEOPS_CREATE: + { + xl_fileops_create *xlrec = (xl_fileops_create *) data; + const char *path = data + SizeOfFileOpsCreate; + + appendStringInfo(buf, "create \"%s\" flags 0x%x mode 0%o", + path, xlrec->flags, xlrec->mode); + } + break; + + case XLOG_FILEOPS_WRITE: + { + xl_fileops_write *xlrec = (xl_fileops_write *) data; + const char *path = data + SizeOfFileOpsWrite; + + appendStringInfo(buf, "write \"%s\" offset %lld len %u", + path, (long long) xlrec->offset, xlrec->len); + } + break; + + case XLOG_FILEOPS_RENAME: + { + xl_fileops_rename *xlrec = (xl_fileops_rename *) data; + const char *oldpath = data + SizeOfFileOpsRename; + const char *newpath = oldpath + xlrec->oldpath_len; + + appendStringInfo(buf, "rename \"%s\" to \"%s\"", + oldpath, newpath); + } + break; + + case XLOG_FILEOPS_DELETE: + { + xl_fileops_delete *xlrec = (xl_fileops_delete *) data; + const char *path = data + SizeOfFileOpsDelete; + + appendStringInfo(buf, "delete \"%s\" at_%s", + path, + xlrec->at_commit ? "commit" : "abort"); + } + break; + + case XLOG_FILEOPS_SYMLINK: + { + xl_fileops_symlink *xlrec = (xl_fileops_symlink *) data; + const char *target = data + SizeOfFileOpsSymlink; + const char *linkpath = target + xlrec->target_len; + + appendStringInfo(buf, "symlink \"%s\" -> \"%s\"", + linkpath, target); + } + break; + + case XLOG_FILEOPS_LINK: + { + xl_fileops_link *xlrec = (xl_fileops_link *) data; + const char *oldpath = data + SizeOfFileOpsLink; + const char *newpath = oldpath + xlrec->oldpath_len; + + appendStringInfo(buf, "link \"%s\" -> \"%s\"", + newpath, oldpath); + } + break; + + case XLOG_FILEOPS_MKDIR: + { + xl_fileops_mkdir *xlrec = (xl_fileops_mkdir *) data; + const char *path = data + SizeOfFileOpsMkdir; + + appendStringInfo(buf, "mkdir \"%s\" mode 0%o", + path, (unsigned int) xlrec->mode); + } + break; + + case XLOG_FILEOPS_RMDIR: + { + xl_fileops_rmdir *xlrec = (xl_fileops_rmdir *) data; + const char *path = data + SizeOfFileOpsRmdir; + + appendStringInfo(buf, "rmdir \"%s\" at_%s", + path, + xlrec->at_commit ? "commit" : "abort"); + } + break; + + case XLOG_FILEOPS_CHMOD: + { + xl_fileops_chmod *xlrec = (xl_fileops_chmod *) data; + const char *path = data + SizeOfFileOpsChmod; + + appendStringInfo(buf, "chmod \"%s\" mode 0%o", + path, (unsigned int) xlrec->mode); + } + break; + + case XLOG_FILEOPS_CHOWN: + { + xl_fileops_chown *xlrec = (xl_fileops_chown *) data; + const char *path = data + SizeOfFileOpsChown; + + appendStringInfo(buf, "chown \"%s\" uid %d gid %d", + path, (int) xlrec->uid, (int) xlrec->gid); + } + break; + + case XLOG_FILEOPS_TRUNCATE: + { + xl_fileops_truncate *xlrec = (xl_fileops_truncate *) data; + const char *path = data + SizeOfFileOpsTruncate; + + appendStringInfo(buf, "truncate \"%s\" to %lld bytes", + path, (long long) xlrec->length); + } + break; + + case XLOG_FILEOPS_SETXATTR: + { + xl_fileops_setxattr *xlrec = (xl_fileops_setxattr *) data; + const char *path = data + SizeOfFileOpsSetxattr; + const char *name = path + xlrec->path_len; + + appendStringInfo(buf, "setxattr \"%s\" name \"%s\" len %u", + path, name, xlrec->value_len); + } + break; + + case XLOG_FILEOPS_REMOVEXATTR: + { + xl_fileops_removexattr *xlrec = + (xl_fileops_removexattr *) data; + const char *path = data + SizeOfFileOpsRemovexattr; + const char *name = path + xlrec->path_len; + + appendStringInfo(buf, "removexattr \"%s\" name \"%s\"", + path, name); + } + break; + + default: + appendStringInfo(buf, "unknown fileops op code %u", info); + break; + } +} + +const char * +fileops_identify(uint8 info) +{ + const char *id = NULL; + + switch (info & ~XLR_INFO_MASK) + { + case XLOG_FILEOPS_CREATE: + id = "CREATE"; + break; + case XLOG_FILEOPS_DELETE: + id = "DELETE"; + break; + case XLOG_FILEOPS_RENAME: + id = "RENAME"; + break; + case XLOG_FILEOPS_WRITE: + id = "WRITE"; + break; + case XLOG_FILEOPS_TRUNCATE: + id = "TRUNCATE"; + break; + case XLOG_FILEOPS_CHMOD: + id = "CHMOD"; + break; + case XLOG_FILEOPS_CHOWN: + id = "CHOWN"; + break; + case XLOG_FILEOPS_MKDIR: + id = "MKDIR"; + break; + case XLOG_FILEOPS_RMDIR: + id = "RMDIR"; + break; + case XLOG_FILEOPS_SYMLINK: + id = "SYMLINK"; + break; + case XLOG_FILEOPS_LINK: + id = "LINK"; + break; + case XLOG_FILEOPS_SETXATTR: + id = "SETXATTR"; + break; + case XLOG_FILEOPS_REMOVEXATTR: + id = "REMOVEXATTR"; + break; + } + + return id; +} diff --git a/src/backend/access/rmgrdesc/meson.build b/src/backend/access/rmgrdesc/meson.build index d0dc4cb229a..c8af1ce82dd 100644 --- a/src/backend/access/rmgrdesc/meson.build +++ b/src/backend/access/rmgrdesc/meson.build @@ -27,4 +27,10 @@ rmgr_desc_sources = files( 'xlogdesc.c', ) +# FILEOPS is always built (USE_FILEOPS); rmgrlist.h registers +# fileops_desc/fileops_identify, so fileopsdesc.c must always be compiled or +# pg_waldump fails to link. The autoconf build picks this up via its +# *desc*.c wildcard; meson needs it listed explicitly. +rmgr_desc_sources += files('fileopsdesc.c') + backend_sources += rmgr_desc_sources diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c index 130eb06bee3..7441cf2a716 100644 --- a/src/backend/access/transam/rmgr.c +++ b/src/backend/access/transam/rmgr.c @@ -6,6 +6,7 @@ * src/backend/access/transam/rmgr.c */ #include "postgres.h" +#include "storage/fileops.h" #include "access/rmgr.h" #include "access/xlog_internal.h" diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 9463c065e1f..55c553339d6 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -100,6 +100,7 @@ #include "replication/origin.h" #include "replication/syncrep.h" #include "storage/fd.h" +#include "storage/fileops.h" #include "storage/ipc.h" #include "storage/md.h" #include "storage/predicate.h" @@ -150,6 +151,15 @@ int max_prepared_xacts = 0; * twophase.h */ +/* + * FILEOPS 2PC prepare hook (fileops.c). Declared locally by prototype to + * avoid pulling storage/fileops.h into this file. Serializes this + * transaction's pending file operations into the 2PC state file so + * COMMIT/ROLLBACK PREPARED -- possibly from another backend after a crash -- + * can perform the deferred commit-time ops or the abort-time filesystem undo. + */ +extern void FileOpsAtPrepare(void); + typedef struct GlobalTransactionData { GlobalTransaction next; /* list link for free list */ @@ -996,11 +1006,17 @@ TwoPhaseFilePath(char *path, FullTransactionId fxid) * * TWOPHASE_MAGIC must be bumped whenever xl_xact_prepare changes layout. * The struct gained last_batch_lsn[NUndoPersistenceLevels] (24 bytes) for - * UNDO chain tracking across 2PC boundaries, requiring this bump from + * UNDO chain tracking across 2PC boundaries, requiring a bump from * 0x57F94534 to 0x57F94535 to prevent old servers from silently misreading * the variable-length arrays that follow the fixed header at the wrong offsets. + * + * Bumped again to 0x57F94536: FILEOPS now registers TWOPHASE_RM_FILEOPS_ID + * records so deferred commit-time file operations (RENAME/DELETE/RMDIR/RMTREE) + * and abort-time filesystem undo survive PREPARE. This adds a new 2PC record + * type (not a header field), but a bump still keeps a mixed-version cluster + * from replaying a state file whose record stream it does not understand. */ -#define TWOPHASE_MAGIC 0x57F94535 /* format identifier */ +#define TWOPHASE_MAGIC 0x57F94536 /* format identifier */ typedef xl_xact_prepare TwoPhaseFileHeader; @@ -1179,6 +1195,13 @@ EndPrepare(GlobalTransaction gxact) StateFileChunk *record; bool replorigin; + /* + * FILEOPS: persist this transaction's pending file operations into the + * 2PC state file, so COMMIT PREPARED can run them after a crash. Must + * run before the END sentinel while the record chain is still open. A no-op if this transaction registered no file operations. + */ + FileOpsAtPrepare(); + /* Add the end sentinel to the list of 2PC records */ RegisterTwoPhaseRecord(TWOPHASE_RM_END_ID, 0, NULL, 0); diff --git a/src/backend/access/transam/twophase_rmgr.c b/src/backend/access/transam/twophase_rmgr.c index fae254c6e23..687e77f7e67 100644 --- a/src/backend/access/transam/twophase_rmgr.c +++ b/src/backend/access/transam/twophase_rmgr.c @@ -19,6 +19,14 @@ #include "pgstat.h" #include "storage/lock.h" #include "storage/predicate.h" +#include "storage/fileops.h" + +/* + * FILEOPS 2PC callbacks (fileops.c). Declared locally by prototype to avoid + * pulling storage/fileops.h in here; the signature is the generic + * TwoPhaseCallback. Postcommit performs the deferred commit-time file ops; + * postabort performs the abort-time filesystem undo. + */ const TwoPhaseCallback twophase_recover_callbacks[TWOPHASE_RM_MAX_ID + 1] = @@ -27,7 +35,8 @@ const TwoPhaseCallback twophase_recover_callbacks[TWOPHASE_RM_MAX_ID + 1] = lock_twophase_recover, /* Lock */ NULL, /* pgstat */ multixact_twophase_recover, /* MultiXact */ - predicatelock_twophase_recover /* PredicateLock */ + predicatelock_twophase_recover, /* PredicateLock */ + NULL /* FILEOPS (post-commit/abort only) */ }; const TwoPhaseCallback twophase_postcommit_callbacks[TWOPHASE_RM_MAX_ID + 1] = @@ -36,7 +45,8 @@ const TwoPhaseCallback twophase_postcommit_callbacks[TWOPHASE_RM_MAX_ID + 1] = lock_twophase_postcommit, /* Lock */ pgstat_twophase_postcommit, /* pgstat */ multixact_twophase_postcommit, /* MultiXact */ - NULL /* PredicateLock */ + NULL, /* PredicateLock */ + fileops_twophase_postcommit /* FILEOPS */ }; const TwoPhaseCallback twophase_postabort_callbacks[TWOPHASE_RM_MAX_ID + 1] = @@ -45,7 +55,8 @@ const TwoPhaseCallback twophase_postabort_callbacks[TWOPHASE_RM_MAX_ID + 1] = lock_twophase_postabort, /* Lock */ pgstat_twophase_postabort, /* pgstat */ multixact_twophase_postabort, /* MultiXact */ - NULL /* PredicateLock */ + NULL, /* PredicateLock */ + fileops_twophase_postabort /* FILEOPS */ }; const TwoPhaseCallback twophase_standby_recover_callbacks[TWOPHASE_RM_MAX_ID + 1] = @@ -54,5 +65,6 @@ const TwoPhaseCallback twophase_standby_recover_callbacks[TWOPHASE_RM_MAX_ID + 1 lock_twophase_standby_recover, /* Lock */ NULL, /* pgstat */ NULL, /* MultiXact */ - NULL /* PredicateLock */ + NULL, /* PredicateLock */ + NULL /* FILEOPS */ }; diff --git a/src/backend/storage/file/Makefile b/src/backend/storage/file/Makefile index 660ac51807e..8bcfafec7cf 100644 --- a/src/backend/storage/file/Makefile +++ b/src/backend/storage/file/Makefile @@ -16,6 +16,8 @@ OBJS = \ buffile.o \ copydir.o \ fd.o \ + fileops.o \ + fileops_undo.o \ fileset.o \ reinit.o \ sharedfileset.o diff --git a/src/backend/storage/file/README b/src/backend/storage/file/README new file mode 100644 index 00000000000..233db9bbbf3 --- /dev/null +++ b/src/backend/storage/file/README @@ -0,0 +1,201 @@ +src/backend/storage/file/README + +Transactional File Operations (FILEOPS) +======================================= + +This directory's fileops.c implements FILEOPS: a set of filesystem +operations that participate in PostgreSQL transactions. An ordinary +unlink() or rename() issued in the middle of a transaction is not covered by +WAL and is not reversed if the transaction aborts or the server crashes. +FILEOPS closes that gap for the filesystem mutations the server performs +outside the buffer manager. + +FILEOPS is NOT used for relation files. Creating and dropping tables, +indexes, TRUNCATE, and CREATE/DROP DATABASE|TABLESPACE at the catalog/smgr +level go through smgr and catalog/storage.c, which have zero FileOps calls. +The only in-core callers of the FileOps* API are: + + - dbcommands.c (CREATE/ALTER/DROP DATABASE: directory + PG_VERSION, + ALTER DATABASE ... SET TABLESPACE) + - tablespace.c (CREATE/DROP TABLESPACE: the per-tablespace directory + and symlink; the LOCATION paths are absolute by design) + - copydir.c (the destination tree of a directory copy) + + +Two op classes: constructive (immediate) vs destructive (deferred) +------------------------------------------------------------------ + +CONSTRUCTIVE operations run IMMEDIATELY when the API is called, so the file +or directory can be used within the same transaction, and each registers an +abort-time undo action that reverses it if the transaction rolls back: + + FileOpsCreate -- create() now; UNDO/pending-delete on abort + FileOpsMkdir -- mkdir() now; UNDO rmdir on abort + FileOpsChmod -- chmod() now; UNDO restore old mode on abort + FileOpsChown -- chown() now; UNDO restore old uid/gid on abort + FileOpsTruncate -- ftruncate() now; UNDO restore old length on abort + FileOpsSymlink -- symlink() now; UNDO unlink on abort + FileOpsLink -- link() now; UNDO unlink on abort + FileOpsWrite -- pwrite() now (content, WAL-logged; no UNDO) + FileOpsSetXattr / FileOpsRemoveXattr -- now; UNDO restore old value on abort + +DESTRUCTIVE operations are DEFERRED: the API only records a PendingFileOp, +and the filesystem is touched at transaction end: + + FileOpsRename -- deferred, always at_commit + FileOpsDelete -- deferred; at_commit chooses commit vs abort + FileOpsRmdir / FileOpsRmdirRecursive -- deferred; at_commit + FileOpsRmtree -- deferred; walked at execute time (copydir abort cleanup) + +FileOpsDoPendingOps(isCommit) runs the pending list at transaction end: the +at_commit=true ops fire on commit, the at_commit=false ops fire on abort. + + +When FILEOPS is durable: only at wal_level >= replica +----------------------------------------------------- + +FILEOPS writes WAL and UNDO records ONLY when XLogIsNeeded() is true, i.e. +wal_level >= replica. At wal_level = minimal it emits no WAL and no UNDO: + + - There is NO crash safety at minimal. The immediate constructive ops + mutate the filesystem with no WAL record to redo, and the only reversal + for a crash-mid-transaction is the in-memory pendingFileOps list, which + does not survive a crash. + + - There is NO 2PC safety at minimal. With nothing serialized, a PREPARE'd + transaction's file operations cannot be reproduced after a crash. + +Every FILEOPS + crash-recovery + 2PC guarantee below assumes wal_level >= +replica, which is what any deployment relying on those guarantees must set. + + +Revert and recovery model (NO fileops_cleanup) +---------------------------------------------- + +There is NO fileops_cleanup() function. The RM_FILEOPS_ID WAL resource +manager registers only rm_redo/rm_desc/rm_identify; its rm_startup and +rm_cleanup slots are NULL (see access/rmgrlist.h). Reversal rides two +mechanisms, both on the COMMON WAL stream: + + 1. Live-transaction abort (in-memory). ROLLBACK runs + FileOpsDoPendingOps(false), executing each constructive op's registered + abort-time undo (delete the created file, restore the old + mode/length/xattr, rmdir the created directory, ...). No WAL needed; + the in-process list has the state. + + 2. Crash recovery (common-WAL UNDO records). Each constructive op + also prepares an UNDO record (PrepareXactUndoData(UNDO_RMID_FILEOPS,...)) + and inserts it (InsertXactUndoData) in the same critical section as its + forward WAL record. On crash, a loser transaction's UNDO chain is + applied by the cluster-wide UNDO engine's recovery pass + (PerformUndoRecovery, at end of redo); the apply side is + fileops_undo_apply() in fileops_undo.c, registered through the UNDO rmgr + registry (FileopsUndoRmgrInit), NOT through a WAL rm_cleanup callback. + +Deferred DESTRUCTIVE ops emit their forward WAL record at commit-EXECUTION +time, inside FileOpsDoPendingOps() when the op actually happens on the +primary, so a physical standby reproduces it by redo. Their redo handlers +are idempotent (delete of an already-absent file, rename whose source is +gone, rmdir of an already-absent directory all count as success). Only the +commit path is logged; an at_commit=false delete/rmdir run during abort emits +no WAL, because a standby must never apply an aborted transaction's effect. + +Two deliberate exceptions: + + RMTREE (FileOpsRmtree) writes no WAL and has no redo. Its only core caller + (copydir.c) registers it at_commit=false, so it fires only on ABORT to + clean up a partially-copied tree; aborts are never replayed as committed, + so it cannot diverge a standby. + + SYMLINK redo is an intentional no-op. FileOpsSymlink is used only for + tablespace symlinks; standby/crash-recovery replay of those is owned by + RM_TBLSPC, which applies the recovery-time layout remap. + + +Two-phase commit +---------------- + +PREPARE TRANSACTION serializes the transaction's pending FileOps into the +2PC state file so they survive to COMMIT/ROLLBACK PREPARED, including from a +different backend after a crash between PREPARE and the final decision: + + - FileOpsAtPrepare(), called from EndPrepare() between StartPrepare() and + the END sentinel, writes one TWOPHASE_RM_FILEOPS_ID record per pending + at_commit=true op into the durable 2PC state. + + - The FILEOPS 2PC post-commit callback performs those deferred commit-time + ops (rename/delete/rmdir/rmtree), reusing the same per-op executor as + FileOpsDoPendingOps(). COMMIT PREPARED thus applies them; a crash before + it re-drives them from the durable 2PC state. + + - The abort-time ops (at_commit=false, the immediate constructive ops' undo) + are NOT serialized here. They are reversed on ROLLBACK PREPARED by the + constructive op's cluster-wide UNDO record, which + FinishPreparedTransaction() applies synchronously. Duplicating them in + the 2PC record stream would revert them twice. The FILEOPS post-abort + callback is therefore effectively a no-op. + + - PostPrepare_FileOps() then discards this backend's in-memory list; the + durable 2PC state (for commit-time ops) and the UNDO chain (for abort-time + reversal), not this backend, drive the eventual work. + +Before this, PostPrepare_FileOps() freed the pending list without recording +it, so deferred commit-time ops were lost across PREPARE. Adding the +TWOPHASE_RM_FILEOPS_ID record type bumped TWOPHASE_MAGIC (twophase.c). + + +How to use the API +------------------ + +The public API is declared in src/include/storage/fileops.h. Call these in +place of the raw libc/syscall equivalents whenever the operation must be +transactional. + + register_delete (FileOpsCreate). When true, the newly created file is + registered for deletion if the transaction aborts. + + at_commit (FileOpsDelete, FileOpsRmdir, FileOpsRmdirRecursive, + FileOpsRmtree). at_commit = true removes on commit (dropping state the + transaction is logically deleting); at_commit = false removes only on abort + (cleaning up something the transaction just created, e.g. a copy tree). + +FileOpsRmtree defers its tree walk to execute time (removing whatever ended +up on disk by commit/abort), where FileOpsRmdirRecursive enumerates contents +at registration time. + +The API hides platform differences: PG_O_DIRECT for unbuffered I/O, +pg_fsync() for the correct per-platform flush, fsync_parent_path() for +directory durability on Unix, and durable_rename()/durable_unlink() for +crash-safe rename and unlink ordering. + + +Transaction lifecycle integration +---------------------------------- + +The deferred operations are driven by AM-neutral hooks that xact.c calls +(wired to the FileOps* functions in fileops_undo.c): + + FileOpsDoPendingOps(isCommit) -- executes the pending list at end of xact. + AtSubCommit_FileOps() -- reparents a subxact's pending ops. + AtSubAbort_FileOps() -- runs a subxact's abort-time ops. + FileOpsAtPrepare() -- serializes pending ops into 2PC state. + PostPrepare_FileOps() -- discards the in-memory list after PREPARE. + FileOpsCancelPendingDelete() -- removes a registered delete entry. + FileOpsSync() -- forces durability of a path immediately. + + +WAL replay +---------- + +fileops_redo() dispatches each XLOG_FILEOPS_* record to the matching action +during recovery; fileops_desc()/fileops_identify() render records for +pg_waldump. Because the redo handler is itself the recovery action, code +paths that run under WAL replay (the "redo" arms in dbcommands.c, +tablespace.c, and copydir.c) call the raw syscalls directly rather than +re-entering the FileOps* API. + +Tests: src/test/recovery/t/062_fileops_recovery.pl (crash recovery), +063_fileops_2pc.pl (constructive-op 2PC via chmod), +064_fileops_standby_divergence.pl (physical-standby convergence), +061_fileops_2pc_deferred.pl (deferred commit-time ops through PREPARE + +crash), and the test_fileops module's regression test. diff --git a/src/backend/storage/file/fileops.c b/src/backend/storage/file/fileops.c new file mode 100644 index 00000000000..aab81341a96 --- /dev/null +++ b/src/backend/storage/file/fileops.c @@ -0,0 +1,2561 @@ +/*------------------------------------------------------------------------- + * + * fileops.c + * Transactional file operations with WAL logging + * + * This module provides transactional filesystem operations that integrate + * with PostgreSQL's WAL and transaction management. It is NOT used for + * relation files (smgr/catalog handle those); the only in-core callers are + * dbcommands.c, tablespace.c, and copydir.c. + * + * Two op classes: + * - CONSTRUCTIVE ops (create, mkdir, chmod, chown, truncate, symlink, link, + * write, setxattr) run IMMEDIATELY, and each registers an abort-time undo + * action (a common-WAL UNDO record plus, where relevant, a pending op) + * that reverses it on rollback. + * - DESTRUCTIVE ops (rename, delete, rmdir, rmtree) are DEFERRED: a + * PendingFileOp is recorded now and the filesystem is touched at + * transaction end by FileOpsDoPendingOps(). + * + * WAL and UNDO are written only when XLogIsNeeded() (wal_level >= replica); + * at wal_level = minimal there is no crash or 2PC safety. + * + * The deferred-operations pattern works as follows: + * 1. A destructive API call records a PendingFileOp entry in a linked list + * 2. At commit/abort time, FileOpsDoPendingOps() executes or discards + * the pending operations based on transaction outcome (a commit-time + * destructive op emits its WAL record at execution time) + * 3. At PREPARE, FileOpsAtPrepare() serializes the pending ops into the + * 2PC state file so COMMIT/ROLLBACK PREPARED can perform them + * + * Subtransaction support: + * - At subtransaction commit, entries are reassigned to the parent level + * - At subtransaction abort, abort-time actions execute immediately + * + * Platform-specific handling: + * - O_DIRECT: Uses PG_O_DIRECT abstraction (Linux native O_DIRECT, + * macOS F_NOCACHE via fcntl, Windows FILE_FLAG_NO_BUFFERING) + * - fsync: Uses pg_fsync() which selects the appropriate mechanism + * (Linux fdatasync, macOS F_FULLFSYNC, Windows FlushFileBuffers, + * BSD fsync) + * - Directory sync: Uses fsync_fname()/fsync_parent_path() which + * handle directory fsync on Unix platforms (not needed on Windows) + * - Durable operations: Uses durable_rename()/durable_unlink() which + * ensure operations persist across crashes via proper fsync ordering + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/storage/file/fileops.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include +#include +#ifdef HAVE_SYS_FCNTL_H +#include +#endif + +/* + * ENODATA is Linux-specific; FreeBSD and other BSDs don't define it. + * When removing extended attributes, ENODATA means "attribute does not + * exist" -- equivalent to ENOATTR on BSDs. + */ +#ifndef ENODATA +#ifdef ENOATTR +#define ENODATA ENOATTR +#else +#define ENODATA ENOENT +#endif +#endif + +#include "access/fileops_xlog.h" +#include "access/rmgr.h" +#include "access/twophase_rmgr.h" +#include "access/undolog.h" +#include "access/undorecord.h" +#include "access/undormgr.h" +#include "access/xact.h" +#include "access/xactundo.h" +#include "access/xlog.h" +#include "access/xloginsert.h" +#include "catalog/pg_class.h" +#include "miscadmin.h" +#include "port/pg_xattr.h" +#include "storage/fd.h" +#include "storage/fileops.h" +#include "utils/injection_point.h" +#include "utils/memutils.h" + + +/* Head of the pending file operations linked list */ +static PendingFileOp *pendingFileOps = NULL; + +static void FileOpsExecOne(PendingFileOp *pending, bool isCommit, + XLogRecPtr *max_lsn); + +/* + * fileops_fsync_parent -- fsync the parent directory of a file path + * + * This ensures that directory entry changes (create, delete, rename) + * are durable. On Windows, directory fsync is not needed because NTFS + * journals directory entries; fsync_fname_ext() handles this by being + * a no-op for directories on Windows. + */ +static void +fileops_fsync_parent(const char *fname, int elevel) +{ + char parentpath[MAXPGPATH]; + char *sep; + + strlcpy(parentpath, fname, MAXPGPATH); + + sep = strrchr(parentpath, '/'); + if (sep != NULL) + { + /* Got a path component, fsync the directory portion */ + if (sep == parentpath) + parentpath[1] = '\0'; /* root directory */ + else + *sep = '\0'; + + fsync_fname_ext(parentpath, true, true, elevel); + } +} + +/* + * AddPendingFileOp - Add a new pending file operation to the list + * + * All fields are deep-copied into TopMemoryContext to survive + * until transaction end, following the PendingRelDelete pattern. + */ +static void +AddPendingFileOp(PendingFileOpType type, const char *path, + const char *newpath, off_t length, bool at_commit) +{ + PendingFileOp *pending; + MemoryContext oldcxt; + + oldcxt = MemoryContextSwitchTo(TopMemoryContext); + + pending = (PendingFileOp *) palloc(sizeof(PendingFileOp)); + pending->type = type; + pending->path = pstrdup(path); + pending->newpath = newpath ? pstrdup(newpath) : NULL; + pending->length = length; + pending->data = NULL; + pending->data_len = 0; + pending->at_commit = at_commit; + pending->nestLevel = GetCurrentTransactionNestLevel(); + pending->next = pendingFileOps; + pendingFileOps = pending; + + MemoryContextSwitchTo(oldcxt); +} + +/* + * AddPendingFileOpWithData - Add a pending file operation with extra data + * + * Like AddPendingFileOp but also stores arbitrary data (e.g., original + * xattr value for restore on abort). + */ +static void +AddPendingFileOpWithData(PendingFileOpType type, const char *path, + const char *newpath, off_t length, + const void *data, size_t data_len, + bool at_commit) +{ + PendingFileOp *pending; + MemoryContext oldcxt; + + oldcxt = MemoryContextSwitchTo(TopMemoryContext); + + pending = (PendingFileOp *) palloc(sizeof(PendingFileOp)); + pending->type = type; + pending->path = pstrdup(path); + pending->newpath = newpath ? pstrdup(newpath) : NULL; + pending->length = length; + if (data && data_len > 0) + { + pending->data = palloc(data_len); + memcpy(pending->data, data, data_len); + pending->data_len = data_len; + } + else + { + pending->data = NULL; + pending->data_len = 0; + } + pending->at_commit = at_commit; + pending->nestLevel = GetCurrentTransactionNestLevel(); + pending->next = pendingFileOps; + pendingFileOps = pending; + + MemoryContextSwitchTo(oldcxt); +} + +/* + * FreePendingFileOp - Free a pending file operation entry + */ +static void +FreePendingFileOp(PendingFileOp *pending) +{ + if (pending->path) + pfree(pending->path); + if (pending->newpath) + pfree(pending->newpath); + if (pending->data) + pfree(pending->data); + pfree(pending); +} + +/* + * FileOpsCancelPendingDelete - Cancel a pending file deletion + * + * This removes matching DELETE entries from the pendingFileOps list. + * It is called by RelationPreserveStorage() to ensure that when a + * relation's storage is preserved (e.g., during index reuse in ALTER TABLE), + * the corresponding FileOps DELETE entry is also cancelled, preventing + * FileOpsDoPendingOps from deleting the file at commit time. + */ +void +FileOpsCancelPendingDelete(const char *path, bool at_commit) +{ + PendingFileOp *pending; + PendingFileOp *prev; + PendingFileOp *next; + + prev = NULL; + for (pending = pendingFileOps; pending != NULL; pending = next) + { + next = pending->next; + if (pending->type == PENDING_FILEOP_DELETE && + pending->at_commit == at_commit && + strcmp(pending->path, path) == 0) + { + /* unlink and free list entry */ + if (prev) + prev->next = next; + else + pendingFileOps = next; + FreePendingFileOp(pending); + /* prev does not change */ + } + else + { + prev = pending; + } + } +} + +/* + * FileOpsSync - Ensure a file's data is durably written to disk + * + * This is a convenience wrapper around fsync_fname() that uses the + * platform-appropriate sync mechanism: + * - Linux: fdatasync() (only flushes data, not metadata unless needed) + * - macOS: fcntl(F_FULLFSYNC) (flushes disk write cache) + * - FreeBSD: fsync() + * - Windows: FlushFileBuffers() + * + * An ERROR is raised if the sync fails. + */ +/* + * FileOpsCreate - Create a file within a transaction + * + * Creates the file immediately (so it can be used within the transaction) + * and logs the creation to WAL. If register_delete is true, the file will + * be deleted if the transaction aborts. + * + * Platform handling: + * - Linux/FreeBSD: O_DIRECT passed directly to open() + * - macOS: F_NOCACHE fcntl applied after open() + * - Windows: FILE_FLAG_NO_BUFFERING (handled by port layer) + * + * Returns the file descriptor on success, or -1 on failure. + */ +int +FileOpsCreate(const char *path, int flags, mode_t mode, bool register_delete) +{ + int fd; + bool need_wal = XLogIsNeeded(); + xl_fileops_create xlrec; + int pathlen = 0; + char undo_payload[sizeof(FileopsUndoCreate) + MAXPGPATH]; + XactUndoContext undo_ctx; + + Assert(!IsInParallelMode()); + + /* + * Write-ahead ordering: the WAL record (and UNDO record) must be durably + * on disk BEFORE the file is physically created. Once that is true, the + * create+fsync below is contractually obligated to happen -- a failure at + * that point would leave a durable record of a creation that never + * occurred, so it runs inside a critical section and any failure is + * promoted to PANIC by the elog machinery (see errfinish() in elog.c). + * This mirrors RelationTruncate() in catalog/storage.c and the (already + * correct) FileOpsTruncate() below. + */ + if (need_wal) + { + FileopsUndoCreate *hdr = (FileopsUndoCreate *) undo_payload; + + xlrec.flags = flags; + xlrec.mode = mode; + xlrec.register_delete = register_delete; + + pathlen = (int) strlen(path) + 1; + + /* + * The path length is stored in a uint16 (hdr->path_len) and the + * payload buffer is sized for MAXPGPATH. Paths are MAXPGPATH-bounded + * everywhere upstream, so this is defensive, but a + * filesystem-mutation WAL record must never silently truncate an + * over-long path into a uint16 -- fail loudly instead. + */ + if (pathlen > MAXPGPATH) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("file operation path length %d exceeds maximum %d", + pathlen, MAXPGPATH))); + + hdr->path_len = (uint16) pathlen; + memcpy(undo_payload + sizeof(FileopsUndoCreate), path, pathlen); + + /* + * PrepareXactUndoData() may palloc (first UNDO record of this + * persistence level in the transaction allocates an UndoRecordSet), + * so it must run before the critical section; palloc is forbidden + * inside one. + */ + PrepareXactUndoData(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_CREATE, + InvalidOid, undo_payload, + sizeof(FileopsUndoCreate) + pathlen); + + /* + * Load the injection point cache entry before the critical section; + * looking it up can palloc, which is forbidden once we + * START_CRIT_SECTION. + */ + INJECTION_POINT_LOAD("fileops-create-after-wal-flush"); + } + + if (need_wal) + START_CRIT_SECTION(); + + if (need_wal) + { + XLogRecPtr lsn; + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsCreate); + XLogRegisterData(path, pathlen); + lsn = XLogInsert(RM_FILEOPS_ID, XLOG_FILEOPS_CREATE); + + InsertXactUndoData(&undo_ctx); + CleanupXactUndoInsertion(&undo_ctx); + + XLogFlush(lsn); + + INJECTION_POINT_CACHED("fileops-create-after-wal-flush", NULL); + } + + fd = OpenTransientFilePerm(path, flags | O_CREAT, mode); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create file \"%s\": %m", path))); + + if (enableFsync) + { + if (pg_fsync(fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", path))); + fileops_fsync_parent(path, data_sync_elevel(ERROR)); + } + + if (need_wal) + END_CRIT_SECTION(); + + if (register_delete) + AddPendingFileOp(PENDING_FILEOP_DELETE, path, NULL, 0, false); + + return fd; +} + +/* + * FileOpsDelete - Schedule a file deletion within a transaction + * + * The file is not deleted immediately. Instead, the deletion is deferred + * to transaction commit (if at_commit is true) or abort (if false). + * On Unix: unlink() with parent dir fsync. + * On Windows: pgunlink() with retry on EACCES. + */ +void +FileOpsDelete(const char *path, bool at_commit) +{ + Assert(!IsInParallelMode()); + + /* + * no WAL is emitted at registration time. The deferred delete is + * WAL-logged in FileOpsDoPendingOps() at commit-execution time, when it + * actually happens on the primary, so a physical standby reproduces it + * via redo. Registration-time logging would be a no-op on redo (the + * standby never runs FileOpsDoPendingOps()) and, for at_commit=false + * delete-on-abort registrations, would wrongly apply an aborted + * transaction's effect on the standby. + */ + AddPendingFileOp(PENDING_FILEOP_DELETE, path, NULL, 0, at_commit); +} + +/* + * FileOpsRename - Rename a file within a transaction + * + * The rename is deferred to commit time. Uses durable_rename() internally + * which handles all platform differences: + * - Unix: rename() with fsync of both old and new parent dirs + * - Windows: MoveFileEx(MOVEFILE_REPLACE_EXISTING) with retry + */ +int +FileOpsRename(const char *oldpath, const char *newpath) +{ + Assert(!IsInParallelMode()); + + /* + * WAL is emitted at commit-execution time in FileOpsDoPendingOps(), not + * at registration, so the standby reproduces the rename via redo. + */ + AddPendingFileOp(PENDING_FILEOP_RENAME, oldpath, newpath, 0, true); + + return 0; +} + +/* + * FileOpsWrite - Write data to a file at a specific offset + * + * Immediate execution, WAL-logged for crash recovery replay. + * Uses pwrite() on POSIX. On Windows: SetFilePointerEx + WriteFile. + * + * Returns 0 on success, -1 on failure. + */ +int +FileOpsWrite(const char *path, off_t offset, const void *data, uint32 len) +{ + int fd; + ssize_t nbytes; + bool need_wal = XLogIsNeeded(); + + Assert(!IsInParallelMode()); + + fd = OpenTransientFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\" for writing: %m", path))); + + /* + * Write-ahead ordering: WAL must be durable before the physical write + * that it describes. There is no UNDO record here (unlike the other + * Category-A operations) because FileOpsWrite has no "original value" + * concept to restore on abort -- callers that need rollback protection + * pair it with FileOpsCreate's delete-on-abort or FileOpsTruncate. Once + * the WAL record is flushed, the write+fsync below is contractually + * obligated to happen; a failure at that point is escalated to PANIC by + * the surrounding critical section (see errfinish() in elog.c), matching + * FileOpsTruncate() below and RelationTruncate() in catalog/storage.c. + */ + if (need_wal) + { + xl_fileops_write xlrec; + int pathlen; + XLogRecPtr lsn; + + pathlen = (int) strlen(path) + 1; + + xlrec.offset = offset; + xlrec.len = len; + xlrec.path_len = (uint16) pathlen; + + START_CRIT_SECTION(); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsWrite); + XLogRegisterData(path, pathlen); + XLogRegisterData(data, len); + lsn = XLogInsert(RM_FILEOPS_ID, XLOG_FILEOPS_WRITE); + + XLogFlush(lsn); + } + + nbytes = pg_pwrite(fd, data, len, offset); + if (nbytes != (ssize_t) len) + { + int save_errno = errno; + + CloseTransientFile(fd); + errno = save_errno; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write %u bytes to file \"%s\" at offset %lld: %m", + len, path, (long long) offset))); + } + + if (enableFsync && pg_fsync(fd) != 0) + { + int save_errno = errno; + + CloseTransientFile(fd); + errno = save_errno; + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\" after write: %m", path))); + } + + if (need_wal) + END_CRIT_SECTION(); + + if (CloseTransientFile(fd) != 0) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", path))); + + return 0; +} + +/* + * FileOpsTruncate - Truncate a file within a transaction + * + * Executed immediately and WAL-logged. Uses ftruncate() on POSIX, + * SetEndOfFile() on Windows. File is fsynced after truncation. + */ +void +FileOpsTruncate(const char *path, off_t length) +{ + int fd; + struct stat st; + off_t orig_length = 0; + bool need_wal = XLogIsNeeded(); + + Assert(!IsInParallelMode()); + + /* Capture original length for UNDO before truncation */ + if (stat(path, &st) == 0) + orig_length = st.st_size; + + if (need_wal) + { + xl_fileops_truncate xlrec; + int pathlen; + XactUndoContext undo_ctx; + char payload[sizeof(FileopsUndoTruncate) + MAXPGPATH]; + FileopsUndoTruncate *hdr = (FileopsUndoTruncate *) payload; + XLogRecPtr lsn; + + xlrec.length = length; + xlrec.orig_length = orig_length; + pathlen = (int) strlen(path) + 1; + + hdr->orig_length = orig_length; + hdr->path_len = (uint16) pathlen; + memcpy(payload + sizeof(FileopsUndoTruncate), path, pathlen); + + /* + * PrepareXactUndoData() may palloc, so it must happen before the + * critical section below (palloc is forbidden inside one). + */ + PrepareXactUndoData(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_TRUNCATE, + InvalidOid, payload, + sizeof(FileopsUndoTruncate) + pathlen); + + /* + * Once the WAL/UNDO record is durably flushed below, the physical + * ftruncate()+fsync() is contractually obligated to happen: a + * durably-recorded truncation that never occurs on disk would leave + * recovery/rollback with nothing to reconcile against. Wrap the + * whole WAL-insert-through-syscall sequence in a critical section so + * that any failure past this point is promoted to PANIC by the elog + * machinery (errfinish() in elog.c), instead of silently diverging + * via an ordinary transaction abort. This retrofits the same + * protection RelationTruncate() already has in catalog/storage.c, + * which this function was previously missing. + */ + START_CRIT_SECTION(); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsTruncate); + XLogRegisterData(path, pathlen); + lsn = XLogInsert(RM_FILEOPS_ID, XLOG_FILEOPS_TRUNCATE); + + InsertXactUndoData(&undo_ctx); + CleanupXactUndoInsertion(&undo_ctx); + + /* + * Flush the redo and UNDO records to durable storage before mutating + * the file below. ftruncate()+pg_fsync() makes the physical change + * durable immediately; if we crashed after that but before the WAL + * reached disk, recovery would have no record of the operation and no + * UNDO batch to reverse it on rollback. This mirrors the write-ahead + * ordering in RelationTruncate() (see catalog/storage.c). Flush the + * LSN XLogInsert() itself returned, not a separately-fetched "current + * insert position" -- under concurrency the latter can point into the + * middle of a record another backend has reserved but not finished + * copying, which makes XLogFlush() fail with "xlog flush request is + * not satisfied". + */ + XLogFlush(lsn); + } + + fd = OpenTransientFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\" for truncation: %m", path))); + + if (ftruncate(fd, length) < 0) + { + int save_errno = errno; + + CloseTransientFile(fd); + errno = save_errno; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not truncate file \"%s\" to %lld bytes: %m", + path, (long long) length))); + } + + if (enableFsync && pg_fsync(fd) != 0) + { + int save_errno = errno; + + CloseTransientFile(fd); + errno = save_errno; + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\" after truncation: %m", + path))); + } + + if (need_wal) + END_CRIT_SECTION(); + + if (CloseTransientFile(fd) != 0) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", path))); + + /* + * Register abort-time pending op to restore original size on rollback. + * This provides immediate rollback without waiting for the background + * UNDO worker (which is deferred due to BumpContext limitations). + */ + AddPendingFileOp(PENDING_FILEOP_TRUNCATE, path, NULL, orig_length, false); +} + +/* + * FileOpsChmod - Change file permissions within a transaction + * + * Immediate execution, WAL-logged. + * On POSIX: chmod(). On Windows: _chmod() with limited mode bits + * (only _S_IREAD/_S_IWRITE; no group/other). Logs WARNING for + * unsupported mode bits on Windows. + * + * Returns 0 on success, -1 on failure with errno left set so the caller + * can distinguish ENOENT from other errors. No WAL, UNDO record, or + * pending op is registered on the failure path because the on-disk mode + * was not changed. + * + * Design note (write-ordering vs. tolerable-failure contract): WAL/UNDO + * must be durable before chmod() runs, but tablespace.c's caller relies on + * getting a plain -1/ENOENT back (not a crash) when the target directory + * does not exist -- a real caller-observed case, not a hypothetical one. + * We resolve the conflict by reusing the stat() this function already + * needs (to capture the original mode for UNDO) as a preflight: if the + * path cannot be stat()ed, chmod() would fail the same way, so we return + * -1 with stat()'s errno before writing anything to WAL. This is not a + * "trial run" of the mutating syscall itself (no double-mutation hazard), + * and existence does not flip in the narrow window between this stat() + * and the chmod() below for paths FileOps callers own/manage. Once that + * preflight passes, the WAL/UNDO record becomes durable and chmod() is + * contractually obligated to happen: any failure past that point (a + * genuine race, read-only remount, immutable-flag, etc.) is escalated to + * PANIC by the surrounding critical section, per errfinish() in elog.c. + */ +int +FileOpsChmod(const char *path, mode_t mode) +{ + struct stat st; + mode_t orig_mode; + bool need_wal; + + Assert(!IsInParallelMode()); + + /* Capture original mode for UNDO before chmod, and preflight ENOENT */ + if (stat(path, &st) != 0) + return -1; /* leave errno set by stat() for the caller */ + orig_mode = st.st_mode & 07777; + + need_wal = XLogIsNeeded(); + + if (need_wal) + { + xl_fileops_chmod xlrec; + int pathlen; + XactUndoContext undo_ctx; + char payload[sizeof(FileopsUndoChmod) + MAXPGPATH]; + FileopsUndoChmod *hdr = (FileopsUndoChmod *) payload; + XLogRecPtr lsn; + + xlrec.mode = mode; + xlrec.orig_mode = orig_mode; + pathlen = (int) strlen(path) + 1; + + hdr->orig_mode = orig_mode; + hdr->path_len = (uint16) pathlen; + memcpy(payload + sizeof(FileopsUndoChmod), path, pathlen); + + /* PrepareXactUndoData() may palloc; must run before the crit section */ + PrepareXactUndoData(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_CHMOD, + InvalidOid, payload, + sizeof(FileopsUndoChmod) + pathlen); + + /* + * Load the injection point cache entry before the critical section; + * looking it up can palloc, which is forbidden once we + * START_CRIT_SECTION. + */ + INJECTION_POINT_LOAD("fileops-chmod-after-wal-flush"); + + START_CRIT_SECTION(); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsChmod); + XLogRegisterData(path, pathlen); + lsn = XLogInsert(RM_FILEOPS_ID, XLOG_FILEOPS_CHMOD); + + InsertXactUndoData(&undo_ctx); + CleanupXactUndoInsertion(&undo_ctx); + + XLogFlush(lsn); + + INJECTION_POINT_CACHED("fileops-chmod-after-wal-flush", NULL); + } + + if (chmod(path, mode) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not chmod file \"%s\": %m", path))); + + if (need_wal) + END_CRIT_SECTION(); + + /* + * Register abort-time pending op to restore original mode on rollback. + * Store mode in the length field (mode_t fits in off_t). + */ + AddPendingFileOp(PENDING_FILEOP_CHMOD, path, NULL, (off_t) orig_mode, false); + + return 0; +} + +/* + * FileOpsChown - Change file ownership within a transaction + * + * Immediate execution, WAL-logged. + * On POSIX: chown(). On Windows: no-op with WARNING (Windows uses + * ACLs for ownership, not uid/gid). + * + * Returns 0 on success. + */ +int +FileOpsChown(const char *path, uid_t uid, gid_t gid) +{ + struct stat st; + uid_t orig_uid = (uid_t) -1; + gid_t orig_gid = (gid_t) -1; + bool need_wal = XLogIsNeeded(); + + Assert(!IsInParallelMode()); + + /* Capture original ownership for UNDO before chown */ + if (stat(path, &st) == 0) + { + orig_uid = st.st_uid; + orig_gid = st.st_gid; + } + + /* + * Write-ahead ordering: WAL/UNDO must be durable before the physical + * chown(). Once durable, chown() is contractually obligated to happen; a + * failure past that point is escalated to PANIC by the surrounding + * critical section (see errfinish() in elog.c), matching + * FileOpsTruncate() above and RelationTruncate() in catalog/storage.c. + */ + if (need_wal) + { + xl_fileops_chown xlrec; + int pathlen; + XactUndoContext undo_ctx; + char payload[sizeof(FileopsUndoChown) + MAXPGPATH]; + FileopsUndoChown *hdr = (FileopsUndoChown *) payload; + XLogRecPtr lsn; + + xlrec.uid = uid; + xlrec.gid = gid; + xlrec.orig_uid = orig_uid; + xlrec.orig_gid = orig_gid; + pathlen = (int) strlen(path) + 1; + + hdr->orig_uid = orig_uid; + hdr->orig_gid = orig_gid; + hdr->path_len = (uint16) pathlen; + memcpy(payload + sizeof(FileopsUndoChown), path, pathlen); + + /* PrepareXactUndoData() may palloc; must run before the crit section */ + PrepareXactUndoData(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_CHOWN, + InvalidOid, payload, + sizeof(FileopsUndoChown) + pathlen); + + START_CRIT_SECTION(); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsChown); + XLogRegisterData(path, pathlen); + lsn = XLogInsert(RM_FILEOPS_ID, XLOG_FILEOPS_CHOWN); + + InsertXactUndoData(&undo_ctx); + CleanupXactUndoInsertion(&undo_ctx); + + XLogFlush(lsn); + } + +#ifndef WIN32 + if (chown(path, uid, gid) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not chown file \"%s\" to %d:%d: %m", + path, (int) uid, (int) gid))); +#else + ereport(WARNING, + (errmsg("chown is not supported on Windows, skipping for \"%s\"", + path))); +#endif + + if (need_wal) + END_CRIT_SECTION(); + + return 0; +} + +void +FileOpsSync(const char *path) +{ + fsync_fname(path, false); +} + +/* + * FileOpsMkdir - Create a directory within a transaction + * + * Immediate execution. Optionally registers rmdir-on-abort. + * Uses MakePGDirectory() pattern (mkdir with pg_dir_create_mode). + * On Windows: _mkdir() (no mode parameter, permissions from parent). + * + * Returns 0 on success. + */ +int +FileOpsMkdir(const char *path, mode_t mode) +{ + bool need_wal = XLogIsNeeded(); + + Assert(!IsInParallelMode()); + + /* + * Write-ahead ordering: WAL/UNDO must be durable before the physical + * mkdir()+parent-dir-fsync. Once durable, the mkdir is contractually + * obligated to happen; a failure past that point is escalated to PANIC by + * the surrounding critical section (see errfinish() in elog.c), matching + * FileOpsTruncate() above and RelationTruncate() in catalog/storage.c. + */ + if (need_wal) + { + xl_fileops_mkdir xlrec; + int pathlen; + XactUndoContext undo_ctx; + char payload[sizeof(FileopsUndoMkdir) + MAXPGPATH]; + FileopsUndoMkdir *hdr = (FileopsUndoMkdir *) payload; + XLogRecPtr lsn; + + xlrec.mode = mode; + pathlen = (int) strlen(path) + 1; + + hdr->path_len = (uint16) pathlen; + memcpy(payload + sizeof(FileopsUndoMkdir), path, pathlen); + + /* PrepareXactUndoData() may palloc; must run before the crit section */ + PrepareXactUndoData(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_MKDIR, + InvalidOid, payload, + sizeof(FileopsUndoMkdir) + pathlen); + + START_CRIT_SECTION(); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsMkdir); + XLogRegisterData(path, pathlen); + lsn = XLogInsert(RM_FILEOPS_ID, XLOG_FILEOPS_MKDIR); + + InsertXactUndoData(&undo_ctx); + CleanupXactUndoInsertion(&undo_ctx); + + XLogFlush(lsn); + } + + if (MakePGDirectory(path) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\": %m", path))); + + if (enableFsync) + fileops_fsync_parent(path, data_sync_elevel(ERROR)); + + if (need_wal) + END_CRIT_SECTION(); + + /* Register rmdir-on-abort so directory is cleaned up on rollback */ + AddPendingFileOp(PENDING_FILEOP_RMDIR, path, NULL, 0, false); + + return 0; +} + +/* + * FileOpsRmdir - Remove a directory within a transaction + * + * Deferred to commit time (like DELETE). Uses rmdir() on all platforms. + * On Windows: _rmdir(). + */ +void +FileOpsRmdir(const char *path, bool at_commit) +{ + Assert(!IsInParallelMode()); + + /* + * WAL is emitted at commit-execution time in FileOpsDoPendingOps(), not + * at registration, so the standby reproduces the rmdir via redo. + */ + AddPendingFileOp(PENDING_FILEOP_RMDIR, path, NULL, 0, at_commit); +} + +/* + * FileOpsRmtree - Schedule a recursive directory removal at commit/abort time. + * + * Unlike FileOpsRmdirRecursive (which enumerates the tree at registration + * time and queues per-entry FILEOPS_DELETE/RMDIR ops), this enqueues a + * single PENDING_FILEOP_RMTREE that defers the tree walk to execute time. + * That is the right semantic for callers that register the destination of + * an in-progress copy: the directory is empty (or nonexistent) when the + * registration happens, but contains files by the time of abort. + * + * No WAL is written because rmtree is not a directly recoverable + * operation; the abort case relies on the surrounding DDL having already + * recorded enough state to make the partial tree disposable. + */ +void +FileOpsRmtree(const char *path, bool at_commit) +{ + Assert(!IsInParallelMode()); + AddPendingFileOp(PENDING_FILEOP_RMTREE, path, NULL, 0, at_commit); +} + +/* + * FileOpsRmdirRecursive - Schedule recursive directory removal. + * + * Enumerates directory tree and schedules: + * - FileOpsDelete() for each file (at_commit) + * - FileOpsRmdir() for each subdirectory (at_commit, leaf-first order) + * + * Permissive: ENOENT is logged but not an error. + */ +void +FileOpsRmdirRecursive(const char *path, bool at_commit) +{ + DIR *dir; + struct dirent *de; + + dir = AllocateDir(path); + if (dir == NULL) + { + if (errno == ENOENT) + { + elog(LOG, "FileOpsRmdirRecursive: directory \"%s\" does not exist", + path); + return; + } + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open directory \"%s\": %m", path))); + } + + while ((de = ReadDirExtended(dir, path, LOG)) != NULL) + { + char subpath[MAXPGPATH]; + struct stat st; + + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) + continue; + + snprintf(subpath, MAXPGPATH, "%s/%s", path, de->d_name); + if (lstat(subpath, &st) == 0) + { + if (S_ISDIR(st.st_mode)) + FileOpsRmdirRecursive(subpath, at_commit); + else + FileOpsDelete(subpath, at_commit); + } + } + FreeDir(dir); + FileOpsRmdir(path, at_commit); +} + +/* + * FileOpsSymlink - Create a symbolic link within a transaction + * + * Immediate execution. Registers delete-on-abort for cleanup. + * On POSIX: symlink(). On Windows: pgsymlink() which creates NTFS + * junction points via DeviceIoControl(). Note: junction points only + * work for directories on Windows. + * + * Returns 0 on success. + */ +int +FileOpsSymlink(const char *target, const char *linkpath) +{ + bool need_wal = XLogIsNeeded(); + + Assert(!IsInParallelMode()); + + /* + * Write-ahead ordering: WAL/UNDO must be durable before the physical + * symlink()+parent-dir-fsync. Once durable, the symlink is contractually + * obligated to happen; a failure past that point is escalated to PANIC by + * the surrounding critical section (see errfinish() in elog.c), matching + * FileOpsTruncate() above and RelationTruncate() in catalog/storage.c. + */ + if (need_wal) + { + xl_fileops_symlink xlrec; + int targetlen; + int linkpathlen; + XactUndoContext undo_ctx; + char payload[sizeof(FileopsUndoSymlink) + MAXPGPATH]; + FileopsUndoSymlink *hdr = (FileopsUndoSymlink *) payload; + XLogRecPtr lsn; + + targetlen = (int) strlen(target) + 1; + linkpathlen = (int) strlen(linkpath) + 1; + xlrec.target_len = (uint16) targetlen; + + hdr->linkpath_len = (uint16) linkpathlen; + memcpy(payload + sizeof(FileopsUndoSymlink), linkpath, linkpathlen); + + /* PrepareXactUndoData() may palloc; must run before the crit section */ + PrepareXactUndoData(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_SYMLINK, + InvalidOid, payload, + sizeof(FileopsUndoSymlink) + linkpathlen); + + START_CRIT_SECTION(); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsSymlink); + XLogRegisterData(target, targetlen); + XLogRegisterData(linkpath, linkpathlen); + lsn = XLogInsert(RM_FILEOPS_ID, XLOG_FILEOPS_SYMLINK); + + InsertXactUndoData(&undo_ctx); + CleanupXactUndoInsertion(&undo_ctx); + + XLogFlush(lsn); + } + + if (symlink(target, linkpath) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create symbolic link \"%s\" -> \"%s\": %m", + linkpath, target))); + + if (enableFsync) + fileops_fsync_parent(linkpath, data_sync_elevel(ERROR)); + + if (need_wal) + END_CRIT_SECTION(); + + /* Register delete-on-abort to clean up the symlink on rollback */ + AddPendingFileOp(PENDING_FILEOP_DELETE, linkpath, NULL, 0, false); + + return 0; +} + +/* + * FileOpsLink - Create a hard link within a transaction + * + * Immediate execution. Registers delete-on-abort for cleanup. + * On POSIX: link(). On Windows: CreateHardLinkA() (NTFS only). + * + * Returns 0 on success. + */ +int +FileOpsLink(const char *oldpath, const char *newpath) +{ + bool need_wal = XLogIsNeeded(); + + Assert(!IsInParallelMode()); + + /* + * Write-ahead ordering: WAL/UNDO must be durable before the physical + * link()+parent-dir-fsync. Once durable, the link is contractually + * obligated to happen; a failure past that point is escalated to PANIC by + * the surrounding critical section (see errfinish() in elog.c), matching + * FileOpsTruncate() above and RelationTruncate() in catalog/storage.c. + */ + if (need_wal) + { + xl_fileops_link xlrec; + int oldpathlen; + int newpathlen; + XactUndoContext undo_ctx; + char payload[sizeof(FileopsUndoLink) + MAXPGPATH]; + FileopsUndoLink *hdr = (FileopsUndoLink *) payload; + XLogRecPtr lsn; + + oldpathlen = (int) strlen(oldpath) + 1; + newpathlen = (int) strlen(newpath) + 1; + xlrec.oldpath_len = (uint16) oldpathlen; + + hdr->newpath_len = (uint16) newpathlen; + memcpy(payload + sizeof(FileopsUndoLink), newpath, newpathlen); + + /* PrepareXactUndoData() may palloc; must run before the crit section */ + PrepareXactUndoData(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_LINK, + InvalidOid, payload, + sizeof(FileopsUndoLink) + newpathlen); + + START_CRIT_SECTION(); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsLink); + XLogRegisterData(oldpath, oldpathlen); + XLogRegisterData(newpath, newpathlen); + lsn = XLogInsert(RM_FILEOPS_ID, XLOG_FILEOPS_LINK); + + InsertXactUndoData(&undo_ctx); + CleanupXactUndoInsertion(&undo_ctx); + + XLogFlush(lsn); + } + + if (link(oldpath, newpath) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create hard link \"%s\" -> \"%s\": %m", + newpath, oldpath))); + + if (enableFsync) + fileops_fsync_parent(newpath, data_sync_elevel(ERROR)); + + if (need_wal) + END_CRIT_SECTION(); + + /* Register delete-on-abort to clean up the link on rollback */ + AddPendingFileOp(PENDING_FILEOP_DELETE, newpath, NULL, 0, false); + + return 0; +} + +/* + * FileOpsSetXattr - Set an extended attribute on a file + * + * Immediate execution, WAL-logged. + * Uses pg_setxattr() portability layer which handles: + * - Linux: setxattr() + * - macOS: setxattr() with extra options parameter + * - FreeBSD: extattr_set_file() + * - Windows: NTFS Alternate Data Streams + * - Other: returns ENOTSUP + * + * Returns 0 on success, -1 if extended attributes are not supported on + * this filesystem/platform (errno left as ENOTSUP/EOPNOTSUPP). + * + * Design note (write-ordering vs. tolerable-failure conflict): the analysis + * that preceded this fix flagged that WAL-before-syscall conflicts with the + * old "return -1 on ENOTSUP/EOPNOTSUPP/EPERM/EACCES" contract, since a + * durable WAL/UNDO record asserting a xattr change that never happened is + * exactly the bug being fixed. We resolve it by splitting the old tolerable + * set into two buckets: + * + * - ENOTSUP/EOPNOTSUPP ("this filesystem doesn't implement xattrs at + * all") is a whole-filesystem capability question, not a per-call race: + * it cannot flip between a preflight check and the real syscall a few + * instructions later. The pg_getxattr() probe below (already needed to + * capture the original value for UNDO) surfaces this same errno for the + * same reason before any WAL is written, so we check it there and + * return -1 pre-WAL -- no new syscall, no trial run of the mutating + * operation itself. + * + * - EPERM/EACCES ("security policy denies this specific write") is a + * genuine permission decision that CAN legitimately differ between a + * read probe and a write attempt (e.g. SELinux, immutable flag), so it + * cannot be preflighted without attempting the mutating syscall itself + * (which the prior analysis correctly flagged as fragile/racy and not + * attempted here). grep of the tree found zero production callers of + * FileOpsSetXattr; the only consumer (test_fileops) exists to probe + * whole-filesystem xattr support, which the ENOTSUP/EOPNOTSUPP preflight + * above already covers. We therefore treat a post-WAL EPERM/EACCES the + * same as any other post-WAL syscall failure: it escalates to PANIC via + * the surrounding critical section (errfinish() in elog.c), because a + * durable WAL/UNDO record with no corresponding physical change is the + * load-bearing invariant this whole fix protects. If a real caller ever + * needs graceful EPERM/EACCES handling, that is a signal this decision + * needs revisiting (see FILEOPS-ANALYSIS-UNIMPLEMENTED.md point 3). + */ +int +FileOpsSetXattr(const char *path, const char *name, + const void *value, size_t len) +{ + bool had_value = false; + char *orig_value = NULL; + ssize_t orig_value_len = 0; + bool need_wal; + struct stat st; + + Assert(!IsInParallelMode()); + + /* + * Preflight existence check: FileOpsSetXattr on a nonexistent path is a + * caller error (ENOENT was never in the tolerable set below), and must be + * caught here, before any WAL is written -- otherwise it would be caught + * by the mutating syscall itself AFTER WAL is durable, escalating an + * ordinary caller mistake to PANIC. + */ + if (stat(path, &st) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", path))); + + /* + * Capture existing xattr value before overwriting. This is needed by the + * abort-time pending op (clean rollback) regardless of wal_level, so it + * must run unconditionally -- not gated on XLogIsNeeded() like the + * WAL/UNDO emission below. If the attribute doesn't exist, had_value + * stays false. + * + * This call doubles as the ENOTSUP/EOPNOTSUPP capability preflight + * described above: on every supported platform, a filesystem/inode that + * doesn't implement extended attributes fails an attribute-value probe + * the same way it would fail a set, regardless of whether the named + * attribute happens to exist. + */ + { + ssize_t vlen; + + vlen = pg_getxattr(path, name, NULL, 0); + if (vlen > 0) + { + orig_value = (char *) palloc(vlen); + orig_value_len = pg_getxattr(path, name, orig_value, vlen); + if (orig_value_len >= 0) + had_value = true; + else + { + pfree(orig_value); + orig_value = NULL; + orig_value_len = 0; + } + } + else if (vlen == 0) + { + /* Attribute exists but has zero-length value */ + had_value = true; + } + else if (errno == ENOTSUP || errno == EOPNOTSUPP) + { + /* + * Whole-filesystem capability failure; safe to preflight (see + * above) + */ + return -1; + } + + /* + * else: ENODATA/PG_ENOATTR (attribute absent) or other; + * had_value=false + */ + } + + need_wal = XLogIsNeeded(); + + /* + * Write-ahead ordering: WAL/UNDO must be durable before the physical + * setxattr(). Once durable, setxattr() is contractually obligated to + * succeed; any failure past this point (including EPERM/EACCES, per the + * design note above) is escalated to PANIC by the surrounding critical + * section. + */ + if (need_wal) + { + xl_fileops_setxattr xlrec; + int pathlen; + int namelen; + XactUndoContext undo_ctx; + Size payload_len; + char *payload; + FileopsUndoSetxattr *hdr; + XLogRecPtr lsn; + + pathlen = (int) strlen(path) + 1; + namelen = (int) strlen(name) + 1; + + xlrec.path_len = (uint16) pathlen; + xlrec.name_len = (uint16) namelen; + xlrec.value_len = (uint32) len; + xlrec.had_value = had_value; + xlrec.orig_value_len = (uint32) (had_value ? orig_value_len : 0); + + payload_len = sizeof(FileopsUndoSetxattr) + pathlen + namelen + + (had_value ? orig_value_len : 0); + + /* + * PrepareXactUndoData() may palloc; this palloc must run before the + * crit section too. + */ + payload = (char *) palloc(payload_len); + hdr = (FileopsUndoSetxattr *) payload; + + hdr->path_len = (uint16) pathlen; + hdr->name_len = (uint16) namelen; + hdr->orig_value_len = (uint32) (had_value ? orig_value_len : 0); + hdr->had_value = had_value; + + memcpy(payload + sizeof(FileopsUndoSetxattr), path, pathlen); + memcpy(payload + sizeof(FileopsUndoSetxattr) + pathlen, name, namelen); + if (had_value && orig_value_len > 0) + memcpy(payload + sizeof(FileopsUndoSetxattr) + pathlen + namelen, + orig_value, orig_value_len); + + PrepareXactUndoData(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_SETXATTR, + InvalidOid, payload, payload_len); + + START_CRIT_SECTION(); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsSetxattr); + XLogRegisterData(path, pathlen); + XLogRegisterData(name, namelen); + XLogRegisterData(value, (uint32) len); + if (had_value && orig_value_len > 0) + XLogRegisterData(orig_value, (uint32) orig_value_len); + lsn = XLogInsert(RM_FILEOPS_ID, XLOG_FILEOPS_SETXATTR); + + InsertXactUndoData(&undo_ctx); + CleanupXactUndoInsertion(&undo_ctx); + pfree(payload); + + XLogFlush(lsn); + } + + if (pg_setxattr(path, name, value, len) < 0) + { + if (orig_value) + pfree(orig_value); + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not set extended attribute \"%s\" on \"%s\": %m", + name, path))); + } + + if (need_wal) + END_CRIT_SECTION(); + + /* + * Register abort-time pending op to restore/remove xattr on rollback. If + * had_value: restore original value. If !had_value: remove the xattr. + */ + if (had_value) + AddPendingFileOpWithData(PENDING_FILEOP_SETXATTR, path, name, + 0, orig_value, (size_t) orig_value_len, false); + else + AddPendingFileOp(PENDING_FILEOP_REMOVEXATTR, path, name, 0, false); + + if (orig_value) + pfree(orig_value); + + return 0; +} + +/* + * FileOpsRemoveXattr - Remove an extended attribute from a file + * + * Immediate execution, WAL-logged. + * Uses pg_removexattr() portability layer. + * + * Returns 0 on success, -1 if extended attributes are not supported on + * this filesystem/platform (errno left as ENOTSUP/EOPNOTSUPP). + * + * Design note: see FileOpsSetXattr's design note above for the general + * reasoning (write-ahead ordering vs. the old tolerable-failure contract). + * REMOVEXATTR's tolerable set is narrower than SETXATTR's: only + * ENOTSUP/EOPNOTSUPP (whole-filesystem incapability, preflighted the same + * way, via the pg_getxattr() probe already needed to save the original + * value for UNDO) and "attribute does not exist" (PG_ENOATTR/ENODATA). + * The latter is not a permission decision like SETXATTR's EPERM/EACCES -- + * it is symmetric between a read probe and the write attempt (if + * pg_getxattr() just reported the attribute absent, pg_removexattr() will + * fail the same way for the same reason microseconds later), and it is + * information we already captured for the UNDO decision, not a new trial + * run of the mutating syscall. We therefore tolerate exactly that one + * errno post-WAL, inside the critical section, and PANIC on anything else, + * matching fileops_redo()'s own existing tolerance for this same errno. + */ +int +FileOpsRemoveXattr(const char *path, const char *name) +{ + char *saved_value = NULL; + ssize_t saved_value_len = 0; + bool need_wal; + + Assert(!IsInParallelMode()); + + /* + * Capture existing xattr value before removal. This is needed by the + * abort-time pending op (clean rollback) regardless of wal_level, so it + * must run unconditionally -- not gated on XLogIsNeeded() like the + * WAL/UNDO emission below. We need it to restore the attribute on + * rollback. It also doubles as the ENOTSUP/EOPNOTSUPP capability + * preflight (see design note above): a filesystem/inode without xattr + * support fails this probe the same way it would fail a removal. + */ + { + ssize_t vlen; + + vlen = pg_getxattr(path, name, NULL, 0); + if (vlen > 0) + { + saved_value = (char *) palloc(vlen); + saved_value_len = pg_getxattr(path, name, saved_value, vlen); + if (saved_value_len < 0) + { + pfree(saved_value); + saved_value = NULL; + saved_value_len = 0; + } + } + else if (vlen == 0) + { + /* Attribute exists but has zero-length value */ + saved_value_len = 0; + saved_value = (char *) palloc(1); /* non-NULL marker */ + } + else if (errno == ENOTSUP || errno == EOPNOTSUPP) + { + /* + * Whole-filesystem capability failure; safe to preflight (see + * above) + */ + return -1; + } + + /* + * else: ENODATA/PG_ENOATTR (attribute absent) or other; + * saved_value=NULL + */ + } + + need_wal = XLogIsNeeded(); + + /* + * Write-ahead ordering: WAL (and UNDO, if there was a value to restore) + * must be durable before the physical removal. Once durable, + * pg_removexattr() is contractually obligated to either succeed or fail + * with the already-anticipated PG_ENOATTR (see design note); any other + * failure escalates to PANIC via the surrounding critical section. + */ + if (need_wal) + { + xl_fileops_removexattr xlrec; + int pathlen; + int namelen; + XactUndoContext undo_ctx; + Size payload_len = 0; + char *payload = NULL; + bool had_value = (saved_value != NULL); + XLogRecPtr lsn; + + pathlen = (int) strlen(path) + 1; + namelen = (int) strlen(name) + 1; + + xlrec.path_len = (uint16) pathlen; + xlrec.name_len = (uint16) namelen; + xlrec.had_value = had_value; + xlrec.orig_value_len = (uint32) (had_value ? saved_value_len : 0); + + if (saved_value != NULL) + { + FileopsUndoRemovexattr *hdr; + + payload_len = sizeof(FileopsUndoRemovexattr) + pathlen + namelen + + saved_value_len; + + /* + * PrepareXactUndoData() may palloc; must run before the crit + * section, so this palloc does too. + */ + payload = (char *) palloc(payload_len); + hdr = (FileopsUndoRemovexattr *) payload; + + hdr->path_len = (uint16) pathlen; + hdr->name_len = (uint16) namelen; + hdr->value_len = (uint32) saved_value_len; + + memcpy(payload + sizeof(FileopsUndoRemovexattr), path, pathlen); + memcpy(payload + sizeof(FileopsUndoRemovexattr) + pathlen, + name, namelen); + if (saved_value_len > 0) + memcpy(payload + sizeof(FileopsUndoRemovexattr) + pathlen + namelen, + saved_value, saved_value_len); + + PrepareXactUndoData(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_REMOVEXATTR, + InvalidOid, payload, payload_len); + } + + START_CRIT_SECTION(); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsRemovexattr); + XLogRegisterData(path, pathlen); + XLogRegisterData(name, namelen); + if (had_value && saved_value_len > 0) + XLogRegisterData(saved_value, (uint32) saved_value_len); + lsn = XLogInsert(RM_FILEOPS_ID, XLOG_FILEOPS_REMOVEXATTR); + + if (saved_value != NULL) + { + InsertXactUndoData(&undo_ctx); + CleanupXactUndoInsertion(&undo_ctx); + pfree(payload); + } + + XLogFlush(lsn); + } + + if (pg_removexattr(path, name) < 0 && errno != PG_ENOATTR) + { + if (saved_value) + pfree(saved_value); + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not remove extended attribute \"%s\" from \"%s\": %m", + name, path))); + } + + if (need_wal) + END_CRIT_SECTION(); + + /* + * Register abort-time pending op to restore the removed xattr on + * rollback. Use SETXATTR type: on abort, set the xattr back to its saved + * value. + */ + if (saved_value != NULL) + AddPendingFileOpWithData(PENDING_FILEOP_SETXATTR, path, name, + 0, saved_value, (size_t) saved_value_len, false); + + if (saved_value) + pfree(saved_value); + + return 0; +} + +/* + * FileOpsDoPendingOps - Execute pending file operations at transaction end + * + * At commit, operations with at_commit=true are executed. + * At abort, operations with at_commit=false are executed. + * + * This is called from xact.c at transaction commit/abort, analogous + * to smgrDoPendingDeletes(). + */ +void +FileOpsDoPendingOps(bool isCommit) +{ + int nestLevel = GetCurrentTransactionNestLevel(); + PendingFileOp *pending; + PendingFileOp *prev; + PendingFileOp *next; + PendingFileOp *exec_head = NULL; + PendingFileOp *exec_tail = NULL; + XLogRecPtr max_lsn = InvalidXLogRecPtr; + + /* + * Pass 1: detach the entries belonging to this (or an inner) nesting + * level from the pending list. Those matching the transaction outcome + * are queued for execution; the rest are discarded. Outer-level entries + * stay on the list for a later call. The queue is built in pending-list + * order, which is newest-first because AddPendingFileOp prepends. + */ + prev = NULL; + for (pending = pendingFileOps; pending != NULL; pending = next) + { + next = pending->next; + + if (pending->nestLevel < nestLevel) + { + /* outer-level entries should not be processed yet */ + prev = pending; + continue; + } + + /* unlink from list first, so we don't retry on failure */ + if (prev) + prev->next = next; + else + pendingFileOps = next; + + if (pending->at_commit == isCommit) + { + /* + * queue for execution, preserving pending-list (newest-first) + * order + */ + pending->next = NULL; + if (exec_tail != NULL) + exec_tail->next = pending; + else + exec_head = pending; + exec_tail = pending; + } + else + FreePendingFileOp(pending); + } + + /* + * Commit-time ops must run in registration order (oldest-first) so that + * nested directory removals proceed deepest-first. The pending list is + * LIFO, so the queue collected above is newest-first; reverse it for + * commit. Abort-time ops must keep LIFO order, which the collected queue + * already preserves, so they are left as-is. + */ + if (isCommit) + { + PendingFileOp *rev = NULL; + + while (exec_head != NULL) + { + next = exec_head->next; + exec_head->next = rev; + rev = exec_head; + exec_head = next; + } + exec_head = rev; + } + + /* Commit record written, nothing executed yet (for tests) */ + if (isCommit && exec_head != NULL) + INJECTION_POINT("fileops-commit-before-pending-ops", NULL); + + /* Pass 2: execute the queued ops in order, freeing each when done. */ + for (pending = exec_head; pending != NULL; pending = next) + { + next = pending->next; + + FileOpsExecOne(pending, isCommit, &max_lsn); + + FreePendingFileOp(pending); + } + + /* + * Each destructive commit op above XLogFlush()es its own record BEFORE + * performing the filesystem mutation (WAL-before-mutation), so no primary + * FS change can outrun its WAL record. This trailing flush is therefore + * redundant, kept only as a no-op safety backstop (max_lsn is already + * durable). Only the commit path ever sets max_lsn. + */ + if (!XLogRecPtrIsInvalid(max_lsn)) + XLogFlush(max_lsn); +} + +/* + * FileOpsExecOne - Execute one pending file operation + * + * Performs a single deferred op (commit- or abort-time) exactly as it would + * run in an ordinary transaction end. Shared by FileOpsDoPendingOps() and by + * the 2PC post-commit/post-abort callbacks, so an op prepared with PREPARE + * TRANSACTION runs identically to one that never left this backend. + * + * *max_lsn accumulates the highest WAL LSN written by a commit-time op so the + * caller can flush it. Each destructive commit op also flushes its own record + * before mutating the filesystem (WAL-before-mutation), so *max_lsn is only a + * backstop. + */ +static void +FileOpsExecOne(PendingFileOp *pending, bool isCommit, XLogRecPtr *max_lsn) +{ + { + switch (pending->type) + { + case PENDING_FILEOP_CREATE: + /* Creates are executed immediately, nothing to do */ + break; + + case PENDING_FILEOP_RENAME: + + /* + * log-at-commit-execution. Emit the WAL record now, when the + * rename actually happens on the primary, so a physical + * standby reproduces it via redo (which is now a real, + * idempotent handler). WAL-before-op preserves write-ahead + * ordering; the commit record that follows orders this after + * the transaction's other WAL. Only commit-time execution is + * logged: an abort never reaches a PENDING_FILEOP_RENAME + * (rename is always at_commit). + */ + if (isCommit && XLogIsNeeded()) + { + xl_fileops_rename xlrec; + int oldpathlen = (int) strlen(pending->path) + 1; + int newpathlen = (int) strlen(pending->newpath) + 1; + + xlrec.oldpath_len = (uint16) oldpathlen; + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsRename); + XLogRegisterData(pending->path, oldpathlen); + XLogRegisterData(pending->newpath, newpathlen); + *max_lsn = XLogInsert(RM_FILEOPS_ID, XLOG_FILEOPS_RENAME); + + /* + * WAL-before-mutation durability: flush this record + * BEFORE touching the filesystem. A crash after the + * rename but before a batched flush would mutate the + * primary's FS without the record ever reaching the + * standby, diverging the two. Mirrors the create/write + * paths that flush inside their crit section before + * mutating. + */ + XLogFlush(*max_lsn); + } + (void) durable_rename(pending->path, pending->newpath, + WARNING); + break; + + case PENDING_FILEOP_DELETE: + + /* + * log-at-commit-execution, but ONLY for the commit path. A + * delete-on-abort registration (at_commit=false) executes + * here during abort; that must NOT be WAL-logged, because a + * standby must never apply an aborted transaction's + * filesystem effect. + */ + if (isCommit && XLogIsNeeded()) + { + xl_fileops_delete xlrec; + int pathlen = (int) strlen(pending->path) + 1; + + xlrec.at_commit = true; + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsDelete); + XLogRegisterData(pending->path, pathlen); + *max_lsn = XLogInsert(RM_FILEOPS_ID, XLOG_FILEOPS_DELETE); + + /* WAL-before-mutation durability (see RENAME above). */ + XLogFlush(*max_lsn); + } + if (unlink(pending->path) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove file \"%s\": %m", + pending->path))); + } + else + { + if (enableFsync) + fileops_fsync_parent(pending->path, WARNING); + } + break; + + case PENDING_FILEOP_RMDIR: + + /* + * log-at-commit-execution, commit path only (see the DELETE + * case above for the abort-path rationale). + */ + if (isCommit && XLogIsNeeded()) + { + xl_fileops_rmdir xlrec; + int pathlen = (int) strlen(pending->path) + 1; + + xlrec.at_commit = true; + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsRmdir); + XLogRegisterData(pending->path, pathlen); + *max_lsn = XLogInsert(RM_FILEOPS_ID, XLOG_FILEOPS_RMDIR); + + /* WAL-before-mutation durability (see RENAME above). */ + XLogFlush(*max_lsn); + } + if (rmdir(pending->path) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove directory \"%s\": %m", + pending->path))); + } + else + { + if (enableFsync) + fileops_fsync_parent(pending->path, WARNING); + } + break; + + case PENDING_FILEOP_RMTREE: + + /* + * Recursive removal. We defer the actual tree walk to + * execute time so that a copy operation that registered the + * destination tree at copy-start can have the abort path + * remove whatever ended up on disk by the time of abort. + * Enumerating at registration time would only see the empty + * destination directory created moments earlier. + */ + if (!rmtree(pending->path, true)) + ereport(WARNING, + (errmsg("could not remove directory tree \"%s\"", + pending->path))); + else if (enableFsync) + fileops_fsync_parent(pending->path, WARNING); + break; + + case PENDING_FILEOP_TRUNCATE: + { + /* + * Restore original file size on abort. The original + * length is stored in pending->length. + */ + int trunc_fd; + + trunc_fd = OpenTransientFile(pending->path, + O_RDWR | PG_BINARY); + if (trunc_fd >= 0) + { + if (ftruncate(trunc_fd, pending->length) < 0) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not restore file \"%s\" to original size %lld: %m", + pending->path, + (long long) pending->length))); + else if (enableFsync && pg_fsync(trunc_fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\" after restoring size: %m", + pending->path))); + CloseTransientFile(trunc_fd); + } + else + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not open file \"%s\" to restore size: %m", + pending->path))); + } + } + break; + + case PENDING_FILEOP_CHMOD: + { + /* + * Restore original file mode on abort. The original mode + * is stored in pending->length (cast from mode_t). + */ + mode_t restore_mode = (mode_t) pending->length; + + if (chmod(pending->path, restore_mode) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not restore file \"%s\" to original mode 0%o: %m", + pending->path, + (unsigned int) restore_mode))); + } + } + break; + + case PENDING_FILEOP_SETXATTR: + { + /* + * Restore an xattr to its original value on abort. path = + * file, newpath = attr name, data = orig value. + */ + if (pending->data && pending->data_len > 0) + { + if (pg_setxattr(pending->path, pending->newpath, + pending->data, pending->data_len) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not restore extended attribute \"%s\" on \"%s\": %m", + pending->newpath, + pending->path))); + } + } + else + { + /* Zero-length original value */ + if (pg_setxattr(pending->path, pending->newpath, + "", 0) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not restore extended attribute \"%s\" on \"%s\": %m", + pending->newpath, + pending->path))); + } + } + } + break; + + case PENDING_FILEOP_REMOVEXATTR: + { + /* + * Remove an xattr on abort (attribute was newly created + * in the aborted transaction). + */ + if (pg_removexattr(pending->path, pending->newpath) < 0) + { + /* + * "Attribute already absent" is success for a + * remove-on-abort (idempotent, like our unlink/rmdir + * redo). Use PG_ENOATTR, the portable + * absent-attribute errno: on Linux that is ENODATA, + * on macOS/BSD ENOATTR -- the raw ENODATA/ENOENT + * check missed the macOS case and warned spuriously + * (sec3 portability bug). + */ + if (errno != ENOENT && errno != PG_ENOATTR) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove extended attribute \"%s\" from \"%s\": %m", + pending->newpath, + pending->path))); + } + } + break; + + default: + break; + } + } +} + +/* + * AtSubCommit_FileOps - Handle subtransaction commit + * + * Reassign all pending ops from the current nesting level to the parent. + */ +void +AtSubCommit_FileOps(void) +{ + int nestLevel = GetCurrentTransactionNestLevel(); + PendingFileOp *pending; + + for (pending = pendingFileOps; pending != NULL; pending = pending->next) + { + if (pending->nestLevel >= nestLevel) + pending->nestLevel = nestLevel - 1; + } +} + +/* + * AtSubAbort_FileOps - Handle subtransaction abort + * + * Execute abort-time actions for the current nesting level immediately. + */ +void +AtSubAbort_FileOps(void) +{ + FileOpsDoPendingOps(false); +} + +/* + * PostPrepare_FileOps - Clean up after PREPARE TRANSACTION + * + * FileOpsAtPrepare() (called from EndPrepare, before the 2PC END sentinel) + * has already serialized every pending file operation into the two-phase + * state file, so this backend can now discard its in-memory list. The ops + * are performed later by the FILEOPS 2PC post-commit/post-abort callbacks + * (fileops_twophase_postcommit/postabort), possibly from a different backend + * after a crash, driven from the durable 2PC state. + */ +void +PostPrepare_FileOps(void) +{ + PendingFileOp *pending; + PendingFileOp *next; + + for (pending = pendingFileOps; pending != NULL; pending = next) + { + next = pending->next; + pendingFileOps = next; + FreePendingFileOp(pending); + } +} + +/* + * Serialized form of a PendingFileOp for the 2PC state file. + * + * A fixed header followed by three variable-length blobs (path, newpath, + * data), in that order. Self-describing so a crash-recovered COMMIT/ROLLBACK + * PREPARED -- run by any backend -- can rebuild an equivalent PendingFileOp + * and hand it to FileOpsExecOne(). Only the fields FileOpsExecOne() reads + * are carried; nestLevel is irrelevant once prepared. + */ +typedef struct FileOpsTwoPhaseRecord +{ + int type; /* PendingFileOpType */ + bool at_commit; /* run at COMMIT PREPARED vs ROLLBACK PREPARED */ + off_t length; /* truncate length / write offset / mode */ + uint32 path_len; /* bytes, including NUL, 0 if none */ + uint32 newpath_len; /* bytes, including NUL, 0 if none */ + uint32 data_len; /* bytes, 0 if none */ + /* followed by path, newpath, data */ +} FileOpsTwoPhaseRecord; + +/* + * FileOpsAtPrepare - Serialize deferred commit-time file ops into 2PC state + * + * Called from EndPrepare() between StartPrepare() and the END sentinel. Each + * pending at_commit=true op (RENAME/DELETE/RMDIR/RMTREE) is written as one + * TWOPHASE_RM_FILEOPS_ID record. This is what makes those deferred + * commit-time ops survive PREPARE for COMMIT PREPARED across a crash between + * PREPARE and the final decision. + * + * Abort-time ops (at_commit=false) are intentionally NOT serialized: they are + * the immediate constructive ops' undo, already covered for ROLLBACK PREPARED + * by the cluster-wide UNDO chain, which FinishPreparedTransaction() applies + * synchronously. + */ +void +FileOpsAtPrepare(void) +{ + PendingFileOp *pending; + + for (pending = pendingFileOps; pending != NULL; pending = pending->next) + { + FileOpsTwoPhaseRecord rec; + char *buf; + uint32 path_len; + uint32 newpath_len; + uint32 data_len; + uint32 total; + char *p; + + /* + * Only the deferred commit-time ops (at_commit=true: RENAME/DELETE/ + * RMDIR/RMTREE) need to be carried in the 2PC state: they have no + * other durable representation, so without this they were silently + * lost at PREPARE. The abort-time ops (at_commit=false, registered + * by the immediate constructive ops) are ALREADY covered for ROLLBACK + * PREPARED by the constructive op's cluster-wide UNDO record, which + * FinishPreparedTransaction() applies. Serializing them here too + * would revert them twice, so skip them. + */ + if (!pending->at_commit) + continue; + + path_len = pending->path ? (uint32) strlen(pending->path) + 1 : 0; + newpath_len = pending->newpath ? (uint32) strlen(pending->newpath) + 1 : 0; + data_len = (pending->data && pending->data_len > 0) ? + (uint32) pending->data_len : 0; + + rec.type = (int) pending->type; + rec.at_commit = pending->at_commit; + rec.length = pending->length; + rec.path_len = path_len; + rec.newpath_len = newpath_len; + rec.data_len = data_len; + + total = sizeof(FileOpsTwoPhaseRecord) + path_len + newpath_len + data_len; + buf = palloc(total); + memcpy(buf, &rec, sizeof(FileOpsTwoPhaseRecord)); + p = buf + sizeof(FileOpsTwoPhaseRecord); + if (path_len > 0) + { + memcpy(p, pending->path, path_len); + p += path_len; + } + if (newpath_len > 0) + { + memcpy(p, pending->newpath, newpath_len); + p += newpath_len; + } + if (data_len > 0) + memcpy(p, pending->data, data_len); + + RegisterTwoPhaseRecord(TWOPHASE_RM_FILEOPS_ID, 0, buf, total); + pfree(buf); + } +} + +/* + * fileops_twophase_do - Shared body for the FILEOPS 2PC callbacks + * + * Rebuild a PendingFileOp from the serialized 2PC record and, if it matches + * the transaction outcome, perform it exactly as FileOpsDoPendingOps() would. + * A commit-time op emits its own WAL (idempotent redo on a standby); the + * trailing flush makes any accumulated LSN durable. + */ +static void +fileops_twophase_do(bool isCommit, void *recdata, uint32 len) +{ + FileOpsTwoPhaseRecord rec; + PendingFileOp pending; + const char *p; + XLogRecPtr max_lsn = InvalidXLogRecPtr; + + if (len < sizeof(FileOpsTwoPhaseRecord)) + { + elog(WARNING, "FILEOPS 2PC record too short (%u bytes)", len); + return; + } + memcpy(&rec, recdata, sizeof(FileOpsTwoPhaseRecord)); + + /* Only ops matching this outcome run; the rest are simply discarded. */ + if (rec.at_commit != isCommit) + return; + + memset(&pending, 0, sizeof(pending)); + pending.type = (PendingFileOpType) rec.type; + pending.at_commit = rec.at_commit; + pending.length = rec.length; + + p = (const char *) recdata + sizeof(FileOpsTwoPhaseRecord); + if (rec.path_len > 0) + { + pending.path = (char *) p; + p += rec.path_len; + } + if (rec.newpath_len > 0) + { + pending.newpath = (char *) p; + p += rec.newpath_len; + } + if (rec.data_len > 0) + { + pending.data = (void *) p; + pending.data_len = rec.data_len; + } + + FileOpsExecOne(&pending, isCommit, &max_lsn); + + if (!XLogRecPtrIsInvalid(max_lsn)) + XLogFlush(max_lsn); +} + +/* + * fileops_twophase_postcommit - COMMIT PREPARED callback (perform at_commit ops) + */ +void +fileops_twophase_postcommit(FullTransactionId fxid, uint16 info, + void *recdata, uint32 len) +{ + fileops_twophase_do(true, recdata, len); +} + +/* + * fileops_twophase_postabort - ROLLBACK PREPARED callback + * + * A no-op in practice: only at_commit=true ops are serialized at PREPARE, and + * those are not performed on abort. Kept for symmetry and to defensively + * discard any at_commit=true record on the abort path. + */ +void +fileops_twophase_postabort(FullTransactionId fxid, uint16 info, + void *recdata, uint32 len) +{ + fileops_twophase_do(false, recdata, len); +} + +/* + * fileops_redo - WAL redo function for FILEOPS records + * + * Replay file operations during crash recovery or standby apply. + * Each operation type has its own redo handler added in separate commits. + */ +void +fileops_redo(XLogReaderState *record) +{ + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + char *data = XLogRecGetData(record); + + switch (info) + { + case XLOG_FILEOPS_CREATE: + { + xl_fileops_create *xlrec = (xl_fileops_create *) data; + const char *path = data + SizeOfFileOpsCreate; + int fd; + + fd = BasicOpenFilePerm(path, + (xlrec->flags & ~PG_O_DIRECT) | O_CREAT, + xlrec->mode); + if (fd < 0) + { + if (errno == ENOENT) + { + char parentpath[MAXPGPATH]; + char *sep; + + strlcpy(parentpath, path, MAXPGPATH); + sep = strrchr(parentpath, '/'); + if (sep != NULL) + { + *sep = '\0'; + if (MakePGDirectory(parentpath) < 0 && + errno != EEXIST) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\" during WAL replay: %m", + parentpath))); + } + + fd = BasicOpenFilePerm(path, + (xlrec->flags & ~PG_O_DIRECT) | O_CREAT, + xlrec->mode); + } + + if (fd < 0 && errno != EEXIST) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not create file \"%s\" during WAL replay: %m", + path))); + } + + if (fd >= 0) + { + if (enableFsync && pg_fsync(fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\" during WAL replay: %m", + path))); + close(fd); + if (enableFsync) + fileops_fsync_parent(path, data_sync_elevel(ERROR)); + } + } + break; + + case XLOG_FILEOPS_WRITE: + { + xl_fileops_write *xlrec = (xl_fileops_write *) data; + const char *path = data + SizeOfFileOpsWrite; + const char *wdata = path + xlrec->path_len; + int fd; + + fd = BasicOpenFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not open file \"%s\" for write during WAL replay: %m", + path))); + } + else + { + if (pg_pwrite(fd, wdata, xlrec->len, xlrec->offset) != + (ssize_t) xlrec->len) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not write to file \"%s\" during WAL replay: %m", + path))); + else if (enableFsync && pg_fsync(fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\" during WAL replay: %m", + path))); + close(fd); + } + } + break; + + case XLOG_FILEOPS_RENAME: + + /* + * idempotent redo of a deferred rename executed at commit on the + * primary. The record carries oldpath then newpath. A standby + * (or a re-replay after a primary crash mid-commit) may find the + * rename already applied: source gone. In that case treat it as + * success rather than failing on a missing source -- this is what + * makes redo converge whether the crash happened before or after + * the physical rename. + */ + { + xl_fileops_rename *xlrec = (xl_fileops_rename *) data; + const char *oldpath = data + SizeOfFileOpsRename; + const char *newpath = oldpath + xlrec->oldpath_len; + + if (rename(oldpath, newpath) < 0) + { + /* + * ENOENT on the source means the rename already took + * effect (or the source never existed on this standby); + * either way the desired end state -- newpath present, + * oldpath absent -- is what we want, so it is not an + * error. + */ + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not rename file \"%s\" to \"%s\" during WAL replay: %m", + oldpath, newpath))); + } + else if (enableFsync) + { + fileops_fsync_parent(newpath, data_sync_elevel(ERROR)); + fileops_fsync_parent(oldpath, data_sync_elevel(ERROR)); + } + } + break; + + case XLOG_FILEOPS_DELETE: + + /* + * idempotent redo of a deferred delete executed at commit on the + * primary. An already-absent file (ENOENT) means the delete + * already took effect -- treat as success. Only commit-path + * deletes are ever logged (see FileOpsDoPendingOps), so redo of + * an aborted transaction's delete-on-abort can never occur here. + */ + { + xl_fileops_delete *xlrec pg_attribute_unused() = + (xl_fileops_delete *) data; + const char *path = data + SizeOfFileOpsDelete; + + if (unlink(path) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove file \"%s\" during WAL replay: %m", + path))); + } + else if (enableFsync) + fileops_fsync_parent(path, data_sync_elevel(ERROR)); + } + break; + + case XLOG_FILEOPS_SYMLINK: + + /* + * sec2: intentional no-op during redo. + * + * FileOpsSymlink() has exactly one caller, + * create_tablespace_directories() in tablespace.c, and it is used + * ONLY for tablespace symlinks under $PGDATA/pg_tblspc. Standby + * (and primary crash-recovery) replay of those symlinks is OWNED + * by RM_TBLSPC: tblspc_redo() -> create_tablespace_directories() + * runs with InRecovery set, first removing any stale symlink + * (remove_tablespace_symlink) and then re-creating it with the + * recovery-time layout logic that honors the standby's own + * tablespace layout / allow_in_place_tablespaces / tablespace_map + * remap. + * + * Replaying the FILEOPS record here would be redundant (RM_TBLSPC + * already creates the link) and, worse, actively wrong on a + * remapped standby: it would unlink the correctly-remapped link + * and re-point it at the primary's LITERAL absolute target, which + * may not exist or may belong to a different location on the + * standby. So FILEOPS defers tablespace-symlink replay to + * RM_TBLSPC and does nothing here. + * + * If FileOpsSymlink ever gains a non-tablespace caller, this must + * be revisited to replay those (and only those) symlinks; that + * would require distinguishing the two cases in the record. + */ + break; + + case XLOG_FILEOPS_LINK: + { + xl_fileops_link *xlrec = (xl_fileops_link *) data; + const char *oldpath = data + SizeOfFileOpsLink; + const char *newpath = oldpath + xlrec->oldpath_len; + + /* Remove existing link first for idempotent redo */ + unlink(newpath); + if (link(oldpath, newpath) < 0 && errno != EEXIST) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not create hard link \"%s\" during WAL replay: %m", + newpath))); + else if (enableFsync) + fileops_fsync_parent(newpath, data_sync_elevel(ERROR)); + } + break; + + case XLOG_FILEOPS_MKDIR: + { + xl_fileops_mkdir *xlrec pg_attribute_unused() = + (xl_fileops_mkdir *) data; + const char *path = data + SizeOfFileOpsMkdir; + + if (MakePGDirectory(path) < 0 && errno != EEXIST) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\" during WAL replay: %m", + path))); + else if (enableFsync) + fileops_fsync_parent(path, data_sync_elevel(ERROR)); + } + break; + + case XLOG_FILEOPS_RMDIR: + + /* + * idempotent redo of a deferred rmdir executed at commit on the + * primary. An already-absent directory (ENOENT) means the rmdir + * already took effect -- treat as success. + */ + { + xl_fileops_rmdir *xlrec pg_attribute_unused() = + (xl_fileops_rmdir *) data; + const char *path = data + SizeOfFileOpsRmdir; + + if (rmdir(path) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove directory \"%s\" during WAL replay: %m", + path))); + } + else if (enableFsync) + fileops_fsync_parent(path, data_sync_elevel(ERROR)); + } + break; + + case XLOG_FILEOPS_CHMOD: + { + xl_fileops_chmod *xlrec = (xl_fileops_chmod *) data; + const char *path = data + SizeOfFileOpsChmod; + + if (chmod(path, xlrec->mode) < 0 && errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not chmod file \"%s\" during WAL replay: %m", + path))); + } + break; + + case XLOG_FILEOPS_CHOWN: + { +#ifndef WIN32 + xl_fileops_chown *xlrec = (xl_fileops_chown *) data; + const char *path = data + SizeOfFileOpsChown; + + if (chown(path, xlrec->uid, xlrec->gid) < 0 && + errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not chown file \"%s\" during WAL replay: %m", + path))); +#endif + } + break; + + case XLOG_FILEOPS_TRUNCATE: + { + xl_fileops_truncate *xlrec = (xl_fileops_truncate *) data; + const char *path = data + SizeOfFileOpsTruncate; + int fd; + + XLogFlush(record->EndRecPtr); + + fd = BasicOpenFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not open file \"%s\" for truncation during WAL replay: %m", + path))); + } + else + { + if (ftruncate(fd, xlrec->length) < 0) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not truncate file \"%s\" to %lld bytes during WAL replay: %m", + path, (long long) xlrec->length))); + else if (enableFsync && pg_fsync(fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\" during WAL replay: %m", + path))); + close(fd); + } + } + break; + + case XLOG_FILEOPS_SETXATTR: + { + xl_fileops_setxattr *xlrec = (xl_fileops_setxattr *) data; + const char *path = data + SizeOfFileOpsSetxattr; + const char *name = path + xlrec->path_len; + const void *value = name + xlrec->name_len; + + if (pg_setxattr(path, name, value, xlrec->value_len) < 0 && + errno != ENOTSUP && errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not set extended attribute \"%s\" on \"%s\" during WAL replay: %m", + name, path))); + } + break; + + case XLOG_FILEOPS_REMOVEXATTR: + { + xl_fileops_removexattr *xlrec = + (xl_fileops_removexattr *) data; + const char *path = data + SizeOfFileOpsRemovexattr; + const char *name = path + xlrec->path_len; + + if (pg_removexattr(path, name) < 0 && + errno != ENOTSUP && errno != ENOENT && + errno != PG_ENOATTR) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove extended attribute \"%s\" from \"%s\" during WAL replay: %m", + name, path))); + } + break; + + default: + elog(PANIC, "fileops_redo: unknown op code %u", info); + break; + } +} diff --git a/src/backend/storage/file/fileops_undo.c b/src/backend/storage/file/fileops_undo.c new file mode 100644 index 00000000000..3afdb0b03a8 --- /dev/null +++ b/src/backend/storage/file/fileops_undo.c @@ -0,0 +1,431 @@ +/*------------------------------------------------------------------------- + * + * fileops_undo.c + * FILEOPS UNDO resource manager + * + * This module implements the UNDO apply callbacks for transactional file + * operations. UNDO covers structural filesystem operations (file existence, + * permissions, directory structure). File content recovery uses traditional + * WAL (via FileOpsWrite), not FILEOPS UNDO. + * + * On transaction abort, each FILEOPS UNDO record reverses the structural + * change made by the corresponding FileOps* function: + * - CREATE: unlink the created file + * - RENAME: rename back to the original path + * - TRUNCATE: ftruncate to original length + * - CHMOD: chmod to original mode + * - CHOWN: chown to original uid/gid + * - MKDIR: rmdir the created directory + * - SYMLINK: unlink the symlink + * - LINK: unlink the hard link + * - SETXATTR: restore/remove the original xattr + * - REMOVEXATTR: restore the original xattr value + * + * DELETE and RMDIR are handled by PendingFileOps (deferred execution) + * and do not need UNDO records since they execute only at commit time. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/storage/file/fileops_undo.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include + +#include "access/undormgr.h" +#include "access/xact.h" +#include "port/pg_xattr.h" +#include "storage/fd.h" +#include "storage/fileops.h" +#include "utils/injection_point.h" + +/* + * UNDO subtype constants and payload structures are defined in fileops.h, + * shared between this file (apply side) and fileops.c (insert side). + */ + +/* Forward declarations */ +static UndoApplyResult fileops_undo_apply(uint8 rmid, uint16 info, + TransactionId xid, Oid reloid, + const char *payload, Size payload_len, + UndoRecPtr urec_ptr); +static void fileops_undo_desc(StringInfo buf, uint8 rmid, uint16 info, + const char *payload, Size payload_len); + +/* + * Report an xattr restore that failed, and return UNDO_APPLY_ERROR so the + * caller counts it as a record that could not be applied. Like the other + * subtypes, a file that no longer exists is not a failure. + */ +static UndoApplyResult +fileops_undo_xattr_failed(const char *op, const char *path, const char *name) +{ + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO %s: could not restore attribute \"%s\" of \"%s\": %m", + op, name, path))); + return UNDO_APPLY_ERROR; +} + +/* The FILEOPS UNDO RM registration entry */ +static const UndoRmgrData fileops_undo_rmgr = { + .rm_name = "fileops", + .rm_undo = fileops_undo_apply, + .rm_desc = fileops_undo_desc, +}; + +/* + * FileopsUndoRmgrInit - Register the FILEOPS UNDO resource manager + */ +void +FileopsUndoRmgrInit(void) +{ + RegisterUndoRmgr(UNDO_RMID_FILEOPS, &fileops_undo_rmgr); + + /* + * Wire FILEOPS's pending-structural-file-operation callbacks into the + * AM-neutral hooks in xact.c (see access/xact.h for why xact.c cannot + * call these by name directly). + */ + PendingPhysOpsDo_hook = FileOpsDoPendingOps; + PendingPhysOpsPostPrepare_hook = PostPrepare_FileOps; + PendingPhysOpsAtSubCommit_hook = AtSubCommit_FileOps; + PendingPhysOpsAtSubAbort_hook = AtSubAbort_FileOps; +} + +/* + * fileops_undo_apply - Apply a single FILEOPS UNDO record + * + * Reverses the structural filesystem operation. + */ +static UndoApplyResult +fileops_undo_apply(uint8 rmid, uint16 info, TransactionId xid, Oid reloid, + const char *payload, Size payload_len, UndoRecPtr urec_ptr) +{ + UndoApplyResult result = UNDO_APPLY_SUCCESS; + + Assert(rmid == UNDO_RMID_FILEOPS); + + INJECTION_POINT("fileops-undo-apply-begin", NULL); + + switch (info) + { + case FILEOPS_UNDO_CREATE: + { + FileopsUndoCreate hdr; + const char *path; + + if (payload_len < sizeof(FileopsUndoCreate)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoCreate)); + path = payload + sizeof(FileopsUndoCreate); + + if (unlink(path) < 0 && errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO CREATE: could not unlink \"%s\": %m", + path))); + result = UNDO_APPLY_ERROR; + } + else + ereport(DEBUG2, + (errmsg("FILEOPS UNDO CREATE: unlinked \"%s\"", + path))); + } + break; + + case FILEOPS_UNDO_RENAME: + { + FileopsUndoRename hdr; + const char *oldpath; + const char *newpath; + + if (payload_len < sizeof(FileopsUndoRename)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoRename)); + oldpath = payload + sizeof(FileopsUndoRename); + newpath = oldpath + hdr.oldpath_len; + + /* Reverse the rename: newpath -> oldpath */ + if (rename(newpath, oldpath) < 0) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO RENAME: could not rename \"%s\" back to \"%s\": %m", + newpath, oldpath))); + result = UNDO_APPLY_ERROR; + } + else + ereport(DEBUG2, + (errmsg("FILEOPS UNDO RENAME: renamed \"%s\" back to \"%s\"", + newpath, oldpath))); + } + break; + + case FILEOPS_UNDO_TRUNCATE: + { + FileopsUndoTruncate hdr; + const char *path; + int fd; + + if (payload_len < sizeof(FileopsUndoTruncate)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoTruncate)); + path = payload + sizeof(FileopsUndoTruncate); + + fd = BasicOpenFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + { + if (errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO TRUNCATE: could not open \"%s\": %m", + path))); + result = UNDO_APPLY_ERROR; + } + } + else + { + if (ftruncate(fd, hdr.orig_length) < 0) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO TRUNCATE: could not restore length of \"%s\": %m", + path))); + result = UNDO_APPLY_ERROR; + } + close(fd); + } + } + break; + + case FILEOPS_UNDO_CHMOD: + { + FileopsUndoChmod hdr; + const char *path; + + if (payload_len < sizeof(FileopsUndoChmod)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoChmod)); + path = payload + sizeof(FileopsUndoChmod); + + if (chmod(path, hdr.orig_mode) < 0 && errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO CHMOD: could not restore mode of \"%s\": %m", + path))); + result = UNDO_APPLY_ERROR; + } + } + break; + + case FILEOPS_UNDO_CHOWN: + { +#ifndef WIN32 + FileopsUndoChown hdr; + const char *path; + + if (payload_len < sizeof(FileopsUndoChown)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoChown)); + path = payload + sizeof(FileopsUndoChown); + + if (chown(path, hdr.orig_uid, hdr.orig_gid) < 0 && + errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO CHOWN: could not restore ownership of \"%s\": %m", + path))); + result = UNDO_APPLY_ERROR; + } +#endif + } + break; + + case FILEOPS_UNDO_MKDIR: + { + FileopsUndoMkdir hdr; + const char *path; + + if (payload_len < sizeof(FileopsUndoMkdir)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoMkdir)); + path = payload + sizeof(FileopsUndoMkdir); + + if (rmdir(path) < 0 && errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO MKDIR: could not rmdir \"%s\": %m", + path))); + result = UNDO_APPLY_ERROR; + } + } + break; + + case FILEOPS_UNDO_SYMLINK: + { + FileopsUndoSymlink hdr; + const char *linkpath; + + if (payload_len < sizeof(FileopsUndoSymlink)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoSymlink)); + linkpath = payload + sizeof(FileopsUndoSymlink); + + if (unlink(linkpath) < 0 && errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO SYMLINK: could not unlink \"%s\": %m", + linkpath))); + result = UNDO_APPLY_ERROR; + } + } + break; + + case FILEOPS_UNDO_LINK: + { + FileopsUndoLink hdr; + const char *newpath; + + if (payload_len < sizeof(FileopsUndoLink)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoLink)); + newpath = payload + sizeof(FileopsUndoLink); + + if (unlink(newpath) < 0 && errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO LINK: could not unlink \"%s\": %m", + newpath))); + result = UNDO_APPLY_ERROR; + } + } + break; + + case FILEOPS_UNDO_SETXATTR: + { + FileopsUndoSetxattr hdr; + const char *path; + const char *name; + + if (payload_len < sizeof(FileopsUndoSetxattr)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoSetxattr)); + path = payload + sizeof(FileopsUndoSetxattr); + name = path + hdr.path_len; + + if (hdr.had_value) + { + /* Restore original value */ + const void *orig_value = name + hdr.name_len; + + if (pg_setxattr(path, name, orig_value, + hdr.orig_value_len) < 0 && + errno != ENOENT) + result = fileops_undo_xattr_failed("SETXATTR", path, + name); + } + else + { + /* xattr didn't exist before, remove it */ + if (pg_removexattr(path, name) < 0 && + errno != ENOENT && errno != PG_ENOATTR) + result = fileops_undo_xattr_failed("SETXATTR", path, + name); + } + } + break; + + case FILEOPS_UNDO_REMOVEXATTR: + { + FileopsUndoRemovexattr hdr; + const char *path; + const char *name; + const void *value; + + if (payload_len < sizeof(FileopsUndoRemovexattr)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoRemovexattr)); + path = payload + sizeof(FileopsUndoRemovexattr); + name = path + hdr.path_len; + value = name + hdr.name_len; + + /* Restore the removed xattr */ + if (pg_setxattr(path, name, value, hdr.value_len) < 0 && + errno != ENOENT) + result = fileops_undo_xattr_failed("REMOVEXATTR", path, + name); + } + break; + + default: + ereport(WARNING, + (errmsg("FILEOPS UNDO: unknown subtype %u", info))); + return UNDO_APPLY_ERROR; + } + + INJECTION_POINT("fileops-undo-apply-end", NULL); + + return result; +} + +/* + * fileops_undo_desc - Describe a FILEOPS UNDO record for debugging + */ +static void +fileops_undo_desc(StringInfo buf, uint8 rmid, uint16 info, + const char *payload, Size payload_len) +{ + const char *opname; + + switch (info) + { + case FILEOPS_UNDO_CREATE: + opname = "CREATE"; + break; + case FILEOPS_UNDO_RENAME: + opname = "RENAME"; + break; + case FILEOPS_UNDO_TRUNCATE: + opname = "TRUNCATE"; + break; + case FILEOPS_UNDO_CHMOD: + opname = "CHMOD"; + break; + case FILEOPS_UNDO_CHOWN: + opname = "CHOWN"; + break; + case FILEOPS_UNDO_MKDIR: + opname = "MKDIR"; + break; + case FILEOPS_UNDO_SYMLINK: + opname = "SYMLINK"; + break; + case FILEOPS_UNDO_LINK: + opname = "LINK"; + break; + case FILEOPS_UNDO_SETXATTR: + opname = "SETXATTR"; + break; + case FILEOPS_UNDO_REMOVEXATTR: + opname = "REMOVEXATTR"; + break; + default: + opname = "UNKNOWN"; + break; + } + + appendStringInfo(buf, "fileops %s", opname); +} diff --git a/src/backend/storage/file/meson.build b/src/backend/storage/file/meson.build index 795402589b0..7e487324c5f 100644 --- a/src/backend/storage/file/meson.build +++ b/src/backend/storage/file/meson.build @@ -4,6 +4,8 @@ backend_sources += files( 'buffile.c', 'copydir.c', 'fd.c', + 'fileops.c', + 'fileops_undo.c', 'fileset.c', 'reinit.c', 'sharedfileset.c', diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c index 8570f17916f..ed27a243feb 100644 --- a/src/bin/pg_waldump/rmgrdesc.c +++ b/src/bin/pg_waldump/rmgrdesc.c @@ -17,6 +17,7 @@ #include "access/hash_xlog.h" #include "access/heapam_xlog.h" #include "access/multixact.h" +#include "access/fileops_xlog.h" #include "access/nbtxlog.h" #include "access/rmgr.h" #include "access/spgxlog.h" diff --git a/src/bin/pg_waldump/t/001_basic.pl b/src/bin/pg_waldump/t/001_basic.pl index fbeb19ee4a4..ded5e37feda 100644 --- a/src/bin/pg_waldump/t/001_basic.pl +++ b/src/bin/pg_waldump/t/001_basic.pl @@ -97,7 +97,8 @@ ReplicationOrigin Generic LogicalMessage XLOG2 -Undo$/, +Undo +FileOps$/, 'rmgr list'); diff --git a/src/include/access/fileops_xlog.h b/src/include/access/fileops_xlog.h new file mode 100644 index 00000000000..b3ece9a2686 --- /dev/null +++ b/src/include/access/fileops_xlog.h @@ -0,0 +1,40 @@ +/*------------------------------------------------------------------------- + * + * fileops_xlog.h + * Transactional file operations XLOG resource manager definitions + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/include/access/fileops_xlog.h + * + *------------------------------------------------------------------------- + */ +#ifndef FILEOPS_XLOG_H +#define FILEOPS_XLOG_H + +#include "access/xlogreader.h" +#include "lib/stringinfo.h" + +/* XLOG stuff - all record types defined upfront for WAL compatibility */ +#define XLOG_FILEOPS_CREATE 0x00 +#define XLOG_FILEOPS_DELETE 0x10 +#define XLOG_FILEOPS_RENAME 0x20 +#define XLOG_FILEOPS_WRITE 0x30 +#define XLOG_FILEOPS_TRUNCATE 0x40 +#define XLOG_FILEOPS_CHMOD 0x50 +#define XLOG_FILEOPS_CHOWN 0x60 +#define XLOG_FILEOPS_MKDIR 0x70 +#define XLOG_FILEOPS_RMDIR 0x80 +#define XLOG_FILEOPS_SYMLINK 0x90 +#define XLOG_FILEOPS_LINK 0xA0 +#define XLOG_FILEOPS_SETXATTR 0xB0 +#define XLOG_FILEOPS_REMOVEXATTR 0xC0 + +/* Resource manager functions */ +extern void fileops_redo(XLogReaderState *record); +extern void fileops_desc(StringInfo buf, XLogReaderState *record); +extern const char *fileops_identify(uint8 info); + +#endif /* FILEOPS_XLOG_H */ diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h index 046ba45cb25..a34394a1e23 100644 --- a/src/include/access/rmgrlist.h +++ b/src/include/access/rmgrlist.h @@ -49,3 +49,6 @@ PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL, logicalmsg_decode) PG_RMGR(RM_XLOG2_ID, "XLOG2", xlog2_redo, xlog2_desc, xlog2_identify, NULL, NULL, NULL, xlog2_decode) PG_RMGR(RM_UNDO_ID, "Undo", undo_redo, undo_desc, undo_identify, NULL, NULL, NULL, NULL) +#ifdef USE_FILEOPS +PG_RMGR(RM_FILEOPS_ID, "FileOps", fileops_redo, fileops_desc, fileops_identify, NULL, NULL, NULL, NULL) +#endif diff --git a/src/include/access/twophase_rmgr.h b/src/include/access/twophase_rmgr.h index 8927f369c39..37be3c7b290 100644 --- a/src/include/access/twophase_rmgr.h +++ b/src/include/access/twophase_rmgr.h @@ -28,7 +28,8 @@ typedef uint8 TwoPhaseRmgrId; #define TWOPHASE_RM_PGSTAT_ID 2 #define TWOPHASE_RM_MULTIXACT_ID 3 #define TWOPHASE_RM_PREDICATELOCK_ID 4 -#define TWOPHASE_RM_MAX_ID TWOPHASE_RM_PREDICATELOCK_ID +#define TWOPHASE_RM_FILEOPS_ID 5 +#define TWOPHASE_RM_MAX_ID TWOPHASE_RM_FILEOPS_ID extern PGDLLIMPORT const TwoPhaseCallback twophase_recover_callbacks[]; extern PGDLLIMPORT const TwoPhaseCallback twophase_postcommit_callbacks[]; @@ -39,4 +40,10 @@ extern PGDLLIMPORT const TwoPhaseCallback twophase_standby_recover_callbacks[]; extern void RegisterTwoPhaseRecord(TwoPhaseRmgrId rmid, uint16 info, const void *data, uint32 len); +/* FILEOPS two-phase commit callbacks (defined in storage/file/fileops.c) */ +extern void fileops_twophase_postcommit(FullTransactionId fxid, uint16 info, + void *recdata, uint32 len); +extern void fileops_twophase_postabort(FullTransactionId fxid, uint16 info, + void *recdata, uint32 len); + #endif /* TWOPHASE_RMGR_H */ diff --git a/src/include/access/undormgrlist.h b/src/include/access/undormgrlist.h index 4ab60e21418..a9f8ec5afd7 100644 --- a/src/include/access/undormgrlist.h +++ b/src/include/access/undormgrlist.h @@ -31,3 +31,6 @@ */ /* there is deliberately not an #ifndef UNDORMGRLIST_H here */ + +/* FILEOPS: transactional filesystem operations UNDO rmgr (UNDO_RMID_FILEOPS=2) */ +UNDO_RMGR_INIT(FileopsUndoRmgrInit) diff --git a/src/include/port/pg_xattr.h b/src/include/port/pg_xattr.h new file mode 100644 index 00000000000..7159fc27a26 --- /dev/null +++ b/src/include/port/pg_xattr.h @@ -0,0 +1,40 @@ +/*------------------------------------------------------------------------- + * + * pg_xattr.h + * Cross-platform extended attribute abstraction + * + * Provides pg_setxattr() and pg_removexattr() that work across: + * - Linux: setxattr/removexattr + * - macOS: setxattr/removexattr (extra options param) + * - FreeBSD: extattr_set_file/extattr_delete_file + * - Windows: NTFS Alternate Data Streams + * - Fallback: returns ENOTSUP with WARNING + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * + * src/include/port/pg_xattr.h + * + *------------------------------------------------------------------------- + */ +#ifndef PG_XATTR_H +#define PG_XATTR_H + +/* + * ENODATA is Linux-specific. FreeBSD/macOS use ENOATTR for "attribute not + * found". Provide a portable PG_ENOATTR so callers don't need #ifdefs. + */ +#if defined(ENOATTR) +#define PG_ENOATTR ENOATTR +#elif defined(ENODATA) +#define PG_ENOATTR ENODATA +#else +#define PG_ENOATTR ENOENT /* last-resort fallback */ +#endif + +extern int pg_setxattr(const char *path, const char *name, + const void *value, size_t size); +extern ssize_t pg_getxattr(const char *path, const char *name, + void *value, size_t size); +extern int pg_removexattr(const char *path, const char *name); + +#endif /* PG_XATTR_H */ diff --git a/src/include/storage/fileops.h b/src/include/storage/fileops.h new file mode 100644 index 00000000000..e8ab366acd5 --- /dev/null +++ b/src/include/storage/fileops.h @@ -0,0 +1,463 @@ +/*------------------------------------------------------------------------- + * + * fileops.h + * Transactional file operations API + * + * This module provides transactional filesystem operations that are + * WAL-logged and integrated with PostgreSQL's transaction management. + * File operations are deferred until transaction commit/abort, ensuring + * atomicity with the rest of the transaction. + * + * The RM_FILEOPS_ID resource manager handles WAL replay for these + * operations, ensuring correct behavior during crash recovery and + * standby replay. + * + * The operation set follows the Berkeley DB fileops.src model: each + * filesystem operation is a composable unit with its own WAL record + * type, redo handler, and descriptor. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/storage/fileops.h + * + *------------------------------------------------------------------------- + */ +#ifndef FILEOPS_H +#define FILEOPS_H + +#include "access/xlogreader.h" +#include "lib/stringinfo.h" + +/* + * WAL record types for FILEOPS operations. + * + * The high 4 bits of the info byte are used for record type, + * leaving the low bits for flags (following PostgreSQL convention). + * + * Following the Berkeley DB fileops.src model, each filesystem + * operation has its own WAL record type for independent redo. + */ +#define XLOG_FILEOPS_CREATE 0x00 +#define XLOG_FILEOPS_DELETE 0x10 +#define XLOG_FILEOPS_RENAME 0x20 +#define XLOG_FILEOPS_WRITE 0x30 +#define XLOG_FILEOPS_TRUNCATE 0x40 +#define XLOG_FILEOPS_CHMOD 0x50 +#define XLOG_FILEOPS_CHOWN 0x60 +#define XLOG_FILEOPS_MKDIR 0x70 +#define XLOG_FILEOPS_RMDIR 0x80 +#define XLOG_FILEOPS_SYMLINK 0x90 +#define XLOG_FILEOPS_LINK 0xA0 +#define XLOG_FILEOPS_SETXATTR 0xB0 +#define XLOG_FILEOPS_REMOVEXATTR 0xC0 + +/* + * PendingFileOp - Deferred file operation entry + * + * File operations are collected in a linked list during a transaction + * and executed at commit or abort time. This follows the same pattern + * used by PendingRelDelete in catalog/storage.c. + */ +typedef enum PendingFileOpType +{ + PENDING_FILEOP_CREATE, + PENDING_FILEOP_DELETE, + PENDING_FILEOP_RENAME, + PENDING_FILEOP_WRITE, + PENDING_FILEOP_TRUNCATE, + PENDING_FILEOP_CHMOD, + PENDING_FILEOP_CHOWN, + PENDING_FILEOP_MKDIR, + PENDING_FILEOP_RMDIR, + PENDING_FILEOP_RMTREE, /* recursive removal at execute time */ + PENDING_FILEOP_SYMLINK, + PENDING_FILEOP_LINK, + PENDING_FILEOP_SETXATTR, + PENDING_FILEOP_REMOVEXATTR +} PendingFileOpType; + +typedef struct PendingFileOp +{ + PendingFileOpType type; /* operation type */ + char *path; /* primary file path */ + char *newpath; /* new path (RENAME/SYMLINK/LINK), or xattr + * name */ + off_t length; /* truncation length, write offset, or mode */ + void *data; /* generic data (e.g., original xattr value) */ + size_t data_len; /* length of data */ + bool at_commit; /* execute at commit (true) or abort (false) */ + int nestLevel; /* transaction nesting level */ + struct PendingFileOp *next; /* linked list link */ +} PendingFileOp; + +/* + * Public API for transactional file operations + * + * These functions handle platform-specific differences automatically: + * - O_DIRECT: PG_O_DIRECT (Linux/FreeBSD native, macOS F_NOCACHE, + * Windows FILE_FLAG_NO_BUFFERING) + * - fsync: pg_fsync() (Linux fdatasync, macOS F_FULLFSYNC, + * BSD fsync, Windows FlushFileBuffers) + * - Directory sync: fsync_parent_path() (Unix only, no-op on Windows) + * - Durable ops: durable_rename()/durable_unlink() with proper + * fsync ordering for crash safety + * + * Operation-specific API functions are declared below their WAL + * record structures in subsequent sections. + */ + +/* Utility functions */ +extern void FileOpsCancelPendingDelete(const char *path, bool at_commit); +extern void FileOpsSync(const char *path); + +/* Transaction lifecycle hooks */ +extern void FileOpsDoPendingOps(bool isCommit); +extern void AtSubCommit_FileOps(void); +extern void AtSubAbort_FileOps(void); +extern void PostPrepare_FileOps(void); +extern void FileOpsAtPrepare(void); + +/* + * xl_fileops_create - WAL record for file creation + * + * Records that a file was created within a transaction. If the transaction + * aborts, the file will be deleted. The path is stored as variable-length + * data following the fixed header. + */ +typedef struct xl_fileops_create +{ + int flags; /* open flags used for creation */ + mode_t mode; /* file permission mode */ + bool register_delete; /* register for delete-on-abort */ + /* variable-length path follows */ +} xl_fileops_create; + +#define SizeOfFileOpsCreate (offsetof(xl_fileops_create, register_delete) + sizeof(bool)) + +/* File creation API */ +extern int FileOpsCreate(const char *path, int flags, mode_t mode, + bool register_delete); + +/* + * xl_fileops_delete - WAL record for file deletion + * + * Records that a file deletion was requested. The at_commit flag indicates + * whether the deletion should happen at commit (true) or was registered + * as a delete-on-abort from a prior create (false). + */ +typedef struct xl_fileops_delete +{ + bool at_commit; /* true = delete at commit, false = at abort */ + /* variable-length path follows */ +} xl_fileops_delete; + +#define SizeOfFileOpsDelete (offsetof(xl_fileops_delete, at_commit) + sizeof(bool)) + +/* File deletion API */ +extern void FileOpsDelete(const char *path, bool at_commit); + +/* + * xl_fileops_rename - WAL record for file rename + * + * Records that a file was renamed. Both old and new paths are stored + * as variable-length data: oldpath_len bytes of old path, then the + * new path follows. + */ +typedef struct xl_fileops_rename +{ + uint16 oldpath_len; /* length of old path (including NUL) */ + /* variable-length old path follows, then new path */ +} xl_fileops_rename; + +#define SizeOfFileOpsRename (offsetof(xl_fileops_rename, oldpath_len) + sizeof(uint16)) + +/* File rename API */ +extern int FileOpsRename(const char *oldpath, const char *newpath); + +/* + * xl_fileops_write - WAL record for file write at offset + * + * Records that data was written to a file at a specific offset. + * The path and data are stored as variable-length data following + * the fixed header. + */ +typedef struct xl_fileops_write +{ + off_t offset; /* write offset in file */ + uint32 len; /* data length */ + uint16 path_len; /* length of path (including NUL) */ + /* variable-length path follows, then data */ +} xl_fileops_write; + +#define SizeOfFileOpsWrite (offsetof(xl_fileops_write, path_len) + sizeof(uint16)) + +/* File write API */ +extern int FileOpsWrite(const char *path, off_t offset, + const void *data, uint32 len); + +/* + * xl_fileops_truncate - WAL record for file truncation + */ +typedef struct xl_fileops_truncate +{ + off_t length; /* new file length */ + off_t orig_length; /* pre-truncate length (crash-revert + * before-state) */ + /* variable-length path follows */ +} xl_fileops_truncate; + +#define SizeOfFileOpsTruncate (offsetof(xl_fileops_truncate, orig_length) + sizeof(off_t)) + +/* File truncation API */ +extern void FileOpsTruncate(const char *path, off_t length); + +/* + * xl_fileops_chmod - WAL record for file permission change + */ +typedef struct xl_fileops_chmod +{ + mode_t mode; /* new permission mode */ + mode_t orig_mode; /* pre-chmod mode (crash-revert before-state) */ + /* variable-length path follows */ +} xl_fileops_chmod; + +#define SizeOfFileOpsChmod (offsetof(xl_fileops_chmod, orig_mode) + sizeof(mode_t)) + +/* + * xl_fileops_chown - WAL record for file ownership change + */ +typedef struct xl_fileops_chown +{ + uid_t uid; /* new owner user id */ + gid_t gid; /* new owner group id */ + uid_t orig_uid; /* pre-chown uid (crash-revert before-state) */ + gid_t orig_gid; /* pre-chown gid (crash-revert before-state) */ + /* variable-length path follows */ +} xl_fileops_chown; + +#define SizeOfFileOpsChown (offsetof(xl_fileops_chown, orig_gid) + sizeof(gid_t)) + +/* File metadata API */ +extern int FileOpsChmod(const char *path, mode_t mode); +extern int FileOpsChown(const char *path, uid_t uid, gid_t gid); + +/* + * xl_fileops_mkdir - WAL record for directory creation + */ +typedef struct xl_fileops_mkdir +{ + mode_t mode; /* directory permission mode */ + /* variable-length path follows */ +} xl_fileops_mkdir; + +#define SizeOfFileOpsMkdir (offsetof(xl_fileops_mkdir, mode) + sizeof(mode_t)) + +/* + * xl_fileops_rmdir - WAL record for directory removal + */ +typedef struct xl_fileops_rmdir +{ + bool at_commit; /* true = rmdir at commit, false = at abort */ + /* variable-length path follows */ +} xl_fileops_rmdir; + +#define SizeOfFileOpsRmdir (offsetof(xl_fileops_rmdir, at_commit) + sizeof(bool)) + +/* Directory lifecycle API */ +extern int FileOpsMkdir(const char *path, mode_t mode); +extern void FileOpsRmdir(const char *path, bool at_commit); +extern void FileOpsRmdirRecursive(const char *path, bool at_commit); + +/* + * FileOpsRmtree - schedule deferred recursive directory removal. + * + * Use this when the destination tree is empty (or nonexistent) at + * registration time but is expected to contain files by the time of + * commit or abort -- e.g., when registering the destination of an + * in-progress copydir() for delete-on-abort. Unlike + * FileOpsRmdirRecursive, the tree walk is deferred to execute time, + * so whatever ended up on disk gets removed. + */ +extern void FileOpsRmtree(const char *path, bool at_commit); + +/* + * xl_fileops_symlink - WAL record for symbolic link creation + */ +typedef struct xl_fileops_symlink +{ + uint16 target_len; /* length of target (including NUL) */ + /* variable-length target follows, then linkpath */ +} xl_fileops_symlink; + +#define SizeOfFileOpsSymlink (offsetof(xl_fileops_symlink, target_len) + sizeof(uint16)) + +/* + * xl_fileops_link - WAL record for hard link creation + */ +typedef struct xl_fileops_link +{ + uint16 oldpath_len; /* length of old path (including NUL) */ + /* variable-length old path follows, then new path */ +} xl_fileops_link; + +#define SizeOfFileOpsLink (offsetof(xl_fileops_link, oldpath_len) + sizeof(uint16)) + +/* Link operations API */ +extern int FileOpsSymlink(const char *target, const char *linkpath); +extern int FileOpsLink(const char *oldpath, const char *newpath); + +/* + * xl_fileops_setxattr - WAL record for setting an extended attribute + */ +typedef struct xl_fileops_setxattr +{ + uint16 name_len; /* attribute name length (including NUL) */ + uint32 value_len; /* attribute value length */ + uint16 path_len; /* file path length (including NUL) */ + uint32 orig_value_len; /* pre-set value length (crash-revert) */ + bool had_value; /* true if xattr existed before this set */ + /* variable-length: path, name, value, orig_value (if had_value) */ +} xl_fileops_setxattr; + +#define SizeOfFileOpsSetxattr (offsetof(xl_fileops_setxattr, had_value) + sizeof(bool)) + +/* + * xl_fileops_removexattr - WAL record for removing an extended attribute + */ +typedef struct xl_fileops_removexattr +{ + uint16 name_len; /* attribute name length (including NUL) */ + uint16 path_len; /* file path length (including NUL) */ + uint32 orig_value_len; /* removed value length (crash-revert); 0 = + * none */ + bool had_value; /* true if xattr existed before this remove */ + /* variable-length: path, name, orig_value (if had_value) */ +} xl_fileops_removexattr; + +#define SizeOfFileOpsRemovexattr (offsetof(xl_fileops_removexattr, had_value) + sizeof(bool)) + +/* Extended attribute API */ +extern int FileOpsSetXattr(const char *path, const char *name, + const void *value, size_t len); +extern int FileOpsRemoveXattr(const char *path, const char *name); + +/* WAL redo and descriptor functions */ +extern void fileops_redo(XLogReaderState *record); +extern void fileops_desc(StringInfo buf, XLogReaderState *record); +extern const char *fileops_identify(uint8 info); + +/* FILEOPS UNDO RM (fileops_undo.c) */ +extern void FileopsUndoRmgrInit(void); + +/* + * UNDO resource-manager ID for FILEOPS (stored in urec_rmid). Defined here, + * with FILEOPS, rather than in the generic access/undormgr.h, so the UNDO + * core names no specific consumer. See access/undormgr.h for the shared, + * WAL-durable ID number space. + */ +#define UNDO_RMID_FILEOPS 2 + +/* FILEOPS UNDO subtypes (stored in urec_info) */ +#define FILEOPS_UNDO_CREATE 0x0001 +#define FILEOPS_UNDO_RENAME 0x0002 +#define FILEOPS_UNDO_TRUNCATE 0x0003 +#define FILEOPS_UNDO_CHMOD 0x0004 +#define FILEOPS_UNDO_CHOWN 0x0005 +#define FILEOPS_UNDO_MKDIR 0x0006 +#define FILEOPS_UNDO_SYMLINK 0x0007 +#define FILEOPS_UNDO_LINK 0x0008 +#define FILEOPS_UNDO_SETXATTR 0x0009 +#define FILEOPS_UNDO_REMOVEXATTR 0x000A + +/* + * FILEOPS UNDO payload structures + * + * These structs define the on-disk layout of UNDO record payloads for each + * FILEOPS operation. Variable-length paths and data follow the fixed header. + */ + +/* CREATE undo payload: just the path to unlink */ +typedef struct FileopsUndoCreate +{ + uint16 path_len; /* including NUL */ + /* followed by path */ +} FileopsUndoCreate; + +/* RENAME undo payload: oldpath and newpath for reverse rename */ +typedef struct FileopsUndoRename +{ + uint16 oldpath_len; + uint16 newpath_len; + /* followed by oldpath, then newpath */ +} FileopsUndoRename; + +/* TRUNCATE undo payload: path + original length */ +typedef struct FileopsUndoTruncate +{ + off_t orig_length; + uint16 path_len; + /* followed by path */ +} FileopsUndoTruncate; + +/* CHMOD undo payload: path + original mode */ +typedef struct FileopsUndoChmod +{ + mode_t orig_mode; + uint16 path_len; + /* followed by path */ +} FileopsUndoChmod; + +/* CHOWN undo payload: path + original uid/gid */ +typedef struct FileopsUndoChown +{ + uid_t orig_uid; + gid_t orig_gid; + uint16 path_len; + /* followed by path */ +} FileopsUndoChown; + +/* MKDIR undo payload: just the path to rmdir */ +typedef struct FileopsUndoMkdir +{ + uint16 path_len; + /* followed by path */ +} FileopsUndoMkdir; + +/* SYMLINK undo payload: just the linkpath to unlink */ +typedef struct FileopsUndoSymlink +{ + uint16 linkpath_len; + /* followed by linkpath */ +} FileopsUndoSymlink; + +/* LINK undo payload: just the newpath to unlink */ +typedef struct FileopsUndoLink +{ + uint16 newpath_len; + /* followed by newpath */ +} FileopsUndoLink; + +/* SETXATTR undo payload: path + name + original value (or empty if new) */ +typedef struct FileopsUndoSetxattr +{ + uint16 path_len; + uint16 name_len; + uint32 orig_value_len; /* 0 if xattr didn't exist before */ + bool had_value; /* true if xattr existed before setxattr */ + /* followed by path, name, original value (if had_value) */ +} FileopsUndoSetxattr; + +/* REMOVEXATTR undo payload: path + name + removed value */ +typedef struct FileopsUndoRemovexattr +{ + uint16 path_len; + uint16 name_len; + uint32 value_len; + /* followed by path, name, value */ +} FileopsUndoRemovexattr; + +/* Recursive directory removal */ +extern void FileOpsRmdirRecursive(const char *path, bool at_commit); + +#endif /* FILEOPS_H */ diff --git a/src/port/Makefile b/src/port/Makefile index 7e9b5877652..39e1388227a 100644 --- a/src/port/Makefile +++ b/src/port/Makefile @@ -51,6 +51,7 @@ OBJS = \ pg_popcount_aarch64.o \ pg_popcount_x86.o \ pg_strong_random.o \ + pg_xattr.o \ pgcheckdir.o \ pgmkdirp.o \ pgsleep.o \ diff --git a/src/port/meson.build b/src/port/meson.build index 922b3f64676..20e1ca00431 100644 --- a/src/port/meson.build +++ b/src/port/meson.build @@ -14,6 +14,7 @@ pgport_sources = [ 'pg_popcount_aarch64.c', 'pg_popcount_x86.c', 'pg_strong_random.c', + 'pg_xattr.c', 'pgcheckdir.c', 'pgmkdirp.c', 'pgsleep.c', diff --git a/src/port/pg_xattr.c b/src/port/pg_xattr.c new file mode 100644 index 00000000000..58a3d422817 --- /dev/null +++ b/src/port/pg_xattr.c @@ -0,0 +1,217 @@ +/*------------------------------------------------------------------------- + * + * pg_xattr.c + * Cross-platform extended attribute abstraction + * + * Platform detection uses compiler-defined macros rather than + * configure-time checks, avoiding meson.build/configure.ac changes. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/port/pg_xattr.c + * + *------------------------------------------------------------------------- + */ +#include "c.h" + +#include +#include "port/pg_xattr.h" + +/* + * Platform detection via compiler macros. + * Linux and macOS both provide but with different APIs. + * FreeBSD uses . Windows uses NTFS Alternate Data Streams. + */ +#if defined(__linux__) || defined(__APPLE__) +#include +#define PG_HAVE_XATTR 1 +#elif defined(__FreeBSD__) +#include +#include +#define PG_HAVE_EXTATTR 1 +#elif defined(WIN32) +#define PG_HAVE_ADS 1 +#endif + +#if defined(PG_HAVE_ADS) +/* + * Set errno for a failed operation on the stream "path:name". A missing + * stream fails with ERROR_FILE_NOT_FOUND, like a missing file; report it as + * PG_ENOATTR when the file itself exists, as the other platforms do. + */ +static void +pg_ads_set_errno(const char *path, DWORD err) +{ + _dosmaperr(err); + if (errno == ENOENT && GetFileAttributesA(path) != INVALID_FILE_ATTRIBUTES) + errno = PG_ENOATTR; +} +#endif + +/* + * pg_setxattr - Set an extended attribute on a file + * + * Returns 0 on success, -1 on failure (errno set). + */ +int +pg_setxattr(const char *path, const char *name, + const void *value, size_t size) +{ +#if defined(PG_HAVE_XATTR) +#if defined(__APPLE__) + return setxattr(path, name, value, size, 0, 0); +#else + return setxattr(path, name, value, size, 0); +#endif + +#elif defined(PG_HAVE_EXTATTR) + ssize_t ret; + + ret = extattr_set_file(path, EXTATTR_NAMESPACE_USER, + name, value, size); + return (ret >= 0) ? 0 : -1; + +#elif defined(PG_HAVE_ADS) + char ads_path[MAXPGPATH]; + HANDLE hFile; + DWORD written; + + snprintf(ads_path, sizeof(ads_path), "%s:%s", path, name); + + hFile = CreateFileA(ads_path, GENERIC_WRITE, 0, NULL, + CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (hFile == INVALID_HANDLE_VALUE) + { + _dosmaperr(GetLastError()); + return -1; + } + + if (!WriteFile(hFile, value, (DWORD) size, &written, NULL) || + written != (DWORD) size) + { + _dosmaperr(GetLastError()); + CloseHandle(hFile); + return -1; + } + + CloseHandle(hFile); + return 0; + +#else + /* Unsupported platform: succeed in WAL but no-op locally */ + (void) path; + (void) name; + (void) value; + (void) size; + errno = ENOTSUP; + return -1; +#endif +} + +/* + * pg_getxattr - Get an extended attribute value from a file + * + * Returns the number of bytes placed in value on success, + * or -1 on failure (errno set). If value is NULL or size is 0, + * returns the size of the attribute value without reading it. + */ +ssize_t +pg_getxattr(const char *path, const char *name, + void *value, size_t size) +{ +#if defined(PG_HAVE_XATTR) +#if defined(__APPLE__) + return getxattr(path, name, value, size, 0, 0); +#else + return getxattr(path, name, value, size); +#endif + +#elif defined(PG_HAVE_EXTATTR) + return extattr_get_file(path, EXTATTR_NAMESPACE_USER, + name, value, size); + +#elif defined(PG_HAVE_ADS) + char ads_path[MAXPGPATH]; + HANDLE hFile; + DWORD bytesRead; + LARGE_INTEGER fileSize; + + snprintf(ads_path, sizeof(ads_path), "%s:%s", path, name); + + hFile = CreateFileA(ads_path, GENERIC_READ, FILE_SHARE_READ, NULL, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (hFile == INVALID_HANDLE_VALUE) + { + pg_ads_set_errno(path, GetLastError()); + return -1; + } + + if (!GetFileSizeEx(hFile, &fileSize)) + { + _dosmaperr(GetLastError()); + CloseHandle(hFile); + return -1; + } + + if (value == NULL || size == 0) + { + CloseHandle(hFile); + return (ssize_t) fileSize.QuadPart; + } + + if (!ReadFile(hFile, value, (DWORD) size, &bytesRead, NULL)) + { + _dosmaperr(GetLastError()); + CloseHandle(hFile); + return -1; + } + + CloseHandle(hFile); + return (ssize_t) bytesRead; + +#else + (void) path; + (void) name; + (void) value; + (void) size; + errno = ENOTSUP; + return -1; +#endif +} + +/* + * pg_removexattr - Remove an extended attribute from a file + * + * Returns 0 on success, -1 on failure (errno set). + */ +int +pg_removexattr(const char *path, const char *name) +{ +#if defined(PG_HAVE_XATTR) +#if defined(__APPLE__) + return removexattr(path, name, 0); +#else + return removexattr(path, name); +#endif + +#elif defined(PG_HAVE_EXTATTR) + return extattr_delete_file(path, EXTATTR_NAMESPACE_USER, name); + +#elif defined(PG_HAVE_ADS) + char ads_path[MAXPGPATH]; + + snprintf(ads_path, sizeof(ads_path), "%s:%s", path, name); + if (DeleteFileA(ads_path)) + return 0; + + pg_ads_set_errno(path, GetLastError()); + return -1; + +#else + (void) path; + (void) name; + errno = ENOTSUP; + return -1; +#endif +} diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index 6c1504ac932..f16d803fa2c 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -32,6 +32,7 @@ SUBDIRS = \ test_escape \ test_extensible \ test_extensions \ + test_fileops \ test_ginpostinglist \ test_int128 \ test_integerset \ diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build index ef3cdb8faf9..49cc09d041b 100644 --- a/src/test/modules/meson.build +++ b/src/test/modules/meson.build @@ -34,6 +34,7 @@ subdir('test_dsm_registry') subdir('test_escape') subdir('test_extensible') subdir('test_extensions') +subdir('test_fileops') subdir('test_ginpostinglist') subdir('test_int128') subdir('test_integerset') diff --git a/src/test/modules/test_fileops/Makefile b/src/test/modules/test_fileops/Makefile new file mode 100644 index 00000000000..69100c75272 --- /dev/null +++ b/src/test/modules/test_fileops/Makefile @@ -0,0 +1,19 @@ +MODULE_big = test_fileops +OBJS = test_fileops.o +EXTENSION = test_fileops +DATA = test_fileops--1.0.sql +PGFILEDESC = "test_fileops - test transactional file operations" +REGRESS = test_fileops +TAP_TESTS = 1 +EXTRA_INSTALL = src/test/modules/injection_points +export enable_injection_points +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/test_fileops +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/test_fileops/expected/test_fileops.out b/src/test/modules/test_fileops/expected/test_fileops.out new file mode 100644 index 00000000000..b529cfe3bb2 --- /dev/null +++ b/src/test/modules/test_fileops/expected/test_fileops.out @@ -0,0 +1,413 @@ +-- +-- Tests for FILEOPS UNDO rollback of direct file operations. +-- +-- This test module provides SQL-callable wrappers around FileOps C functions +-- that have no DDL-level callers, enabling direct testing of UNDO rollback +-- for: FileOpsTruncate, FileOpsChmod, FileOpsLink, FileOpsSetXattr, +-- FileOpsRemoveXattr. +-- +-- REQUIRES: UNDO subsystem (always active) +-- +-- These are POSIX file semantics: modes, hard links that do not share the +-- target's attributes, extended attributes. Windows has none of them (chmod +-- only toggles the read-only attribute, which a hard link shares), so the test +-- does not run there; the TAP tests cover the operations FILEOPS supports on +-- Windows. +SELECT version() ~ '(Visual C\+\+|mingw32|msvc|windows)' AS skip_test \gset +\if :skip_test +\quit +\endif +CREATE EXTENSION test_fileops; +-- ================================================================ +-- Setup: create test files in the data directory +-- ================================================================ +SELECT test_fileops_create_tempfile('test_fileops_a.dat') AS filepath_a \gset +SELECT test_fileops_create_tempfile('test_fileops_b.dat') AS filepath_b \gset +-- Record initial state (umask may affect mode) +SELECT test_fileops_file_size(:'filepath_a') AS initial_size; + initial_size +-------------- + 1024 +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS initial_mode; + initial_mode +-------------- + 384 +(1 row) + +-- ================================================================ +-- Test 1: FileOpsTruncate UNDO - rollback restores original size +-- ================================================================ +BEGIN; +SELECT test_fileops_truncate(:'filepath_a', 256); + test_fileops_truncate +----------------------- + +(1 row) + +SELECT test_fileops_file_size(:'filepath_a') AS size_during_txn; + size_during_txn +----------------- + 256 +(1 row) + +ROLLBACK; +-- After rollback, UNDO should restore original size (1024) +SELECT test_fileops_file_size(:'filepath_a') AS size_after_rollback; + size_after_rollback +--------------------- + 1024 +(1 row) + +-- ================================================================ +-- Test 2: FileOpsTruncate commit - change persists +-- ================================================================ +BEGIN; +SELECT test_fileops_truncate(:'filepath_b', 512); + test_fileops_truncate +----------------------- + +(1 row) + +COMMIT; +SELECT test_fileops_file_size(:'filepath_b') AS size_after_commit; + size_after_commit +------------------- + 512 +(1 row) + +-- ================================================================ +-- Test 3: FileOpsChmod UNDO - rollback restores original permissions +-- ================================================================ +-- Change to a known starting mode first (committed) +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 = 420 decimal + test_fileops_chmod +-------------------- + +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_baseline; + mode_baseline +--------------- + 420 +(1 row) + +BEGIN; +-- Change to 0600 = 384 decimal +SELECT test_fileops_chmod(:'filepath_a', 384); + test_fileops_chmod +-------------------- + +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_during_txn; + mode_during_txn +----------------- + 384 +(1 row) + +ROLLBACK; +-- After rollback, UNDO should restore to 0644 = 420 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_rollback; + mode_after_rollback +--------------------- + 420 +(1 row) + +-- ================================================================ +-- Test 4: FileOpsChmod commit - change persists +-- ================================================================ +BEGIN; +SELECT test_fileops_chmod(:'filepath_a', 448); -- 0700 = 448 decimal + test_fileops_chmod +-------------------- + +(1 row) + +COMMIT; +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_commit; + mode_after_commit +------------------- + 448 +(1 row) + +-- Restore for subsequent tests +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 + test_fileops_chmod +-------------------- + +(1 row) + +-- ================================================================ +-- Test 5: FileOpsLink UNDO - rollback removes the hard link +-- ================================================================ +SELECT test_fileops_data_dir() || '/test_fileops_link.dat' AS linkpath \gset +BEGIN; +SELECT test_fileops_link(:'filepath_a', :'linkpath'); + test_fileops_link +------------------- + +(1 row) + +SELECT test_fileops_file_exists(:'linkpath') AS link_during_txn; + link_during_txn +----------------- + t +(1 row) + +ROLLBACK; +-- After rollback, UNDO should remove the hard link +SELECT test_fileops_file_exists(:'linkpath') AS link_after_rollback; + link_after_rollback +--------------------- + f +(1 row) + +-- ================================================================ +-- Test 6: FileOpsLink commit - link persists +-- ================================================================ +BEGIN; +SELECT test_fileops_link(:'filepath_a', :'linkpath'); + test_fileops_link +------------------- + +(1 row) + +COMMIT; +SELECT test_fileops_file_exists(:'linkpath') AS link_after_commit; + link_after_commit +------------------- + t +(1 row) + +-- ================================================================ +-- Test 7: FileOpsTruncate in subtransaction - ROLLBACK TO undoes it +-- ================================================================ +-- Ensure file is 1024 bytes +SELECT test_fileops_file_size(:'filepath_a') AS size_before_test7; + size_before_test7 +------------------- + 1024 +(1 row) + +BEGIN; +-- Outer truncate (should persist after commit) +SELECT test_fileops_truncate(:'filepath_a', 800); + test_fileops_truncate +----------------------- + +(1 row) + +SAVEPOINT sp1; +-- Inner truncate (should be rolled back) +SELECT test_fileops_truncate(:'filepath_a', 100); + test_fileops_truncate +----------------------- + +(1 row) + +SELECT test_fileops_file_size(:'filepath_a') AS size_in_savepoint; + size_in_savepoint +------------------- + 100 +(1 row) + +ROLLBACK TO sp1; +-- After ROLLBACK TO, should be back to 800 (the outer truncate value) +SELECT test_fileops_file_size(:'filepath_a') AS size_after_rollback_to; + size_after_rollback_to +------------------------ + 800 +(1 row) + +COMMIT; +-- After commit, the outer truncate to 800 should persist +SELECT test_fileops_file_size(:'filepath_a') AS size_after_commit; + size_after_commit +------------------- + 800 +(1 row) + +-- ================================================================ +-- Test 8: FileOpsChmod in subtransaction - ROLLBACK TO restores mode +-- ================================================================ +-- Set a known mode baseline +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 + test_fileops_chmod +-------------------- + +(1 row) + +BEGIN; +-- Outer chmod (persists) +SELECT test_fileops_chmod(:'filepath_a', 448); -- 0700 + test_fileops_chmod +-------------------- + +(1 row) + +SAVEPOINT sp1; +-- Inner chmod (rolled back) +SELECT test_fileops_chmod(:'filepath_a', 256); -- 0400 + test_fileops_chmod +-------------------- + +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_in_savepoint; + mode_in_savepoint +------------------- + 256 +(1 row) + +ROLLBACK TO sp1; +-- After ROLLBACK TO, should be back to 0700 = 448 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_rollback_to; + mode_after_rollback_to +------------------------ + 448 +(1 row) + +COMMIT; +-- After commit, outer chmod (0700 = 448) persists +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_commit; + mode_after_commit +------------------- + 448 +(1 row) + +-- ================================================================ +-- Test 9: FileOpsLink in subtransaction - ROLLBACK TO removes link +-- ================================================================ +SELECT test_fileops_data_dir() || '/test_fileops_link2.dat' AS linkpath2 \gset +BEGIN; +SAVEPOINT sp1; +SELECT test_fileops_link(:'filepath_a', :'linkpath2'); + test_fileops_link +------------------- + +(1 row) + +SELECT test_fileops_file_exists(:'linkpath2') AS link2_in_savepoint; + link2_in_savepoint +-------------------- + t +(1 row) + +ROLLBACK TO sp1; +-- Link should be removed by UNDO +SELECT test_fileops_file_exists(:'linkpath2') AS link2_after_rollback_to; + link2_after_rollback_to +------------------------- + f +(1 row) + +COMMIT; +SELECT test_fileops_file_exists(:'linkpath2') AS link2_after_commit; + link2_after_commit +-------------------- + f +(1 row) + +-- ================================================================ +-- Test 10: FileOpsSetXattr / FileOpsRemoveXattr UNDO +-- (skipped on platforms without xattr support) +-- ================================================================ +-- Try to set an xattr; returns false if unsupported (ENOTSUP/EPERM/EACCES) +SELECT test_fileops_setxattr(:'filepath_a', 'user.test_key', 'initial_value') + AS xattr_supported \gset +\if :xattr_supported +-- Test 10a: SetXattr UNDO - rollback removes newly-set xattr +BEGIN; +SELECT test_fileops_setxattr(:'filepath_a', 'user.rollback_test', 'will_vanish'); + test_fileops_setxattr +----------------------- + t +(1 row) + +SELECT test_fileops_getxattr(:'filepath_a', 'user.rollback_test') AS xattr_during_txn; + xattr_during_txn +------------------ + will_vanish +(1 row) + +ROLLBACK; +SELECT test_fileops_getxattr(:'filepath_a', 'user.rollback_test') AS xattr_after_rollback; + xattr_after_rollback +---------------------- + +(1 row) + +-- Test 10b: SetXattr overwrite UNDO - rollback restores old value +BEGIN; +SELECT test_fileops_setxattr(:'filepath_a', 'user.test_key', 'overwritten'); + test_fileops_setxattr +----------------------- + t +(1 row) + +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_overwritten; + xattr_overwritten +------------------- + overwritten +(1 row) + +ROLLBACK; +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_restored; + xattr_restored +---------------- + initial_value +(1 row) + +-- Test 10c: RemoveXattr UNDO - rollback restores removed xattr +BEGIN; +SELECT test_fileops_removexattr(:'filepath_a', 'user.test_key'); + test_fileops_removexattr +-------------------------- + t +(1 row) + +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_after_remove; + xattr_after_remove +-------------------- + +(1 row) + +ROLLBACK; +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_after_remove_rollback; + xattr_after_remove_rollback +----------------------------- + initial_value +(1 row) + +-- Test 10d: Xattr in subtransaction +BEGIN; +SAVEPOINT sp1; +SELECT test_fileops_setxattr(:'filepath_a', 'user.sub_key', 'sub_value'); + test_fileops_setxattr +----------------------- + t +(1 row) + +SELECT test_fileops_getxattr(:'filepath_a', 'user.sub_key') AS xattr_in_savepoint; + xattr_in_savepoint +-------------------- + sub_value +(1 row) + +ROLLBACK TO sp1; +SELECT test_fileops_getxattr(:'filepath_a', 'user.sub_key') AS xattr_after_sp_rollback; + xattr_after_sp_rollback +------------------------- + +(1 row) + +COMMIT; +\else +SELECT 'xattr tests skipped (not supported on this platform/filesystem)' AS notice; +\endif +-- ================================================================ +-- Cleanup +-- ================================================================ +DROP EXTENSION test_fileops; diff --git a/src/test/modules/test_fileops/expected/test_fileops_1.out b/src/test/modules/test_fileops/expected/test_fileops_1.out new file mode 100644 index 00000000000..41193304cd6 --- /dev/null +++ b/src/test/modules/test_fileops/expected/test_fileops_1.out @@ -0,0 +1,332 @@ +-- +-- Tests for FILEOPS UNDO rollback of direct file operations. +-- +-- This test module provides SQL-callable wrappers around FileOps C functions +-- that have no DDL-level callers, enabling direct testing of UNDO rollback +-- for: FileOpsTruncate, FileOpsChmod, FileOpsLink, FileOpsSetXattr, +-- FileOpsRemoveXattr. +-- +-- REQUIRES: UNDO subsystem (always active) +-- +-- These are POSIX file semantics: modes, hard links that do not share the +-- target's attributes, extended attributes. Windows has none of them (chmod +-- only toggles the read-only attribute, which a hard link shares), so the test +-- does not run there; the TAP tests cover the operations FILEOPS supports on +-- Windows. +SELECT version() ~ '(Visual C\+\+|mingw32|msvc|windows)' AS skip_test \gset +\if :skip_test +\quit +\endif +CREATE EXTENSION test_fileops; +-- ================================================================ +-- Setup: create test files in the data directory +-- ================================================================ +SELECT test_fileops_create_tempfile('test_fileops_a.dat') AS filepath_a \gset +SELECT test_fileops_create_tempfile('test_fileops_b.dat') AS filepath_b \gset +-- Record initial state (umask may affect mode) +SELECT test_fileops_file_size(:'filepath_a') AS initial_size; + initial_size +-------------- + 1024 +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS initial_mode; + initial_mode +-------------- + 384 +(1 row) + +-- ================================================================ +-- Test 1: FileOpsTruncate UNDO - rollback restores original size +-- ================================================================ +BEGIN; +SELECT test_fileops_truncate(:'filepath_a', 256); + test_fileops_truncate +----------------------- + +(1 row) + +SELECT test_fileops_file_size(:'filepath_a') AS size_during_txn; + size_during_txn +----------------- + 256 +(1 row) + +ROLLBACK; +-- After rollback, UNDO should restore original size (1024) +SELECT test_fileops_file_size(:'filepath_a') AS size_after_rollback; + size_after_rollback +--------------------- + 1024 +(1 row) + +-- ================================================================ +-- Test 2: FileOpsTruncate commit - change persists +-- ================================================================ +BEGIN; +SELECT test_fileops_truncate(:'filepath_b', 512); + test_fileops_truncate +----------------------- + +(1 row) + +COMMIT; +SELECT test_fileops_file_size(:'filepath_b') AS size_after_commit; + size_after_commit +------------------- + 512 +(1 row) + +-- ================================================================ +-- Test 3: FileOpsChmod UNDO - rollback restores original permissions +-- ================================================================ +-- Change to a known starting mode first (committed) +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 = 420 decimal + test_fileops_chmod +-------------------- + +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_baseline; + mode_baseline +--------------- + 420 +(1 row) + +BEGIN; +-- Change to 0600 = 384 decimal +SELECT test_fileops_chmod(:'filepath_a', 384); + test_fileops_chmod +-------------------- + +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_during_txn; + mode_during_txn +----------------- + 384 +(1 row) + +ROLLBACK; +-- After rollback, UNDO should restore to 0644 = 420 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_rollback; + mode_after_rollback +--------------------- + 420 +(1 row) + +-- ================================================================ +-- Test 4: FileOpsChmod commit - change persists +-- ================================================================ +BEGIN; +SELECT test_fileops_chmod(:'filepath_a', 448); -- 0700 = 448 decimal + test_fileops_chmod +-------------------- + +(1 row) + +COMMIT; +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_commit; + mode_after_commit +------------------- + 448 +(1 row) + +-- Restore for subsequent tests +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 + test_fileops_chmod +-------------------- + +(1 row) + +-- ================================================================ +-- Test 5: FileOpsLink UNDO - rollback removes the hard link +-- ================================================================ +SELECT test_fileops_data_dir() || '/test_fileops_link.dat' AS linkpath \gset +BEGIN; +SELECT test_fileops_link(:'filepath_a', :'linkpath'); + test_fileops_link +------------------- + +(1 row) + +SELECT test_fileops_file_exists(:'linkpath') AS link_during_txn; + link_during_txn +----------------- + t +(1 row) + +ROLLBACK; +-- After rollback, UNDO should remove the hard link +SELECT test_fileops_file_exists(:'linkpath') AS link_after_rollback; + link_after_rollback +--------------------- + f +(1 row) + +-- ================================================================ +-- Test 6: FileOpsLink commit - link persists +-- ================================================================ +BEGIN; +SELECT test_fileops_link(:'filepath_a', :'linkpath'); + test_fileops_link +------------------- + +(1 row) + +COMMIT; +SELECT test_fileops_file_exists(:'linkpath') AS link_after_commit; + link_after_commit +------------------- + t +(1 row) + +-- ================================================================ +-- Test 7: FileOpsTruncate in subtransaction - ROLLBACK TO undoes it +-- ================================================================ +-- Ensure file is 1024 bytes +SELECT test_fileops_file_size(:'filepath_a') AS size_before_test7; + size_before_test7 +------------------- + 1024 +(1 row) + +BEGIN; +-- Outer truncate (should persist after commit) +SELECT test_fileops_truncate(:'filepath_a', 800); + test_fileops_truncate +----------------------- + +(1 row) + +SAVEPOINT sp1; +-- Inner truncate (should be rolled back) +SELECT test_fileops_truncate(:'filepath_a', 100); + test_fileops_truncate +----------------------- + +(1 row) + +SELECT test_fileops_file_size(:'filepath_a') AS size_in_savepoint; + size_in_savepoint +------------------- + 100 +(1 row) + +ROLLBACK TO sp1; +-- After ROLLBACK TO, should be back to 800 (the outer truncate value) +SELECT test_fileops_file_size(:'filepath_a') AS size_after_rollback_to; + size_after_rollback_to +------------------------ + 800 +(1 row) + +COMMIT; +-- After commit, the outer truncate to 800 should persist +SELECT test_fileops_file_size(:'filepath_a') AS size_after_commit; + size_after_commit +------------------- + 800 +(1 row) + +-- ================================================================ +-- Test 8: FileOpsChmod in subtransaction - ROLLBACK TO restores mode +-- ================================================================ +-- Set a known mode baseline +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 + test_fileops_chmod +-------------------- + +(1 row) + +BEGIN; +-- Outer chmod (persists) +SELECT test_fileops_chmod(:'filepath_a', 448); -- 0700 + test_fileops_chmod +-------------------- + +(1 row) + +SAVEPOINT sp1; +-- Inner chmod (rolled back) +SELECT test_fileops_chmod(:'filepath_a', 256); -- 0400 + test_fileops_chmod +-------------------- + +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_in_savepoint; + mode_in_savepoint +------------------- + 256 +(1 row) + +ROLLBACK TO sp1; +-- After ROLLBACK TO, should be back to 0700 = 448 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_rollback_to; + mode_after_rollback_to +------------------------ + 448 +(1 row) + +COMMIT; +-- After commit, outer chmod (0700 = 448) persists +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_commit; + mode_after_commit +------------------- + 448 +(1 row) + +-- ================================================================ +-- Test 9: FileOpsLink in subtransaction - ROLLBACK TO removes link +-- ================================================================ +SELECT test_fileops_data_dir() || '/test_fileops_link2.dat' AS linkpath2 \gset +BEGIN; +SAVEPOINT sp1; +SELECT test_fileops_link(:'filepath_a', :'linkpath2'); + test_fileops_link +------------------- + +(1 row) + +SELECT test_fileops_file_exists(:'linkpath2') AS link2_in_savepoint; + link2_in_savepoint +-------------------- + t +(1 row) + +ROLLBACK TO sp1; +-- Link should be removed by UNDO +SELECT test_fileops_file_exists(:'linkpath2') AS link2_after_rollback_to; + link2_after_rollback_to +------------------------- + f +(1 row) + +COMMIT; +SELECT test_fileops_file_exists(:'linkpath2') AS link2_after_commit; + link2_after_commit +-------------------- + f +(1 row) + +-- ================================================================ +-- Test 10: FileOpsSetXattr / FileOpsRemoveXattr UNDO +-- (skipped on platforms without xattr support) +-- ================================================================ +-- Try to set an xattr; returns false if unsupported (ENOTSUP/EPERM/EACCES) +SELECT test_fileops_setxattr(:'filepath_a', 'user.test_key', 'initial_value') + AS xattr_supported \gset +\if :xattr_supported +\else +SELECT 'xattr tests skipped (not supported on this platform/filesystem)' AS notice; + notice +----------------------------------------------------------------- + xattr tests skipped (not supported on this platform/filesystem) +(1 row) + +\endif +-- ================================================================ +-- Cleanup +-- ================================================================ +DROP EXTENSION test_fileops; diff --git a/src/test/modules/test_fileops/expected/test_fileops_2.out b/src/test/modules/test_fileops/expected/test_fileops_2.out new file mode 100644 index 00000000000..d25aa81d54b --- /dev/null +++ b/src/test/modules/test_fileops/expected/test_fileops_2.out @@ -0,0 +1,18 @@ +-- +-- Tests for FILEOPS UNDO rollback of direct file operations. +-- +-- This test module provides SQL-callable wrappers around FileOps C functions +-- that have no DDL-level callers, enabling direct testing of UNDO rollback +-- for: FileOpsTruncate, FileOpsChmod, FileOpsLink, FileOpsSetXattr, +-- FileOpsRemoveXattr. +-- +-- REQUIRES: UNDO subsystem (always active) +-- +-- These are POSIX file semantics: modes, hard links that do not share the +-- target's attributes, extended attributes. Windows has none of them (chmod +-- only toggles the read-only attribute, which a hard link shares), so the test +-- does not run there; the TAP tests cover the operations FILEOPS supports on +-- Windows. +SELECT version() ~ '(Visual C\+\+|mingw32|msvc|windows)' AS skip_test \gset +\if :skip_test +\quit diff --git a/src/test/modules/test_fileops/meson.build b/src/test/modules/test_fileops/meson.build new file mode 100644 index 00000000000..adda18e624e --- /dev/null +++ b/src/test/modules/test_fileops/meson.build @@ -0,0 +1,44 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group + +test_fileops_sources = files( + 'test_fileops.c', +) + +if host_system == 'windows' + test_fileops_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'test_fileops', + '--FILEDESC', 'test_fileops - regression testing of FILEOPS subsystem',]) +endif + +test_fileops = shared_module('test_fileops', + test_fileops_sources, + kwargs: pg_test_mod_args, +) +test_install_libs += test_fileops + +test_install_data += files( + 'test_fileops.control', + 'test_fileops--1.0.sql', +) + +tests += { + 'name': 'test_fileops', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'regress': { + 'sql': [ + 'test_fileops', + ], + }, + 'tap': { + 'env': { + 'enable_injection_points': get_option('injection_points') ? 'yes' : 'no', + }, + 'tests': [ + 't/001_fileops_apply_failure.pl', + 't/007_fileops_commit_crash.pl', + 't/023_fileops_inline_abort_ctx.pl', + 't/024_fileops_recovery_apply_failure.pl', + ], + }, +} diff --git a/src/test/modules/test_fileops/sql/test_fileops.sql b/src/test/modules/test_fileops/sql/test_fileops.sql new file mode 100644 index 00000000000..ef1a10523d1 --- /dev/null +++ b/src/test/modules/test_fileops/sql/test_fileops.sql @@ -0,0 +1,229 @@ +-- +-- Tests for FILEOPS UNDO rollback of direct file operations. +-- +-- This test module provides SQL-callable wrappers around FileOps C functions +-- that have no DDL-level callers, enabling direct testing of UNDO rollback +-- for: FileOpsTruncate, FileOpsChmod, FileOpsLink, FileOpsSetXattr, +-- FileOpsRemoveXattr. +-- +-- REQUIRES: UNDO subsystem (always active) +-- +-- These are POSIX file semantics: modes, hard links that do not share the +-- target's attributes, extended attributes. Windows has none of them (chmod +-- only toggles the read-only attribute, which a hard link shares), so the test +-- does not run there; the TAP tests cover the operations FILEOPS supports on +-- Windows. +SELECT version() ~ '(Visual C\+\+|mingw32|msvc|windows)' AS skip_test \gset +\if :skip_test +\quit +\endif + +CREATE EXTENSION test_fileops; + +-- ================================================================ +-- Setup: create test files in the data directory +-- ================================================================ + +SELECT test_fileops_create_tempfile('test_fileops_a.dat') AS filepath_a \gset +SELECT test_fileops_create_tempfile('test_fileops_b.dat') AS filepath_b \gset + +-- Record initial state (umask may affect mode) +SELECT test_fileops_file_size(:'filepath_a') AS initial_size; +SELECT test_fileops_get_mode(:'filepath_a') AS initial_mode; + +-- ================================================================ +-- Test 1: FileOpsTruncate UNDO - rollback restores original size +-- ================================================================ + +BEGIN; +SELECT test_fileops_truncate(:'filepath_a', 256); +SELECT test_fileops_file_size(:'filepath_a') AS size_during_txn; +ROLLBACK; + +-- After rollback, UNDO should restore original size (1024) +SELECT test_fileops_file_size(:'filepath_a') AS size_after_rollback; + +-- ================================================================ +-- Test 2: FileOpsTruncate commit - change persists +-- ================================================================ + +BEGIN; +SELECT test_fileops_truncate(:'filepath_b', 512); +COMMIT; + +SELECT test_fileops_file_size(:'filepath_b') AS size_after_commit; + +-- ================================================================ +-- Test 3: FileOpsChmod UNDO - rollback restores original permissions +-- ================================================================ + +-- Change to a known starting mode first (committed) +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 = 420 decimal +SELECT test_fileops_get_mode(:'filepath_a') AS mode_baseline; + +BEGIN; +-- Change to 0600 = 384 decimal +SELECT test_fileops_chmod(:'filepath_a', 384); +SELECT test_fileops_get_mode(:'filepath_a') AS mode_during_txn; +ROLLBACK; + +-- After rollback, UNDO should restore to 0644 = 420 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_rollback; + +-- ================================================================ +-- Test 4: FileOpsChmod commit - change persists +-- ================================================================ + +BEGIN; +SELECT test_fileops_chmod(:'filepath_a', 448); -- 0700 = 448 decimal +COMMIT; + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_commit; + +-- Restore for subsequent tests +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 + +-- ================================================================ +-- Test 5: FileOpsLink UNDO - rollback removes the hard link +-- ================================================================ + +SELECT test_fileops_data_dir() || '/test_fileops_link.dat' AS linkpath \gset + +BEGIN; +SELECT test_fileops_link(:'filepath_a', :'linkpath'); +SELECT test_fileops_file_exists(:'linkpath') AS link_during_txn; +ROLLBACK; + +-- After rollback, UNDO should remove the hard link +SELECT test_fileops_file_exists(:'linkpath') AS link_after_rollback; + +-- ================================================================ +-- Test 6: FileOpsLink commit - link persists +-- ================================================================ + +BEGIN; +SELECT test_fileops_link(:'filepath_a', :'linkpath'); +COMMIT; + +SELECT test_fileops_file_exists(:'linkpath') AS link_after_commit; + +-- ================================================================ +-- Test 7: FileOpsTruncate in subtransaction - ROLLBACK TO undoes it +-- ================================================================ + +-- Ensure file is 1024 bytes +SELECT test_fileops_file_size(:'filepath_a') AS size_before_test7; + +BEGIN; +-- Outer truncate (should persist after commit) +SELECT test_fileops_truncate(:'filepath_a', 800); + +SAVEPOINT sp1; +-- Inner truncate (should be rolled back) +SELECT test_fileops_truncate(:'filepath_a', 100); +SELECT test_fileops_file_size(:'filepath_a') AS size_in_savepoint; +ROLLBACK TO sp1; + +-- After ROLLBACK TO, should be back to 800 (the outer truncate value) +SELECT test_fileops_file_size(:'filepath_a') AS size_after_rollback_to; +COMMIT; + +-- After commit, the outer truncate to 800 should persist +SELECT test_fileops_file_size(:'filepath_a') AS size_after_commit; + +-- ================================================================ +-- Test 8: FileOpsChmod in subtransaction - ROLLBACK TO restores mode +-- ================================================================ + +-- Set a known mode baseline +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 + +BEGIN; +-- Outer chmod (persists) +SELECT test_fileops_chmod(:'filepath_a', 448); -- 0700 + +SAVEPOINT sp1; +-- Inner chmod (rolled back) +SELECT test_fileops_chmod(:'filepath_a', 256); -- 0400 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_in_savepoint; +ROLLBACK TO sp1; + +-- After ROLLBACK TO, should be back to 0700 = 448 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_rollback_to; +COMMIT; + +-- After commit, outer chmod (0700 = 448) persists +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_commit; + +-- ================================================================ +-- Test 9: FileOpsLink in subtransaction - ROLLBACK TO removes link +-- ================================================================ + +SELECT test_fileops_data_dir() || '/test_fileops_link2.dat' AS linkpath2 \gset + +BEGIN; +SAVEPOINT sp1; +SELECT test_fileops_link(:'filepath_a', :'linkpath2'); +SELECT test_fileops_file_exists(:'linkpath2') AS link2_in_savepoint; +ROLLBACK TO sp1; + +-- Link should be removed by UNDO +SELECT test_fileops_file_exists(:'linkpath2') AS link2_after_rollback_to; +COMMIT; + +SELECT test_fileops_file_exists(:'linkpath2') AS link2_after_commit; + +-- ================================================================ +-- Test 10: FileOpsSetXattr / FileOpsRemoveXattr UNDO +-- (skipped on platforms without xattr support) +-- ================================================================ + +-- Try to set an xattr; returns false if unsupported (ENOTSUP/EPERM/EACCES) +SELECT test_fileops_setxattr(:'filepath_a', 'user.test_key', 'initial_value') + AS xattr_supported \gset + +\if :xattr_supported + +-- Test 10a: SetXattr UNDO - rollback removes newly-set xattr +BEGIN; +SELECT test_fileops_setxattr(:'filepath_a', 'user.rollback_test', 'will_vanish'); +SELECT test_fileops_getxattr(:'filepath_a', 'user.rollback_test') AS xattr_during_txn; +ROLLBACK; + +SELECT test_fileops_getxattr(:'filepath_a', 'user.rollback_test') AS xattr_after_rollback; + +-- Test 10b: SetXattr overwrite UNDO - rollback restores old value +BEGIN; +SELECT test_fileops_setxattr(:'filepath_a', 'user.test_key', 'overwritten'); +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_overwritten; +ROLLBACK; + +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_restored; + +-- Test 10c: RemoveXattr UNDO - rollback restores removed xattr +BEGIN; +SELECT test_fileops_removexattr(:'filepath_a', 'user.test_key'); +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_after_remove; +ROLLBACK; + +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_after_remove_rollback; + +-- Test 10d: Xattr in subtransaction +BEGIN; +SAVEPOINT sp1; +SELECT test_fileops_setxattr(:'filepath_a', 'user.sub_key', 'sub_value'); +SELECT test_fileops_getxattr(:'filepath_a', 'user.sub_key') AS xattr_in_savepoint; +ROLLBACK TO sp1; + +SELECT test_fileops_getxattr(:'filepath_a', 'user.sub_key') AS xattr_after_sp_rollback; +COMMIT; + +\else +SELECT 'xattr tests skipped (not supported on this platform/filesystem)' AS notice; +\endif + +-- ================================================================ +-- Cleanup +-- ================================================================ + +DROP EXTENSION test_fileops; diff --git a/src/test/modules/test_fileops/t/001_fileops_apply_failure.pl b/src/test/modules/test_fileops/t/001_fileops_apply_failure.pl new file mode 100644 index 00000000000..d3a80282118 --- /dev/null +++ b/src/test/modules/test_fileops/t/001_fileops_apply_failure.pl @@ -0,0 +1,42 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +# A FILEOPS UNDO record that cannot be applied must be reported as a failure +# instead of being treated as rolled back. +# +# The transaction truncates a file, which is then replaced behind its back by a +# directory of the same name, so the truncation's UNDO cannot reopen it. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('fileops_apply_failure'); +$node->init; +# FILEOPS writes UNDO only when WAL is needed. +$node->append_conf( + 'postgresql.conf', q{ +wal_level = replica +}); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $path = $node->safe_psql('postgres', + "SELECT test_fileops_create_tempfile('fileops_fail.dat')"); + +my $bg = $node->background_psql('postgres'); +$bg->query_safe('BEGIN'); +$bg->query_safe("SELECT test_fileops_truncate('$path', 10)"); +unlink($path) or die "unlink $path: $!"; +mkdir($path) or die "mkdir $path: $!"; +$bg->query('ROLLBACK'); +$bg->quit; + +like( + slurp_file($node->logfile), + qr/FILEOPS UNDO TRUNCATE: could not open/, + 'the failure was reported'); + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/007_fileops_commit_crash.pl b/src/test/modules/test_fileops/t/007_fileops_commit_crash.pl new file mode 100644 index 00000000000..0ff3e80890f --- /dev/null +++ b/src/test/modules/test_fileops/t/007_fileops_commit_crash.pl @@ -0,0 +1,77 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group + +# A deferred commit-time file operation must survive a crash right after +# its transaction's commit record: the commit is durable, so the deletion, +# rename or rmdir it implies must happen too, on restart if not before. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $node = PostgreSQL::Test::Cluster->new('fileops_commit_crash'); +$node->init; +$node->append_conf( + 'postgresql.conf', q{ +shared_preload_libraries = 'injection_points' +wal_level = replica +}); +$node->start; +if (!$node->check_extension('injection_points')) +{ + plan skip_all => 'Extension injection_points not installed'; +} +$node->safe_psql('postgres', 'CREATE EXTENSION injection_points'); +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +my $datadir = $node->data_dir; + +# Commit $sql, crash right after the commit record, restart. +sub commit_and_crash +{ + my ($sql) = @_; + $node->safe_psql('postgres', + "SELECT injection_points_attach('fileops-commit-before-pending-ops', 'wait')" + ); + my $bg = $node->background_psql('postgres', on_error_stop => 0); + $bg->query_until(qr/start/, "\\echo start\n$sql\n"); + $node->wait_for_event('client backend', + 'fileops-commit-before-pending-ops'); + $node->stop('immediate'); + $bg->{run}->finish; + $node->start; +} + +$node->safe_psql('postgres', + "SELECT test_fileops_create_tempfile('del.dat')"); +$node->safe_psql('postgres', + "SELECT test_fileops_create_tempfile('ren.dat')"); +$node->safe_psql('postgres', "SELECT test_fileops_mkdir('rmd', 448)"); +commit_and_crash( + q{BEGIN; +SELECT test_fileops_delete('del.dat'); +SELECT test_fileops_rename('ren.dat', 'ren2.dat'); +SELECT test_fileops_rmdir('rmd'); +COMMIT;}); +TODO: +{ + # The deferred ops are WAL-logged only when executed, after the commit + # record, so this crash loses them. Their intent must be durable before + # the commit record and executed by commit redo. + local $TODO = + 'commit-time FILEOPS ops are logged after the commit record'; + ok(!-e "$datadir/del.dat", + 'committed FileOpsDelete happened despite the crash'); + ok( !-e "$datadir/ren.dat" && -e "$datadir/ren2.dat", + 'committed FileOpsRename happened despite the crash'); + ok(!-e "$datadir/rmd", + 'committed FileOpsRmdir happened despite the crash'); +} + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/023_fileops_inline_abort_ctx.pl b/src/test/modules/test_fileops/t/023_fileops_inline_abort_ctx.pl new file mode 100644 index 00000000000..edce8683773 --- /dev/null +++ b/src/test/modules/test_fileops/t/023_fileops_inline_abort_ctx.pl @@ -0,0 +1,60 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# AtAbort_XactUndo()'s inline-abort path must restore the memory context and +# free its working context on every exit, including the early exit taken when +# the batch LSN does not point at a valid UNDO batch. +# +# That exit jumped over the MemoryContextSwitchTo(old_ctx) / +# MemoryContextDelete(undo_ctx) pair, so the "Inline UNDO Apply" context (a +# child of TopMemoryContext) leaked on every such abort, and the rest of the +# abort ran in, and returned into, that context. One leaked context is +# enough to show it: it survives into the next statement of the same +# backend, where pg_backend_memory_contexts lists it. +# +# The invalid-batch exit is forced with the "inline-undo-invalid-batch" +# injection point. A FILEOPS chmod puts one UNDO record in the +# transaction's chain so the inline path runs at all. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $node = PostgreSQL::Test::Cluster->new('fileops_inline_abort_ctx'); +$node->init; +# FILEOPS writes UNDO only when WAL is needed. +$node->append_conf('postgresql.conf', 'wal_level = replica'); +$node->start; +$node->safe_psql('postgres', + 'CREATE EXTENSION test_fileops; CREATE EXTENSION injection_points;'); + +my $path = $node->safe_psql('postgres', + "SELECT test_fileops_create_tempfile('inline_abort_ctx.dat')"); + +# One backend: the leaked context has to be looked for in the backend that +# aborted, after its transaction has ended. +my $bg = $node->background_psql('postgres'); +$bg->query_safe('SELECT injection_points_set_local()'); +$bg->query_safe( + "SELECT injection_points_attach('inline-undo-invalid-batch', 'error')"); + +$bg->query_safe('BEGIN'); +$bg->query_safe("SELECT test_fileops_chmod('$path', 384)"); +$bg->query('ROLLBACK'); + +# The ROLLBACK warns "inline UNDO failed"; query() leaves that on stderr. +my $leaked = $bg->query( + "SELECT count(*) FROM pg_backend_memory_contexts " + . "WHERE name = 'Inline UNDO Apply'"); +chomp $leaked; +is($leaked, '0', + 'the inline-abort working context is freed on the invalid-batch exit'); + +$bg->quit; +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/024_fileops_recovery_apply_failure.pl b/src/test/modules/test_fileops/t/024_fileops_recovery_apply_failure.pl new file mode 100644 index 00000000000..62eb302b20f --- /dev/null +++ b/src/test/modules/test_fileops/t/024_fileops_recovery_apply_failure.pl @@ -0,0 +1,59 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# The startup undo phase must not report a crash loser as rolled back when one +# of its UNDO records could not be applied. +# +# A crash loser changed a file's mode through FILEOPS. Before the restart the +# file's directory loses its search permission, so the inverse chmod() fails +# with a real EACCES. Nothing retries it, so the mode stays changed; the +# summary at the end of the undo phase has to say so. +# +# Must not run as root (root ignores the directory permission). +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +plan skip_all => 'running as root: EACCES cannot be produced' if $> == 0; +plan skip_all => 'directory search permission is POSIX-only' + if $PostgreSQL::Test::Utils::windows_os; + +my $node = PostgreSQL::Test::Cluster->new('fileops_recovery_apply_failure'); +$node->init; +# FILEOPS writes UNDO only when WAL is needed. +$node->append_conf('postgresql.conf', "wal_level = replica\nautovacuum = off\n"); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $dir = PostgreSQL::Test::Utils::tempdir('eacces'); +my $path = "$dir/m.dat"; +open(my $fh, '>', $path) or die "open $path: $!"; +close($fh); +chmod(0644, $path) or die "chmod $path: $!"; + +# A chmod in flight when the server crashes: a crash loser. +my $bg = $node->background_psql('postgres', on_error_stop => 0); +$bg->query_safe('BEGIN'); +$bg->query_safe("SELECT test_fileops_chmod('$path', 384)"); # 0600 +$node->safe_psql('postgres', 'SELECT pg_switch_wal()'); +my $offset = -s $node->logfile; +$node->stop('immediate'); +$bg->{run}->finish; + +chmod(0600, $dir) or die "chmod $dir: $!"; +$node->start; +chmod(0700, $dir) or die "chmod $dir: $!"; + +my $log = slurp_file($node->logfile, $offset); +like($log, qr/FILEOPS UNDO CHMOD: could not restore mode of .*: Permission denied/, + 'the inverse failed with a real EACCES'); +like($log, qr/UNDO recovery complete: 1 transactions processed, 0 records applied, 1 records could not be applied/, + 'the undo phase reports the record it could not apply'); +unlike($log, qr/UNDO recovery complete: \d+ transactions rolled back/, + 'the loser is not reported as rolled back'); +is(sprintf('%04o', (stat($path))[2] & 07777), '0600', + 'the mode change was not rolled back'); + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/test_fileops--1.0.sql b/src/test/modules/test_fileops/test_fileops--1.0.sql new file mode 100644 index 00000000000..80545574727 --- /dev/null +++ b/src/test/modules/test_fileops/test_fileops--1.0.sql @@ -0,0 +1,99 @@ +/* src/test/modules/test_fileops/test_fileops--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION test_fileops" to load this file. \quit + +-- Create a temporary file and return its path +CREATE FUNCTION test_fileops_create_tempfile(filename text) + RETURNS text + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Create a file via FileOpsCreate (exercises the WAL-before-syscall path) +CREATE FUNCTION test_fileops_create(filepath text, mode int) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Truncate a file to the given length (uses FileOpsTruncate) +CREATE FUNCTION test_fileops_truncate(filepath text, length bigint) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Get file size +CREATE FUNCTION test_fileops_file_size(filepath text) + RETURNS bigint + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Chmod a file (uses FileOpsChmod) +CREATE FUNCTION test_fileops_chmod(filepath text, mode int) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Get file permissions (mode) +CREATE FUNCTION test_fileops_get_mode(filepath text) + RETURNS int + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Create a hard link (uses FileOpsLink) +CREATE FUNCTION test_fileops_link(oldpath text, newpath text) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Check if a file exists +CREATE FUNCTION test_fileops_file_exists(filepath text) + RETURNS boolean + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Set extended attribute (uses FileOpsSetXattr) +CREATE FUNCTION test_fileops_setxattr(filepath text, attrname text, attrvalue text) + RETURNS boolean + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Get extended attribute value +CREATE FUNCTION test_fileops_getxattr(filepath text, attrname text) + RETURNS text + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Remove extended attribute (uses FileOpsRemoveXattr) +CREATE FUNCTION test_fileops_removexattr(filepath text, attrname text) + RETURNS boolean + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Get the data directory path (for constructing absolute paths) +CREATE FUNCTION test_fileops_data_dir() + RETURNS text + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Deferred delete at commit (uses FileOpsDelete, at_commit=true) +CREATE FUNCTION test_fileops_delete(filepath text) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Deferred rename at commit (uses FileOpsRename) +CREATE FUNCTION test_fileops_rename(oldpath text, newpath text) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Create a directory (uses FileOpsMkdir) +CREATE FUNCTION test_fileops_mkdir(path text, mode int) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Deferred rmdir at commit (uses FileOpsRmdir, at_commit=true) +CREATE FUNCTION test_fileops_rmdir(path text) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Write a string at an offset (uses FileOpsWrite) +CREATE FUNCTION test_fileops_write(filepath text, "offset" bigint, data text) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Change ownership (uses FileOpsChown) +CREATE FUNCTION test_fileops_chown(filepath text, uid int, gid int) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Create a symbolic link (uses FileOpsSymlink) +CREATE FUNCTION test_fileops_symlink(target text, linkpath text) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; diff --git a/src/test/modules/test_fileops/test_fileops.c b/src/test/modules/test_fileops/test_fileops.c new file mode 100644 index 00000000000..c871d974427 --- /dev/null +++ b/src/test/modules/test_fileops/test_fileops.c @@ -0,0 +1,471 @@ +/*------------------------------------------------------------------------- + * + * test_fileops.c + * Test module exposing FileOps C API to SQL for regression testing. + * + * This module provides SQL-callable wrappers around the FileOps functions + * that have no DDL-level callers, enabling direct testing of: + * - FileOpsTruncate (with UNDO rollback of file size) + * - FileOpsChmod (with UNDO rollback of permissions) + * - FileOpsLink (with UNDO rollback of hard links) + * - FileOpsSetXattr / FileOpsRemoveXattr (with UNDO rollback) + * + * Copyright (c) 2024-2026, PostgreSQL Global Development Group + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include +#include + +#include "fmgr.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "port/pg_xattr.h" +#include "access/xlogdefs.h" +#include "storage/fd.h" +#include "storage/fileops.h" +#include "utils/builtins.h" +#include "utils/pg_lsn.h" + +PG_MODULE_MAGIC; + +PG_FUNCTION_INFO_V1(test_fileops_create_tempfile); +PG_FUNCTION_INFO_V1(test_fileops_create); +PG_FUNCTION_INFO_V1(test_fileops_truncate); +PG_FUNCTION_INFO_V1(test_fileops_file_size); +PG_FUNCTION_INFO_V1(test_fileops_chmod); +PG_FUNCTION_INFO_V1(test_fileops_get_mode); +PG_FUNCTION_INFO_V1(test_fileops_link); +PG_FUNCTION_INFO_V1(test_fileops_file_exists); +PG_FUNCTION_INFO_V1(test_fileops_setxattr); +PG_FUNCTION_INFO_V1(test_fileops_getxattr); +PG_FUNCTION_INFO_V1(test_fileops_removexattr); +PG_FUNCTION_INFO_V1(test_fileops_data_dir); +PG_FUNCTION_INFO_V1(test_fileops_write); +PG_FUNCTION_INFO_V1(test_fileops_chown); +PG_FUNCTION_INFO_V1(test_fileops_symlink); + +PG_FUNCTION_INFO_V1(test_fileops_delete); +PG_FUNCTION_INFO_V1(test_fileops_rename); +PG_FUNCTION_INFO_V1(test_fileops_mkdir); +PG_FUNCTION_INFO_V1(test_fileops_rmdir); + +/* + * test_fileops_create_tempfile - Create a test file in data directory. + * + * Creates a file with some content using FileOpsCreate + write, returning + * the absolute path. The file is created with 1024 bytes of content. + */ +Datum +test_fileops_create_tempfile(PG_FUNCTION_ARGS) +{ + text *filename = PG_GETARG_TEXT_PP(0); + char *fname = text_to_cstring(filename); + char filepath[MAXPGPATH]; + int fd; + char buf[1024]; + + snprintf(filepath, MAXPGPATH, "%s/%s", DataDir, fname); + + /* Use raw create + write so we have a file to manipulate */ + fd = open(filepath, O_CREAT | O_WRONLY | O_TRUNC | PG_BINARY, 0644); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create file \"%s\": %m", filepath))); + + /* Fill with 1024 bytes */ + memset(buf, 'X', sizeof(buf)); + if (write(fd, buf, sizeof(buf)) != sizeof(buf)) + { + int save_errno = errno; + + close(fd); + errno = save_errno; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write to file \"%s\": %m", filepath))); + } + close(fd); + + PG_RETURN_TEXT_P(cstring_to_text(filepath)); +} + +/* + * test_fileops_create - Create a file using FileOpsCreate. + * + * Unlike test_fileops_create_tempfile (which uses raw open()/write() to set + * up fixture files outside any transactional path), this calls FileOpsCreate + * directly so tests can exercise its WAL-before-syscall crash-recovery + * behavior. + */ +Datum +test_fileops_create(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + int mode = PG_GETARG_INT32(1); + char *filepath = text_to_cstring(filepath_text); + int fd; + + fd = FileOpsCreate(filepath, O_WRONLY | PG_BINARY, (mode_t) mode, true); + if (fd >= 0) + CloseTransientFile(fd); + + pfree(filepath); + PG_RETURN_VOID(); +} + +/* + * test_fileops_truncate - Truncate a file using FileOpsTruncate. + * + * This exercises the UNDO path: on rollback, the original file size + * should be restored. + */ +Datum +test_fileops_truncate(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + int64 length = PG_GETARG_INT64(1); + char *filepath = text_to_cstring(filepath_text); + + FileOpsTruncate(filepath, (off_t) length); + + pfree(filepath); + PG_RETURN_VOID(); +} + +/* + * test_fileops_file_size - Get the current size of a file. + */ +Datum +test_fileops_file_size(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + char *filepath = text_to_cstring(filepath_text); + struct stat st; + + if (stat(filepath, &st) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", filepath))); + + pfree(filepath); + PG_RETURN_INT64((int64) st.st_size); +} + +/* + * test_fileops_chmod - Change file permissions using FileOpsChmod. + * + * This exercises the UNDO path: on rollback, the original permissions + * should be restored. + */ +Datum +test_fileops_chmod(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + int mode = PG_GETARG_INT32(1); + char *filepath = text_to_cstring(filepath_text); + + if (FileOpsChmod(filepath, (mode_t) mode) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not chmod file \"%s\": %m", filepath))); + + pfree(filepath); + PG_RETURN_VOID(); +} + +/* + * test_fileops_get_mode - Get file permission bits. + */ +Datum +test_fileops_get_mode(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + char *filepath = text_to_cstring(filepath_text); + struct stat st; + + if (stat(filepath, &st) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", filepath))); + + pfree(filepath); + PG_RETURN_INT32((int32) (st.st_mode & 0777)); +} + +/* + * test_fileops_link - Create a hard link using FileOpsLink. + * + * This exercises the UNDO path: on rollback, the link should be removed. + */ +Datum +test_fileops_link(PG_FUNCTION_ARGS) +{ + text *oldpath_text = PG_GETARG_TEXT_PP(0); + text *newpath_text = PG_GETARG_TEXT_PP(1); + char *oldpath = text_to_cstring(oldpath_text); + char *newpath = text_to_cstring(newpath_text); + + if (FileOpsLink(oldpath, newpath) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not link \"%s\" to \"%s\": %m", + oldpath, newpath))); + + pfree(oldpath); + pfree(newpath); + PG_RETURN_VOID(); +} + +/* + * test_fileops_file_exists - Check whether a file exists. + */ +Datum +test_fileops_file_exists(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + char *filepath = text_to_cstring(filepath_text); + struct stat st; + bool exists; + + exists = (stat(filepath, &st) == 0); + + pfree(filepath); + PG_RETURN_BOOL(exists); +} + +/* + * test_fileops_setxattr - Set an extended attribute using FileOpsSetXattr. + * + * Returns true if the platform supports xattrs, false if ENOTSUP. + * This exercises the UNDO path: on rollback, the original xattr value + * (or absence) should be restored. + */ +Datum +test_fileops_setxattr(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + text *name_text = PG_GETARG_TEXT_PP(1); + text *value_text = PG_GETARG_TEXT_PP(2); + char *filepath = text_to_cstring(filepath_text); + char *name = text_to_cstring(name_text); + char *value = text_to_cstring(value_text); + int ret; + + ret = FileOpsSetXattr(filepath, name, value, strlen(value)); + if (ret != 0) + { + if (errno == ENOTSUP || errno == EOPNOTSUPP || + errno == EPERM || errno == EACCES) + { + pfree(filepath); + pfree(name); + pfree(value); + PG_RETURN_BOOL(false); + } + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not set xattr \"%s\" on \"%s\": %m", + name, filepath))); + } + + pfree(filepath); + pfree(name); + pfree(value); + PG_RETURN_BOOL(true); +} + +/* + * test_fileops_getxattr - Get an extended attribute value. + * + * Returns NULL if the attribute doesn't exist or xattrs are unsupported. + */ +Datum +test_fileops_getxattr(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + text *name_text = PG_GETARG_TEXT_PP(1); + char *filepath = text_to_cstring(filepath_text); + char *name = text_to_cstring(name_text); + char buf[1024]; + ssize_t len; + + len = pg_getxattr(filepath, name, buf, sizeof(buf) - 1); + if (len < 0) + { + pfree(filepath); + pfree(name); + PG_RETURN_NULL(); + } + + buf[len] = '\0'; + + pfree(filepath); + pfree(name); + PG_RETURN_TEXT_P(cstring_to_text(buf)); +} + +/* + * test_fileops_removexattr - Remove an extended attribute using + * FileOpsRemoveXattr. + * + * Returns true if successful, false if xattrs are unsupported. + * This exercises the UNDO path: on rollback, the removed xattr should + * be restored. + */ +Datum +test_fileops_removexattr(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + text *name_text = PG_GETARG_TEXT_PP(1); + char *filepath = text_to_cstring(filepath_text); + char *name = text_to_cstring(name_text); + int ret; + + ret = FileOpsRemoveXattr(filepath, name); + if (ret != 0) + { + if (errno == ENOTSUP || errno == EOPNOTSUPP) + { + pfree(filepath); + pfree(name); + PG_RETURN_BOOL(false); + } + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not remove xattr \"%s\" from \"%s\": %m", + name, filepath))); + } + + pfree(filepath); + pfree(name); + PG_RETURN_BOOL(true); +} + +/* + * test_fileops_data_dir - Return the data directory path. + */ +Datum +test_fileops_data_dir(PG_FUNCTION_ARGS) +{ + PG_RETURN_TEXT_P(cstring_to_text(DataDir)); +} + +/* + * test_fileops_delete - Deferred delete via FileOpsDelete(at_commit=true). + * + * Exercises the log-at-commit-execution path: the delete happens (and is + * WAL-logged) in FileOpsDoPendingOps() at commit, so a physical standby + * reproduces it by replay. + */ +Datum +test_fileops_delete(PG_FUNCTION_ARGS) +{ + char *filepath = text_to_cstring(PG_GETARG_TEXT_PP(0)); + + FileOpsDelete(filepath, true); + pfree(filepath); + PG_RETURN_VOID(); +} + +/* + * test_fileops_rename - Deferred rename via FileOpsRename (always at_commit). + */ +Datum +test_fileops_rename(PG_FUNCTION_ARGS) +{ + char *oldpath = text_to_cstring(PG_GETARG_TEXT_PP(0)); + char *newpath = text_to_cstring(PG_GETARG_TEXT_PP(1)); + + if (FileOpsRename(oldpath, newpath) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not rename \"%s\" to \"%s\": %m", + oldpath, newpath))); + pfree(oldpath); + pfree(newpath); + PG_RETURN_VOID(); +} + +/* + * test_fileops_mkdir - Create a directory via FileOpsMkdir. + * + * FileOpsMkdir creates the directory immediately and registers an + * rmdir-on-abort; on commit the directory persists. Used to set up a + * directory the test can then remove with test_fileops_rmdir. + */ +Datum +test_fileops_mkdir(PG_FUNCTION_ARGS) +{ + char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + int mode = PG_GETARG_INT32(1); + + if (FileOpsMkdir(path, (mode_t) mode) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\": %m", path))); + pfree(path); + PG_RETURN_VOID(); +} + +/* + * test_fileops_rmdir - Deferred rmdir via FileOpsRmdir(at_commit=true). + */ +Datum +test_fileops_rmdir(PG_FUNCTION_ARGS) +{ + char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + + FileOpsRmdir(path, true); + pfree(path); + PG_RETURN_VOID(); +} + +/* + * test_fileops_write - Write a string at an offset via FileOpsWrite. + */ +Datum +test_fileops_write(PG_FUNCTION_ARGS) +{ + char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + int64 offset = PG_GETARG_INT64(1); + char *data = text_to_cstring(PG_GETARG_TEXT_PP(2)); + + (void) FileOpsWrite(path, (off_t) offset, data, (uint32) strlen(data)); + pfree(path); + pfree(data); + PG_RETURN_VOID(); +} + +/* + * test_fileops_chown - Change ownership via FileOpsChown. + */ +Datum +test_fileops_chown(PG_FUNCTION_ARGS) +{ + char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + + (void) FileOpsChown(path, (uid_t) PG_GETARG_INT32(1), + (gid_t) PG_GETARG_INT32(2)); + pfree(path); + PG_RETURN_VOID(); +} + +/* + * test_fileops_symlink - Create a symbolic link via FileOpsSymlink. + */ +Datum +test_fileops_symlink(PG_FUNCTION_ARGS) +{ + char *target = text_to_cstring(PG_GETARG_TEXT_PP(0)); + char *linkpath = text_to_cstring(PG_GETARG_TEXT_PP(1)); + + (void) FileOpsSymlink(target, linkpath); + pfree(target); + pfree(linkpath); + PG_RETURN_VOID(); +} diff --git a/src/test/modules/test_fileops/test_fileops.control b/src/test/modules/test_fileops/test_fileops.control new file mode 100644 index 00000000000..1d39ab2a916 --- /dev/null +++ b/src/test/modules/test_fileops/test_fileops.control @@ -0,0 +1,4 @@ +comment = 'Test module for transactional file operations (FILEOPS)' +default_version = '1.0' +module_pathname = '$libdir/test_fileops' +relocatable = true diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile index 9c4102b6b2c..cb169662fb6 100644 --- a/src/test/recovery/Makefile +++ b/src/test/recovery/Makefile @@ -13,6 +13,7 @@ EXTRA_INSTALL=contrib/pg_prewarm \ contrib/pg_stat_statements \ contrib/test_decoding \ src/test/modules/injection_points \ + src/test/modules/test_fileops \ src/test/modules/test_wait_lsn subdir = src/test/recovery diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build index b6ac9dd07a7..daa07c38560 100644 --- a/src/test/recovery/meson.build +++ b/src/test/recovery/meson.build @@ -69,6 +69,11 @@ tests += { 't/058_shutdown_crash_restart.pl', 't/059_remote_apply_status_interval.pl', 't/060_multixact_crashed_updater.pl', + 't/064_fileops_standby_divergence.pl', + 't/061_fileops_2pc_deferred.pl', + 't/062_fileops_recovery.pl', + 't/063_fileops_2pc.pl', + 't/065_fileops_crash_undo.pl', ], }, } diff --git a/src/test/recovery/t/061_fileops_2pc_deferred.pl b/src/test/recovery/t/061_fileops_2pc_deferred.pl new file mode 100644 index 00000000000..670cdb9b720 --- /dev/null +++ b/src/test/recovery/t/061_fileops_2pc_deferred.pl @@ -0,0 +1,167 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group +# +# Test that DEFERRED commit-time FILEOPS (delete/rename) and abort-time +# filesystem undo survive PREPARE TRANSACTION across a crash. +# +# The bug: PostPrepare_FileOps() used to free the pending-op list without +# recording it anywhere, so a transaction PREPARE'd with a deferred delete or +# rename (at_commit=true) lost that op entirely -- COMMIT PREPARED performed +# nothing -- and an immediate create's abort-time undo was lost for ROLLBACK +# PREPARED. The fix serializes the pending ops into the 2PC state file at +# PREPARE (TWOPHASE_RM_FILEOPS_ID records) and performs them in the FILEOPS +# post-commit / post-abort callbacks, so they survive a crash between PREPARE +# and the final decision, driven from the durable 2PC state. +# +# Unlike 063_fileops_2pc.pl (which only exercises chmod, a constructive op +# reversed via the cluster-wide UNDO chain), this test drives the deferred +# commit-time delete/rename path -- the one that was silently dropped. +# +# NB: FILEOPS writes WAL/UNDO only when XLogIsNeeded() (wal_level >= replica), +# so 2PC durability requires wal_level = replica; this test runs there. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('fileops_2pc_deferred'); +$node->init; +$node->append_conf( + "postgresql.conf", qq( +autovacuum = off +max_prepared_transactions = 10 +wal_level = replica +)); +$node->start; + +if (!$node->check_extension('test_fileops')) +{ + plan skip_all => 'Extension test_fileops not installed'; +} +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $datadir = $node->data_dir; + +sub file_exists +{ + my ($path) = @_; + return $node->safe_psql('postgres', + qq{SELECT test_fileops_file_exists('$path')}); +} + +# Poll until the file's existence matches $want ('t'/'f'). +sub wait_for_exists +{ + my ($path, $want, $desc) = @_; + my $ok = $node->poll_query_until('postgres', + qq{SELECT test_fileops_file_exists('$path') = '$want'}); + is($ok, '1', $desc); +} + +# ================================================================ +# Scenario A: COMMIT PREPARED performs the deferred commit-time ops +# - deferred delete of an existing file +# - deferred rename of an existing file +# - immediate create of a new file (register_delete) +# PREPARE, crash, restart, then COMMIT PREPARED. All three must take effect. +# ================================================================ +my $del_a = "$datadir/fo2pc_del_a.dat"; +my $ren_a = "$datadir/fo2pc_ren_a.dat"; +my $ren_a2 = "$datadir/fo2pc_ren_a.renamed"; +my $new_a = "$datadir/fo2pc_new_a.dat"; + +$node->safe_psql('postgres', + qq{SELECT test_fileops_create_tempfile('fo2pc_del_a.dat')}); +$node->safe_psql('postgres', + qq{SELECT test_fileops_create_tempfile('fo2pc_ren_a.dat')}); + +is(file_exists($del_a), 't', 'A: delete target exists before txn'); +is(file_exists($ren_a), 't', 'A: rename source exists before txn'); + +$node->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_delete('$del_a'); +SELECT test_fileops_rename('$ren_a', '$ren_a2'); +SELECT test_fileops_create('$new_a', 384); +PREPARE TRANSACTION 'fo2pc_commit'; +)); + +# While prepared, deferred ops have NOT run yet; the immediate create has. +is(file_exists($del_a), 't', + 'A: delete deferred -- file still present while prepared'); +is(file_exists($ren_a), 't', + 'A: rename deferred -- source still present while prepared'); +is(file_exists($new_a), 't', 'A: immediate create present while prepared'); + +# Crash between PREPARE and COMMIT PREPARED. +$node->stop('immediate'); +$node->start; + +is($node->safe_psql('postgres', q{SELECT gid FROM pg_prepared_xacts}), + 'fo2pc_commit', 'A: prepared xact survives crash'); + +$node->safe_psql('postgres', q{COMMIT PREPARED 'fo2pc_commit'}); + +is(file_exists($del_a), 'f', + 'A: COMMIT PREPARED performed the deferred delete'); +is(file_exists($ren_a), 'f', + 'A: COMMIT PREPARED performed the rename (source gone)'); +is(file_exists($ren_a2), 't', + 'A: COMMIT PREPARED performed the rename (dest present)'); +is(file_exists($new_a), 't', + 'A: created file persists after COMMIT PREPARED'); + +# ================================================================ +# Scenario B: ROLLBACK PREPARED does NOT perform the deferred commit-time +# ops, and DOES undo the immediate create. +# PREPARE, crash, restart, then ROLLBACK PREPARED. +# ================================================================ +my $del_b = "$datadir/fo2pc_del_b.dat"; +my $ren_b = "$datadir/fo2pc_ren_b.dat"; +my $ren_b2 = "$datadir/fo2pc_ren_b.renamed"; +my $new_b = "$datadir/fo2pc_new_b.dat"; + +$node->safe_psql('postgres', + qq{SELECT test_fileops_create_tempfile('fo2pc_del_b.dat')}); +$node->safe_psql('postgres', + qq{SELECT test_fileops_create_tempfile('fo2pc_ren_b.dat')}); + +$node->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_delete('$del_b'); +SELECT test_fileops_rename('$ren_b', '$ren_b2'); +SELECT test_fileops_create('$new_b', 384); +PREPARE TRANSACTION 'fo2pc_rollback'; +)); + +is(file_exists($new_b), 't', 'B: immediate create present while prepared'); + +# Crash between PREPARE and ROLLBACK PREPARED. +$node->stop('immediate'); +$node->start; + +is($node->safe_psql('postgres', q{SELECT gid FROM pg_prepared_xacts}), + 'fo2pc_rollback', 'B: prepared xact survives crash'); + +$node->safe_psql('postgres', q{ROLLBACK PREPARED 'fo2pc_rollback'}); + +is(file_exists($del_b), 't', + 'B: ROLLBACK PREPARED did NOT delete (file still present)'); +is(file_exists($ren_b), 't', + 'B: ROLLBACK PREPARED did NOT rename (source still present)'); +is(file_exists($ren_b2), 'f', + 'B: ROLLBACK PREPARED did NOT rename (dest absent)'); +# The immediate create is undone by the constructive op's cluster-wide UNDO +# record on ROLLBACK PREPARED. +wait_for_exists($new_b, 'f', + 'B: ROLLBACK PREPARED undid the immediate create'); + +# Server healthy at the end. +is($node->safe_psql('postgres', 'SELECT 1'), + '1', 'server operational after all deferred-2PC scenarios'); + +$node->stop; +done_testing(); diff --git a/src/test/recovery/t/062_fileops_recovery.pl b/src/test/recovery/t/062_fileops_recovery.pl new file mode 100644 index 00000000000..306fc19883a --- /dev/null +++ b/src/test/recovery/t/062_fileops_recovery.pl @@ -0,0 +1,446 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group +# +# Test crash recovery for transactional file operations (FILEOPS). +# +# These tests verify that FILEOPS WAL replay correctly handles: +# - Crash during file creation (with delete-on-abort) +# - Crash during deferred file deletion +# - Crash during file operations on standby +# - Multiple sequential crashes + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('fileops_recovery'); +$node->init; +$node->append_conf( + "postgresql.conf", qq( +autovacuum = off +log_min_messages = debug2 +)); +$node->start; + +# ================================================================ +# Test 1: CREATE TABLE survives crash +# ================================================================ + +$node->safe_psql( + "postgres", qq( +CREATE TABLE fileops_test (id int, data text); +INSERT INTO fileops_test VALUES (1, 'created_table'); +)); + +$node->stop('immediate'); +$node->start; + +my $result = + $node->safe_psql("postgres", "SELECT data FROM fileops_test WHERE id = 1"); +is($result, 'created_table', 'CREATE TABLE survives crash'); + +# ================================================================ +# Test 2: DROP TABLE is properly handled after crash +# ================================================================ + +$node->safe_psql( + "postgres", qq( +CREATE TABLE drop_me (id int); +INSERT INTO drop_me VALUES (1); +)); + +# Get the relfilenode before dropping +my $relpath = + $node->safe_psql("postgres", "SELECT pg_relation_filepath('drop_me')"); + +$node->safe_psql("postgres", "DROP TABLE drop_me"); + +$node->stop('immediate'); +$node->start; + +# Table should be gone +my ($ret, $stdout, $stderr) = + $node->psql("postgres", "SELECT * FROM drop_me"); +isnt($ret, 0, 'dropped table is gone after crash recovery'); + +# ================================================================ +# Test 3: Crash during transaction with CREATE TABLE (uncommitted) +# ================================================================ + +# This table is committed +$node->safe_psql( + "postgres", qq( +CREATE TABLE committed_table (id int); +INSERT INTO committed_table VALUES (42); +)); + +# Crash the server +$node->stop('immediate'); +$node->start; + +# Committed table should exist +$result = $node->safe_psql("postgres", "SELECT id FROM committed_table"); +is($result, '42', 'committed CREATE TABLE survives crash'); + +# ================================================================ +# Test 4: Multiple CREATE and DROP operations then crash +# ================================================================ + +$node->safe_psql( + "postgres", qq( +CREATE TABLE t1 (id int); +CREATE TABLE t2 (id int); +CREATE TABLE t3 (id int); +INSERT INTO t1 VALUES (1); +INSERT INTO t2 VALUES (2); +INSERT INTO t3 VALUES (3); +DROP TABLE t2; +)); + +$node->stop('immediate'); +$node->start; + +$result = $node->safe_psql("postgres", "SELECT id FROM t1"); +is($result, '1', 't1 survives crash'); + +($ret, $stdout, $stderr) = $node->psql("postgres", "SELECT * FROM t2"); +isnt($ret, 0, 't2 (dropped) is gone after crash'); + +$result = $node->safe_psql("postgres", "SELECT id FROM t3"); +is($result, '3', 't3 survives crash'); + +# ================================================================ +# Test 5: Crash after checkpoint with file operations +# ================================================================ + +$node->safe_psql( + "postgres", qq( +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t3; +CREATE TABLE checkpoint_test (id int); +INSERT INTO checkpoint_test VALUES (1); +CHECKPOINT; +INSERT INTO checkpoint_test VALUES (2); +)); + +$node->stop('immediate'); +$node->start; + +$result = + $node->safe_psql("postgres", "SELECT count(*) FROM checkpoint_test"); +is($result, '2', 'data after checkpoint survives crash'); + +# ================================================================ +# Test 6: Multiple crashes in sequence with file operations +# ================================================================ + +$node->safe_psql( + "postgres", qq( +DROP TABLE IF EXISTS checkpoint_test; +CREATE TABLE multi_crash (id int); +INSERT INTO multi_crash VALUES (1); +)); + +$node->stop('immediate'); +$node->start; + +$node->safe_psql( + "postgres", qq( +INSERT INTO multi_crash VALUES (2); +CREATE TABLE multi_crash_2 (id int); +INSERT INTO multi_crash_2 VALUES (10); +)); + +$node->stop('immediate'); +$node->start; + +$result = $node->safe_psql("postgres", "SELECT count(*) FROM multi_crash"); +is($result, '2', 'multi_crash table correct after double crash'); + +$result = $node->safe_psql("postgres", "SELECT id FROM multi_crash_2"); +is($result, '10', 'multi_crash_2 table correct after double crash'); + +# ================================================================ +# Test 7: CREATE TABLESPACE survives crash +# ================================================================ + +$node->safe_psql( + "postgres", qq( +DROP TABLE IF EXISTS multi_crash; +DROP TABLE IF EXISTS multi_crash_2; +SET allow_in_place_tablespaces = on; +CREATE TABLESPACE fileops_crash_ts LOCATION ''; +CREATE TABLE ts_crash_table (id int) TABLESPACE fileops_crash_ts; +INSERT INTO ts_crash_table VALUES (99); +)); + +$node->stop('immediate'); +$node->start; + +$result = $node->safe_psql("postgres", "SELECT id FROM ts_crash_table"); +is($result, '99', 'table in tablespace survives crash'); + +$result = $node->safe_psql("postgres", + "SELECT spcname FROM pg_tablespace WHERE spcname = 'fileops_crash_ts'"); +is($result, 'fileops_crash_ts', 'tablespace survives crash'); + +# ================================================================ +# Test 8: DROP TABLESPACE completes after crash +# ================================================================ + +$node->safe_psql( + "postgres", qq( +DROP TABLE ts_crash_table; +DROP TABLESPACE fileops_crash_ts; +)); + +$node->stop('immediate'); +$node->start; + +$result = $node->safe_psql("postgres", + "SELECT count(*) FROM pg_tablespace WHERE spcname = 'fileops_crash_ts'"); +is($result, '0', 'dropped tablespace is gone after crash'); + +# ================================================================ +# Test 9: CREATE DATABASE survives crash +# ================================================================ + +$node->safe_psql( + "postgres", qq( +CREATE DATABASE fileops_crash_db; +)); + +$node->stop('immediate'); +$node->start; + +$result = $node->safe_psql("postgres", + "SELECT datname FROM pg_database WHERE datname = 'fileops_crash_db'"); +is($result, 'fileops_crash_db', 'CREATE DATABASE survives crash'); + +$node->safe_psql("postgres", "DROP DATABASE fileops_crash_db"); + +# ================================================================ +# Test 10: Standby crash during FILEOPS replay +# ================================================================ + +# Set up primary + standby +my $primary = PostgreSQL::Test::Cluster->new('fileops_primary'); +$primary->init(allows_streaming => 1); +$primary->append_conf( + "postgresql.conf", qq( +autovacuum = off +)); +$primary->start; +$primary->backup('backup'); + +my $standby = PostgreSQL::Test::Cluster->new('fileops_standby'); +$standby->init_from_backup($primary, 'backup', has_streaming => 1); +$standby->start; + +# Create table on primary and wait for standby to catch up +$primary->safe_psql( + "postgres", qq( +CREATE TABLE standby_test (id int); +INSERT INTO standby_test VALUES (1); +)); + +$primary->wait_for_catchup($standby); + +# Verify on standby +$result = $standby->safe_psql("postgres", "SELECT id FROM standby_test"); +is($result, '1', 'CREATE TABLE replicated to standby'); + +# Crash the standby +$standby->stop('immediate'); +$standby->start; + +# Add more data on primary +$primary->safe_psql( + "postgres", qq( +INSERT INTO standby_test VALUES (2); +)); + +$primary->wait_for_catchup($standby); + +$result = + $standby->safe_psql("postgres", "SELECT count(*) FROM standby_test"); +is($result, '2', 'standby recovers and catches up after crash'); + +# Clean up primary/standby +$standby->stop; +$primary->stop; + +# Clean up original node +$node->stop; + +# ================================================================ +# Test 11/12: Crash exactly between WAL-flush and the physical syscall +# for the write-ordering fix (FileOpsCreate, FileOpsChmod). +# +# Before the fix, these two functions ran the physical syscall (and, for +# Create, the fsync) BEFORE the WAL record reached disk. After the fix, +# WAL/UNDO is durably flushed first, and only then does the physical +# syscall run, inside a critical section. These tests use injection +# points to pause the backend in that exact window -- after XLogFlush(), +# before open()/chmod() -- then hard-kill the postmaster. +# +# The injection point fires from inside the enclosing single-statement +# implicit transaction, before that transaction's COMMIT record could be +# written. So crashing there means the transaction is, correctly, +# recovered as never-committed: WAL redo physically replays the CREATE/ +# CHMOD unconditionally (redo does not know yet whether the surrounding +# xact committed), and then crash recovery's abort processing applies the +# UNDO record for that not-committed transaction, reversing the physical +# change. The end state must be as if the transaction never ran at all -- +# no orphaned file, no leftover mode change -- which is exactly the +# invariant broken before this fix: with the syscall running before WAL, +# a crash in this window used to leave a physical create()/chmod() with +# NO corresponding WAL or UNDO record for recovery to reconcile against +# (the syscall had already happened, but WAL/UNDO for it had not), +# leaving a permanently orphaned file or an un-reversible mode change even +# though the owning transaction never committed. Now WAL/UNDO is always +# durable before the syscall runs, so recovery always has a consistent +# decision to make -- keep (if committed) or reverse (if not) -- no +# matter which side of the syscall the crash lands on. +# ================================================================ + +if ($ENV{enable_injection_points} eq 'yes') +{ + my $inode = PostgreSQL::Test::Cluster->new('fileops_wal_ordering'); + $inode->init; + $inode->append_conf( + 'postgresql.conf', qq( +shared_preload_libraries = 'injection_points' +autovacuum = off +log_min_messages = debug2 +wal_level = replica +)); + $inode->start; + + if ( $inode->check_extension('injection_points') + && $inode->check_extension('test_fileops')) + { + $inode->safe_psql('postgres', 'CREATE EXTENSION injection_points'); + $inode->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + + my $idatadir = $inode->data_dir; + + # --- Test 11: FileOpsCreate ----------------------------------- + $inode->safe_psql('postgres', + q{SELECT injection_points_attach('fileops-create-after-wal-flush', 'wait')} + ); + + my $create_path = "$idatadir/wal_ordering_create.dat"; + my $bgpsql = $inode->background_psql('postgres'); + $bgpsql->query_until( + qr/starting_create/, + qq(\\echo starting_create +SELECT test_fileops_create('$create_path', 384); +\\q +)); + + $inode->wait_for_event('client backend', + 'fileops-create-after-wal-flush'); + + # The WAL record is durably flushed, but the physical create() has + # not run yet -- the file must not exist on disk right now. + ok( !-e $create_path, + 'Test 11: file does not exist yet at the injection point ' + . '(WAL durable, physical create() not yet run)'); + + # Hard-kill the postmaster from exactly this window. The wrapping + # implicit transaction is still in progress (blocked on the + # injection point), so it never committed. + $inode->stop('immediate'); + $inode->start; + + # Recovery redoes the physical CREATE from the durable WAL record + # unconditionally, then determines the transaction never committed + # and applies its UNDO record to reverse the create. Net effect: + # the file must NOT exist -- exactly as if the transaction had + # never run, with no orphaned state left behind by the crash. + ok( !-e $create_path, + 'Test 11: FileOpsCreate of a not-committed transaction is ' + . 'cleanly reversed after a crash mid-critical-section'); + + # Server is healthy after the reconciliation. + my $result11 = $inode->safe_psql('postgres', 'SELECT 1'); + is($result11, '1', 'Test 11: server operational after recovery'); + + $bgpsql->{run}->finish; + + # --- Test 12: FileOpsChmod ------------------------------------- + $inode->safe_psql('postgres', + qq{SELECT test_fileops_create_tempfile('wal_ordering_chmod.dat')} + ); + my $chmod_path = "$idatadir/wal_ordering_chmod.dat"; + $inode->safe_psql('postgres', + qq{SELECT test_fileops_chmod('$chmod_path', 420)}); # 0644 + my $mode_before = $inode->safe_psql('postgres', + qq{SELECT test_fileops_get_mode('$chmod_path')}); + # Windows has no POSIX file modes: chmod only toggles read-only. + my $posix_modes = !$windows_os; + is($mode_before, '420', 'Test 12: baseline mode is 0644') + if $posix_modes; + + $inode->safe_psql('postgres', + q{SELECT injection_points_attach('fileops-chmod-after-wal-flush', 'wait')} + ); + + $bgpsql = $inode->background_psql('postgres'); + $bgpsql->query_until( + qr/starting_chmod/, + qq(\\echo starting_chmod +SELECT test_fileops_chmod('$chmod_path', 448); -- 0700 +\\q +)); + + $inode->wait_for_event('client backend', + 'fileops-chmod-after-wal-flush'); + + # WAL is durably flushed, but chmod() has not run yet -- the mode + # on disk must still be the baseline 0644. + my $mode_at_injection = $inode->safe_psql('postgres', + qq{SELECT test_fileops_get_mode('$chmod_path')}); + is($mode_at_injection, '420', + 'Test 12: mode unchanged on disk at the injection point ' + . '(WAL durable, physical chmod() not yet run)') + if $posix_modes; + + # Hard-kill mid-critical-section, same as Test 11. The chmod's + # wrapping implicit transaction never committed. + $inode->stop('immediate'); + $inode->start; + + # Recovery redoes the physical CHMOD unconditionally, then reverses + # it via UNDO because the transaction never committed. Net effect: + # mode must be back to the pre-transaction baseline (0644), not the + # attempted 0700, and not some undefined intermediate state. + my $mode_after = $inode->safe_psql('postgres', + qq{SELECT test_fileops_get_mode('$chmod_path')}); + is($mode_after, '420', + 'Test 12: FileOpsChmod of a not-committed transaction is ' + . 'cleanly reversed after a crash mid-critical-section') + if $posix_modes; + + # Server is healthy after the reconciliation. + my $result12 = $inode->safe_psql('postgres', 'SELECT 1'); + is($result12, '1', 'Test 12: server operational after recovery'); + + $bgpsql->{run}->finish; + } + else + { + note + 'skipping Test 11/12 (injection_points or test_fileops extension not installed)'; + } + + $inode->stop; +} +else +{ + note 'skipping Test 11/12 (injection points not supported by this build)'; +} + +done_testing(); diff --git a/src/test/recovery/t/063_fileops_2pc.pl b/src/test/recovery/t/063_fileops_2pc.pl new file mode 100644 index 00000000000..b88c8dd5910 --- /dev/null +++ b/src/test/recovery/t/063_fileops_2pc.pl @@ -0,0 +1,149 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group +# +# Test two-phase commit (PREPARE TRANSACTION / COMMIT PREPARED / ROLLBACK +# PREPARED) for transactions that generated cluster-wide FILEOPS UNDO. +# +# FILEOPS uses the undo-in-common-WAL mechanism. At PREPARE the permanent +# UNDO chain-head LSN is durably saved in the 2PC state (xl_xact_prepare / +# gxact->undo_batch_lsn), and its WAL is retained while the xact stays +# prepared. ROLLBACK PREPARED (even from a different backend, post-crash) +# applies that chain, which reverses the filesystem change. COMMIT PREPARED +# keeps it. +# +# Vehicle: the test_fileops extension's SQL wrappers for FileOpsChmod, which +# are legal inside a transaction block that can then be PREPARE'd. +# +# NB: FILEOPS only writes UNDO WAL when XLogIsNeeded() (wal_level >= replica). +# At wal_level = minimal it relies on the backend-private pending-op list, +# which is discarded at PREPARE and cannot survive 2PC. This test therefore +# runs at wal_level = replica, which is what any FILEOPS + 2PC deployment +# requires. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Every test here is a chmod reversal; Windows has no POSIX file modes. +plan skip_all => 'POSIX permission semantics required' if $windows_os; + +my $node = PostgreSQL::Test::Cluster->new('fileops_2pc'); +$node->init; +$node->append_conf( + "postgresql.conf", qq( +autovacuum = off +max_prepared_transactions = 10 +wal_level = replica +)); +$node->start; + +if (!$node->check_extension('test_fileops')) +{ + plan skip_all => 'Extension test_fileops not installed'; +} +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $datadir = $node->data_dir; +my $file = "$datadir/twopc.dat"; + +# Poll until the file mode equals $want, or fail. +sub wait_for_mode +{ + my ($want, $desc) = @_; + my $ok = $node->poll_query_until('postgres', + qq{SELECT test_fileops_get_mode('$file') = $want}); + is($ok, '1', $desc); +} + +# 0644 = 420, 0700 = 448, 0600 = 384 (decimal) +$node->safe_psql('postgres', + qq{SELECT test_fileops_create_tempfile('twopc.dat')}); +$node->safe_psql('postgres', qq{SELECT test_fileops_chmod('$file', 420)}); +my $baseline = + $node->safe_psql('postgres', qq{SELECT test_fileops_get_mode('$file')}); +is($baseline, '420', 'baseline mode is 0644'); + +# ================================================================ +# Test 1: ROLLBACK PREPARED reverses the chmod +# ================================================================ +$node->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_chmod('$file', 448); -- 0700 +PREPARE TRANSACTION 'fo_rb'; +)); +my $m = + $node->safe_psql('postgres', qq{SELECT test_fileops_get_mode('$file')}); +is($m, '448', 'Test 1: chmod applied and still visible while prepared'); + +$node->safe_psql('postgres', q{ROLLBACK PREPARED 'fo_rb'}); +wait_for_mode(420, 'Test 1: ROLLBACK PREPARED reverses chmod (0700 -> 0644)'); + +# ================================================================ +# Test 2: COMMIT PREPARED keeps the chmod +# ================================================================ +$node->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_chmod('$file', 448); -- 0700 +PREPARE TRANSACTION 'fo_cp'; +)); +$node->safe_psql('postgres', q{COMMIT PREPARED 'fo_cp'}); +$m = $node->safe_psql('postgres', qq{SELECT test_fileops_get_mode('$file')}); +is($m, '448', 'Test 2: COMMIT PREPARED keeps chmod (0700)'); + +# Reset to baseline for the crash test. +$node->safe_psql('postgres', qq{SELECT test_fileops_chmod('$file', 420)}); + +# ================================================================ +# Test 3: WAL retention across checkpoints while prepared +# ================================================================ +$node->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_chmod('$file', 384); -- 0600 +PREPARE TRANSACTION 'fo_ckpt'; +)); +# Churn WAL and checkpoint; the prepared xact's UNDO batch must be retained. +$node->safe_psql('postgres', 'CHECKPOINT'); +for my $i (1 .. 20) +{ + $node->safe_psql('postgres', + "CREATE TABLE junk_$i(x int); " + . "INSERT INTO junk_$i SELECT g FROM generate_series(1,5000) g; " + . "DROP TABLE junk_$i;"); +} +$node->safe_psql('postgres', 'CHECKPOINT'); +$node->safe_psql('postgres', q{ROLLBACK PREPARED 'fo_ckpt'}); +wait_for_mode(420, + 'Test 3: ROLLBACK PREPARED still reverses after WAL churn + checkpoints'); + +# ================================================================ +# Test 4: crash recovery -- prepared UNDO xact survives, then reverts +# ================================================================ +$node->safe_psql('postgres', qq{SELECT test_fileops_chmod('$file', 420)}); +$node->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_chmod('$file', 448); -- 0700 +PREPARE TRANSACTION 'fo_crash'; +)); +$node->stop('immediate'); +$node->start; + +my $prep = $node->safe_psql('postgres', q{SELECT gid FROM pg_prepared_xacts}); +is($prep, 'fo_crash', 'Test 4: prepared xact survives crash recovery'); +$m = $node->safe_psql('postgres', qq{SELECT test_fileops_get_mode('$file')}); +is($m, '448', 'Test 4: chmod still applied (prepared, not yet rolled back)'); + +$node->safe_psql('postgres', q{ROLLBACK PREPARED 'fo_crash'}); +wait_for_mode(420, + 'Test 4: ROLLBACK PREPARED after crash+restart reverses chmod'); + +# Server healthy at the end. +is($node->safe_psql('postgres', 'SELECT 1'), + '1', 'server operational after all 2PC UNDO scenarios'); + +$node->stop; +done_testing(); diff --git a/src/test/recovery/t/064_fileops_standby_divergence.pl b/src/test/recovery/t/064_fileops_standby_divergence.pl new file mode 100644 index 00000000000..65a07fb873e --- /dev/null +++ b/src/test/recovery/t/064_fileops_standby_divergence.pl @@ -0,0 +1,124 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group +# +# Regression test: physical-standby reproduction of FILEOPS deferred +# DESTRUCTIVE operations (DELETE, RENAME, RMDIR). +# +# These three ops execute on the primary at commit inside +# FileOpsDoPendingOps(). Each is logged at commit-EXECUTION time with a +# real, idempotent redo handler, so a physical standby (which only replays +# WAL, never running FileOpsDoPendingOps) reproduces them and its filesystem +# converges with the primary's. +# +# This test sets up a streaming physical standby, performs a committed +# deferred DELETE, RENAME, and RMDIR on the primary, waits for catchup, and +# asserts the standby's filesystem MATCHES the primary's. It MUST FAIL on the +# pre-fix (no-op redo) code -- see the header note in fileops.c and the +# "prove non-vacuity" comment below. +# +# All FILEOPS paths are RELATIVE to the data directory, matching every core +# FILEOPS caller (dbcommands.c, tablespace.c): the backend's cwd is PGDATA, so +# a relative path resolves to the right file on both the primary and a standby +# whose absolute data_dir differs. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $primary = PostgreSQL::Test::Cluster->new('fileops_div_primary'); +$primary->init(allows_streaming => 1); +$primary->append_conf( + "postgresql.conf", qq( +autovacuum = off +wal_level = replica +)); +$primary->start; + +if (!$primary->check_extension('test_fileops')) +{ + plan skip_all => 'Extension test_fileops not installed'; +} +$primary->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $primary_dir = $primary->data_dir; + +# Fixture, all committed and WAL-logged so the standby has them before we +# start mutating. Use FileOps CREATE (WAL-logged, standby-reproduced) and +# FileOps MKDIR (WAL-logged) rather than raw open(), so the fixture itself +# replicates. Paths are relative to PGDATA. +$primary->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_create('div_delete_me.dat', 384); -- 0600 +SELECT test_fileops_create('div_rename_src.dat', 384); +SELECT test_fileops_mkdir('div_rmdir_me', 448); -- 0700 +COMMIT; +)); + +# Take the base backup AFTER the fixture exists so init_from_backup gives the +# standby the fixture files, then stream the mutations. +$primary->backup('backup'); +my $standby = PostgreSQL::Test::Cluster->new('fileops_div_standby'); +$standby->init_from_backup($primary, 'backup', has_streaming => 1); +$standby->start; + +my $standby_dir = $standby->data_dir; + +$primary->wait_for_catchup($standby); + +# Sanity: the fixture replicated to the standby. +ok(-e "$standby_dir/div_delete_me.dat", 'fixture: delete target on standby'); +ok(-e "$standby_dir/div_rename_src.dat", 'fixture: rename source on standby'); +ok(-d "$standby_dir/div_rmdir_me", 'fixture: rmdir target on standby'); + +# Perform the three deferred destructive ops in a committed transaction on the +# primary. Each goes through the FileOps deferred path (at_commit=true) and, +# with the fix, is WAL-logged at commit-execution. +$primary->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_delete('div_delete_me.dat'); +SELECT test_fileops_rename('div_rename_src.dat', 'div_rename_dst.dat'); +SELECT test_fileops_rmdir('div_rmdir_me'); +COMMIT; +)); + +# Confirm the primary applied them (the op ran in FileOpsDoPendingOps). +ok(!-e "$primary_dir/div_delete_me.dat", 'primary: delete applied'); +ok(!-e "$primary_dir/div_rename_src.dat", 'primary: rename source gone'); +ok(-e "$primary_dir/div_rename_dst.dat", 'primary: rename dest present'); +ok(!-d "$primary_dir/div_rmdir_me", 'primary: rmdir applied'); + +$primary->wait_for_catchup($standby); + +# The diff the reviewer asked for: the standby's filesystem must MATCH the +# primary's. Pre-fix (no-op redo) the standby still has the original file / +# dir and lacks the renamed file -- these four assertions all fail, which is +# the non-vacuity proof. Post-fix they pass. +ok(!-e "$standby_dir/div_delete_me.dat", + 'deferred DELETE reproduced on standby (file gone)'); +ok(!-e "$standby_dir/div_rename_src.dat", + 'deferred RENAME reproduced on standby (old name gone)'); +ok(-e "$standby_dir/div_rename_dst.dat", + 'deferred RENAME reproduced on standby (new name present)'); +ok(!-d "$standby_dir/div_rmdir_me", + 'deferred RMDIR reproduced on standby (dir gone)'); + +# Idempotent redo across a standby restart: replaying the same records again +# after a crash must not error or diverge. +$standby->stop('immediate'); +$standby->start; +$primary->safe_psql('postgres', 'SELECT txid_current()'); +$primary->wait_for_catchup($standby); +ok(!-e "$standby_dir/div_delete_me.dat", + 'standby end state stable after restart (delete)'); +ok(-e "$standby_dir/div_rename_dst.dat", + 'standby end state stable after restart (rename)'); +ok(!-d "$standby_dir/div_rmdir_me", + 'standby end state stable after restart (rmdir)'); + +$standby->stop; +$primary->stop; + +done_testing(); diff --git a/src/test/recovery/t/065_fileops_crash_undo.pl b/src/test/recovery/t/065_fileops_crash_undo.pl new file mode 100644 index 00000000000..f598c7e7418 --- /dev/null +++ b/src/test/recovery/t/065_fileops_crash_undo.pl @@ -0,0 +1,137 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group +# +# FILEOPS crash-recovery revert via the cluster-wide UNDO engine (common WAL). +# +# FILEOPS is a consumer of the cluster-wide UNDO engine: its structural +# operations (file existence, permissions, directory structure) are +# WAL-durable via RM_FILEOPS_ID forward records (fileops_redo), and each +# reversible op also writes an UNDO record into the common WAL +# (PrepareXactUndoData(UNDO_RMID_FILEOPS,...) / InsertXactUndoData) in the +# same critical section as its forward record. Filesystem structural +# operations are cluster-wide changes whose durability/revert belong in the +# common WAL. This test proves that model is crash-safe. +# +# A transaction produces FILEOPS create/chmod (immediate, WAL-flushed), then the +# server is crashed (immediate stop) BEFORE the transaction commits. On +# recovery: +# - fileops_redo() REDOES the forward create/chmod unconditionally, and +# - PerformUndoRecovery() (end of redo) applies the never-committed +# transaction's FILEOPS UNDO chain via the RM_FILEOPS_ID rm_undo +# callbacks (fileops_undo.c), reversing every op: +# - the created file is unlinked (CREATE revert), and +# - the chmod'd file's mode is restored (CHMOD revert, before-state in WAL). +# +# FILEOPS only writes WAL at wal_level >= replica (XLogIsNeeded()), so the test +# runs at wal_level = replica, matching any FILEOPS deployment. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('fileops_crash'); +$node->init; +$node->append_conf( + 'postgresql.conf', qq( +autovacuum = off +wal_level = replica +)); +$node->start; + +if (!$node->check_extension('test_fileops')) +{ + plan skip_all => 'Extension test_fileops not installed'; +} +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $datadir = $node->data_dir; +my $create_path = "$datadir/fileops_crash_create.dat"; +my $chmod_path = "$datadir/fileops_crash_chmod.dat"; + +# --------------------------------------------------------------------------- +# Set up a pre-existing file to chmod (this part is committed). 0644 = 420. +# --------------------------------------------------------------------------- +$node->safe_psql('postgres', + q{SELECT test_fileops_create_tempfile('fileops_crash_chmod.dat')}); +$node->safe_psql('postgres', + qq{SELECT test_fileops_chmod('$chmod_path', 420)}); +SKIP: +{ + skip 'POSIX permission semantics required', 1 if $windows_os; + is( $node->safe_psql( + 'postgres', qq{SELECT test_fileops_get_mode('$chmod_path')}), + '420', + 'baseline mode of pre-existing file is 0644'); +} + +# --------------------------------------------------------------------------- +# Open a transaction that produces FILEOPS UNDO (create a new file and +# chmod the existing one) and DO NOT commit it. The FILEOPS forward operations +# execute and are WAL-flushed immediately (the file exists on disk, the mode is +# changed), but the wrapping transaction stays in progress. +# --------------------------------------------------------------------------- +my $bg = $node->background_psql('postgres'); +$bg->query_until( + qr/fileops_ready/, qq( +BEGIN; +SELECT test_fileops_create('$create_path', 384); -- 0600, new file +SELECT test_fileops_chmod('$chmod_path', 448); -- 0700, existing file +\\echo fileops_ready +)); + +# The forward ops are visible on disk while the xact is in progress. +ok(-e $create_path, + 'created file exists on disk before crash (in-progress xact)'); +SKIP: +{ + skip 'POSIX permission semantics required', 1 if $windows_os; + is( $node->safe_psql( + 'postgres', qq{SELECT test_fileops_get_mode('$chmod_path')}), + '448', + 'chmod applied on disk before crash (in-progress xact)'); +} + +# --------------------------------------------------------------------------- +# Crash immediately: the transaction never commits. Stop the server before +# closing the client: closing it first would let the backend roll the +# transaction back online, which is a different case. The background psql +# connection dies with the server; abandon it (do not try to drain/finish it, +# which would hang waiting on the dead connection). +# --------------------------------------------------------------------------- +$node->stop('immediate'); +$bg->{run}->kill_kill; +$node->start; + +# --------------------------------------------------------------------------- +# Recovery redoes the forward ops from WAL, then PerformUndoRecovery() reverses +# the never-committed transaction's ops from its common-WAL UNDO chain: +# - the created file is unlinked, and +# - the chmod is restored to the original 0644. +# --------------------------------------------------------------------------- +ok( !-e $create_path, + 'FILEOPS COMMON-WAL CRASH RECOVERY: created file was REVERTED (unlinked) after crash' +); +SKIP: +{ + skip 'POSIX permission semantics required', 1 if $windows_os; + is( $node->safe_psql( + 'postgres', qq{SELECT test_fileops_get_mode('$chmod_path')}), + '420', + 'FILEOPS COMMON-WAL CRASH RECOVERY: chmod was REVERTED (0700 -> 0644) after crash' + ); +} + +# Recovery must be clean. +my $rlog = slurp_file($node->logfile); +unlike( + $rlog, + qr/PANIC|inconsistent|corrupt/i, + 'clean crash recovery, no PANIC/corruption'); + +# Server healthy afterwards. +is($node->safe_psql('postgres', 'SELECT 1'), + '1', 'server operational after crash-recovery undo'); + +$node->stop; +done_testing(); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 336028f7d8d..ca8d7b8a92e 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -834,6 +834,17 @@ ExtensionVersionInfo FDWCollateState FD_SET FILE +FileOpsTwoPhaseRecord +FileopsUndoChmod +FileopsUndoChown +FileopsUndoCreate +FileopsUndoLink +FileopsUndoMkdir +FileopsUndoRemovexattr +FileopsUndoRename +FileopsUndoSetxattr +FileopsUndoSymlink +FileopsUndoTruncate FILETIME FPI FSMAddress @@ -1909,6 +1920,8 @@ PACE_HEADER PACL PATH PCtxtHandle +PendingFileOp +PendingFileOpType PERL_CONTEXT PERL_SI PFN @@ -3510,6 +3523,19 @@ X86FeatureId XactUndoContext XactUndoData XactUndoSubTransactionState +xl_fileops_chmod +xl_fileops_chown +xl_fileops_create +xl_fileops_delete +xl_fileops_link +xl_fileops_mkdir +xl_fileops_removexattr +xl_fileops_rename +xl_fileops_rmdir +xl_fileops_setxattr +xl_fileops_symlink +xl_fileops_truncate +xl_fileops_write xl_undo_allocate xl_undo_apply xl_undo_batch -- 2.54.0