From 387597a8b9bd90a6d6729d5974af152e2e690432 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 14 Sep 2026 13:03:40 +0000 Subject: [PATCH v9 3/3] Route filesystem mutations through FILEOPS Route the server's transactional DDL/storage callers that mutate the filesystem outside the buffer manager through the FILEOPS resource manager added in the previous commit, so an abort or crash recovery recursively reverts a partial filesystem change instead of leaving orphan trees behind. - copydir() gains a register_for_abort_cleanup argument: forward-path callers register the destination tree with FILEOPS (FileOpsRmtree at abort), while WAL-replay callers pass false because the redo handler is itself the recovery action. - CREATE DATABASE (STRATEGY=FILE_COPY) and ALTER DATABASE SET TABLESPACE copy their directory trees with abort cleanup enabled, closing the long-standing crash-asymmetry where a rolled-back copy left an orphan tree. - CREATE/DROP TABLESPACE route their directory create/remove through FileOps* so tablespace symlink and directory changes are transactional. Author: Greg Burd --- src/backend/commands/dbcommands.c | 144 +++++++++----- src/backend/commands/tablespace.c | 175 +++++++++++++----- src/backend/storage/file/copydir.c | 33 +++- src/include/storage/copydir.h | 14 +- .../t/007_fileops_commit_crash.pl | 18 ++ 5 files changed, 289 insertions(+), 95 deletions(-) diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c index 7e3fc59eafd..99549a79e74 100644 --- a/src/backend/commands/dbcommands.c +++ b/src/backend/commands/dbcommands.c @@ -56,6 +56,7 @@ #include "replication/slot.h" #include "storage/copydir.h" #include "storage/fd.h" +#include "storage/fileops.h" #include "storage/ipc.h" #include "storage/lmgr.h" #include "storage/md.h" @@ -473,55 +474,91 @@ CreateDirAndVersionFile(char *dbpath, Oid dbid, Oid tsid, bool isRedo) nbytes = strlen(PG_MAJORVERSION) + 1; /* Create database directory. */ - if (MakePGDirectory(dbpath) < 0) + if (!isRedo) { - /* Failure other than already exists or not in WAL replay? */ - if (errno != EEXIST || !isRedo) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not create directory \"%s\": %m", dbpath))); + FileOpsMkdir(dbpath, pg_dir_create_mode); + } + else + { + if (MakePGDirectory(dbpath) < 0) + { + /* Failure other than already exists or not in WAL replay? */ + if (errno != EEXIST) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\": %m", dbpath))); + } } /* - * Create PG_VERSION file in the database path. If the file already - * exists and we are in WAL replay then try again to open it in write - * mode. + * Create PG_VERSION file in the database path. */ snprintf(versionfile, sizeof(versionfile), "%s/%s", dbpath, "PG_VERSION"); - fd = OpenTransientFile(versionfile, O_WRONLY | O_CREAT | O_EXCL | PG_BINARY); - if (fd < 0 && errno == EEXIST && isRedo) - fd = OpenTransientFile(versionfile, O_WRONLY | O_TRUNC | PG_BINARY); + if (!isRedo) + { + /* + * Use FileOpsCreate + FileOpsWrite for transactional version file + * creation with crash-safe rollback support. + */ + fd = FileOpsCreate(versionfile, O_WRONLY | PG_BINARY, + pg_file_create_mode, true); + pgstat_report_wait_start(WAIT_EVENT_VERSION_FILE_WRITE); + errno = 0; + if ((int) write(fd, buf, nbytes) != nbytes) + { + if (errno == 0) + errno = ENOSPC; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write to file \"%s\": %m", versionfile))); + } + pgstat_report_wait_end(); - if (fd < 0) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not create file \"%s\": %m", versionfile))); + pgstat_report_wait_start(WAIT_EVENT_VERSION_FILE_SYNC); + if (pg_fsync(fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", versionfile))); + fsync_fname(dbpath, true); + pgstat_report_wait_end(); - /* Write PG_MAJORVERSION in the PG_VERSION file. */ - pgstat_report_wait_start(WAIT_EVENT_VERSION_FILE_WRITE); - errno = 0; - if (write(fd, buf, nbytes) != nbytes) - { - /* If write didn't set errno, assume problem is no disk space. */ - if (errno == 0) - errno = ENOSPC; - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not write to file \"%s\": %m", versionfile))); + CloseTransientFile(fd); } - pgstat_report_wait_end(); + else + { + /* WAL replay: use raw file operations */ + fd = OpenTransientFile(versionfile, O_WRONLY | O_CREAT | O_EXCL | PG_BINARY); + if (fd < 0 && errno == EEXIST) + fd = OpenTransientFile(versionfile, O_WRONLY | O_TRUNC | PG_BINARY); - pgstat_report_wait_start(WAIT_EVENT_VERSION_FILE_SYNC); - if (pg_fsync(fd) != 0) - ereport(data_sync_elevel(ERROR), - (errcode_for_file_access(), - errmsg("could not fsync file \"%s\": %m", versionfile))); - fsync_fname(dbpath, true); - pgstat_report_wait_end(); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create file \"%s\": %m", versionfile))); - /* Close the version file. */ - CloseTransientFile(fd); + pgstat_report_wait_start(WAIT_EVENT_VERSION_FILE_WRITE); + errno = 0; + if ((int) write(fd, buf, nbytes) != nbytes) + { + if (errno == 0) + errno = ENOSPC; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write to file \"%s\": %m", versionfile))); + } + pgstat_report_wait_end(); + + pgstat_report_wait_start(WAIT_EVENT_VERSION_FILE_SYNC); + if (pg_fsync(fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", versionfile))); + fsync_fname(dbpath, true); + pgstat_report_wait_end(); + + CloseTransientFile(fd); + } /* If we are not in WAL replay then write the WAL. */ if (!isRedo) @@ -633,9 +670,11 @@ CreateDatabaseUsingFileCopy(Oid src_dboid, Oid dst_dboid, Oid src_tsid, /* * Copy this subdirectory to the new location * - * We don't need to copy subdirectories + * We don't need to copy subdirectories. Register the destination for + * delete-on-abort so a transaction abort recursively removes the + * partial copy. */ - copydir(srcpath, dstpath, false); + copydir(srcpath, dstpath, false, true); /* Record the filesystem change in XLOG */ { @@ -2221,8 +2260,18 @@ movedb(const char *dbname, const char *tblspcname) FreeDir(dstdir); /* - * The directory exists but is empty. We must remove it before using + * The directory exists but is empty. We must remove it before using * the copydir function. + * + * NB: this rmdir is deliberately NOT routed through FILEOPS. The + * destination tablespace path is not yet referenced by any catalog + * row (the pg_database.dattablespace update happens later, and is + * what binds dst_dbpath to the database on commit), so an abort that + * leaves dst_dbpath missing does not orphan any catalog state -- the + * database remains in src_tblspcoid. copydir() below will recreate + * dst_dbpath via MakePGDirectory in its first action and registers + * the destination tree for delete-on-abort, so the partial-copy + * window is covered. */ if (rmdir(dst_dbpath) != 0) elog(ERROR, "could not remove directory \"%s\": %m", @@ -2245,9 +2294,13 @@ movedb(const char *dbname, const char *tblspcname) bool new_record_repl[Natts_pg_database] = {0}; /* - * Copy files from the old tablespace to the new one + * Copy files from the old tablespace to the new one. Register the + * destination for delete-on-abort so a transaction abort recursively + * removes the partial copy (closing the long-standing crash-asymmetry + * with the source tree, which is preserved until commit by movedb's + * post-commit cleanup). */ - copydir(src_dbpath, dst_dbpath, false); + copydir(src_dbpath, dst_dbpath, false, true); /* * Record the filesystem change in XLOG @@ -3407,9 +3460,12 @@ dbase_redo(XLogReaderState *record) /* * Copy this subdirectory to the new location * - * We don't need to copy subdirectories + * We don't need to copy subdirectories. Pass false for + * register_for_abort_cleanup: this is a WAL replay path with no + * surrounding transaction; the redo handler is itself the recovery + * action. */ - copydir(src_path, dst_path, false); + copydir(src_path, dst_path, false, false); pfree(src_path); pfree(dst_path); diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c index 7c40176b7b9..0ba99f635af 100644 --- a/src/backend/commands/tablespace.c +++ b/src/backend/commands/tablespace.c @@ -70,6 +70,7 @@ #include "miscadmin.h" #include "postmaster/bgwriter.h" #include "storage/fd.h" +#include "storage/fileops.h" #include "storage/lmgr.h" #include "storage/lwlock.h" #include "storage/procsignal.h" @@ -600,11 +601,18 @@ create_tablespace_directories(const char *location, const Oid tablespaceoid) if (in_place) { - if (MakePGDirectory(linkloc) < 0 && errno != EEXIST) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not create directory \"%s\": %m", - linkloc))); + if (!InRecovery) + { + FileOpsMkdir(linkloc, pg_dir_create_mode); + } + else + { + if (MakePGDirectory(linkloc) < 0 && errno != EEXIST) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\": %m", + linkloc))); + } } location_with_version_dir = psprintf("%s/%s", in_place ? linkloc : location, @@ -615,20 +623,36 @@ create_tablespace_directories(const char *location, const Oid tablespaceoid) * it doesn't exist or has the wrong owner. Not needed for in-place mode, * because in that case we created the directory with the desired * permissions. + * + * Outside recovery, route through FileOpsChmod so a transaction abort + * restores the prior mode via the FILEOPS UNDO record. In recovery + * (where there is no surrounding transaction) issue chmod directly. On + * Windows, chmod is partial (only the read-only bit is meaningful); + * FileOpsChmod handles platform differences. */ - if (!in_place && chmod(location, pg_dir_create_mode) != 0) + if (!in_place) { - if (errno == ENOENT) - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_FILE), - errmsg("directory \"%s\" does not exist", location), - InRecovery ? errhint("Create this directory for the tablespace before " - "restarting the server.") : 0)); + int rc; + + if (InRecovery) + rc = chmod(location, pg_dir_create_mode); else - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not set permissions on directory \"%s\": %m", - location))); + rc = FileOpsChmod(location, pg_dir_create_mode); + + if (rc != 0) + { + if (errno == ENOENT) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FILE), + errmsg("directory \"%s\" does not exist", location), + InRecovery ? errhint("Create this directory for the tablespace before " + "restarting the server.") : 0)); + else + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not set permissions on directory \"%s\": %m", + location))); + } } /* @@ -645,6 +669,8 @@ create_tablespace_directories(const char *location, const Oid tablespaceoid) (errcode_for_file_access(), errmsg("could not stat directory \"%s\": %m", location_with_version_dir))); + else if (!InRecovery) + FileOpsMkdir(location_with_version_dir, pg_dir_create_mode); else if (MakePGDirectory(location_with_version_dir) < 0) ereport(ERROR, (errcode_for_file_access(), @@ -671,11 +697,16 @@ create_tablespace_directories(const char *location, const Oid tablespaceoid) /* * Create the symlink under PGDATA */ - if (!in_place && symlink(location, linkloc) < 0) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not create symbolic link \"%s\": %m", - linkloc))); + if (!in_place) + { + if (!InRecovery) + FileOpsSymlink(location, linkloc); + else if (symlink(location, linkloc) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create symbolic link \"%s\": %m", + linkloc))); + } pfree(linkloc); pfree(location_with_version_dir); @@ -773,27 +804,50 @@ destroy_tablespace_directories(Oid tablespaceoid, bool redo) return false; } - /* remove empty directory */ - if (rmdir(subfile) < 0) - ereport(redo ? LOG : ERROR, - (errcode_for_file_access(), - errmsg("could not remove directory \"%s\": %m", - subfile))); + /* + * Remove empty directory. Outside recovery, defer to commit time so + * a transaction abort leaves the per-DB tablespace subdirectory + * intact. In recovery, issue rmdir directly because the redo handler + * is itself the recovery action. + */ + if (redo) + { + if (rmdir(subfile) < 0) + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not remove directory \"%s\": %m", + subfile))); + } + else + { + FileOpsRmdir(subfile, true /* at_commit */ ); + } pfree(subfile); } FreeDir(dirdesc); - /* remove version directory */ - if (rmdir(linkloc_with_version_dir) < 0) + /* + * Remove version directory. Same forward / redo split as above: defer to + * commit outside recovery so abort restores the version dir; issue rmdir + * directly during redo. + */ + if (redo) { - ereport(redo ? LOG : ERROR, - (errcode_for_file_access(), - errmsg("could not remove directory \"%s\": %m", - linkloc_with_version_dir))); - pfree(linkloc_with_version_dir); - return false; + if (rmdir(linkloc_with_version_dir) < 0) + { + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not remove directory \"%s\": %m", + linkloc_with_version_dir))); + pfree(linkloc_with_version_dir); + return false; + } + } + else + { + FileOpsRmdir(linkloc_with_version_dir, true /* at_commit */ ); } /* @@ -819,26 +873,53 @@ remove_symlink: } else if (S_ISDIR(st.st_mode)) { - if (rmdir(linkloc) < 0) + /* + * Junction-style entry (Windows) or stale directory (any platform). + * In redo, remove now; outside redo, defer to commit so abort + * restores the entry. + */ + if (redo) { - int saved_errno = errno; + if (rmdir(linkloc) < 0) + { + int saved_errno = errno; - ereport(redo ? LOG : (saved_errno == ENOENT ? WARNING : ERROR), - (errcode_for_file_access(), - errmsg("could not remove directory \"%s\": %m", - linkloc))); + ereport(saved_errno == ENOENT ? LOG : LOG, + (errcode_for_file_access(), + errmsg("could not remove directory \"%s\": %m", + linkloc))); + } + } + else + { + FileOpsRmdir(linkloc, true /* at_commit */ ); } } else if (S_ISLNK(st.st_mode)) { - if (unlink(linkloc) < 0) + /* + * Tablespace symlink under PGDATA/pg_tblspc/. Symmetric with + * FileOpsSymlink on the create path: defer the unlink to commit so + * abort restores the symlink, closing the long-standing + * crash-asymmetry between CREATE TABLESPACE and DROP TABLESPACE. In + * redo, issue unlink directly because the redo handler is itself the + * recovery action. + */ + if (redo) { - int saved_errno = errno; + if (unlink(linkloc) < 0) + { + int saved_errno = errno; - ereport(redo ? LOG : (saved_errno == ENOENT ? WARNING : ERROR), - (errcode_for_file_access(), - errmsg("could not remove symbolic link \"%s\": %m", - linkloc))); + ereport(saved_errno == ENOENT ? LOG : LOG, + (errcode_for_file_access(), + errmsg("could not remove symbolic link \"%s\": %m", + linkloc))); + } + } + else + { + FileOpsDelete(linkloc, true /* at_commit */ ); } } else diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c index ee42c796f77..3c716c84bb0 100644 --- a/src/backend/storage/file/copydir.c +++ b/src/backend/storage/file/copydir.c @@ -24,11 +24,13 @@ #include #include +#include "access/xlog.h" #include "common/file_utils.h" #include "miscadmin.h" #include "pgstat.h" #include "storage/copydir.h" #include "storage/fd.h" +#include "storage/fileops.h" #include "utils/wait_event.h" /* GUCs */ @@ -44,9 +46,17 @@ static void clone_file(const char *fromfile, const char *tofile); * * This function uses the file_copy_method GUC. New uses of this function must * be documented in doc/src/sgml/config.sgml. + * + * If register_for_abort_cleanup is true and we are not in WAL replay, the + * destination tree is registered with FILEOPS so that a transaction abort + * recursively removes the partial copy. This closes the long-standing + * crash-asymmetry where CREATE DATABASE FILE_COPY and ALTER DATABASE SET + * TABLESPACE leave orphan trees behind on rollback. Recovery callers pass + * false because they are themselves the recovery action. */ void -copydir(const char *fromdir, const char *todir, bool recurse) +copydir(const char *fromdir, const char *todir, bool recurse, + bool register_for_abort_cleanup) { DIR *xldir; struct dirent *xlde; @@ -58,6 +68,16 @@ copydir(const char *fromdir, const char *todir, bool recurse) (errcode_for_file_access(), errmsg("could not create directory \"%s\": %m", todir))); + /* + * Register the destination tree for delete-on-abort BEFORE we start + * copying. If the surrounding transaction aborts (or this function + * itself errors out), FILEOPS will recursively unlink the partial copy. + * Skip this in WAL replay: the redo handler IS the recovery action and + * has no surrounding transaction context. + */ + if (register_for_abort_cleanup && !RecoveryInProgress()) + FileOpsRmtree(todir, false /* at_commit -> at abort */ ); + xldir = AllocateDir(fromdir); while ((xlde = ReadDir(xldir, fromdir)) != NULL) @@ -78,9 +98,16 @@ copydir(const char *fromdir, const char *todir, bool recurse) if (xlde_type == PGFILETYPE_DIR) { - /* recurse to handle subdirectories */ + /* + * Recurse to handle subdirectories. Pass false for + * register_for_abort_cleanup: the parent call (if any) already + * registered the top-level destination tree, and + * FileOpsRmdirRecursive walks the tree at abort time, so + * registering subdirectories separately would only duplicate + * work. + */ if (recurse) - copydir(fromfile, tofile, true); + copydir(fromfile, tofile, true, false); } else if (xlde_type == PGFILETYPE_REG) { diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h index 40849c2ef99..31faa51cd6a 100644 --- a/src/include/storage/copydir.h +++ b/src/include/storage/copydir.h @@ -22,7 +22,19 @@ typedef enum FileCopyMethod /* GUC parameters */ extern PGDLLIMPORT int file_copy_method; -extern void copydir(const char *fromdir, const char *todir, bool recurse); +/* + * copydir(): copy a directory tree. + * + * If register_for_abort_cleanup is true, the destination tree is registered + * with FILEOPS for delete-on-abort. Forward-path callers in transactional + * commands (CREATE DATABASE STRATEGY=FILE_COPY, ALTER DATABASE SET + * TABLESPACE) should pass true so that a transaction abort removes the + * partial copy. Recovery callers (XLOG_DBASE_CREATE_FILE_COPY redo) must + * pass false because there is no surrounding transaction context and the + * redo handler is itself the recovery action. + */ +extern void copydir(const char *fromdir, const char *todir, bool recurse, + bool register_for_abort_cleanup); extern void copy_file(const char *fromfile, const char *tofile); #endif /* COPYDIR_H */ diff --git a/src/test/modules/test_fileops/t/007_fileops_commit_crash.pl b/src/test/modules/test_fileops/t/007_fileops_commit_crash.pl index 0ff3e80890f..f40dcd271b0 100644 --- a/src/test/modules/test_fileops/t/007_fileops_commit_crash.pl +++ b/src/test/modules/test_fileops/t/007_fileops_commit_crash.pl @@ -73,5 +73,23 @@ TODO: 'committed FileOpsRmdir happened despite the crash'); } +# DROP TABLESPACE is safe: its XLOG_TBLSPC_DROP precedes the commit record +# and its redo removes the directories. (In-place, so they are under +# pg_tblspc.) +$node->safe_psql('postgres', + 'SET allow_in_place_tablespaces = on; CREATE TABLESPACE ts LOCATION \'\'' +); +my $tsoid = $node->safe_psql('postgres', + "SELECT oid FROM pg_tablespace WHERE spcname = 'ts'"); +ok(-d "$datadir/pg_tblspc/$tsoid", 'tablespace directory exists'); +commit_and_crash('DROP TABLESPACE ts;'); +is( $node->safe_psql( + 'postgres', "SELECT count(*) FROM pg_tablespace WHERE spcname = 'ts'" + ), + '0', + 'DROP TABLESPACE committed'); +ok(!-e "$datadir/pg_tblspc/$tsoid", + 'committed DROP TABLESPACE removed its directory despite the crash'); + $node->stop; done_testing(); -- 2.54.0