From 38f05eb964718898ded7950f95c4a2e4a75a330f Mon Sep 17 00:00:00 2001 From: Ayush Tiwari Date: Thu, 19 Nov 2026 17:18:24 +0530 Subject: [PATCH v5 1/2] Batch fsyncs when removing old WAL segments RemoveOldXlogFiles() uses durable_unlink() for every old WAL file it cannot recycle. Each call fsyncs pg_wal separately. Unlink all such files first, fsync pg_wal once after the scan, then remove their archive-status files. If a crash precedes the directory fsync, obsolete files may reappear; retaining the status files lets a later checkpoint remove them again. Report a failure of the batched pg_wal fsync through data_sync_elevel(), like other directory fsyncs, since the error may belong to any recent change in pg_wal, not just these unlinks. If it is reported as an ERROR, the archive-status files are kept. Leave RemoveNonParentXlogFiles() unbatched. The segments it removes may have no archive-status file, and after promotion it is not re-run, so a removal lost in a crash could leave a garbage segment to be archived later. --- src/backend/access/transam/xlog.c | 49 +++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index fd37ea4f607..ac740725a48 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -768,7 +768,7 @@ static void RemoveOldXlogFiles(XLogSegNo segno, XLogRecPtr lastredoptr, XLogRecPtr endptr, TimeLineID insertTLI); static void RemoveXlogFile(const struct dirent *segment_de, XLogSegNo recycleSegNo, XLogSegNo *endlogSegNo, - TimeLineID insertTLI); + TimeLineID insertTLI, List **cleanup_names); static void UpdateLastRemovedPtr(char *filename); static void ValidateXLOGDirectoryStructure(void); static void CleanupBackupHistory(void); @@ -3988,6 +3988,8 @@ RemoveOldXlogFiles(XLogSegNo segno, XLogRecPtr lastredoptr, XLogRecPtr endptr, char lastoff[MAXFNAMELEN]; XLogSegNo endlogSegNo; XLogSegNo recycleSegNo; + List *cleanup_names = NIL; + ListCell *lc; /* Initialize info about where to try to recycle to */ XLByteToSeg(endptr, endlogSegNo, wal_segment_size); @@ -4030,12 +4032,23 @@ RemoveOldXlogFiles(XLogSegNo segno, XLogRecPtr lastredoptr, XLogRecPtr endptr, /* Update the last removed location in shared memory first */ UpdateLastRemovedPtr(xlde->d_name); - RemoveXlogFile(xlde, recycleSegNo, &endlogSegNo, insertTLI); + RemoveXlogFile(xlde, recycleSegNo, &endlogSegNo, insertTLI, + &cleanup_names); } } } FreeDir(xldir); + + /* Make all removals durable with one directory fsync. */ + if (cleanup_names != NIL) + fsync_fname(XLOGDIR, true); + + /* Remove archive status only after the WAL-file removals are durable. */ + foreach(lc, cleanup_names) + XLogArchiveCleanup((char *) lfirst(lc)); + + list_free_deep(cleanup_names); } /* @@ -4103,7 +4116,7 @@ RemoveNonParentXlogFiles(XLogRecPtr switchpoint, TimeLineID newTLI) * - but seems safer to let them be archived and removed later. */ if (!XLogArchiveIsReady(xlde->d_name)) - RemoveXlogFile(xlde, recycleSegNo, &endLogSegNo, newTLI); + RemoveXlogFile(xlde, recycleSegNo, &endLogSegNo, newTLI, NULL); } } @@ -4122,19 +4135,26 @@ RemoveNonParentXlogFiles(XLogRecPtr switchpoint, TimeLineID newTLI) * * insertTLI is the current timeline for XLOG insertion. Any recycled segments * should be used for this timeline. + * + * If cleanup_names is not NULL, a removed segment's pg_wal fsync and + * archive-status cleanup are left to the caller, and its name is appended + * to *cleanup_names. */ static void RemoveXlogFile(const struct dirent *segment_de, XLogSegNo recycleSegNo, XLogSegNo *endlogSegNo, - TimeLineID insertTLI) + TimeLineID insertTLI, List **cleanup_names) { char path[MAXPGPATH]; + bool removed = false; + const char *unlinkpath; #ifdef WIN32 char newpath[MAXPGPATH]; #endif const char *segname = segment_de->d_name; snprintf(path, MAXPGPATH, XLOGDIR "/%s", segname); + unlinkpath = path; /* * Before deleting the file, see if it can be recycled as a future log @@ -4185,19 +4205,30 @@ RemoveXlogFile(const struct dirent *segment_de, path))); return; } - rc = durable_unlink(newpath, LOG); -#else - rc = durable_unlink(path, LOG); + unlinkpath = newpath; #endif + + if (cleanup_names != NULL) + rc = unlink(unlinkpath); + else + rc = durable_unlink(unlinkpath, LOG); if (rc != 0) { - /* Message already logged by durable_unlink() */ + if (cleanup_names != NULL) + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not remove file \"%s\": %m", + unlinkpath))); return; } CheckpointStats.ckpt_segs_removed++; + removed = true; } - XLogArchiveCleanup(segname); + if (removed && cleanup_names != NULL) + *cleanup_names = lappend(*cleanup_names, pstrdup(segname)); + else + XLogArchiveCleanup(segname); } /* base-commit: 6ec71e1ad8c26b3a93e7c5739e797a2d353dc5d6 -- 2.34.1