From 1490ce000e87bdca7edcb9e3e952a04ffdea8335 Mon Sep 17 00:00:00 2001 From: Nathan Bossart Date: Sun, 12 Dec 2021 22:07:11 -0800 Subject: [PATCH v13 6/6] Move removal of old logical rewrite mapping files to custodian. If there are many such files to remove, checkpoints can take much longer. To avoid this, move this work to the newly-introduced custodian process. Since the mapping files include 32-bit transaction IDs, there is a risk of wraparound if the files are not cleaned up fast enough. Removing these files in checkpoints offered decent wraparound protection simply due to the relatively high frequency of checkpointing. With this change, servers should still clean up mappings files with decently high frequency, but in theory the wraparound risk might worsen for some (e.g., if the custodian is spending a lot of time on a different task). Given this is an existing problem, this change makes no effort to handle the wraparound risk, and it is left as a future exercise. --- src/backend/access/heap/rewriteheap.c | 80 +++++++++++++++++++++++---- src/backend/postmaster/custodian.c | 43 ++++++++++++++ src/include/access/rewriteheap.h | 1 + src/include/postmaster/custodian.h | 4 ++ 4 files changed, 118 insertions(+), 10 deletions(-) diff --git a/src/backend/access/heap/rewriteheap.c b/src/backend/access/heap/rewriteheap.c index 2fe9e48e50..07976504cc 100644 --- a/src/backend/access/heap/rewriteheap.c +++ b/src/backend/access/heap/rewriteheap.c @@ -116,6 +116,7 @@ #include "lib/ilist.h" #include "miscadmin.h" #include "pgstat.h" +#include "postmaster/custodian.h" #include "replication/logical.h" #include "replication/slot.h" #include "storage/bufmgr.h" @@ -123,6 +124,7 @@ #include "storage/procarray.h" #include "storage/smgr.h" #include "utils/memutils.h" +#include "utils/pg_lsn.h" #include "utils/rel.h" /* @@ -1179,7 +1181,8 @@ heap_xlog_logical_rewrite(XLogReaderState *r) * Perform a checkpoint for logical rewrite mappings * * This serves two tasks: - * 1) Remove all mappings not needed anymore based on the logical restart LSN + * 1) Alert the custodian to remove all mappings not needed anymore based on the + * logical restart LSN * 2) Flush all remaining mappings to disk, so that replay after a checkpoint * only has to deal with the parts of a mapping that have been written out * after the checkpoint started. @@ -1207,6 +1210,11 @@ CheckPointLogicalRewriteHeap(void) if (cutoff != InvalidXLogRecPtr && redo < cutoff) cutoff = redo; + /* let the custodian know what it can remove */ + RequestCustodian(CUSTODIAN_REMOVE_REWRITE_MAPPINGS, + !IsUnderPostmaster, + LSNGetDatum(cutoff)); + mappings_dir = AllocateDir("pg_logical/mappings"); while ((mapping_de = ReadDir(mappings_dir, "pg_logical/mappings")) != NULL) { @@ -1239,15 +1247,7 @@ CheckPointLogicalRewriteHeap(void) lsn = ((uint64) hi) << 32 | lo; - if (lsn < cutoff || cutoff == InvalidXLogRecPtr) - { - elog(DEBUG1, "removing logical rewrite file \"%s\"", path); - if (unlink(path) < 0) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not remove file \"%s\": %m", path))); - } - else + if (lsn >= cutoff && cutoff != InvalidXLogRecPtr) { /* on some operating systems fsyncing a file requires O_RDWR */ int fd = OpenTransientFile(path, O_RDWR | PG_BINARY); @@ -1285,3 +1285,63 @@ CheckPointLogicalRewriteHeap(void) /* persist directory entries to disk */ fsync_fname("pg_logical/mappings", true); } + +/* + * Remove all mappings not needed anymore based on the logical restart LSN saved + * by the checkpointer. We use this saved value instead of calling + * ReplicationSlotsComputeLogicalRestartLSN() so that we don't try to remove + * files that a concurrent call to CheckPointLogicalRewriteHeap() is trying to + * flush to disk. + */ +void +RemoveOldLogicalRewriteMappings(void) +{ + XLogRecPtr cutoff; + DIR *mappings_dir; + struct dirent *mapping_de; + char path[MAXPGPATH + 20]; + + cutoff = CustodianGetLogicalRewriteCutoff(); + + mappings_dir = AllocateDir("pg_logical/mappings"); + while ((mapping_de = ReadDir(mappings_dir, "pg_logical/mappings")) != NULL) + { + Oid dboid; + Oid relid; + XLogRecPtr lsn; + TransactionId rewrite_xid; + TransactionId create_xid; + uint32 hi, + lo; + PGFileType de_type; + + if (strcmp(mapping_de->d_name, ".") == 0 || + strcmp(mapping_de->d_name, "..") == 0) + continue; + + snprintf(path, sizeof(path), "pg_logical/mappings/%s", mapping_de->d_name); + de_type = get_dirent_type(path, mapping_de, false, DEBUG1); + + if (de_type != PGFILETYPE_ERROR && de_type != PGFILETYPE_REG) + continue; + + /* Skip over files that cannot be ours. */ + if (strncmp(mapping_de->d_name, "map-", 4) != 0) + continue; + + if (sscanf(mapping_de->d_name, LOGICAL_REWRITE_FORMAT, + &dboid, &relid, &hi, &lo, &rewrite_xid, &create_xid) != 6) + elog(ERROR, "could not parse filename \"%s\"", mapping_de->d_name); + + lsn = ((uint64) hi) << 32 | lo; + if (lsn >= cutoff && cutoff != InvalidXLogRecPtr) + continue; + + elog(DEBUG1, "removing logical rewrite file \"%s\"", path); + if (unlink(path) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not remove file \"%s\": %m", path))); + } + FreeDir(mappings_dir); +} diff --git a/src/backend/postmaster/custodian.c b/src/backend/postmaster/custodian.c index 855a756ca0..d4be19e5de 100644 --- a/src/backend/postmaster/custodian.c +++ b/src/backend/postmaster/custodian.c @@ -21,6 +21,7 @@ */ #include "postgres.h" +#include "access/rewriteheap.h" #include "libpq/pqsignal.h" #include "pgstat.h" #include "postmaster/custodian.h" @@ -33,11 +34,13 @@ #include "storage/procsignal.h" #include "storage/smgr.h" #include "utils/memutils.h" +#include "utils/pg_lsn.h" static void DoCustodianTasks(bool retry); static CustodianTask CustodianGetNextTask(void); static void CustodianEnqueueTask(CustodianTask task); static const struct cust_task_funcs_entry *LookupCustodianFunctions(CustodianTask task); +static void CustodianSetLogicalRewriteCutoff(Datum arg); typedef struct { @@ -45,6 +48,8 @@ typedef struct CustodianTask task_queue_elems[NUM_CUSTODIAN_TASKS]; int task_queue_head; + + XLogRecPtr logical_rewrite_mappings_cutoff; /* can remove older mappings */ } CustodianShmemStruct; static CustodianShmemStruct *CustodianShmem; @@ -73,6 +78,7 @@ struct cust_task_funcs_entry static const struct cust_task_funcs_entry cust_task_functions[] = { {CUSTODIAN_REMOVE_TEMP_FILES, RemovePgTempFiles, NULL}, {CUSTODIAN_REMOVE_SERIALIZED_SNAPSHOTS, RemoveOldSerializedSnapshots, NULL}, + {CUSTODIAN_REMOVE_REWRITE_MAPPINGS, RemoveOldLogicalRewriteMappings, CustodianSetLogicalRewriteCutoff}, {INVALID_CUSTODIAN_TASK, NULL, NULL} /* must be last */ }; @@ -384,3 +390,40 @@ LookupCustodianFunctions(CustodianTask task) elog(ERROR, "could not lookup functions for custodian task %d", task); pg_unreachable(); } + +/* + * Stores the provided cutoff LSN in the custodian's shared memory. + * + * It's okay if the cutoff LSN is updated before a previously set cutoff has + * been used for cleaning up files. If that happens, it just means that the + * next invocation of RemoveOldLogicalRewriteMappings() will use a more accurate + * cutoff. + */ +static void +CustodianSetLogicalRewriteCutoff(Datum arg) +{ + SpinLockAcquire(&CustodianShmem->cust_lck); + CustodianShmem->logical_rewrite_mappings_cutoff = DatumGetLSN(arg); + SpinLockRelease(&CustodianShmem->cust_lck); + + /* if pass-by-ref, free Datum memory */ +#ifndef USE_FLOAT8_BYVAL + pfree(DatumGetPointer(arg)); +#endif +} + +/* + * Used by the custodian to determine which logical rewrite mapping files it can + * remove. + */ +XLogRecPtr +CustodianGetLogicalRewriteCutoff(void) +{ + XLogRecPtr cutoff; + + SpinLockAcquire(&CustodianShmem->cust_lck); + cutoff = CustodianShmem->logical_rewrite_mappings_cutoff; + SpinLockRelease(&CustodianShmem->cust_lck); + + return cutoff; +} diff --git a/src/include/access/rewriteheap.h b/src/include/access/rewriteheap.h index 5cc04756a5..bc875330d7 100644 --- a/src/include/access/rewriteheap.h +++ b/src/include/access/rewriteheap.h @@ -53,5 +53,6 @@ typedef struct LogicalRewriteMappingData */ #define LOGICAL_REWRITE_FORMAT "map-%x-%x-%X_%X-%x-%x" extern void CheckPointLogicalRewriteHeap(void); +extern void RemoveOldLogicalRewriteMappings(void); #endif /* REWRITE_HEAP_H */ diff --git a/src/include/postmaster/custodian.h b/src/include/postmaster/custodian.h index 37334941cc..f177d55159 100644 --- a/src/include/postmaster/custodian.h +++ b/src/include/postmaster/custodian.h @@ -12,6 +12,8 @@ #ifndef _CUSTODIAN_H #define _CUSTODIAN_H +#include "access/xlogdefs.h" + /* * If you add a new task here, be sure to add its corresponding function * pointers to cust_task_functions in custodian.c. @@ -20,6 +22,7 @@ typedef enum CustodianTask { CUSTODIAN_REMOVE_TEMP_FILES, CUSTODIAN_REMOVE_SERIALIZED_SNAPSHOTS, + CUSTODIAN_REMOVE_REWRITE_MAPPINGS, NUM_CUSTODIAN_TASKS, /* new tasks go above */ INVALID_CUSTODIAN_TASK @@ -29,5 +32,6 @@ extern void CustodianMain(void) pg_attribute_noreturn(); extern Size CustodianShmemSize(void); extern void CustodianShmemInit(void); extern void RequestCustodian(CustodianTask task, bool immediate, Datum arg); +extern XLogRecPtr CustodianGetLogicalRewriteCutoff(void); #endif /* _CUSTODIAN_H */ -- 2.25.1