From ad86daed90d25d247b7278f4d69dd45d5414152a Mon Sep 17 00:00:00 2001 From: Bertrand Drouvot Date: Thu, 14 Aug 2025 07:12:17 +0000 Subject: [PATCH v1] Adding Log messages related to ReorderBuffer size Add log messages related to ReorderBuffer size: 1. Size of the buffer before logical decoding starts to create spilled files on disk (should be > logical_decoding_work_mem but by how much?) 2. Size of the buffer when the logical decoder starts to read from the spilled files. We report the size every 100MB (controlled by ReorderBuffer_Size_Log) as well as information about xid, sub xid and LSNs being concerned. We could report messages at a frequency higher that 100MB but then increasing the risk of loosing information in case of OOM. 100MB should not produce too much writes in the log file (a 10GB ReorderBuffer would produce "only" about 100 lines). --- .../replication/logical/reorderbuffer.c | 24 +++++++++++++++++++ src/backend/utils/misc/guc_tables.c | 11 +++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + src/include/replication/reorderbuffer.h | 1 + 4 files changed, 37 insertions(+) 74.7% src/backend/replication/logical/ 22.2% src/backend/utils/misc/ 3.0% src/include/replication/ diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index 0adec1fddb8..a8a5fd84883 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -224,6 +224,9 @@ typedef struct ReorderBufferDiskChange */ int logical_decoding_work_mem; static const Size max_changes_in_memory = 4096; /* XXX for restore only */ +int reorderbuffer_size_log; +static Size max_logical_decoding_work_mem_used_before_serialize = 0; +static Size reorderbuffer_size_log_modulo = 0; /* GUC variable */ int debug_logical_replication_streaming = DEBUG_LOGICAL_REP_STREAMING_BUFFERED; @@ -1221,6 +1224,8 @@ ReorderBufferCommitChild(ReorderBuffer *rb, TransactionId xid, XLogRecPtr end_lsn) { ReorderBufferTXN *subtxn; + reorderbuffer_size_log_modulo = 0; + max_logical_decoding_work_mem_used_before_serialize = 0; subtxn = ReorderBufferTXNByXid(rb, subxid, false, NULL, InvalidXLogRecPtr, false); @@ -3906,6 +3911,13 @@ ReorderBufferCheckMemoryLimit(ReorderBuffer *rb) rb->size < logical_decoding_work_mem * (Size) 1024) return; + if (rb->size > max_logical_decoding_work_mem_used_before_serialize) + { + max_logical_decoding_work_mem_used_before_serialize = rb->size; + ereport(LOG,(errmsg("ReorderBuffer Size before creating spill file(s): %zu", rb->size), + errcontext("xid %u", rb->by_txn_last_xid))); + } + /* * If debug_logical_replication_streaming is immediate, loop until there's * no change. Otherwise, loop until we reach under the memory limit. One @@ -4531,6 +4543,18 @@ ReorderBufferRestoreChanges(ReorderBuffer *rb, ReorderBufferTXN *txn, Assert(txn->first_lsn != InvalidXLogRecPtr); Assert(txn->final_lsn != InvalidXLogRecPtr); + if (rb->size / (reorderbuffer_size_log * 1024L) > reorderbuffer_size_log_modulo) + { + reorderbuffer_size_log_modulo = rb->size / (reorderbuffer_size_log * 1024L); + ereport(LOG,(errmsg("ReorderBuffer Size while restoring from spilled files: %lu", + rb->size), + errcontext("Restoring top level xid %u, xid %u at first_lsn %X/%08X, final_lsn %X/%08X and end_lsn %X/%08X", + txn->toplevel_xid, + txn->xid, + (uint32) (txn->first_lsn >> 32), (uint32) txn->first_lsn, + (uint32) (txn->final_lsn >> 32), (uint32) txn->final_lsn, + (uint32) (txn->end_lsn >> 32), (uint32) txn->end_lsn))); + } /* free current entries, so we have memory for more */ dlist_foreach_modify(cleanup_iter, &txn->changes) diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index d14b1678e7f..c84df6c844c 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -2603,6 +2603,17 @@ struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + { + {"reorderbuffer_size_log", PGC_USERSET, LOGGING_WHEN, + gettext_noop("Sets the frequency in bytes to display information while restoring from spilled files during logical decoding."), + NULL, + GUC_UNIT_KB + }, + &reorderbuffer_size_log, + 102400, 10240, MAX_KILOBYTES, + NULL, NULL, NULL + }, + /* * We use the hopefully-safely-small value of 100kB as the compiled-in * default for max_stack_depth. InitializeGUCOptions will increase it if diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index a9d8293474a..48b7c1eaa77 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -145,6 +145,7 @@ #maintenance_work_mem = 64MB # min 64kB #autovacuum_work_mem = -1 # min 64kB, or -1 to use maintenance_work_mem #logical_decoding_work_mem = 64MB # min 64kB +#reorderbuffer_size_log = 100MB # min 10MB #max_stack_depth = 2MB # min 100kB #shared_memory_type = mmap # the default is the first option # supported by the operating system: diff --git a/src/include/replication/reorderbuffer.h b/src/include/replication/reorderbuffer.h index 932ce87b8c2..6385b82ec7a 100644 --- a/src/include/replication/reorderbuffer.h +++ b/src/include/replication/reorderbuffer.h @@ -26,6 +26,7 @@ /* GUC variables */ extern PGDLLIMPORT int logical_decoding_work_mem; extern PGDLLIMPORT int debug_logical_replication_streaming; +extern PGDLLIMPORT int reorderbuffer_size_log; /* possible values for debug_logical_replication_streaming */ typedef enum -- 2.34.1