From 111575015f513f4b2678a84eee387798ea6e5969 Mon Sep 17 00:00:00 2001 From: Bertrand Drouvot Date: Tue, 26 Aug 2025 13:08:13 +0000 Subject: [PATCH v2] Add memory_limit_hits to pg_stat_replication_slots It's currently not always possible to determine how many times the logical_decoding_work_mem has been reached. So adding a new counter, memory_limit_hits to report the number of times the logical_decoding_work_mem has been reached while decoding. With such a counter one could get a ratio like total_txns/memory_limit_hits. That could help to see if reaching logical_decoding_work_mem is rare or frequent enough. If frequent, then maybe there is a need to adjust logical_decoding_work_mem. XXXX: Bump catversion. --- contrib/test_decoding/expected/stats.out | 68 +++++++++---------- contrib/test_decoding/sql/stats.sql | 10 +-- doc/src/sgml/monitoring.sgml | 11 +++ src/backend/catalog/system_views.sql | 1 + src/backend/replication/logical/logical.c | 7 +- .../replication/logical/reorderbuffer.c | 7 +- src/backend/utils/activity/pgstat_replslot.c | 1 + src/backend/utils/adt/pgstatfuncs.c | 11 +-- src/include/catalog/pg_proc.dat | 6 +- src/include/pgstat.h | 1 + src/include/replication/reorderbuffer.h | 3 + src/test/regress/expected/rules.out | 3 +- 12 files changed, 79 insertions(+), 50 deletions(-) 47.8% contrib/test_decoding/expected/ 16.9% contrib/test_decoding/sql/ 6.7% doc/src/sgml/ 8.2% src/backend/replication/logical/ 9.7% src/backend/utils/adt/ 3.7% src/include/catalog/ 3.1% src/test/regress/expected/ 3.5% src/ diff --git a/contrib/test_decoding/expected/stats.out b/contrib/test_decoding/expected/stats.out index de6dc416130..642ebe1ee5e 100644 --- a/contrib/test_decoding/expected/stats.out +++ b/contrib/test_decoding/expected/stats.out @@ -37,12 +37,12 @@ SELECT pg_stat_force_next_flush(); (1 row) -SELECT slot_name, spill_txns = 0 AS spill_txns, spill_count = 0 AS spill_count, total_txns > 0 AS total_txns, total_bytes > 0 AS total_bytes FROM pg_stat_replication_slots ORDER BY slot_name; - slot_name | spill_txns | spill_count | total_txns | total_bytes -------------------------+------------+-------------+------------+------------- - regression_slot_stats1 | t | t | t | t - regression_slot_stats2 | t | t | t | t - regression_slot_stats3 | t | t | t | t +SELECT slot_name, spill_txns = 0 AS spill_txns, spill_count = 0 AS spill_count, total_txns > 0 AS total_txns, total_bytes > 0 AS total_bytes, memory_limit_hits = 0 AS memory_limit_hits FROM pg_stat_replication_slots ORDER BY slot_name; + slot_name | spill_txns | spill_count | total_txns | total_bytes | memory_limit_hits +------------------------+------------+-------------+------------+-------------+------------------- + regression_slot_stats1 | t | t | t | t | t + regression_slot_stats2 | t | t | t | t | t + regression_slot_stats3 | t | t | t | t | t (3 rows) RESET logical_decoding_work_mem; @@ -53,12 +53,12 @@ SELECT pg_stat_reset_replication_slot('regression_slot_stats1'); (1 row) -SELECT slot_name, spill_txns = 0 AS spill_txns, spill_count = 0 AS spill_count, total_txns > 0 AS total_txns, total_bytes > 0 AS total_bytes FROM pg_stat_replication_slots ORDER BY slot_name; - slot_name | spill_txns | spill_count | total_txns | total_bytes -------------------------+------------+-------------+------------+------------- - regression_slot_stats1 | t | t | f | f - regression_slot_stats2 | t | t | t | t - regression_slot_stats3 | t | t | t | t +SELECT slot_name, spill_txns = 0 AS spill_txns, spill_count = 0 AS spill_count, total_txns > 0 AS total_txns, total_bytes > 0 AS total_bytes, memory_limit_hits = 0 AS memory_limit_hits FROM pg_stat_replication_slots ORDER BY slot_name; + slot_name | spill_txns | spill_count | total_txns | total_bytes | memory_limit_hits +------------------------+------------+-------------+------------+-------------+------------------- + regression_slot_stats1 | t | t | f | f | t + regression_slot_stats2 | t | t | t | t | t + regression_slot_stats3 | t | t | t | t | t (3 rows) -- reset stats for all slots @@ -68,27 +68,27 @@ SELECT pg_stat_reset_replication_slot(NULL); (1 row) -SELECT slot_name, spill_txns = 0 AS spill_txns, spill_count = 0 AS spill_count, total_txns > 0 AS total_txns, total_bytes > 0 AS total_bytes FROM pg_stat_replication_slots ORDER BY slot_name; - slot_name | spill_txns | spill_count | total_txns | total_bytes -------------------------+------------+-------------+------------+------------- - regression_slot_stats1 | t | t | f | f - regression_slot_stats2 | t | t | f | f - regression_slot_stats3 | t | t | f | f +SELECT slot_name, spill_txns = 0 AS spill_txns, spill_count = 0 AS spill_count, total_txns > 0 AS total_txns, total_bytes > 0 AS total_bytes, memory_limit_hits = 0 AS memory_limit_hits FROM pg_stat_replication_slots ORDER BY slot_name; + slot_name | spill_txns | spill_count | total_txns | total_bytes | memory_limit_hits +------------------------+------------+-------------+------------+-------------+------------------- + regression_slot_stats1 | t | t | f | f | t + regression_slot_stats2 | t | t | f | f | t + regression_slot_stats3 | t | t | f | f | t (3 rows) -- verify accessing/resetting stats for non-existent slot does something reasonable SELECT * FROM pg_stat_get_replication_slot('do-not-exist'); - slot_name | spill_txns | spill_count | spill_bytes | stream_txns | stream_count | stream_bytes | total_txns | total_bytes | stats_reset ---------------+------------+-------------+-------------+-------------+--------------+--------------+------------+-------------+------------- - do-not-exist | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | + slot_name | spill_txns | spill_count | spill_bytes | stream_txns | stream_count | stream_bytes | total_txns | total_bytes | memory_limit_hits | stats_reset +--------------+------------+-------------+-------------+-------------+--------------+--------------+------------+-------------+-------------------+------------- + do-not-exist | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | (1 row) SELECT pg_stat_reset_replication_slot('do-not-exist'); ERROR: replication slot "do-not-exist" does not exist SELECT * FROM pg_stat_get_replication_slot('do-not-exist'); - slot_name | spill_txns | spill_count | spill_bytes | stream_txns | stream_count | stream_bytes | total_txns | total_bytes | stats_reset ---------------+------------+-------------+-------------+-------------+--------------+--------------+------------+-------------+------------- - do-not-exist | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | + slot_name | spill_txns | spill_count | spill_bytes | stream_txns | stream_count | stream_bytes | total_txns | total_bytes | memory_limit_hits | stats_reset +--------------+------------+-------------+-------------+-------------+--------------+--------------+------------+-------------+-------------------+------------- + do-not-exist | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | (1 row) -- spilling the xact @@ -110,12 +110,12 @@ SELECT pg_stat_force_next_flush(); (1 row) -SELECT slot_name, spill_txns > 0 AS spill_txns, spill_count > 0 AS spill_count FROM pg_stat_replication_slots; - slot_name | spill_txns | spill_count -------------------------+------------+------------- - regression_slot_stats1 | t | t - regression_slot_stats2 | f | f - regression_slot_stats3 | f | f +SELECT slot_name, spill_txns > 0 AS spill_txns, spill_count > 0 AS spill_count, memory_limit_hits > 0 AS memory_limit_hits FROM pg_stat_replication_slots; + slot_name | spill_txns | spill_count | memory_limit_hits +------------------------+------------+-------------+------------------- + regression_slot_stats1 | t | t | t + regression_slot_stats2 | f | f | f + regression_slot_stats3 | f | f | f (3 rows) -- Ensure stats can be repeatedly accessed using the same stats snapshot. See @@ -165,10 +165,10 @@ SELECT pg_stat_force_next_flush(); (1 row) -SELECT slot_name, spill_txns, spill_count FROM pg_stat_replication_slots WHERE slot_name = 'regression_slot_stats4_twophase'; - slot_name | spill_txns | spill_count ----------------------------------+------------+------------- - regression_slot_stats4_twophase | 0 | 0 +SELECT slot_name, spill_txns, spill_count, memory_limit_hits FROM pg_stat_replication_slots WHERE slot_name = 'regression_slot_stats4_twophase'; + slot_name | spill_txns | spill_count | memory_limit_hits +---------------------------------+------------+-------------+------------------- + regression_slot_stats4_twophase | 0 | 0 | 0 (1 row) DROP TABLE stats_test; diff --git a/contrib/test_decoding/sql/stats.sql b/contrib/test_decoding/sql/stats.sql index a022fe1bf07..8a58d05a764 100644 --- a/contrib/test_decoding/sql/stats.sql +++ b/contrib/test_decoding/sql/stats.sql @@ -15,16 +15,16 @@ SELECT count(*) FROM pg_logical_slot_get_changes('regression_slot_stats1', NULL, SELECT count(*) FROM pg_logical_slot_get_changes('regression_slot_stats2', NULL, NULL, 'skip-empty-xacts', '1'); SELECT count(*) FROM pg_logical_slot_get_changes('regression_slot_stats3', NULL, NULL, 'skip-empty-xacts', '1'); SELECT pg_stat_force_next_flush(); -SELECT slot_name, spill_txns = 0 AS spill_txns, spill_count = 0 AS spill_count, total_txns > 0 AS total_txns, total_bytes > 0 AS total_bytes FROM pg_stat_replication_slots ORDER BY slot_name; +SELECT slot_name, spill_txns = 0 AS spill_txns, spill_count = 0 AS spill_count, total_txns > 0 AS total_txns, total_bytes > 0 AS total_bytes, memory_limit_hits = 0 AS memory_limit_hits FROM pg_stat_replication_slots ORDER BY slot_name; RESET logical_decoding_work_mem; -- reset stats for one slot, others should be unaffected SELECT pg_stat_reset_replication_slot('regression_slot_stats1'); -SELECT slot_name, spill_txns = 0 AS spill_txns, spill_count = 0 AS spill_count, total_txns > 0 AS total_txns, total_bytes > 0 AS total_bytes FROM pg_stat_replication_slots ORDER BY slot_name; +SELECT slot_name, spill_txns = 0 AS spill_txns, spill_count = 0 AS spill_count, total_txns > 0 AS total_txns, total_bytes > 0 AS total_bytes, memory_limit_hits = 0 AS memory_limit_hits FROM pg_stat_replication_slots ORDER BY slot_name; -- reset stats for all slots SELECT pg_stat_reset_replication_slot(NULL); -SELECT slot_name, spill_txns = 0 AS spill_txns, spill_count = 0 AS spill_count, total_txns > 0 AS total_txns, total_bytes > 0 AS total_bytes FROM pg_stat_replication_slots ORDER BY slot_name; +SELECT slot_name, spill_txns = 0 AS spill_txns, spill_count = 0 AS spill_count, total_txns > 0 AS total_txns, total_bytes > 0 AS total_bytes, memory_limit_hits = 0 AS memory_limit_hits FROM pg_stat_replication_slots ORDER BY slot_name; -- verify accessing/resetting stats for non-existent slot does something reasonable SELECT * FROM pg_stat_get_replication_slot('do-not-exist'); @@ -41,7 +41,7 @@ SELECT count(*) FROM pg_logical_slot_peek_changes('regression_slot_stats1', NULL -- background transaction (say by autovacuum) happens in parallel to the main -- transaction. SELECT pg_stat_force_next_flush(); -SELECT slot_name, spill_txns > 0 AS spill_txns, spill_count > 0 AS spill_count FROM pg_stat_replication_slots; +SELECT slot_name, spill_txns > 0 AS spill_txns, spill_count > 0 AS spill_count, memory_limit_hits > 0 AS memory_limit_hits FROM pg_stat_replication_slots; -- Ensure stats can be repeatedly accessed using the same stats snapshot. See -- https://postgr.es/m/20210317230447.c7uc4g3vbs4wi32i%40alap3.anarazel.de @@ -65,7 +65,7 @@ SELECT count(*) FROM pg_logical_slot_get_changes('regression_slot_stats4_twophas -- Verify that the decoding doesn't spill already-aborted transaction's changes. SELECT pg_stat_force_next_flush(); -SELECT slot_name, spill_txns, spill_count FROM pg_stat_replication_slots WHERE slot_name = 'regression_slot_stats4_twophase'; +SELECT slot_name, spill_txns, spill_count, memory_limit_hits FROM pg_stat_replication_slots WHERE slot_name = 'regression_slot_stats4_twophase'; DROP TABLE stats_test; SELECT pg_drop_replication_slot('regression_slot_stats1'), diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 3f4a27a736e..3b175a1d3e1 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -1644,6 +1644,17 @@ description | Waiting for a newly initialized WAL file to reach durable storage + + + memory_limit_hitsbigint + + + Number of times logical_decoding_work_mem has been + reached while decoding changes from WAL for this slot. + + + + stats_reset timestamp with time zone diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index c77fa0234bb..8a2b9eec340 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1061,6 +1061,7 @@ CREATE VIEW pg_stat_replication_slots AS s.stream_bytes, s.total_txns, s.total_bytes, + s.memory_limit_hits, s.stats_reset FROM pg_replication_slots as r, LATERAL pg_stat_get_replication_slot(slot_name) as s diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index c68c0481f42..fdc0887caae 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -1958,7 +1958,7 @@ UpdateDecodingStats(LogicalDecodingContext *ctx) if (rb->spillBytes <= 0 && rb->streamBytes <= 0 && rb->totalBytes <= 0) return; - elog(DEBUG2, "UpdateDecodingStats: updating stats %p %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64, + elog(DEBUG2, "UpdateDecodingStats: updating stats %p %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64, rb, rb->spillTxns, rb->spillCount, @@ -1967,7 +1967,8 @@ UpdateDecodingStats(LogicalDecodingContext *ctx) rb->streamCount, rb->streamBytes, rb->totalTxns, - rb->totalBytes); + rb->totalBytes, + rb->memoryLimitHits); repSlotStat.spill_txns = rb->spillTxns; repSlotStat.spill_count = rb->spillCount; @@ -1977,6 +1978,7 @@ UpdateDecodingStats(LogicalDecodingContext *ctx) repSlotStat.stream_bytes = rb->streamBytes; repSlotStat.total_txns = rb->totalTxns; repSlotStat.total_bytes = rb->totalBytes; + repSlotStat.memory_limit_hits = rb->memoryLimitHits; pgstat_report_replslot(ctx->slot, &repSlotStat); @@ -1988,6 +1990,7 @@ UpdateDecodingStats(LogicalDecodingContext *ctx) rb->streamBytes = 0; rb->totalTxns = 0; rb->totalBytes = 0; + rb->memoryLimitHits = 0; } /* diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index 4736f993c37..1775da5d4be 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -392,6 +392,7 @@ ReorderBufferAllocate(void) buffer->streamBytes = 0; buffer->totalTxns = 0; buffer->totalBytes = 0; + buffer->memoryLimitHits = 0; buffer->current_restart_decoding_lsn = InvalidXLogRecPtr; @@ -3898,13 +3899,17 @@ static void ReorderBufferCheckMemoryLimit(ReorderBuffer *rb) { ReorderBufferTXN *txn; + bool memory_limit_reached = (rb->size >= logical_decoding_work_mem * (Size) 1024); + + if (memory_limit_reached) + rb->memoryLimitHits += 1; /* * Bail out if debug_logical_replication_streaming is buffered and we * haven't exceeded the memory limit. */ if (debug_logical_replication_streaming == DEBUG_LOGICAL_REP_STREAMING_BUFFERED && - rb->size < logical_decoding_work_mem * (Size) 1024) + !memory_limit_reached) return; /* diff --git a/src/backend/utils/activity/pgstat_replslot.c b/src/backend/utils/activity/pgstat_replslot.c index ccfb11c49bf..27e9384f590 100644 --- a/src/backend/utils/activity/pgstat_replslot.c +++ b/src/backend/utils/activity/pgstat_replslot.c @@ -96,6 +96,7 @@ pgstat_report_replslot(ReplicationSlot *slot, const PgStat_StatReplSlotEntry *re REPLSLOT_ACC(stream_bytes); REPLSLOT_ACC(total_txns); REPLSLOT_ACC(total_bytes); + REPLSLOT_ACC(memory_limit_hits); #undef REPLSLOT_ACC pgstat_unlock_entry(entry_ref); diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index c756c2bebaa..b400794e9f8 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2100,7 +2100,7 @@ pg_stat_get_archiver(PG_FUNCTION_ARGS) Datum pg_stat_get_replication_slot(PG_FUNCTION_ARGS) { -#define PG_STAT_GET_REPLICATION_SLOT_COLS 10 +#define PG_STAT_GET_REPLICATION_SLOT_COLS 11 text *slotname_text = PG_GETARG_TEXT_P(0); NameData slotname; TupleDesc tupdesc; @@ -2129,7 +2129,9 @@ pg_stat_get_replication_slot(PG_FUNCTION_ARGS) INT8OID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 9, "total_bytes", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 10, "stats_reset", + TupleDescInitEntry(tupdesc, (AttrNumber) 10, "memory_limit_hits", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 11, "stats_reset", TIMESTAMPTZOID, -1, 0); BlessTupleDesc(tupdesc); @@ -2154,11 +2156,12 @@ pg_stat_get_replication_slot(PG_FUNCTION_ARGS) values[6] = Int64GetDatum(slotent->stream_bytes); values[7] = Int64GetDatum(slotent->total_txns); values[8] = Int64GetDatum(slotent->total_bytes); + values[9] = Int64GetDatum(slotent->memory_limit_hits); if (slotent->stat_reset_timestamp == 0) - nulls[9] = true; + nulls[10] = true; else - values[9] = TimestampTzGetDatum(slotent->stat_reset_timestamp); + values[10] = TimestampTzGetDatum(slotent->stat_reset_timestamp); /* Returns the record as Datum */ PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls))); diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 01eba3b5a19..1a2d213408b 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5687,9 +5687,9 @@ { oid => '6169', descr => 'statistics: information about replication slot', proname => 'pg_stat_get_replication_slot', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => 'text', - proallargtypes => '{text,text,int8,int8,int8,int8,int8,int8,int8,int8,timestamptz}', - proargmodes => '{i,o,o,o,o,o,o,o,o,o,o}', - proargnames => '{slot_name,slot_name,spill_txns,spill_count,spill_bytes,stream_txns,stream_count,stream_bytes,total_txns,total_bytes,stats_reset}', + proallargtypes => '{text,text,int8,int8,int8,int8,int8,int8,int8,int8,int8,timestamptz}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{slot_name,slot_name,spill_txns,spill_count,spill_bytes,stream_txns,stream_count,stream_bytes,total_txns,total_bytes,memory_limit_hits,stats_reset}', prosrc => 'pg_stat_get_replication_slot' }, { oid => '6230', descr => 'statistics: check if a stats object exists', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index f402b17295c..9215027d0ac 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -395,6 +395,7 @@ typedef struct PgStat_StatReplSlotEntry PgStat_Counter stream_bytes; PgStat_Counter total_txns; PgStat_Counter total_bytes; + PgStat_Counter memory_limit_hits; TimestampTz stat_reset_timestamp; } PgStat_StatReplSlotEntry; diff --git a/src/include/replication/reorderbuffer.h b/src/include/replication/reorderbuffer.h index fa0745552f8..89ac782bfad 100644 --- a/src/include/replication/reorderbuffer.h +++ b/src/include/replication/reorderbuffer.h @@ -696,6 +696,9 @@ struct ReorderBuffer */ int64 totalTxns; /* total number of transactions sent */ int64 totalBytes; /* total amount of data decoded */ + + /* Number of times logical_decoding_work_mem has been reached */ + int64 memoryLimitHits; }; diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 35e8aad7701..8ce27032b87 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2140,9 +2140,10 @@ pg_stat_replication_slots| SELECT s.slot_name, s.stream_bytes, s.total_txns, s.total_bytes, + s.memory_limit_hits, s.stats_reset FROM pg_replication_slots r, - LATERAL pg_stat_get_replication_slot((r.slot_name)::text) s(slot_name, spill_txns, spill_count, spill_bytes, stream_txns, stream_count, stream_bytes, total_txns, total_bytes, stats_reset) + LATERAL pg_stat_get_replication_slot((r.slot_name)::text) s(slot_name, spill_txns, spill_count, spill_bytes, stream_txns, stream_count, stream_bytes, total_txns, total_bytes, memory_limit_hits, stats_reset) WHERE (r.datoid IS NOT NULL); pg_stat_slru| SELECT name, blks_zeroed, -- 2.34.1