From 1d31d5f222c5dbaa563c9e3afee0adaa95163a7e Mon Sep 17 00:00:00 2001 From: Bertrand Drouvot Date: Fri, 17 Jul 2026 13:23:40 +0000 Subject: [PATCH v4 1/6] pgstat: add tests for per-backend statistics Add an isolation test for per-backend WAL statistics. Verify that SNAPSHOT, CACHE and NONE modes behave as expected. Also verify that switching from CACHE to SNAPSHOT discards previously cached entries and builds a fresh snapshot. Use WAL as a representative statistics kind for consistency behavior rather than repeating the same tests for every per-backend statistics kind. Add a TAP test that starts a backend after another session has built a statistics snapshot. Per-backend WAL, IO, and lock queries for the new backend must return no rows rather than fabricated rows containing zeroes. Finally, extend the core statistics regression test for per-backend WAL reset. Check that resetting one backend reduces its counters and sets its reset timestamp without removing that backend's activity from the global WAL statistics. These tests provide compatibility coverage for the per-backend redesign changes introduced by the following commits. Author: Bertrand Drouvot Reviewed-by: Michael Paquier Reviewed-by: Andres Freund Discussion: https://postgr.es/m/anCjoFkGah/uB7lw%40bdtpg --- .../isolation/expected/stats-per-backend.out | 225 ++++++++++++++++++ src/test/isolation/isolation_schedule | 1 + .../isolation/specs/stats-per-backend.spec | 151 ++++++++++++ src/test/recovery/t/029_stats_restart.pl | 36 ++- src/test/regress/expected/stats.out | 29 +++ src/test/regress/sql/stats.sql | 10 + 6 files changed, 450 insertions(+), 2 deletions(-) 46.0% src/test/isolation/expected/ 35.3% src/test/isolation/specs/ 9.2% src/test/recovery/t/ 5.1% src/test/regress/expected/ 3.9% src/test/regress/sql/ diff --git a/src/test/isolation/expected/stats-per-backend.out b/src/test/isolation/expected/stats-per-backend.out new file mode 100644 index 00000000000..5c8f3c990a1 --- /dev/null +++ b/src/test/isolation/expected/stats-per-backend.out @@ -0,0 +1,225 @@ +Parsed test spec with 2 sessions + +starting permutation: s1_fetch_consistency_none s2_generate s1_begin s1_save_backend_stats s2_generate_more s1_check_uncached_backend s1_commit +step s1_fetch_consistency_none: SET stats_fetch_consistency = 'none'; +step s2_generate: + INSERT INTO stats_per_backend_data VALUES (1); + SELECT pg_stat_force_next_flush(); + +pg_stat_force_next_flush +------------------------ + +(1 row) + +step s1_begin: BEGIN; +step s1_save_backend_stats: + INSERT INTO stats_per_backend_saved + SELECT wal_records + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')); + +step s2_generate_more: + INSERT INTO stats_per_backend_data VALUES (2); + SELECT pg_stat_force_next_flush(); + +pg_stat_force_next_flush +------------------------ + +(1 row) + +step s1_check_uncached_backend: + SELECT current.wal_records > saved.wal_records AS none_is_refreshed + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')) AS current + CROSS JOIN stats_per_backend_saved AS saved; + +none_is_refreshed +----------------- +t +(1 row) + +step s1_commit: COMMIT; + +starting permutation: s1_fetch_consistency_snapshot s1_begin s1_build_snapshot s2_generate s1_save_backend_stats s1_commit s1_check_live_backend +step s1_fetch_consistency_snapshot: SET stats_fetch_consistency = 'snapshot'; +step s1_begin: BEGIN; +step s1_build_snapshot: + SELECT wal_records >= 0 AS snapshot_created FROM pg_stat_wal; + +snapshot_created +---------------- +t +(1 row) + +step s2_generate: + INSERT INTO stats_per_backend_data VALUES (1); + SELECT pg_stat_force_next_flush(); + +pg_stat_force_next_flush +------------------------ + +(1 row) + +step s1_save_backend_stats: + INSERT INTO stats_per_backend_saved + SELECT wal_records + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')); + +step s1_commit: COMMIT; +step s1_check_live_backend: + SELECT current.wal_records > saved.wal_records AS snapshot_is_refreshed + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')) AS current + CROSS JOIN stats_per_backend_saved AS saved; + +snapshot_is_refreshed +--------------------- +t +(1 row) + + +starting permutation: s1_fetch_consistency_cache s2_generate s1_begin s1_save_backend_stats s2_generate_more s1_check_cached_backend s1_commit s1_check_refreshed_backend +step s1_fetch_consistency_cache: SET stats_fetch_consistency = 'cache'; +step s2_generate: + INSERT INTO stats_per_backend_data VALUES (1); + SELECT pg_stat_force_next_flush(); + +pg_stat_force_next_flush +------------------------ + +(1 row) + +step s1_begin: BEGIN; +step s1_save_backend_stats: + INSERT INTO stats_per_backend_saved + SELECT wal_records + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')); + +step s2_generate_more: + INSERT INTO stats_per_backend_data VALUES (2); + SELECT pg_stat_force_next_flush(); + +pg_stat_force_next_flush +------------------------ + +(1 row) + +step s1_check_cached_backend: + SELECT current.wal_records = saved.wal_records AS cache_is_stable + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')) AS current + CROSS JOIN stats_per_backend_saved AS saved; + +cache_is_stable +--------------- +t +(1 row) + +step s1_commit: COMMIT; +step s1_check_refreshed_backend: + SELECT current.wal_records > saved.wal_records AS cache_is_refreshed + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')) AS current + CROSS JOIN stats_per_backend_saved AS saved; + +cache_is_refreshed +------------------ +t +(1 row) + + +starting permutation: s1_fetch_consistency_cache s2_generate s1_begin s1_save_backend_stats s2_generate_more s1_fetch_consistency_snapshot s1_check_live_backend s1_commit +step s1_fetch_consistency_cache: SET stats_fetch_consistency = 'cache'; +step s2_generate: + INSERT INTO stats_per_backend_data VALUES (1); + SELECT pg_stat_force_next_flush(); + +pg_stat_force_next_flush +------------------------ + +(1 row) + +step s1_begin: BEGIN; +step s1_save_backend_stats: + INSERT INTO stats_per_backend_saved + SELECT wal_records + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')); + +step s2_generate_more: + INSERT INTO stats_per_backend_data VALUES (2); + SELECT pg_stat_force_next_flush(); + +pg_stat_force_next_flush +------------------------ + +(1 row) + +step s1_fetch_consistency_snapshot: SET stats_fetch_consistency = 'snapshot'; +step s1_check_live_backend: + SELECT current.wal_records > saved.wal_records AS snapshot_is_refreshed + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')) AS current + CROSS JOIN stats_per_backend_saved AS saved; + +snapshot_is_refreshed +--------------------- +t +(1 row) + +step s1_commit: COMMIT; + +starting permutation: s1_reset_backend s1_check_backend_reset +step s1_reset_backend: + SELECT pg_stat_reset_backend_stats( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')); + +pg_stat_reset_backend_stats +--------------------------- + +(1 row) + +step s1_check_backend_reset: + SELECT stats_reset IS NOT NULL AS reset_timestamp_set + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')); + +reset_timestamp_set +------------------- +t +(1 row) + + +starting permutation: s1_reset_shared s1_check_shared_reset +step s1_reset_shared: + SELECT pg_stat_reset_shared('wal'); + +pg_stat_reset_shared +-------------------- + +(1 row) + +step s1_check_shared_reset: + SELECT stats_reset IS NOT NULL AS reset_timestamp_set + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')); + +reset_timestamp_set +------------------- +t +(1 row) + diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule index bf9a2037c70..e8444df2956 100644 --- a/src/test/isolation/isolation_schedule +++ b/src/test/isolation/isolation_schedule @@ -108,6 +108,7 @@ test: vacuum-concurrent-drop test: vacuum-conflict test: vacuum-skip-locked test: stats +test: stats-per-backend test: horizons test: predicate-hash test: predicate-gist diff --git a/src/test/isolation/specs/stats-per-backend.spec b/src/test/isolation/specs/stats-per-backend.spec new file mode 100644 index 00000000000..a4343ff8163 --- /dev/null +++ b/src/test/isolation/specs/stats-per-backend.spec @@ -0,0 +1,151 @@ +# Test snapshot, cache, and reset behavior for per-backend statistics. +# +# Session s2 generates and flushes WAL records, which provide a monotonically +# increasing per-backend counter. WAL is used as a representative statistics +# kind rather than repeating the consistency tests for each kind. Session s1 +# verifies that NONE mode always fetches fresh values, SNAPSHOT mode fixes all +# per-backend entries at the initial statistics snapshot, and CACHE mode fixes +# each entry on first access and refreshes it after the transaction ends. The +# final permutations verify reset timestamps after per-backend and shared +# resets. + +setup +{ + CREATE TABLE stats_per_backend_data(id int); + CREATE TABLE stats_per_backend_saved(wal_records bigint); +} + +teardown +{ + DROP TABLE stats_per_backend_data; + DROP TABLE stats_per_backend_saved; +} + +session s1 +setup { SET stats_fetch_consistency = 'none'; } + +step s1_fetch_consistency_none { SET stats_fetch_consistency = 'none'; } +step s1_fetch_consistency_cache { SET stats_fetch_consistency = 'cache'; } +step s1_fetch_consistency_snapshot { SET stats_fetch_consistency = 'snapshot'; } +step s1_begin { BEGIN; } +step s1_commit { COMMIT; } +step s1_build_snapshot { + SELECT wal_records >= 0 AS snapshot_created FROM pg_stat_wal; +} +step s1_check_live_backend { + SELECT current.wal_records > saved.wal_records AS snapshot_is_refreshed + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')) AS current + CROSS JOIN stats_per_backend_saved AS saved; +} +step s1_save_backend_stats { + INSERT INTO stats_per_backend_saved + SELECT wal_records + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')); +} +step s1_check_cached_backend { + SELECT current.wal_records = saved.wal_records AS cache_is_stable + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')) AS current + CROSS JOIN stats_per_backend_saved AS saved; +} +step s1_check_uncached_backend { + SELECT current.wal_records > saved.wal_records AS none_is_refreshed + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')) AS current + CROSS JOIN stats_per_backend_saved AS saved; +} +step s1_check_refreshed_backend { + SELECT current.wal_records > saved.wal_records AS cache_is_refreshed + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')) AS current + CROSS JOIN stats_per_backend_saved AS saved; +} +step s1_reset_backend { + SELECT pg_stat_reset_backend_stats( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')); +} +step s1_check_backend_reset { + SELECT stats_reset IS NOT NULL AS reset_timestamp_set + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')); +} +step s1_reset_shared { + SELECT pg_stat_reset_shared('wal'); +} +step s1_check_shared_reset { + SELECT stats_reset IS NOT NULL AS reset_timestamp_set + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')); +} + +session s2 +step s2_generate { + INSERT INTO stats_per_backend_data VALUES (1); + SELECT pg_stat_force_next_flush(); +} +step s2_generate_more { + INSERT INTO stats_per_backend_data VALUES (2); + SELECT pg_stat_force_next_flush(); +} + +# with stats_fetch_consistency=none s1 should see flushed changes from s2 +# despite being in a transaction +permutation + s1_fetch_consistency_none + s2_generate + s1_begin + s1_save_backend_stats + s2_generate_more + s1_check_uncached_backend + s1_commit + +# with stats_fetch_consistency=snapshot s1 should not see flushed changes from +# s2 after building the statistics snapshot, but should see them after commit +permutation + s1_fetch_consistency_snapshot + s1_begin + s1_build_snapshot + s2_generate + s1_save_backend_stats + s1_commit + s1_check_live_backend + +# with stats_fetch_consistency=cache s1 should not see flushed changes from s2 +# after the first access, but should see them after commit +permutation + s1_fetch_consistency_cache + s2_generate + s1_begin + s1_save_backend_stats + s2_generate_more + s1_check_cached_backend + s1_commit + s1_check_refreshed_backend + +# changing from cache to snapshot within a transaction should discard the +# per-backend cache before building the full snapshot +permutation + s1_fetch_consistency_cache + s2_generate + s1_begin + s1_save_backend_stats + s2_generate_more + s1_fetch_consistency_snapshot + s1_check_live_backend + s1_commit + +# a per-backend reset should set that backend's reset timestamp +permutation s1_reset_backend s1_check_backend_reset + +# a shared WAL reset should set the reset timestamp for live backends +permutation s1_reset_shared s1_check_shared_reset diff --git a/src/test/recovery/t/029_stats_restart.pl b/src/test/recovery/t/029_stats_restart.pl index cdc427dbc78..27cfdb0b75d 100644 --- a/src/test/recovery/t/029_stats_restart.pl +++ b/src/test/recovery/t/029_stats_restart.pl @@ -1,7 +1,7 @@ # Copyright (c) 2021-2026, PostgreSQL Global Development Group -# Tests statistics handling around restarts, including handling of crashes and -# invalid stats files, as well as restoring stats after "normal" restarts. +# Tests statistics snapshot consistency and handling around restarts, including +# crashes, invalid stats files, and restoring stats after normal restarts. use strict; use warnings FATAL => 'all'; @@ -18,6 +18,38 @@ $node->start; my $connect_db = 'postgres'; my $db_under_test = 'test'; +# Verify that a statistics snapshot excludes backends that started after the +# snapshot was created. +my $snapshot_session = $node->background_psql($connect_db); + +$snapshot_session->query_safe( + q[ + BEGIN; + SET LOCAL stats_fetch_consistency = 'snapshot'; + SELECT count(*) FROM pg_stat_wal; +]); + +# Establish this connection only after the first session built its snapshot. +my $late_session = $node->background_psql($connect_db); +my $late_pid = $late_session->query_safe('SELECT pg_backend_pid()'); + +# WAL returns one scalar composite, so count a field to distinguish a NULL +# result from the fabricated all-zero row. +is( $snapshot_session->query_safe( + qq[ + SELECT + (SELECT count(wal_records) + FROM pg_stat_get_backend_wal($late_pid)), + (SELECT count(*) FROM pg_stat_get_backend_io($late_pid)), + (SELECT count(*) FROM pg_stat_get_backend_lock($late_pid)); + ]), + '0|0|0', + 'statistics snapshot excludes a backend created later'); + +$snapshot_session->query_safe('ROLLBACK'); +$late_session->quit; +$snapshot_session->quit; + my $sect = "startup"; # Check some WAL statistics after a fresh startup. The startup process diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 8b15471248b..e3dd0ce89b0 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1130,6 +1130,35 @@ SELECT wal_bytes > :backend_wal_bytes_before FROM pg_stat_get_backend_wal(pg_bac t (1 row) +-- Test pg_stat_reset_backend_stats() for WAL +SELECT wal_bytes AS wal_bytes_before_reset FROM pg_stat_get_backend_wal(pg_backend_pid()) \gset +SELECT pg_stat_reset_backend_stats(pg_backend_pid()); + pg_stat_reset_backend_stats +----------------------------- + +(1 row) + +SELECT wal_bytes AS wal_bytes_after_reset FROM pg_stat_get_backend_wal(pg_backend_pid()) \gset +SELECT :wal_bytes_after_reset < :wal_bytes_before_reset; + ?column? +---------- + t +(1 row) + +-- global view should still have the data +SELECT wal_bytes >= :wal_bytes_before_reset FROM pg_stat_wal; + ?column? +---------- + t +(1 row) + +-- reset timestamp should be set +SELECT stats_reset IS NOT NULL FROM pg_stat_get_backend_wal(pg_backend_pid()); + ?column? +---------- + t +(1 row) + -- Test pg_stat_get_backend_idset() and some allied functions. -- In particular, verify that their notion of backend ID matches -- our temp schema index. diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index 674637e172b..f4931fd84de 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -528,6 +528,16 @@ SELECT wal_bytes > :wal_bytes_before FROM pg_stat_wal; SELECT pg_stat_force_next_flush(); SELECT wal_bytes > :backend_wal_bytes_before FROM pg_stat_get_backend_wal(pg_backend_pid()); +-- Test pg_stat_reset_backend_stats() for WAL +SELECT wal_bytes AS wal_bytes_before_reset FROM pg_stat_get_backend_wal(pg_backend_pid()) \gset +SELECT pg_stat_reset_backend_stats(pg_backend_pid()); +SELECT wal_bytes AS wal_bytes_after_reset FROM pg_stat_get_backend_wal(pg_backend_pid()) \gset +SELECT :wal_bytes_after_reset < :wal_bytes_before_reset; +-- global view should still have the data +SELECT wal_bytes >= :wal_bytes_before_reset FROM pg_stat_wal; +-- reset timestamp should be set +SELECT stats_reset IS NOT NULL FROM pg_stat_get_backend_wal(pg_backend_pid()); + -- Test pg_stat_get_backend_idset() and some allied functions. -- In particular, verify that their notion of backend ID matches -- our temp schema index. -- 2.34.1