From abc8330baf690a036114ea688fcaa4643ab641c9 Mon Sep 17 00:00:00 2001 From: Bertrand Drouvot Date: Fri, 17 Jul 2026 13:23:40 +0000 Subject: [PATCH v1 1/5] pgstat: add tests for per-backend statistics Add an isolation test for per-backend WAL statistics. Verify that SNAPSHOT, CACHE and NONE modes behave as expected. Add a TAP test that starts a backend after another session has built a statistics snapshot. Per-backend WAL, IO, and Lock queries for the new backend must return no rows, rather than fabricated rows containing zeroes. Extend test_shm_mq with a shared-memory-only worker that reports WAL usage through a routine nonblocking flush and remains alive while the controlling backend verifies that the global WAL counters include it. Finally, extend the core statistics regression test for per-backend WAL reset. Check that resetting one backend reduces its counters and sets its reset timestamp without removing the same activity from the global WAL statistics. These tests will serve as compatibility coverage for the per-backend redesign changes coming in following commits. Author: Bertrand Drouvot Reviewed-by: Discussion: --- .../isolation/expected/stats-per-backend.out | 143 ++++++++++++++++++ src/test/isolation/isolation_schedule | 1 + .../isolation/specs/stats-per-backend.spec | 123 +++++++++++++++ src/test/modules/test_misc/meson.build | 1 + .../modules/test_misc/t/015_stats_snapshot.pl | 48 ++++++ .../test_shm_mq/expected/test_shm_mq.out | 7 + .../modules/test_shm_mq/sql/test_shm_mq.sql | 3 + src/test/modules/test_shm_mq/test.c | 51 +++++++ .../modules/test_shm_mq/test_shm_mq--1.0.sql | 4 + src/test/modules/test_shm_mq/test_shm_mq.h | 4 + src/test/modules/test_shm_mq/worker.c | 10 ++ src/test/regress/expected/stats.out | 29 ++++ src/test/regress/sql/stats.sql | 10 ++ 13 files changed, 434 insertions(+) 30.6% src/test/isolation/expected/ 31.4% src/test/isolation/specs/ 9.7% src/test/modules/test_misc/t/ 17.6% src/test/modules/test_shm_mq/ 5.6% src/test/regress/expected/ 4.3% src/test/regress/sql/ diff --git a/src/test/isolation/expected/stats-per-backend.out b/src/test/isolation/expected/stats-per-backend.out new file mode 100644 index 00000000000..0d85ae99033 --- /dev/null +++ b/src/test/isolation/expected/stats-per-backend.out @@ -0,0 +1,143 @@ +Parsed test spec with 2 sessions + +starting permutation: s1_fetch_consistency_snapshot s1_begin s1_build_snapshot s2_generate s1_check_snapshot_backend s1_commit s1_check_live_backend +step s1_fetch_consistency_snapshot: SET stats_fetch_consistency = 'snapshot'; +step s1_begin: BEGIN; +step s1_build_snapshot: + SELECT wal_records >= 0 AS snapshot_created FROM pg_stat_wal; + +snapshot_created +---------------- +t +(1 row) + +step s2_generate: + INSERT INTO stats_per_backend_data VALUES (1); + SELECT pg_stat_force_next_flush(); + +pg_stat_force_next_flush +------------------------ + +(1 row) + +step s1_check_snapshot_backend: + SELECT wal_records = 0 AS snapshot_excludes_later_update + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')); + +snapshot_excludes_later_update +------------------------------ +t +(1 row) + +step s1_commit: COMMIT; +step s1_check_live_backend: + SELECT wal_records > 0 AS live_entry_exists + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')); + +live_entry_exists +----------------- +t +(1 row) + + +starting permutation: s1_fetch_consistency_cache s2_generate s1_begin s1_save_backend_stats s2_generate_more s1_check_cached_backend s1_commit s1_check_refreshed_backend +step s1_fetch_consistency_cache: SET stats_fetch_consistency = 'cache'; +step s2_generate: + INSERT INTO stats_per_backend_data VALUES (1); + SELECT pg_stat_force_next_flush(); + +pg_stat_force_next_flush +------------------------ + +(1 row) + +step s1_begin: BEGIN; +step s1_save_backend_stats: + INSERT INTO stats_per_backend_saved + SELECT wal_records + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')); + +step s2_generate_more: + INSERT INTO stats_per_backend_data VALUES (2); + SELECT pg_stat_force_next_flush(); + +pg_stat_force_next_flush +------------------------ + +(1 row) + +step s1_check_cached_backend: + SELECT current.wal_records = saved.wal_records AS cache_is_stable + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')) AS current + CROSS JOIN stats_per_backend_saved AS saved; + +cache_is_stable +--------------- +t +(1 row) + +step s1_commit: COMMIT; +step s1_check_refreshed_backend: + SELECT current.wal_records > saved.wal_records AS cache_is_refreshed + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')) AS current + CROSS JOIN stats_per_backend_saved AS saved; + +cache_is_refreshed +------------------ +t +(1 row) + + +starting permutation: s1_reset_backend s1_check_backend_reset +step s1_reset_backend: + SELECT pg_stat_reset_backend_stats( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')); + +pg_stat_reset_backend_stats +--------------------------- + +(1 row) + +step s1_check_backend_reset: + SELECT stats_reset IS NOT NULL AS reset_timestamp_set + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')); + +reset_timestamp_set +------------------- +t +(1 row) + + +starting permutation: s1_reset_shared s1_check_shared_reset +step s1_reset_shared: + SELECT pg_stat_reset_shared('wal'); + +pg_stat_reset_shared +-------------------- + +(1 row) + +step s1_check_shared_reset: + SELECT stats_reset IS NOT NULL AS reset_timestamp_set + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')); + +reset_timestamp_set +------------------- +t +(1 row) + diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule index 26abed9f9f0..5d97c56b5d6 100644 --- a/src/test/isolation/isolation_schedule +++ b/src/test/isolation/isolation_schedule @@ -104,6 +104,7 @@ test: vacuum-concurrent-drop test: vacuum-conflict test: vacuum-skip-locked test: stats +test: stats-per-backend test: horizons test: predicate-hash test: predicate-gist diff --git a/src/test/isolation/specs/stats-per-backend.spec b/src/test/isolation/specs/stats-per-backend.spec new file mode 100644 index 00000000000..5dfc8dbe90e --- /dev/null +++ b/src/test/isolation/specs/stats-per-backend.spec @@ -0,0 +1,123 @@ +# Test snapshot, cache, and reset behavior for per-backend statistics. +# +# Session s2 generates and flushes WAL records, which provide a deterministic +# per-backend counter. Session s1 verifies that SNAPSHOT mode fixes all +# per-backend entries at the initial statistics snapshot, while CACHE mode +# fixes each entry on first access and refreshes it after the transaction +# ends. The final permutations verify reset timestamps after per-backend and +# shared resets. + +setup +{ + CREATE TABLE stats_per_backend_data(id int); + CREATE TABLE stats_per_backend_saved(wal_records bigint); +} + +teardown +{ + DROP TABLE stats_per_backend_data; + DROP TABLE stats_per_backend_saved; +} + +session s1 +setup { SET stats_fetch_consistency = 'none'; } + +step s1_fetch_consistency_cache { SET stats_fetch_consistency = 'cache'; } +step s1_fetch_consistency_snapshot { SET stats_fetch_consistency = 'snapshot'; } +step s1_begin { BEGIN; } +step s1_commit { COMMIT; } +step s1_build_snapshot { + SELECT wal_records >= 0 AS snapshot_created FROM pg_stat_wal; +} +step s1_check_snapshot_backend { + SELECT wal_records = 0 AS snapshot_excludes_later_update + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')); +} +step s1_check_live_backend { + SELECT wal_records > 0 AS live_entry_exists + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')); +} +step s1_save_backend_stats { + INSERT INTO stats_per_backend_saved + SELECT wal_records + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')); +} +step s1_check_cached_backend { + SELECT current.wal_records = saved.wal_records AS cache_is_stable + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')) AS current + CROSS JOIN stats_per_backend_saved AS saved; +} +step s1_check_refreshed_backend { + SELECT current.wal_records > saved.wal_records AS cache_is_refreshed + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')) AS current + CROSS JOIN stats_per_backend_saved AS saved; +} +step s1_reset_backend { + SELECT pg_stat_reset_backend_stats( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')); +} +step s1_check_backend_reset { + SELECT stats_reset IS NOT NULL AS reset_timestamp_set + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')); +} +step s1_reset_shared { + SELECT pg_stat_reset_shared('wal'); +} +step s1_check_shared_reset { + SELECT stats_reset IS NOT NULL AS reset_timestamp_set + FROM pg_stat_get_backend_wal( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'isolation/stats-per-backend/s2')); +} + +session s2 +step s2_generate { + INSERT INTO stats_per_backend_data VALUES (1); + SELECT pg_stat_force_next_flush(); +} +step s2_generate_more { + INSERT INTO stats_per_backend_data VALUES (2); + SELECT pg_stat_force_next_flush(); +} + +# with stats_fetch_consistency=snapshot s1 should not see flushed changes from +# s2 after building the statistics snapshot, but should see them after commit +permutation + s1_fetch_consistency_snapshot + s1_begin + s1_build_snapshot + s2_generate + s1_check_snapshot_backend + s1_commit + s1_check_live_backend + +# with stats_fetch_consistency=cache s1 should not see flushed changes from s2 +# after the first access, but should see them after commit +permutation + s1_fetch_consistency_cache + s2_generate + s1_begin + s1_save_backend_stats + s2_generate_more + s1_check_cached_backend + s1_commit + s1_check_refreshed_backend + +# a per-backend reset should set that backend's reset timestamp +permutation s1_reset_backend s1_check_backend_reset + +# a shared WAL reset should set the reset timestamp for live backends +permutation s1_reset_shared s1_check_shared_reset diff --git a/src/test/modules/test_misc/meson.build b/src/test/modules/test_misc/meson.build index ee290698b31..cf4c8558f3d 100644 --- a/src/test/modules/test_misc/meson.build +++ b/src/test/modules/test_misc/meson.build @@ -23,6 +23,7 @@ tests += { 't/012_ddlutils.pl', 't/013_temp_obj_multisession.pl', 't/014_log_statement_max_length.pl', + 't/015_stats_snapshot.pl', ], # The injection points are cluster-wide, so disable installcheck 'runningcheck': false, diff --git a/src/test/modules/test_misc/t/015_stats_snapshot.pl b/src/test/modules/test_misc/t/015_stats_snapshot.pl new file mode 100644 index 00000000000..43c9157bbb4 --- /dev/null +++ b/src/test/modules/test_misc/t/015_stats_snapshot.pl @@ -0,0 +1,48 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Verify that a statistics snapshot excludes backends that started after the +# snapshot was created. + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('stats_snapshot'); +$node->init; +$node->start; + +my $snapshot = $node->background_psql('postgres'); + +$snapshot->query_safe( + q[ + BEGIN; + SET LOCAL stats_fetch_consistency = 'snapshot'; + SELECT count(*) FROM pg_stat_wal; +]); + +# Establish this connection only after the first session built its snapshot. +my $late = $node->background_psql('postgres'); +my $late_pid = $late->query_safe('SELECT pg_backend_pid()'); + +# WAL returns one scalar composite, so count a field to distinguish a NULL +# result from the fabricated all-zero row. +is( $snapshot->query_safe( + qq[ + SELECT + (SELECT count(wal_records) + FROM pg_stat_get_backend_wal($late_pid)), + (SELECT count(*) FROM pg_stat_get_backend_io($late_pid)), + (SELECT count(*) FROM pg_stat_get_backend_lock($late_pid)); + ]), + '0|0|0', + 'statistics snapshot excludes a backend created later'); + +$snapshot->query_safe('ROLLBACK'); +$late->quit; +$snapshot->quit; +$node->stop; + +done_testing(); diff --git a/src/test/modules/test_shm_mq/expected/test_shm_mq.out b/src/test/modules/test_shm_mq/expected/test_shm_mq.out index c4858b0c205..ec3a7d3199e 100644 --- a/src/test/modules/test_shm_mq/expected/test_shm_mq.out +++ b/src/test/modules/test_shm_mq/expected/test_shm_mq.out @@ -34,3 +34,10 @@ SELECT test_shm_mq_pipelined(16384, (select string_agg(chr(32+(random()*95)::int (1 row) +-- A shmem-only worker can publish statistics before its clean exit. +SELECT test_shm_mq_worker_stats(); + test_shm_mq_worker_stats +-------------------------- + t +(1 row) + diff --git a/src/test/modules/test_shm_mq/sql/test_shm_mq.sql b/src/test/modules/test_shm_mq/sql/test_shm_mq.sql index 9de19d304a2..e80e4399881 100644 --- a/src/test/modules/test_shm_mq/sql/test_shm_mq.sql +++ b/src/test/modules/test_shm_mq/sql/test_shm_mq.sql @@ -10,3 +10,6 @@ SELECT test_shm_mq(1024, 'a', 2001, 1); SELECT test_shm_mq(32768, (select string_agg(chr(32+(random()*95)::int), '') from generate_series(1,(100+900*random())::int)), 10000, 1); SELECT test_shm_mq(100, (select string_agg(chr(32+(random()*95)::int), '') from generate_series(1,(100+200*random())::int)), 10000, 1); SELECT test_shm_mq_pipelined(16384, (select string_agg(chr(32+(random()*95)::int), '') from generate_series(1,270000)), 200, 3); + +-- A shmem-only worker can publish statistics before its clean exit. +SELECT test_shm_mq_worker_stats(); diff --git a/src/test/modules/test_shm_mq/test.c b/src/test/modules/test_shm_mq/test.c index 0e55287e510..4464d796ef9 100644 --- a/src/test/modules/test_shm_mq/test.c +++ b/src/test/modules/test_shm_mq/test.c @@ -26,6 +26,7 @@ PG_MODULE_MAGIC; PG_FUNCTION_INFO_V1(test_shm_mq); PG_FUNCTION_INFO_V1(test_shm_mq_pipelined); +PG_FUNCTION_INFO_V1(test_shm_mq_worker_stats); static void verify_message(Size origlen, char *origdata, Size newlen, char *newdata); @@ -253,6 +254,56 @@ test_shm_mq_pipelined(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +/* + * Verify that a shmem-only worker can publish statistics through a routine + * nonblocking flush while it remains alive. + */ +Datum +test_shm_mq_worker_stats(PG_FUNCTION_ARGS) +{ + const char *message = PG_TEST_SHM_MQ_STATS_MESSAGE; + PgStat_Counter before; + PgStat_Counter after; + dsm_segment *seg; + shm_mq_handle *outqh; + shm_mq_handle *inqh; + shm_mq_result res; + Size len; + void *data; + + pgstat_clear_snapshot(); + before = pgstat_fetch_stat_wal()->wal_counters.wal_records; + + test_shm_mq_setup(1024, 1, &seg, &outqh, &inqh); + + res = shm_mq_send(outqh, strlen(message), message, false, true); + + if (res != SHM_MQ_SUCCESS) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("could not send statistics test message"))); + + res = shm_mq_receive(inqh, &len, &data, false); + + if (res != SHM_MQ_SUCCESS) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("could not receive statistics test message"))); + + verify_message(strlen(message), (char *) message, len, data); + + /* + * The worker is waiting for another message here. Check its live entry + * before detaching the queues and triggering its shutdown flush. + */ + pgstat_clear_snapshot(); + after = pgstat_fetch_stat_wal()->wal_counters.wal_records; + + dsm_detach(seg); + + PG_RETURN_BOOL(after >= before + PG_TEST_SHM_MQ_STATS_RECORDS); +} + /* * Verify that two messages are the same. */ diff --git a/src/test/modules/test_shm_mq/test_shm_mq--1.0.sql b/src/test/modules/test_shm_mq/test_shm_mq--1.0.sql index 56db05d93df..dce6f7378a5 100644 --- a/src/test/modules/test_shm_mq/test_shm_mq--1.0.sql +++ b/src/test/modules/test_shm_mq/test_shm_mq--1.0.sql @@ -17,3 +17,7 @@ CREATE FUNCTION test_shm_mq_pipelined(queue_size pg_catalog.int8, verify pg_catalog.bool default true) RETURNS pg_catalog.void STRICT AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_shm_mq_worker_stats() + RETURNS pg_catalog.bool + AS 'MODULE_PATHNAME' LANGUAGE C; diff --git a/src/test/modules/test_shm_mq/test_shm_mq.h b/src/test/modules/test_shm_mq/test_shm_mq.h index b6a0290289c..501be180642 100644 --- a/src/test/modules/test_shm_mq/test_shm_mq.h +++ b/src/test/modules/test_shm_mq/test_shm_mq.h @@ -21,6 +21,10 @@ /* Identifier for shared memory segments used by this extension. */ #define PG_TEST_SHM_MQ_MAGIC 0x79fb2447 +/* Special message used to test statistics from a shmem-only worker. */ +#define PG_TEST_SHM_MQ_STATS_MESSAGE "test_shm_mq worker stats" +#define PG_TEST_SHM_MQ_STATS_RECORDS 1000000 + /* * This structure is stored in the dynamic shared memory segment. We use * it to determine whether all workers started up OK and successfully diff --git a/src/test/modules/test_shm_mq/worker.c b/src/test/modules/test_shm_mq/worker.c index 0ba1cfcac47..afcea7fc0cb 100644 --- a/src/test/modules/test_shm_mq/worker.c +++ b/src/test/modules/test_shm_mq/worker.c @@ -19,6 +19,7 @@ #include "postgres.h" +#include "executor/instrument.h" #include "miscadmin.h" #include "storage/ipc.h" #include "storage/latch.h" @@ -27,6 +28,7 @@ #include "storage/shm_mq.h" #include "storage/shm_toc.h" #include "tcop/tcopprot.h" +#include "utils/pgstat_internal.h" #include "test_shm_mq.h" @@ -185,6 +187,14 @@ copy_messages(shm_mq_handle *inqh, shm_mq_handle *outqh) if (res != SHM_MQ_SUCCESS) break; + if (len == strlen(PG_TEST_SHM_MQ_STATS_MESSAGE) && + memcmp(data, PG_TEST_SHM_MQ_STATS_MESSAGE, len) == 0) + { + pgWalUsage.wal_records += PG_TEST_SHM_MQ_STATS_RECORDS; + pgstat_report_fixed = true; + (void) pgstat_report_stat(false); + } + /* Send it back out. */ res = shm_mq_send(outqh, len, data, false, true); if (res != SHM_MQ_SUCCESS) diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index e230356de13..fae5f939523 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1044,6 +1044,35 @@ SELECT wal_bytes > :backend_wal_bytes_before FROM pg_stat_get_backend_wal(pg_bac t (1 row) +-- Test pg_stat_reset_backend_stats() for WAL +SELECT wal_bytes AS wal_bytes_before_reset FROM pg_stat_get_backend_wal(pg_backend_pid()) \gset +SELECT pg_stat_reset_backend_stats(pg_backend_pid()); + pg_stat_reset_backend_stats +----------------------------- + +(1 row) + +SELECT wal_bytes AS wal_bytes_after_reset FROM pg_stat_get_backend_wal(pg_backend_pid()) \gset +SELECT :wal_bytes_after_reset < :wal_bytes_before_reset; + ?column? +---------- + t +(1 row) + +-- global view should still have the data +SELECT wal_bytes >= :wal_bytes_before_reset FROM pg_stat_wal; + ?column? +---------- + t +(1 row) + +-- reset timestamp should be set +SELECT stats_reset IS NOT NULL FROM pg_stat_get_backend_wal(pg_backend_pid()); + ?column? +---------- + t +(1 row) + -- Test pg_stat_get_backend_idset() and some allied functions. -- In particular, verify that their notion of backend ID matches -- our temp schema index. diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index 4c265d1245c..c17753dcd20 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -482,6 +482,16 @@ SELECT wal_bytes > :wal_bytes_before FROM pg_stat_wal; SELECT pg_stat_force_next_flush(); SELECT wal_bytes > :backend_wal_bytes_before FROM pg_stat_get_backend_wal(pg_backend_pid()); +-- Test pg_stat_reset_backend_stats() for WAL +SELECT wal_bytes AS wal_bytes_before_reset FROM pg_stat_get_backend_wal(pg_backend_pid()) \gset +SELECT pg_stat_reset_backend_stats(pg_backend_pid()); +SELECT wal_bytes AS wal_bytes_after_reset FROM pg_stat_get_backend_wal(pg_backend_pid()) \gset +SELECT :wal_bytes_after_reset < :wal_bytes_before_reset; +-- global view should still have the data +SELECT wal_bytes >= :wal_bytes_before_reset FROM pg_stat_wal; +-- reset timestamp should be set +SELECT stats_reset IS NOT NULL FROM pg_stat_get_backend_wal(pg_backend_pid()); + -- Test pg_stat_get_backend_idset() and some allied functions. -- In particular, verify that their notion of backend ID matches -- our temp schema index. -- 2.34.1