From 7c42b68e7ebeeceb1475502d2ab5e2f9bd543670 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Tue, 3 Mar 2026 00:17:13 +0300 Subject: [PATCH 3/3] ext_vacuum_statistics extension for extended vacuum statistics. This commit introduces the ext_vacuum_statistics extension, which provides extended vacuum statistics through a dedicated schema and views. Statistics are stored via the pgstat custom statistics infrastructure. The extension registers set_report_vacuum_hook to receive vacuum metrics and persists them into custom stats; when the hook is not set, no additional overhead is incurred. Views pg_stats_vacuum_tables, pg_stats_vacuum_indexes and pg_stats_vacuum_database expose per-table, per-index and aggregated per-database vacuum statistics respectively. GUCs control which objects are tracked and how. vacuum_statistics.enabled (default on) turns collection on or off. vacuum_statistics.object_types (default all) restricts tracking to databases only, relations only, or both. When tracking relations, vacuum_statistics.track_relations (default all) filters by system or user tables. vacuum_statistics.track_databases_from_list and vacuum_statistics.track_relations_from_list (both default off) restrict tracking to databases and relations explicitly added via add_track_database and add_track_relation; when off, all objects of the chosen types are tracked. --- contrib/Makefile | 1 + contrib/ext_vacuum_statistics/Makefile | 24 + contrib/ext_vacuum_statistics/README.md | 165 +++ .../expected/ext_vacuum_statistics.out | 52 + .../vacuum-extending-in-repetable-read.out | 52 + .../ext_vacuum_statistics--1.0.sql | 260 +++++ .../ext_vacuum_statistics.conf | 2 + .../ext_vacuum_statistics.control | 5 + contrib/ext_vacuum_statistics/meson.build | 41 + .../vacuum-extending-in-repetable-read.spec | 59 + .../t/052_vacuum_extending_basic_test.pl | 780 +++++++++++++ .../t/053_vacuum_extending_freeze_test.pl | 285 +++++ .../t/054_vacuum_extending_gucs_test.pl | 203 ++++ .../ext_vacuum_statistics/vacuum_statistics.c | 1000 +++++++++++++++++ contrib/meson.build | 1 + doc/src/sgml/contrib.sgml | 1 + doc/src/sgml/extvacuumstatistics.sgml | 502 +++++++++ doc/src/sgml/filelist.sgml | 1 + src/backend/access/heap/vacuumlazy.c | 112 +- src/backend/commands/vacuumparallel.c | 12 +- src/backend/utils/activity/pgstat_relation.c | 24 + src/include/commands/vacuum.h | 1 + src/include/pgstat.h | 11 + 23 files changed, 3576 insertions(+), 18 deletions(-) create mode 100644 contrib/ext_vacuum_statistics/Makefile create mode 100644 contrib/ext_vacuum_statistics/README.md create mode 100644 contrib/ext_vacuum_statistics/expected/ext_vacuum_statistics.out create mode 100644 contrib/ext_vacuum_statistics/expected/vacuum-extending-in-repetable-read.out create mode 100644 contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql create mode 100644 contrib/ext_vacuum_statistics/ext_vacuum_statistics.conf create mode 100644 contrib/ext_vacuum_statistics/ext_vacuum_statistics.control create mode 100644 contrib/ext_vacuum_statistics/meson.build create mode 100644 contrib/ext_vacuum_statistics/specs/vacuum-extending-in-repetable-read.spec create mode 100644 contrib/ext_vacuum_statistics/t/052_vacuum_extending_basic_test.pl create mode 100644 contrib/ext_vacuum_statistics/t/053_vacuum_extending_freeze_test.pl create mode 100644 contrib/ext_vacuum_statistics/t/054_vacuum_extending_gucs_test.pl create mode 100644 contrib/ext_vacuum_statistics/vacuum_statistics.c create mode 100644 doc/src/sgml/extvacuumstatistics.sgml diff --git a/contrib/Makefile b/contrib/Makefile index 2f0a88d3f77..6e064c566aa 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -19,6 +19,7 @@ SUBDIRS = \ dict_int \ dict_xsyn \ earthdistance \ + ext_vacuum_statistics \ file_fdw \ fuzzystrmatch \ hstore \ diff --git a/contrib/ext_vacuum_statistics/Makefile b/contrib/ext_vacuum_statistics/Makefile new file mode 100644 index 00000000000..ed80bdf28d0 --- /dev/null +++ b/contrib/ext_vacuum_statistics/Makefile @@ -0,0 +1,24 @@ +# contrib/ext_vacuum_statistics/Makefile + +EXTENSION = ext_vacuum_statistics +MODULE_big = ext_vacuum_statistics +OBJS = vacuum_statistics.o +DATA = ext_vacuum_statistics--1.0.sql +PGFILEDESC = "ext_vacuum_statistics - convenience views for extended vacuum statistics" + +ISOLATION = vacuum-extending-in-repetable-read +ISOLATION_OPTS = --temp-config=$(top_srcdir)/contrib/ext_vacuum_statistics/ext_vacuum_statistics.conf +TAP_TESTS = 1 + +NO_INSTALLCHECK = 1 + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/ext_vacuum_statistics +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/contrib/ext_vacuum_statistics/README.md b/contrib/ext_vacuum_statistics/README.md new file mode 100644 index 00000000000..51697eab023 --- /dev/null +++ b/contrib/ext_vacuum_statistics/README.md @@ -0,0 +1,165 @@ +# ext_vacuum_statistics + +Extended vacuum statistics extension for PostgreSQL. It collects and exposes detailed per-table, per-index, and per-database vacuum statistics (buffer I/O, WAL, general, timing) via convenient views in the `ext_vacuum_statistics` schema. + +## Installation + +``` +./configure tmp_install="$(pwd)/my/inst" +make clean && make && make install +cd contrib/ext_vacuum_statistics +make && make install +``` + +It is essential that the extension is listed in `shared_preload_libraries` because it registers a vacuum hook at server startup. + +In your `postgresql.conf`: + +``` +shared_preload_libraries = 'ext_vacuum_statistics' +``` + +Restart PostgreSQL. + +In your database: + +```sql +CREATE EXTENSION ext_vacuum_statistics; +``` + +## Usage + +Query vacuum statistics via the provided views: + +```sql +-- Per-table heap vacuum statistics +SELECT * FROM ext_vacuum_statistics.pg_stats_vacuum_tables; + +-- Per-index vacuum statistics +SELECT * FROM ext_vacuum_statistics.pg_stats_vacuum_indexes; + +-- Per-database aggregate vacuum statistics +SELECT * FROM ext_vacuum_statistics.pg_stats_vacuum_database; +``` + +Example output: + +``` + relname | total_blks_read | total_blks_hit | wal_records | tuples_deleted | pages_removed +-----------+-----------------+----------------+-------------+----------------+--------------- + mytable | 120 | 340 | 15 | 500 | 10 +``` + +Reset statistics when needed: + +```sql +SELECT ext_vacuum_statistics.vacuum_statistics_reset(); +``` + +## Configuration (GUCs) + +| GUC | Default | Description | +|-----|---------|-------------| +| `vacuum_statistics.enabled` | on | Enable extended vacuum statistics collection | +| `vacuum_statistics.object_types` | all | Object types for statistics: `all`, `databases`, `relations` | +| `vacuum_statistics.track_relations` | all | When tracking relations: `all`, `system`, `user` | +| `vacuum_statistics.track_databases_from_list` | off | If on, track only databases added via add_track_database | +| `vacuum_statistics.track_relations_from_list` | off | If on, track only relations added via add_track_relation | + +## Memory usage + +Each tracked object (table, index, or database) uses approximately **232 bytes** of shared memory on Linux x86_64 (e.g. Ubuntu): common stats (buffers, WAL, timing) ~144 bytes; type + union ~88 bytes (union holds table-specific or index-specific fields, allocated size is the same for both). + +The exact size depends on the platform. Call `ext_vacuum_statistics.shared_memory_size()` to get the total shared memory used by the extension. The GUCs provided by the extension allow controlling the amount of memory used: `vacuum_statistics.object_types` to track only databases or relations, `vacuum_statistics.track_relations` to restrict to user or system tables/indexes, and `track_*_from_list` to track only selected databases and relations. + +Example: a database with 1000 tables and 2000 indexes, all tracked, uses about **700 KB** on Ubuntu (3001 entries × 232 bytes). Per-database entries add one entry per tracked database. + +## Advanced tuning + +### Track only database-level stats + +```sql +SET vacuum_statistics.object_types = 'databases'; +``` + +Statistics are accumulated per database; per-relation views remain empty. + +### Track only user or system tables + +```sql +SET vacuum_statistics.object_types = 'relations'; +SET vacuum_statistics.track_relations = 'user'; -- skip system catalogs +-- or +SET vacuum_statistics.track_relations = 'system'; -- only system catalogs +``` + +### Filter by database or relation OIDs + +Add OIDs via functions (persisted to `pg_stat/ext_vacuum_statistics_track.oid`) and enable filtering: + +```sql +-- Add databases and relations to track +SELECT ext_vacuum_statistics.add_track_database(16384); +SELECT ext_vacuum_statistics.add_track_relation(16384, 16385); -- dboid, reloid +SELECT ext_vacuum_statistics.add_track_relation(0, 16386); -- rel 16386 in any db + +-- Enable list-based filtering (off = track all) +SET vacuum_statistics.track_databases_from_list = on; +SET vacuum_statistics.track_relations_from_list = on; +``` + +Remove OIDs when no longer needed: + +```sql +SELECT ext_vacuum_statistics.remove_track_database(16384); +SELECT ext_vacuum_statistics.remove_track_relation(16384, 16385); +``` + +Inspect the current tracking configuration: + +```sql +SELECT * FROM ext_vacuum_statistics.track_list(); +``` + +Returns `track_kind`, `dboid`, `reloid`. When `dboid` or `reloid` is NULL, statistics are collected for all. + +## Recipes + +**Reduce overhead by tracking only databases:** + +```sql +SET vacuum_statistics.object_types = 'databases'; +``` + +**Track only a specific table in a specific database:** + +```sql +SELECT ext_vacuum_statistics.add_track_database( + (SELECT oid FROM pg_database WHERE datname = current_database()) +); +SELECT ext_vacuum_statistics.add_track_relation( + (SELECT oid FROM pg_database WHERE datname = current_database()), + 'mytable'::regclass +); +SET vacuum_statistics.track_databases_from_list = on; +SET vacuum_statistics.track_relations_from_list = on; +``` + +**Disable statistics collection temporarily:** + +```sql +SET vacuum_statistics.enabled = off; +``` + +## Views + +| View | Description | +|------|-------------| +| `ext_vacuum_statistics.pg_stats_vacuum_tables` | Per-table heap vacuum stats (pages scanned, tuples deleted, WAL, timing, etc.) | +| `ext_vacuum_statistics.pg_stats_vacuum_indexes` | Per-index vacuum stats | +| `ext_vacuum_statistics.pg_stats_vacuum_database` | Per-database aggregate vacuum stats | + +## Limitations + +- Must be loaded via `shared_preload_libraries`; it cannot be loaded on demand. +- Tracking configuration (`add_track_*`, `remove_track_*`) is stored in a file and shared across all databases in the cluster. diff --git a/contrib/ext_vacuum_statistics/expected/ext_vacuum_statistics.out b/contrib/ext_vacuum_statistics/expected/ext_vacuum_statistics.out new file mode 100644 index 00000000000..89c9594dea8 --- /dev/null +++ b/contrib/ext_vacuum_statistics/expected/ext_vacuum_statistics.out @@ -0,0 +1,52 @@ +-- ext_vacuum_statistics regression test + +-- Create extension +CREATE EXTENSION ext_vacuum_statistics; + +-- Verify schema and views exist +SELECT nspname FROM pg_namespace WHERE nspname = 'ext_vacuum_statistics'; + nspname +------------------ + ext_vacuum_statistics +(1 row) + +-- Views should be queryable (may return empty if no vacuum has run) +SELECT COUNT(*) >= 0 FROM ext_vacuum_statistics.pg_stats_vacuum_tables; + ?column? +---------- + t +(1 row) + +SELECT COUNT(*) >= 0 FROM ext_vacuum_statistics.pg_stats_vacuum_indexes; + ?column? +---------- + t +(1 row) + +SELECT COUNT(*) >= 0 FROM ext_vacuum_statistics.pg_stats_vacuum_database; + ?column? +---------- + t +(1 row) + +-- Verify views have expected columns +SELECT COUNT(*) AS tables_cols FROM information_schema.columns +WHERE table_schema = 'ext_vacuum_statistics' AND table_name = 'tables'; + tables_cols +------------- + 28 +(1 row) + +SELECT COUNT(*) AS indexes_cols FROM information_schema.columns +WHERE table_schema = 'ext_vacuum_statistics' AND table_name = 'indexes'; + indexes_cols +-------------- + 20 +(1 row) + +SELECT COUNT(*) AS database_cols FROM information_schema.columns +WHERE table_schema = 'ext_vacuum_statistics' AND table_name = 'database'; + database_cols +--------------- + 15 +(1 row) diff --git a/contrib/ext_vacuum_statistics/expected/vacuum-extending-in-repetable-read.out b/contrib/ext_vacuum_statistics/expected/vacuum-extending-in-repetable-read.out new file mode 100644 index 00000000000..6b381f9d232 --- /dev/null +++ b/contrib/ext_vacuum_statistics/expected/vacuum-extending-in-repetable-read.out @@ -0,0 +1,52 @@ +unused step name: s2_delete +Parsed test spec with 2 sessions + +starting permutation: s2_insert s2_print_vacuum_stats_table s1_begin_repeatable_read s2_update s2_insert_interrupt s2_vacuum s2_print_vacuum_stats_table s1_commit s2_checkpoint s2_vacuum s2_print_vacuum_stats_table +step s2_insert: INSERT INTO test_vacuum_stat_isolation(id, ival) SELECT ival, ival%10 FROM generate_series(1,1000) As ival; +step s2_print_vacuum_stats_table: + SELECT + vt.relname, vt.tuples_deleted, vt.recently_dead_tuples, vt.missed_dead_tuples, vt.missed_dead_pages, vt.tuples_frozen + FROM ext_vacuum_statistics.pg_stats_vacuum_tables vt, pg_class c + WHERE vt.relname = 'test_vacuum_stat_isolation' AND vt.relid = c.oid; + +relname|tuples_deleted|recently_dead_tuples|missed_dead_tuples|missed_dead_pages|tuples_frozen +-------+--------------+--------------------+------------------+-----------------+------------- +(0 rows) + +step s1_begin_repeatable_read: + BEGIN transaction ISOLATION LEVEL REPEATABLE READ; + select count(ival) from test_vacuum_stat_isolation where id>900; + +count +----- + 100 +(1 row) + +step s2_update: UPDATE test_vacuum_stat_isolation SET ival = ival + 2 where id > 900; +step s2_insert_interrupt: INSERT INTO test_vacuum_stat_isolation values (1,1); +step s2_vacuum: VACUUM test_vacuum_stat_isolation; +step s2_print_vacuum_stats_table: + SELECT + vt.relname, vt.tuples_deleted, vt.recently_dead_tuples, vt.missed_dead_tuples, vt.missed_dead_pages, vt.tuples_frozen + FROM ext_vacuum_statistics.pg_stats_vacuum_tables vt, pg_class c + WHERE vt.relname = 'test_vacuum_stat_isolation' AND vt.relid = c.oid; + +relname |tuples_deleted|recently_dead_tuples|missed_dead_tuples|missed_dead_pages|tuples_frozen +--------------------------+--------------+--------------------+------------------+-----------------+------------- +test_vacuum_stat_isolation| 0| 100| 0| 0| 0 +(1 row) + +step s1_commit: COMMIT; +step s2_checkpoint: CHECKPOINT; +step s2_vacuum: VACUUM test_vacuum_stat_isolation; +step s2_print_vacuum_stats_table: + SELECT + vt.relname, vt.tuples_deleted, vt.recently_dead_tuples, vt.missed_dead_tuples, vt.missed_dead_pages, vt.tuples_frozen + FROM ext_vacuum_statistics.pg_stats_vacuum_tables vt, pg_class c + WHERE vt.relname = 'test_vacuum_stat_isolation' AND vt.relid = c.oid; + +relname |tuples_deleted|recently_dead_tuples|missed_dead_tuples|missed_dead_pages|tuples_frozen +--------------------------+--------------+--------------------+------------------+-----------------+------------- +test_vacuum_stat_isolation| 100| 100| 0| 0| 101 +(1 row) + diff --git a/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql b/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql new file mode 100644 index 00000000000..4f0b1877f90 --- /dev/null +++ b/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql @@ -0,0 +1,260 @@ +/*------------------------------------------------------------------------- + * + * ext_vacuum_statistics--1.0.sql + * Extended vacuum statistics via hook and custom storage + * + * This extension collects extended vacuum statistics via set_report_vacuum_hook + * and stores them in shared memory. + * + *------------------------------------------------------------------------- + */ + +\echo Use "CREATE EXTENSION ext_vacuum_statistics" to load this file. \quit + +CREATE SCHEMA IF NOT EXISTS ext_vacuum_statistics; + +COMMENT ON SCHEMA ext_vacuum_statistics IS + 'Extended vacuum statistics (heap, index, database)'; + +-- Reset functions +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.extvac_reset_entry( + dboid oid, + relid oid, + type int4 +) +RETURNS boolean +AS 'MODULE_PATHNAME', 'extvac_reset_entry' +LANGUAGE C STRICT PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.extvac_reset_db_entry(dboid oid) +RETURNS bigint +AS 'MODULE_PATHNAME', 'extvac_reset_db_entry' +LANGUAGE C STRICT PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.vacuum_statistics_reset() +RETURNS bigint +AS 'MODULE_PATHNAME', 'vacuum_statistics_reset' +LANGUAGE C STRICT PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.shared_memory_size() +RETURNS bigint +AS 'MODULE_PATHNAME', 'extvac_shared_memory_size' +LANGUAGE C STRICT PARALLEL SAFE; + +COMMENT ON FUNCTION ext_vacuum_statistics.shared_memory_size() IS + 'Total shared memory in bytes used by the extension for vacuum statistics.'; + +-- Add/remove OIDs for tracking +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.add_track_database(dboid oid) +RETURNS boolean +AS 'MODULE_PATHNAME', 'evs_add_track_database' +LANGUAGE C STRICT; + +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.remove_track_database(dboid oid) +RETURNS boolean +AS 'MODULE_PATHNAME', 'evs_remove_track_database' +LANGUAGE C STRICT; + +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.add_track_relation(dboid oid, reloid oid) +RETURNS boolean +AS 'MODULE_PATHNAME', 'evs_add_track_relation' +LANGUAGE C STRICT; + +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.remove_track_relation(dboid oid, reloid oid) +RETURNS boolean +AS 'MODULE_PATHNAME', 'evs_remove_track_relation' +LANGUAGE C STRICT; + +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.track_list() +RETURNS TABLE(track_kind text, dboid oid, reloid oid) +AS 'MODULE_PATHNAME', 'evs_track_list' +LANGUAGE C STRICT; + +COMMENT ON FUNCTION ext_vacuum_statistics.track_list() IS + 'List of database and relation OIDs for which vacuum statistics are collected.'; + +-- Internal C function to fetch table vacuum stats +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.pg_stats_get_vacuum_tables( + IN dboid oid, + IN reloid oid, + OUT relid oid, + OUT total_blks_read bigint, + OUT total_blks_hit bigint, + OUT total_blks_dirtied bigint, + OUT total_blks_written bigint, + OUT wal_records bigint, + OUT wal_fpi bigint, + OUT wal_bytes numeric, + OUT blk_read_time double precision, + OUT blk_write_time double precision, + OUT delay_time double precision, + OUT total_time double precision, + OUT wraparound_failsafe_count integer, + OUT rel_blks_read bigint, + OUT rel_blks_hit bigint, + OUT tuples_deleted bigint, + OUT pages_scanned bigint, + OUT pages_removed bigint, + OUT vm_new_frozen_pages bigint, + OUT vm_new_visible_pages bigint, + OUT vm_new_visible_frozen_pages bigint, + OUT tuples_frozen bigint, + OUT recently_dead_tuples bigint, + OUT index_vacuum_count bigint, + OUT missed_dead_pages bigint, + OUT missed_dead_tuples bigint +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_stats_get_vacuum_tables' +LANGUAGE C STRICT VOLATILE; + +-- Internal C function to fetch index vacuum stats +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.pg_stats_get_vacuum_indexes( + IN dboid oid, + IN reloid oid, + OUT relid oid, + OUT total_blks_read bigint, + OUT total_blks_hit bigint, + OUT total_blks_dirtied bigint, + OUT total_blks_written bigint, + OUT wal_records bigint, + OUT wal_fpi bigint, + OUT wal_bytes numeric, + OUT blk_read_time double precision, + OUT blk_write_time double precision, + OUT delay_time double precision, + OUT total_time double precision, + OUT wraparound_failsafe_count integer, + OUT rel_blks_read bigint, + OUT rel_blks_hit bigint, + OUT tuples_deleted bigint, + OUT pages_deleted bigint +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_stats_get_vacuum_indexes' +LANGUAGE C STRICT VOLATILE; + +-- Internal C function to fetch database vacuum stats +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.pg_stats_get_vacuum_database( + IN dboid oid, + OUT dbid oid, + OUT total_blks_read bigint, + OUT total_blks_hit bigint, + OUT total_blks_dirtied bigint, + OUT total_blks_written bigint, + OUT wal_records bigint, + OUT wal_fpi bigint, + OUT wal_bytes numeric, + OUT blk_read_time double precision, + OUT blk_write_time double precision, + OUT delay_time double precision, + OUT total_time double precision, + OUT wraparound_failsafe_count integer, + OUT interrupts_count integer +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_stats_get_vacuum_database' +LANGUAGE C STRICT VOLATILE; + +-- View: vacuum statistics per table (heap) +CREATE VIEW ext_vacuum_statistics.pg_stats_vacuum_tables AS +SELECT + rel.oid AS relid, + ns.nspname AS schema, + rel.relname AS relname, + db.datname AS dbname, + stats.total_blks_read, + stats.total_blks_hit, + stats.total_blks_dirtied, + stats.total_blks_written, + stats.wal_records, + stats.wal_fpi, + stats.wal_bytes, + stats.blk_read_time, + stats.blk_write_time, + stats.delay_time, + stats.total_time, + stats.wraparound_failsafe_count, + stats.rel_blks_read, + stats.rel_blks_hit, + stats.tuples_deleted, + stats.pages_scanned, + stats.pages_removed, + stats.vm_new_frozen_pages, + stats.vm_new_visible_pages, + stats.vm_new_visible_frozen_pages, + stats.tuples_frozen, + stats.recently_dead_tuples, + stats.index_vacuum_count, + stats.missed_dead_pages, + stats.missed_dead_tuples +FROM pg_database db, + pg_class rel, + pg_namespace ns, + LATERAL ext_vacuum_statistics.pg_stats_get_vacuum_tables(db.oid, rel.oid) stats +WHERE db.datname = current_database() + AND rel.relkind = 'r' + AND rel.relnamespace = ns.oid + AND rel.oid = stats.relid; + +COMMENT ON VIEW ext_vacuum_statistics.pg_stats_vacuum_tables IS + 'Extended vacuum statistics per table (heap)'; + +-- View: vacuum statistics per index +CREATE VIEW ext_vacuum_statistics.pg_stats_vacuum_indexes AS +SELECT + rel.oid AS indexrelid, + ns.nspname AS schema, + rel.relname AS indexrelname, + db.datname AS dbname, + stats.total_blks_read, + stats.total_blks_hit, + stats.total_blks_dirtied, + stats.total_blks_written, + stats.wal_records, + stats.wal_fpi, + stats.wal_bytes, + stats.blk_read_time, + stats.blk_write_time, + stats.delay_time, + stats.total_time, + stats.wraparound_failsafe_count, + stats.rel_blks_read, + stats.rel_blks_hit, + stats.tuples_deleted, + stats.pages_deleted +FROM pg_database db, + pg_class rel, + pg_namespace ns, + LATERAL ext_vacuum_statistics.pg_stats_get_vacuum_indexes(db.oid, rel.oid) stats +WHERE db.datname = current_database() + AND rel.relkind = 'i' + AND rel.relnamespace = ns.oid + AND rel.oid = stats.relid; + +COMMENT ON VIEW ext_vacuum_statistics.pg_stats_vacuum_indexes IS + 'Extended vacuum statistics per index'; + +-- View: vacuum statistics per database (aggregate) +CREATE VIEW ext_vacuum_statistics.pg_stats_vacuum_database AS +SELECT + db.oid AS dboid, + db.datname AS dbname, + stats.total_blks_read AS db_blks_read, + stats.total_blks_hit AS db_blks_hit, + stats.total_blks_dirtied AS db_blks_dirtied, + stats.total_blks_written AS db_blks_written, + stats.wal_records AS db_wal_records, + stats.wal_fpi AS db_wal_fpi, + stats.wal_bytes AS db_wal_bytes, + stats.blk_read_time AS db_blk_read_time, + stats.blk_write_time AS db_blk_write_time, + stats.delay_time AS db_delay_time, + stats.total_time AS db_total_time, + stats.wraparound_failsafe_count AS db_wraparound_failsafe_count, + stats.interrupts_count +FROM pg_database db +LEFT JOIN LATERAL ext_vacuum_statistics.pg_stats_get_vacuum_database(db.oid) stats ON db.oid = stats.dbid; + +COMMENT ON VIEW ext_vacuum_statistics.pg_stats_vacuum_database IS + 'Extended vacuum statistics per database (aggregate)'; diff --git a/contrib/ext_vacuum_statistics/ext_vacuum_statistics.conf b/contrib/ext_vacuum_statistics/ext_vacuum_statistics.conf new file mode 100644 index 00000000000..9b711487623 --- /dev/null +++ b/contrib/ext_vacuum_statistics/ext_vacuum_statistics.conf @@ -0,0 +1,2 @@ +# Config for ext_vacuum_statistics regression tests +shared_preload_libraries = 'ext_vacuum_statistics' diff --git a/contrib/ext_vacuum_statistics/ext_vacuum_statistics.control b/contrib/ext_vacuum_statistics/ext_vacuum_statistics.control new file mode 100644 index 00000000000..518350a64b7 --- /dev/null +++ b/contrib/ext_vacuum_statistics/ext_vacuum_statistics.control @@ -0,0 +1,5 @@ +# ext_vacuum_statistics extension +comment = 'Extended vacuum statistics via hook (requires shared_preload_libraries)' +default_version = '1.0' +relocatable = true +module_pathname = '$libdir/ext_vacuum_statistics' diff --git a/contrib/ext_vacuum_statistics/meson.build b/contrib/ext_vacuum_statistics/meson.build new file mode 100644 index 00000000000..72338baa500 --- /dev/null +++ b/contrib/ext_vacuum_statistics/meson.build @@ -0,0 +1,41 @@ +# Copyright (c) 2022-2026, PostgreSQL Global Development Group +# +# ext_vacuum_statistics - extended vacuum statistics via hook +# Requires shared_preload_libraries = 'ext_vacuum_statistics' + +ext_vacuum_statistics_sources = files( + 'vacuum_statistics.c', +) + +ext_vacuum_statistics = shared_module('ext_vacuum_statistics', + ext_vacuum_statistics_sources, + kwargs: contrib_mod_args + { + 'dependencies': contrib_mod_args['dependencies'], + }, +) +contrib_targets += ext_vacuum_statistics + +install_data( + 'ext_vacuum_statistics.control', + 'ext_vacuum_statistics--1.0.sql', + kwargs: contrib_data_args, +) + +tests += { + 'name': 'ext_vacuum_statistics', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'isolation': { + 'specs': [ + 'vacuum-extending-in-repetable-read', + ], + 'regress_args': ['--temp-config', files('ext_vacuum_statistics.conf')], + 'runningcheck': false, + }, + 'tap': { + 'tests': [ + 't/052_vacuum_extending_basic_test.pl', + 't/053_vacuum_extending_freeze_test.pl', + ], + }, +} diff --git a/contrib/ext_vacuum_statistics/specs/vacuum-extending-in-repetable-read.spec b/contrib/ext_vacuum_statistics/specs/vacuum-extending-in-repetable-read.spec new file mode 100644 index 00000000000..4891e248cca --- /dev/null +++ b/contrib/ext_vacuum_statistics/specs/vacuum-extending-in-repetable-read.spec @@ -0,0 +1,59 @@ +# Test for checking recently_dead_tuples, tuples_deleted and frozen tuples in ext_vacuum_statistics.pg_stats_vacuum_tables. +# recently_dead_tuples values are counted when vacuum hasn't cleared tuples because they were deleted recently. +# recently_dead_tuples aren't increased after releasing lock compared with tuples_deleted, which increased +# by the value of the cleared tuples that the vacuum managed to clear. + +setup +{ + CREATE TABLE test_vacuum_stat_isolation(id int, ival int) WITH (autovacuum_enabled = off); + CREATE EXTENSION ext_vacuum_statistics; + SET track_io_timing = on; +} + +teardown +{ + DROP EXTENSION ext_vacuum_statistics CASCADE; + DROP TABLE test_vacuum_stat_isolation CASCADE; + RESET track_io_timing; +} + +session s1 +setup { + SET track_io_timing = on; +} +step s1_begin_repeatable_read { + BEGIN transaction ISOLATION LEVEL REPEATABLE READ; + select count(ival) from test_vacuum_stat_isolation where id>900; +} +step s1_commit { COMMIT; } + +session s2 +setup { + SET track_io_timing = on; +} +step s2_insert { INSERT INTO test_vacuum_stat_isolation(id, ival) SELECT ival, ival%10 FROM generate_series(1,1000) As ival; } +step s2_update { UPDATE test_vacuum_stat_isolation SET ival = ival + 2 where id > 900; } +step s2_delete { DELETE FROM test_vacuum_stat_isolation where id > 900; } +step s2_insert_interrupt { INSERT INTO test_vacuum_stat_isolation values (1,1); } +step s2_vacuum { VACUUM test_vacuum_stat_isolation; } +step s2_checkpoint { CHECKPOINT; } +step s2_print_vacuum_stats_table +{ + SELECT + vt.relname, vt.tuples_deleted, vt.recently_dead_tuples, vt.missed_dead_tuples, vt.missed_dead_pages, vt.tuples_frozen + FROM ext_vacuum_statistics.pg_stats_vacuum_tables vt, pg_class c + WHERE vt.relname = 'test_vacuum_stat_isolation' AND vt.relid = c.oid; +} + +permutation + s2_insert + s2_print_vacuum_stats_table + s1_begin_repeatable_read + s2_update + s2_insert_interrupt + s2_vacuum + s2_print_vacuum_stats_table + s1_commit + s2_checkpoint + s2_vacuum + s2_print_vacuum_stats_table diff --git a/contrib/ext_vacuum_statistics/t/052_vacuum_extending_basic_test.pl b/contrib/ext_vacuum_statistics/t/052_vacuum_extending_basic_test.pl new file mode 100644 index 00000000000..9463d5145f4 --- /dev/null +++ b/contrib/ext_vacuum_statistics/t/052_vacuum_extending_basic_test.pl @@ -0,0 +1,780 @@ +# Copyright (c) 2025 PostgreSQL Global Development Group +# Test cumulative vacuum stats system using TAP +# +# This test validates the accuracy and behavior of cumulative vacuum statistics +# across heap tables, indexes, and databases using: +# +# • ext_vacuum_statistics.pg_stats_vacuum_tables +# • ext_vacuum_statistics.pg_stats_vacuum_indexes +# • ext_vacuum_statistics.pg_stats_vacuum_database +# +# A polling helper function repeatedly checks the stats views until expected +# deltas appear or a configurable timeout expires. This guarantees that +# stats-collector propagation delays do not lead to flaky test behavior. + +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +#------------------------------------------------------------------------------ +# Test harness setup +#------------------------------------------------------------------------------ + +my $node = PostgreSQL::Test::Cluster->new('stat_vacuum'); +$node->init; + +# Configure the server: preload extension and logging level +$node->append_conf('postgresql.conf', q{ + shared_preload_libraries = 'ext_vacuum_statistics' + log_min_messages = notice +}); + +my $stderr; +my $base_stats; +my $wals; +my $ibase_stats; +my $iwals; + +$node->start( + '>' => \$base_stats, + '2>' => \$stderr +); + +#------------------------------------------------------------------------------ +# Database creation and initialization +#------------------------------------------------------------------------------ + +$node->safe_psql('postgres', q{ + CREATE DATABASE statistic_vacuum_database_regression; + CREATE EXTENSION ext_vacuum_statistics; +}); +# Main test database name and number of rows to insert +my $dbname = 'statistic_vacuum_database_regression'; +my $size_tab = 1000; + +# Enable required session settings and force the stats collector to flush next +$node->safe_psql($dbname, q{ + SET track_functions = 'all'; + SELECT pg_stat_force_next_flush(); +}); + +#------------------------------------------------------------------------------ +# Create test table and populate it +#------------------------------------------------------------------------------ + +$node->safe_psql( + $dbname, + "CREATE EXTENSION ext_vacuum_statistics; + CREATE TABLE vestat (x int PRIMARY KEY) + WITH (autovacuum_enabled = off, fillfactor = 10); + INSERT INTO vestat SELECT x FROM generate_series(1, $size_tab) AS g(x); + ANALYZE vestat;" +); + +#------------------------------------------------------------------------------ +# Timing parameters for polling loops +#------------------------------------------------------------------------------ + +my $timeout = 30; # overall wait timeout in seconds +my $interval = 0.015; # poll interval in seconds (15 ms) +my $start_time = time(); +my $updated = 0; + +#------------------------------------------------------------------------------ +# wait_for_vacuum_stats +# +# Polls ext_vacuum_statistics.pg_stats_vacuum_tables and ext_vacuum_statistics.pg_stats_vacuum_indexes until both the +# table-level and index-level counters exceed the provided baselines, or until +# the configured timeout elapses. +# +# Expected named args (baseline values): +# tab_tuples_deleted +# tab_wal_records +# idx_tuples_deleted +# idx_wal_records +# +# Returns: 1 if the condition is met before timeout, 0 otherwise. +#------------------------------------------------------------------------------ + +sub wait_for_vacuum_stats { + my (%args) = @_; + my $tab_tuples_deleted = ($args{tab_tuples_deleted} or 0); + my $tab_wal_records = ($args{tab_wal_records} or 0); + my $idx_tuples_deleted = ($args{idx_tuples_deleted} or 0); + my $idx_wal_records = ($args{idx_wal_records} or 0); + + my $start = time(); + while ((time() - $start) < $timeout) { + + my $result_query = $node->safe_psql( + $dbname, + "VACUUM vestat; + SELECT + (SELECT (tuples_deleted > $tab_tuples_deleted AND wal_records > $tab_wal_records) + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'vestat') + AND + (SELECT (tuples_deleted > $idx_tuples_deleted AND wal_records > $idx_wal_records) + FROM ext_vacuum_statistics.pg_stats_vacuum_indexes + WHERE indexrelname = 'vestat_pkey');" + ); + + return 1 if ($result_query eq 't'); + + sleep($interval); + } + + return 0; +} + +#------------------------------------------------------------------------------ +# Variables to hold vacuum-stat snapshots for later comparisons +#------------------------------------------------------------------------------ + +my $vm_new_visible_frozen_pages = 0; +my $tuples_deleted = 0; +my $pages_scanned = 0; +my $pages_removed = 0; +my $wal_records = 0; +my $wal_bytes = 0; +my $wal_fpi = 0; + +my $index_tuples_deleted = 0; +my $index_pages_deleted = 0; +my $index_wal_records = 0; +my $index_wal_bytes = 0; +my $index_wal_fpi = 0; + +my $vm_new_visible_frozen_pages_prev = 0; +my $tuples_deleted_prev = 0; +my $pages_scanned_prev = 0; +my $pages_removed_prev = 0; +my $wal_records_prev = 0; +my $wal_bytes_prev = 0; +my $wal_fpi_prev = 0; + +my $index_tuples_deleted_prev = 0; +my $index_pages_deleted_prev = 0; +my $index_wal_records_prev = 0; +my $index_wal_bytes_prev = 0; +my $index_wal_fpi_prev = 0; + +#------------------------------------------------------------------------------ +# fetch_vacuum_stats +# +# Reads current values of relevant vacuum counters for the test table and its +# primary index, storing them in package variables for subsequent comparisons. +#------------------------------------------------------------------------------ + +sub fetch_vacuum_stats { + # fetch actual base vacuum statistics + my $base_statistics = $node->safe_psql( + $dbname, + "SELECT vm_new_visible_frozen_pages, tuples_deleted, pages_scanned, pages_removed, wal_records, wal_bytes, wal_fpi + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'vestat';" + ); + + $base_statistics =~ s/\s*\|\s*/ /g; # transform " | " into space + ($vm_new_visible_frozen_pages, $tuples_deleted, $pages_scanned, $pages_removed, $wal_records, $wal_bytes, $wal_fpi) + = split /\s+/, $base_statistics; + + # --- index stats --- + my $index_base_statistics = $node->safe_psql( + $dbname, + "SELECT tuples_deleted, pages_deleted, wal_records, wal_bytes, wal_fpi + FROM ext_vacuum_statistics.pg_stats_vacuum_indexes + WHERE indexrelname = 'vestat_pkey';" + ); + + $index_base_statistics =~ s/\s*\|\s*/ /g; # transform " | " into space + ($index_tuples_deleted, $index_pages_deleted, $index_wal_records, $index_wal_bytes, $index_wal_fpi) + = split /\s+/, $index_base_statistics; +} + +#------------------------------------------------------------------------------ +# save_vacuum_stats +# +# Save current values (previously fetched by fetch_vacuum_stats) so that we +# later fetch new values and compare them. +#------------------------------------------------------------------------------ +sub save_vacuum_stats { + $vm_new_visible_frozen_pages_prev = $vm_new_visible_frozen_pages; + $tuples_deleted_prev = $tuples_deleted; + $pages_scanned_prev = $pages_scanned; + $pages_removed_prev = $pages_removed; + $wal_records_prev = $wal_records; + $wal_bytes_prev = $wal_bytes; + $wal_fpi_prev = $wal_fpi; + + $index_tuples_deleted_prev = $index_tuples_deleted; + $index_pages_deleted_prev = $index_pages_deleted; + $index_wal_records_prev = $index_wal_records; + $index_wal_bytes_prev = $index_wal_bytes; + $index_wal_fpi_prev = $index_wal_fpi; +} + +#------------------------------------------------------------------------------ +# print_vacuum_stats_on_error +# +# Print values in case of an error +#------------------------------------------------------------------------------ +sub print_vacuum_stats_on_error { + diag( + "Statistics in the failed test\n" . + "Table statistics:\n" . + " Before test:\n" . + " vm_new_visible_frozen_pages = $vm_new_visible_frozen_pages_prev\n" . + " tuples_deleted = $tuples_deleted_prev\n" . + " pages_scanned = $pages_scanned_prev\n" . + " pages_removed = $pages_removed_prev\n" . + " wal_records = $wal_records_prev\n" . + " wal_bytes = $wal_bytes_prev\n" . + " wal_fpi = $wal_fpi_prev\n" . + " After test:\n" . + " vm_new_visible_frozen_pages = $vm_new_visible_frozen_pages\n" . + " tuples_deleted = $tuples_deleted\n" . + " pages_scanned = $pages_scanned\n" . + " pages_removed = $pages_removed\n" . + " wal_records = $wal_records\n" . + " wal_bytes = $wal_bytes\n" . + " wal_fpi = $wal_fpi\n" . + "Index statistics:\n" . + " Before test:\n" . + " tuples_deleted = $index_tuples_deleted_prev\n" . + " pages_deleted = $index_pages_deleted_prev\n" . + " wal_records = $index_wal_records_prev\n" . + " wal_bytes = $index_wal_bytes_prev\n" . + " wal_fpi = $index_wal_fpi_prev\n" . + " After test:\n" . + " tuples_deleted = $index_tuples_deleted\n" . + " pages_deleted = $index_pages_deleted\n" . + " wal_records = $index_wal_records\n" . + " wal_bytes = $index_wal_bytes\n" . + " wal_fpi = $index_wal_fpi\n" + ); +}; + +sub fetch_error_base_db_vacuum_statistics { + my (%args) = @_; + + # Validate presence of required args (allow 0 as valid numeric baseline) + die "database name required" + unless exists $args{database_name} && defined $args{database_name}; + my $database_name = $args{database_name}; + + # fetch actual base database vacuum statistics + my $base_statistics = $node->safe_psql( + $database_name, + "SELECT db_blks_hit, db_blks_dirtied, + db_blks_written, db_wal_records, + db_wal_fpi, db_wal_bytes + FROM ext_vacuum_statistics.pg_stats_vacuum_database, pg_database + WHERE pg_database.datname = '$dbname' + AND pg_database.oid = ext_vacuum_statistics.pg_stats_vacuum_database.dboid;" + ); + $base_statistics =~ s/\s*\|\s*/ /g; # transform " | " in space + my ($db_blks_hit, $total_blks_dirtied, $total_blks_written, + $wal_records, $wal_fpi, $wal_bytes) = split /\s+/, $base_statistics; + + diag( + "BASE STATS MISMATCH FOR DATABASE $dbname:\n" . + " db_blks_hit = $db_blks_hit\n" . + " total_blks_dirtied = $total_blks_dirtied\n" . + " total_blks_written = $total_blks_written\n" . + " wal_records = $wal_records\n" . + " wal_fpi = $wal_fpi\n" . + " wal_bytes = $wal_bytes\n" + ); +} + + +#------------------------------------------------------------------------------ +# Test 1: Delete half the rows, run VACUUM, and wait for stats to advance +#------------------------------------------------------------------------------ +subtest 'Test 1: Delete half the rows, run VACUUM' => sub +{ + +$node->safe_psql($dbname, "DELETE FROM vestat WHERE x % 2 = 0;"); +$node->safe_psql($dbname, "VACUUM vestat;"); + +# Poll the stats view until expected deltas appear or timeout +$updated = wait_for_vacuum_stats( + tab_tuples_deleted => 0, + tab_wal_records => 0, + idx_tuples_deleted => 0, + idx_wal_records => 0, +); +ok($updated, 'vacuum stats updated after vacuuming half-deleted table (tuples_deleted and wal_fpi advanced)') + or diag "Timeout waiting for ext_vacuum_statistics update after $timeout seconds after vacuuming half-deleted table"; + +fetch_vacuum_stats(); + +ok($vm_new_visible_frozen_pages == $vm_new_visible_frozen_pages_prev, 'table vm_new_visible_frozen_pages stay the same'); +ok($tuples_deleted > $tuples_deleted_prev, 'table tuples_deleted has increased'); +ok($pages_scanned > $pages_scanned_prev, 'table pages_scanned has increased'); +ok($pages_removed == $pages_removed_prev, 'table pages_removed stay the same'); +ok($wal_records > $wal_records_prev, 'table wal_records has increased'); +ok($wal_bytes > $wal_bytes_prev, 'table wal_bytes has increased'); +ok($wal_fpi > $wal_fpi_prev, 'table wal_fpi has increased'); + +ok($index_pages_deleted == $index_pages_deleted_prev, 'index pages_deleted stay the same'); +ok($index_tuples_deleted > $index_tuples_deleted_prev, 'index tuples_deleted has increased'); +ok($index_wal_records > $index_wal_records_prev, 'index wal_records has increased'); +ok($index_wal_bytes > $index_wal_bytes_prev, 'index wal_bytes has increased'); +ok($index_wal_fpi == $index_wal_fpi_prev, 'index wal_fpi stay the same'); + +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 2: Delete all rows, run VACUUM, and wait for stats to advance +#------------------------------------------------------------------------------ +subtest 'Test 2: Delete all rows, run VACUUM' => sub +{ +save_vacuum_stats(); + +$node->safe_psql($dbname, "DELETE FROM vestat;"); +$node->safe_psql($dbname, "VACUUM vestat;"); + +$updated = wait_for_vacuum_stats( + tab_tuples_deleted => $tuples_deleted_prev, + tab_wal_records => $wal_records_prev, + idx_tuples_deleted => $index_tuples_deleted_prev, + idx_wal_records => $index_wal_records_prev, +); + +ok($updated, 'vacuum stats updated after vacuuming all-deleted table (tuples_deleted and wal_records advanced)') + or diag "Timeout waiting for ext_vacuum_statistics update after $timeout seconds after vacuuming all-deleted table"; + +fetch_vacuum_stats(); + +ok($vm_new_visible_frozen_pages > $vm_new_visible_frozen_pages_prev, 'table vm_new_visible_frozen_pages has increased'); +ok($tuples_deleted > $tuples_deleted_prev, 'table tuples_deleted has increased'); +ok($pages_scanned > $pages_scanned_prev, 'table pages_scanned has increased'); +ok($pages_removed > $pages_removed_prev, 'table pages_removed has increased'); +ok($wal_records > $wal_records_prev, 'table wal_records has increased'); +ok($wal_bytes > $wal_bytes_prev, 'table wal_bytes has increased'); +ok($wal_fpi > 0, 'table wal_fpi has increased'); + +ok($index_pages_deleted > $index_pages_deleted_prev, 'index pages_deleted has increased'); +ok($index_tuples_deleted > $index_tuples_deleted_prev, 'index tuples_deleted has increased'); +ok($index_wal_records > $index_wal_records_prev, 'index wal_records has increased'); +ok($index_wal_bytes > $index_wal_bytes_prev, 'index wal_bytes has increased'); +ok($index_wal_fpi == $index_wal_fpi_prev, 'index wal_fpi stay the same'); + +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 3: Test VACUUM FULL — it should not report to the stats collector +#------------------------------------------------------------------------------ +subtest 'Test 3: Test VACUUM FULL — it should not report to the stats collector' => sub +{ +save_vacuum_stats(); + +$node->safe_psql( + $dbname, + "INSERT INTO vestat SELECT x FROM generate_series(1, $size_tab) AS g(x); + CHECKPOINT; + DELETE FROM vestat; + VACUUM FULL vestat;" +); + +fetch_vacuum_stats(); + +ok($vm_new_visible_frozen_pages == $vm_new_visible_frozen_pages_prev, 'table vm_new_visible_frozen_pages stay the same'); +ok($tuples_deleted == $tuples_deleted_prev, 'table tuples_deleted stay the same'); +ok($pages_scanned == $pages_scanned_prev, 'table pages_scanned stay the same'); +ok($pages_removed == $pages_removed_prev, 'table pages_removed stay the same'); +ok($wal_records == $wal_records_prev, 'table wal_records stay the same'); +ok($wal_bytes == $wal_bytes_prev, 'table wal_bytes stay the same'); +ok($wal_fpi == $wal_fpi_prev, 'table wal_fpi stay the same'); + +ok($index_pages_deleted == $index_pages_deleted_prev, 'index pages_deleted stay the same'); +ok($index_tuples_deleted == $index_tuples_deleted_prev, 'index tuples_deleted stay the same'); +ok($index_wal_records == $index_wal_records_prev, 'index wal_records stay the same'); +ok($index_wal_bytes == $index_wal_bytes_prev, 'index wal_bytes stay the same'); +ok($index_wal_fpi == $index_wal_fpi_prev, 'index wal_fpi stay the same'); + +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 4: Update table, checkpoint, and VACUUM to provoke WAL/FPI accounting +#------------------------------------------------------------------------------ +subtest 'Test 4: Update table, checkpoint, and VACUUM to provoke WAL/FPI accounting' => sub +{ + +save_vacuum_stats(); + +$node->safe_psql( + $dbname, + "INSERT INTO vestat SELECT x FROM generate_series(1, $size_tab) AS g(x); + CHECKPOINT; + UPDATE vestat SET x = x + 1000; + VACUUM vestat;" +); + +$updated = wait_for_vacuum_stats( + tab_tuples_deleted => $tuples_deleted_prev, + tab_wal_records => $wal_records_prev, + idx_tuples_deleted => $index_tuples_deleted_prev, + idx_wal_records => $index_wal_records_prev, +); + +ok($updated, 'vacuum stats updated after updating tuples in the table (tuples_deleted and wal_records advanced)') + or diag "Timeout waiting for ext_vacuum_statistics update after $timeout seconds"; + +fetch_vacuum_stats(); + +ok($vm_new_visible_frozen_pages == $vm_new_visible_frozen_pages_prev, 'table vm_new_visible_frozen_pages stay the same'); +ok($tuples_deleted > $tuples_deleted_prev, 'table tuples_deleted has increased'); +ok($pages_scanned > $pages_scanned_prev, 'table pages_scanned has increased'); +ok($pages_removed == $pages_removed_prev, 'table pages_removed stay the same'); +ok($wal_records > $wal_records_prev, 'table wal_records has increased'); +ok($wal_bytes > $wal_bytes_prev, 'table wal_bytes has increased'); +ok($wal_fpi > $wal_fpi_prev, 'table wal_fpi has increased'); + +ok($index_pages_deleted > $index_pages_deleted_prev, 'index pages_deleted has increased'); +ok($index_tuples_deleted > $index_tuples_deleted_prev, 'index tuples_deleted has increased'); +ok($index_wal_records > $index_wal_records_prev, 'index wal_records has increased'); +ok($index_wal_bytes > $index_wal_bytes_prev, 'index wal_bytes has increased'); +ok($index_wal_fpi > $index_wal_fpi_prev, 'index wal_fpi has increased'); + +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 5: Update table, trancate and vacuuming +#------------------------------------------------------------------------------ +subtest 'Test 5: Update table, trancate and vacuuming' => sub +{ + +save_vacuum_stats(); + +$node->safe_psql( + $dbname, + "INSERT INTO vestat SELECT x FROM generate_series(1, $size_tab) AS g(x); + UPDATE vestat SET x = x + 1000;" +); +$node->safe_psql($dbname, "TRUNCATE vestat;"); +$node->safe_psql($dbname, "CHECKPOINT;"); +$node->safe_psql($dbname, "VACUUM vestat;"); + +$updated = wait_for_vacuum_stats( + tab_wal_records => $wal_records_prev, +); + +ok($updated, 'vacuum stats updated after updating tuples and trancation in the table (wal_records advanced)') + or diag "Timeout waiting for ext_vacuum_statistics update after $timeout seconds"; + +fetch_vacuum_stats(); + +ok($vm_new_visible_frozen_pages == $vm_new_visible_frozen_pages_prev, 'table vm_new_visible_frozen_pages stay the same'); +ok($tuples_deleted == $tuples_deleted_prev, 'table tuples_deleted stay the same'); +ok($pages_scanned == $pages_scanned_prev, 'table pages_scanned stay the same'); +ok($pages_removed == $pages_removed_prev, 'table pages_removed stay the same'); +ok($wal_records > $wal_records_prev, 'table wal_records has increased'); +ok($wal_bytes > $wal_bytes_prev, 'table wal_bytes has increased'); +ok($wal_fpi == $wal_fpi_prev, 'table wal_fpi stay the same'); + +ok($index_pages_deleted == $index_pages_deleted_prev, 'index pages_deleted stay the same'); +ok($index_tuples_deleted == $index_tuples_deleted_prev, 'index tuples_deleted stay the same'); +ok($index_wal_records == $index_wal_records_prev, 'index wal_records stay the same'); +ok($index_wal_bytes == $index_wal_bytes_prev, 'index wal_bytes stay the same'); +ok($index_wal_fpi == $index_wal_fpi_prev, 'index wal_fpi stay the same'); + +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 6: Delete all tuples from table, trancate, and vacuuming +#------------------------------------------------------------------------------ +subtest 'Test 6: Delete all tuples from table, trancate, and vacuuming' => sub +{ + +save_vacuum_stats(); + +$node->safe_psql( + $dbname, + "INSERT INTO vestat SELECT x FROM generate_series(1, $size_tab) AS g(x); + DELETE FROM vestat; + TRUNCATE vestat; + CHECKPOINT; + VACUUM vestat;" +); + +$updated = wait_for_vacuum_stats( + tab_wal_records => $wal_records, +); + +ok($updated, 'vacuum stats updated after deleting all tuples and trancation in the table (wal_records advanced)') + or diag "Timeout waiting for ext_vacuum_statistics update after $timeout seconds"; + +fetch_vacuum_stats(); + +ok($vm_new_visible_frozen_pages == $vm_new_visible_frozen_pages_prev, 'table vm_new_visible_frozen_pages stay the same'); +ok($tuples_deleted == $tuples_deleted_prev, 'table tuples_deleted stay the same'); +ok($pages_scanned == $pages_scanned_prev, 'table pages_scanned stay the same'); +ok($pages_removed == $pages_removed_prev, 'table pages_removed stay the same'); +ok($wal_records > $wal_records_prev, 'table wal_records has increased'); +ok($wal_bytes > $wal_bytes_prev, 'table wal_bytes has increased'); +ok($wal_fpi == $wal_fpi_prev, 'table wal_fpi stay the same'); + +ok($index_pages_deleted == $index_pages_deleted_prev, 'index pages_deleted stay the same'); +ok($index_tuples_deleted == $index_tuples_deleted_prev, 'index tuples_deleted stay the same'); +ok($index_wal_records == $index_wal_records_prev, 'index wal_records stay the same'); +ok($index_wal_bytes == $index_wal_bytes_prev, 'index wal_bytes stay the same'); +ok($index_wal_fpi == $index_wal_fpi_prev, 'index wal_fpi stay the same'); + +} or print_vacuum_stats_on_error(); + +my $dboid = $node->safe_psql( + $dbname, + "SELECT oid FROM pg_database WHERE datname = current_database();" +); + +#------------------------------------------------------------------------------------------------------- +# Test 7: Check if we return single vacuum statistics for particular relation from the current database +#------------------------------------------------------------------------------------------------------- +subtest 'Test 7: Check if we return vacuum statistics from the current database' => sub +{ +save_vacuum_stats(); + +my $reloid = $node->safe_psql( + $dbname, + q{ + SELECT oid FROM pg_class WHERE relname = 'vestat'; + } +); + +# Check if we can get vacuum statistics of particular heap relation in the current database +$base_stats = $node->safe_psql( + $dbname, + "SELECT count(*) FROM ext_vacuum_statistics.pg_stats_get_vacuum_tables((SELECT oid FROM pg_database WHERE datname = current_database()), $reloid);" +); +is($base_stats, 1, 'heap vacuum stats return from the current relation and database as expected'); + +$reloid = $node->safe_psql( + $dbname, + q{ + SELECT oid FROM pg_class WHERE relname = 'vestat_pkey'; + } +); + +# Check if we can get vacuum statistics of particular index relation in the current database +$base_stats = $node->safe_psql( + $dbname, + "SELECT count(*) FROM ext_vacuum_statistics.pg_stats_get_vacuum_indexes((SELECT oid FROM pg_database WHERE datname = current_database()), $reloid);" +); +is($base_stats, 1, 'index vacuum stats return from the current relation and database as expected'); + +# Check if we return empty results if vacuum statistics with particular oid doesn't exist +$base_stats = $node->safe_psql( + $dbname, + "SELECT count(*) FROM ext_vacuum_statistics.pg_stats_get_vacuum_tables((SELECT oid FROM pg_database WHERE datname = current_database()), 1);" +); +is($base_stats, 0, 'table vacuum stats return no rows, as expected'); + +$base_stats = $node->safe_psql( + $dbname, + "SELECT count(*) FROM ext_vacuum_statistics.pg_stats_get_vacuum_indexes((SELECT oid FROM pg_database WHERE datname = current_database()), 1);" +); +is($base_stats, 0, 'index vacuum stats return no rows, as expected'); + +# Check if we can get vacuum statistics of all relations in the current database +$base_stats = $node->safe_psql( + $dbname, + "SELECT count(*) > 0 FROM ext_vacuum_statistics.pg_stats_vacuum_tables;" +); +ok($base_stats eq 't', 'vacuum stats per all heap objects available'); + +$base_stats = $node->safe_psql( + $dbname, + "SELECT count(*) > 0 FROM ext_vacuum_statistics.pg_stats_vacuum_indexes;" +); +ok($base_stats eq 't', 'vacuum stats per all index objects available'); +}; + +#------------------------------------------------------------------------------ +# Test 8: Check relation-level vacuum statistics from another database +#------------------------------------------------------------------------------ +subtest 'Test 8: Check relation-level vacuum statistics from another database' => sub +{ +$base_stats = $node->safe_psql( + 'postgres', + "SELECT count(*) + FROM ext_vacuum_statistics.pg_stats_vacuum_indexes + WHERE indexrelname = 'vestat_pkey';" +); +is($base_stats, 0, 'check the printing index vacuum extended statistics from another database are not available'); + +$base_stats = $node->safe_psql( + 'postgres', + "SELECT count(*) + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'vestat';" +); +is($base_stats, 0, 'check the printing heap vacuum extended statistics from another database are not available'); + +# Check that relations from another database are not visible in the view when querying from postgres +$base_stats = $node->safe_psql( + 'postgres', + "SELECT count(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'vestat';" +); +is($base_stats, 0, 'vacuum stats per all tables objects from another database are not available as expected'); + +$base_stats = $node->safe_psql( + 'postgres', + "SELECT count(*) FROM ext_vacuum_statistics.pg_stats_vacuum_indexes WHERE indexrelname = 'vestat_pkey';" +); +is($base_stats, 0, 'vacuum stats per all index objects from another database are not available as expected'); +}; + +#-------------------------------------------------------------------------------------- +# Test 9: Check database-level vacuum statistics from the current and another database +#-------------------------------------------------------------------------------------- +subtest 'Test 9: Check database-level vacuum statistics from the current and another database' => sub +{ +my $db_blk_hit = 0; +my $total_blks_dirtied = 0; +my $total_blks_written = 0; +my $wal_records = 0; +my $wal_fpi = 0; +my $wal_bytes = 0; +$base_stats = $node->safe_psql( + $dbname, + "SELECT db_blks_hit, db_blks_dirtied, + db_blks_written, db_wal_records, + db_wal_fpi, db_wal_bytes + FROM ext_vacuum_statistics.pg_stats_vacuum_database, pg_database + WHERE pg_database.datname = '$dbname' + AND pg_database.oid = ext_vacuum_statistics.pg_stats_vacuum_database.dboid;" +); +$base_stats =~ s/\s*\|\s*/ /g; # transform " | " into space + ($db_blk_hit, $total_blks_dirtied, $total_blks_written, $wal_records, $wal_fpi, $wal_bytes) + = split /\s+/, $base_stats; + +ok($db_blk_hit > 0, 'db_blks_hit is more than 0'); +ok($total_blks_dirtied > 0, 'total_blks_dirtied is more than 0'); +ok($total_blks_written > 0, 'total_blks_written is more than 0'); +ok($wal_records > 0, 'wal_records is more than 0'); +ok($wal_fpi > 0, 'wal_fpi is more than 0'); +ok($wal_bytes > 0, 'wal_bytes is more than 0'); + +$base_stats = $node->safe_psql( + 'postgres', + "SELECT count(*) = 1 + FROM ext_vacuum_statistics.pg_stats_vacuum_database, pg_database + WHERE pg_database.datname = '$dbname' + AND pg_database.oid = ext_vacuum_statistics.pg_stats_vacuum_database.dboid;" +); +ok($base_stats eq 't', 'check database-level vacuum stats from another database are available'); +}; + +#------------------------------------------------------------------------------ +# Test 10: Cleanup checks: ensure functions return empty sets for OID = 0 +#------------------------------------------------------------------------------ +subtest 'Test 10: Cleanup checks: ensure functions return empty sets for OID = 0' => sub +{ +my $dboid = $node->safe_psql( + $dbname, + "SELECT oid FROM pg_database WHERE datname = current_database();" +); + +# Vacuum statistics for invalid relation OID return empty +$base_stats = $node->safe_psql( + $dbname, + q{ + SELECT COUNT(*) + FROM ext_vacuum_statistics.pg_stats_get_vacuum_tables((SELECT oid FROM pg_database WHERE datname = current_database()), 0); + } +); +is($base_stats, 0, 'vacuum stats per heap from invalid relation OID return empty as expected'); + +$base_stats = $node->safe_psql( + $dbname, + q{ + SELECT COUNT(*) + FROM ext_vacuum_statistics.pg_stats_get_vacuum_indexes((SELECT oid FROM pg_database WHERE datname = current_database()), 0); + } +); +is($base_stats, 0, 'vacuum stats per index from invalid relation OID return empty as expected'); + +$node->safe_psql($dbname, q{ + DROP TABLE vestat CASCADE; + VACUUM; +}); + +# Check that we don't print vacuum statistics for deleted objects +$base_stats = $node->safe_psql( + $dbname, + q{ + SELECT COUNT(*) + FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relid = 0; + } +); +is($base_stats, 0, 'ext_vacuum_statistics.pg_stats_vacuum_tables correctly returns no rows for OID = 0'); + +$base_stats = $node->safe_psql( + $dbname, + q{ + SELECT COUNT(*) + FROM ext_vacuum_statistics.pg_stats_vacuum_indexes WHERE indexrelid = 0; + } +); +is($base_stats, 0, 'ext_vacuum_statistics.pg_stats_vacuum_indexes correctly returns no rows for OID = 0'); + +my $reloid = $node->safe_psql( + $dbname, + q{ + SELECT oid FROM pg_class WHERE relname = 'pg_shdepend'; + } +); + +$node->safe_psql($dbname, "VACUUM pg_shdepend;"); + +# Check if we can get vacuum statistics for cluster relations (shared catalogs) +$base_stats = $node->safe_psql( + $dbname, + qq{ + SELECT count(*) > 0 + FROM ext_vacuum_statistics.pg_stats_get_vacuum_tables((SELECT oid FROM pg_database WHERE datname = current_database()), $reloid); + } +); + +is($base_stats, 't', 'vacuum stats for common heap objects available'); + +my $indoid = $node->safe_psql( + $dbname, + q{ + SELECT oid FROM pg_class WHERE relname = 'pg_shdepend_reference_index'; + } +); + +$base_stats = $node->safe_psql( + $dbname, + qq{ + SELECT count(*) > 0 + FROM ext_vacuum_statistics.pg_stats_get_vacuum_indexes((SELECT oid FROM pg_database WHERE datname = current_database()), $indoid); + } +); + +is($base_stats, 't', 'vacuum stats for common index objects available'); + +$node->safe_psql('postgres', + "DROP DATABASE $dbname; + VACUUM;" +); + +$base_stats = $node->safe_psql( + 'postgres', + q{ + SELECT count(*) = 0 + FROM ext_vacuum_statistics.pg_stats_get_vacuum_database(0); + } +); +is($base_stats, 't', 'vacuum stats from database with invalid database OID return empty, as expected'); +}; + +$node->stop; + +done_testing(); diff --git a/contrib/ext_vacuum_statistics/t/053_vacuum_extending_freeze_test.pl b/contrib/ext_vacuum_statistics/t/053_vacuum_extending_freeze_test.pl new file mode 100644 index 00000000000..0ba52f7988f --- /dev/null +++ b/contrib/ext_vacuum_statistics/t/053_vacuum_extending_freeze_test.pl @@ -0,0 +1,285 @@ +# Copyright (c) 2025 PostgreSQL Global Development Group +# +# Test cumulative vacuum stats using ext_vacuum_statistics extension (TAP) +# +# In short, this test validates the correctness and stability of cumulative +# vacuum statistics accounting around freezing, visibility, and revision +# tracking across multiple VACUUMs and backend operations. + +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +#------------------------------------------------------------------------------ +# Test cluster setup +#------------------------------------------------------------------------------ + +my $node = PostgreSQL::Test::Cluster->new('ext_stat_vacuum'); +$node->init; + +# Configure the server: preload extension and aggressive freezing behavior +$node->append_conf('postgresql.conf', q{ + shared_preload_libraries = 'ext_vacuum_statistics' + log_min_messages = notice + vacuum_freeze_min_age = 0 + vacuum_freeze_table_age = 0 + vacuum_multixact_freeze_min_age = 0 + vacuum_multixact_freeze_table_age = 0 + vacuum_max_eager_freeze_failure_rate = 1.0 + vacuum_failsafe_age = 0 + vacuum_multixact_failsafe_age = 0 + track_functions = 'all' +}); + +$node->start(); + +#------------------------------------------------------------------------------ +# Database creation and initialization +#------------------------------------------------------------------------------ + +$node->safe_psql('postgres', q{ + CREATE DATABASE statistic_vacuum_database_regression; +}); + +# Main test database name +my $dbname = 'statistic_vacuum_database_regression'; + +# Create extension +$node->safe_psql($dbname, q{ + CREATE EXTENSION ext_vacuum_statistics; +}); + +#------------------------------------------------------------------------------ +# Timing parameters for polling loops +#------------------------------------------------------------------------------ + +my $timeout = 30; # overall wait timeout in seconds +my $interval = 0.015; # poll interval in seconds (15 ms) +my $start_time = time(); +my $updated = 0; + +#------------------------------------------------------------------------------ +# wait_for_vacuum_stats +# +# Polls ext_vacuum_statistics.pg_stats_vacuum_tables until the named columns exceed the +# provided baseline values or until timeout. +# +# tab_all_frozen_pages_count => 0 # baseline numeric +# tab_all_visible_pages_count => 0 # baseline numeric +# run_vacuum => 0 # if true, run vacuum before polling +# +# Returns: 1 if the condition is met before timeout, 0 otherwise. +#------------------------------------------------------------------------------ +sub wait_for_vacuum_stats { + my (%args) = @_; + + my $tab_all_frozen_pages_count = $args{tab_all_frozen_pages_count} || 0; + my $tab_all_visible_pages_count = $args{tab_all_visible_pages_count} || 0; + my $run_vacuum = $args{run_vacuum} ? 1 : 0; + my $result_query; + + my $start = time(); + my $sql; + + # Run VACUUM once if requested, before polling + if ($run_vacuum) { + $node->safe_psql($dbname, 'VACUUM (FREEZE, VERBOSE) vestat'); + } + + while ((time() - $start) < $timeout) { + + if ($run_vacuum) { + $sql = " + SELECT (vm_new_visible_frozen_pages > $tab_all_frozen_pages_count) + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'vestat'"; + } + else { + $sql = " + SELECT (pg_stat_get_rev_all_frozen_pages(c.oid) > $tab_all_frozen_pages_count AND + pg_stat_get_rev_all_visible_pages(c.oid) > $tab_all_visible_pages_count) + FROM pg_class c + WHERE relname = 'vestat'"; + } + + $result_query = $node->safe_psql($dbname, $sql); + + return 1 if (defined $result_query && $result_query eq 't'); + + sleep($interval); + } + + return 0; +} + +#------------------------------------------------------------------------------ +# Variables to hold vacuum statistics snapshots for comparisons +#------------------------------------------------------------------------------ + +my $vm_new_visible_frozen_pages = 0; + +my $rev_all_frozen_pages = 0; +my $rev_all_visible_pages = 0; + +my $vm_new_visible_frozen_pages_prev = 0; + +my $rev_all_frozen_pages_prev = 0; +my $rev_all_visible_pages_prev = 0; + +my $res; + +#------------------------------------------------------------------------------ +# fetch_vacuum_stats +# +# Loads current values of the relevant vacuum counters for the test table +# into the package-level variables above so tests can compare later. +#------------------------------------------------------------------------------ + +sub fetch_vacuum_stats { + $vm_new_visible_frozen_pages = $node->safe_psql( + $dbname, + "SELECT vt.vm_new_visible_frozen_pages + FROM ext_vacuum_statistics.pg_stats_vacuum_tables vt + WHERE vt.relname = 'vestat';" + ); + + $rev_all_frozen_pages = $node->safe_psql( + $dbname, + "SELECT pg_stat_get_rev_all_frozen_pages(c.oid) + FROM pg_class c + WHERE c.relname = 'vestat';" + ); + + $rev_all_visible_pages = $node->safe_psql( + $dbname, + "SELECT pg_stat_get_rev_all_visible_pages(c.oid) + FROM pg_class c + WHERE c.relname = 'vestat';" + ); +} + +#------------------------------------------------------------------------------ +# save_vacuum_stats +#------------------------------------------------------------------------------ +sub save_vacuum_stats { + $vm_new_visible_frozen_pages_prev = $vm_new_visible_frozen_pages; + $rev_all_frozen_pages_prev = $rev_all_frozen_pages; + $rev_all_visible_pages_prev = $rev_all_visible_pages; +} + +#------------------------------------------------------------------------------ +# print_vacuum_stats_on_error +#------------------------------------------------------------------------------ +sub print_vacuum_stats_on_error { + diag( + "Statistics in the failed test\n" . + "Table statistics:\n" . + " Before test:\n" . + " vm_new_visible_frozen_pages = $vm_new_visible_frozen_pages_prev\n" . + " rev_all_frozen_pages = $rev_all_frozen_pages_prev\n" . + " rev_all_visible_pages = $rev_all_visible_pages_prev\n" . + " After test:\n" . + " vm_new_visible_frozen_pages = $vm_new_visible_frozen_pages\n" . + " rev_all_frozen_pages = $rev_all_frozen_pages\n" . + " rev_all_visible_pages = $rev_all_visible_pages\n" + ); +}; + +#------------------------------------------------------------------------------ +# Test 1: Create test table, populate it and run an initial vacuum to force freezing +#------------------------------------------------------------------------------ + +subtest 'Test 1: Create test table, populate it and run an initial vacuum to force freezing' => sub +{ +$node->safe_psql($dbname, q{ + CREATE TABLE vestat (x int) + WITH (autovacuum_enabled = off, fillfactor = 10); + INSERT INTO vestat SELECT x FROM generate_series(1, 1000) AS g(x); + ANALYZE vestat; + VACUUM (FREEZE, VERBOSE) vestat; +}); + +$updated = wait_for_vacuum_stats( + tab_all_frozen_pages_count => 0, + tab_all_visible_pages_count => 0, + run_vacuum => 1, +); + +ok($updated, + 'vacuum stats updated after vacuuming the table (vm_new_visible_frozen_pages advanced)') + or diag "Timeout waiting for ext_vacuum_statistics to update after $timeout seconds during vacuum"; + +fetch_vacuum_stats(); + +ok($vm_new_visible_frozen_pages > $vm_new_visible_frozen_pages_prev, 'table vm_new_visible_frozen_pages has increased'); +ok($rev_all_frozen_pages == $rev_all_frozen_pages_prev, 'table rev_all_frozen_pages stay the same'); +ok($rev_all_visible_pages == $rev_all_visible_pages_prev, 'table rev_all_visible_pages stay the same'); +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 2: Trigger backend updates +# Backend activity should reset per-page visibility/freeze marks and increment revision counters +#------------------------------------------------------------------------------ +subtest 'Test 2: Trigger backend updates' => sub +{ +save_vacuum_stats(); + +$node->safe_psql($dbname, q{ + UPDATE vestat SET x = x + 1001; +}); + +$updated = wait_for_vacuum_stats( + tab_all_frozen_pages_count => 0, + tab_all_visible_pages_count => 0, + run_vacuum => 0, +); + +ok($updated, + 'vacuum stats updated after backend tuple updates (rev_all_frozen_pages and rev_all_visible_pages advanced)') + or diag "Timeout waiting for vacuum stats update after $timeout seconds"; + +fetch_vacuum_stats(); + +ok($vm_new_visible_frozen_pages == $vm_new_visible_frozen_pages_prev, 'table vm_new_visible_frozen_pages stay the same'); +ok($rev_all_frozen_pages > $rev_all_frozen_pages_prev, 'table rev_all_frozen_pages has increased'); +ok($rev_all_visible_pages > $rev_all_visible_pages_prev, 'table rev_all_visible_pages has increased'); +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 3: Force another vacuum after backend modifications - vacuum should restore freeze/visibility +#------------------------------------------------------------------------------ +subtest 'Test 3: Force another vacuum after backend modifications - vacuum should restore freeze/visibility' => sub +{ +save_vacuum_stats(); + +$node->safe_psql($dbname, q{ VACUUM vestat; }); + +$updated = wait_for_vacuum_stats( + tab_all_frozen_pages_count => $vm_new_visible_frozen_pages, + tab_all_visible_pages_count => 0, + run_vacuum => 1, +); + +ok($updated, + 'vacuum stats updated after vacuuming the all-updated table (vm_new_visible_frozen_pages advanced)') + or diag "Timeout waiting for ext_vacuum_statistics to update after $timeout seconds during vacuum"; + +fetch_vacuum_stats(); + +ok($vm_new_visible_frozen_pages > $vm_new_visible_frozen_pages_prev, 'table vm_new_visible_frozen_pages has increased'); +ok($rev_all_frozen_pages == $rev_all_frozen_pages_prev, 'table rev_all_frozen_pages stay the same'); +ok($rev_all_visible_pages == $rev_all_visible_pages_prev, 'table rev_all_visible_pages stay the same'); +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Cleanup +#------------------------------------------------------------------------------ + +$node->safe_psql('postgres', q{ + DROP DATABASE statistic_vacuum_database_regression; +}); + +$node->stop; +done_testing(); diff --git a/contrib/ext_vacuum_statistics/t/054_vacuum_extending_gucs_test.pl b/contrib/ext_vacuum_statistics/t/054_vacuum_extending_gucs_test.pl new file mode 100644 index 00000000000..b8d5bf30ecf --- /dev/null +++ b/contrib/ext_vacuum_statistics/t/054_vacuum_extending_gucs_test.pl @@ -0,0 +1,203 @@ +# Copyright (c) 2025 PostgreSQL Global Development Group +# +# Test GUC parameters for ext_vacuum_statistics extension: +# vacuum_statistics.enabled +# vacuum_statistics.object_types (all, databases, relations) +# vacuum_statistics.track_relations (all, system, user) +# vacuum_statistics.track_databases_from_list, add/remove_track_database +# add/remove_track_database, add/remove_track_relation, track_*_from_list + +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +#------------------------------------------------------------------------------ +# Test cluster setup +#------------------------------------------------------------------------------ + +my $node = PostgreSQL::Test::Cluster->new('ext_stat_vacuum_gucs'); +$node->init; + +$node->append_conf('postgresql.conf', q{ + shared_preload_libraries = 'ext_vacuum_statistics' + log_min_messages = notice +}); + +$node->start; + +#------------------------------------------------------------------------------ +# Database creation and initialization +#------------------------------------------------------------------------------ + +$node->safe_psql('postgres', q{ + CREATE DATABASE statistic_vacuum_gucs; +}); + +my $dbname = 'statistic_vacuum_gucs'; + +$node->safe_psql($dbname, q{ + CREATE EXTENSION ext_vacuum_statistics; + CREATE TABLE guc_test (x int PRIMARY KEY) + WITH (autovacuum_enabled = off); + INSERT INTO guc_test SELECT x FROM generate_series(1, 100) AS g(x); + ANALYZE guc_test; +}); + +# Get OIDs for filtering tests +my $dboid = $node->safe_psql($dbname, q{SELECT oid FROM pg_database WHERE datname = current_database()}); +my $reloid = $node->safe_psql($dbname, q{SELECT oid FROM pg_class WHERE relname = 'guc_test'}); + +#------------------------------------------------------------------------------ +# Reset stats and run vacuum (all in one session so GUCs persist) +#------------------------------------------------------------------------------ + +sub reset_and_vacuum { + my ($db, $table, $opts) = @_; + $table ||= 'guc_test'; + my $gucs = $opts && $opts->{gucs} ? $opts->{gucs} : []; + my $modify = $opts && $opts->{modify}; + my $extra = $opts && $opts->{extra_vacuum} ? $opts->{extra_vacuum} : []; + $extra = [$extra] unless ref $extra eq 'ARRAY'; + my $sql = join("\n", (map { "SET $_;" } @$gucs), + "SELECT ext_vacuum_statistics.vacuum_statistics_reset();", + $modify ? ( + "TRUNCATE $table;", + "INSERT INTO $table SELECT x FROM generate_series(1, 100) AS g(x);", + "DELETE FROM $table;", + ) : (), + "VACUUM $table;", + (map { "VACUUM $_;" } @$extra)); + $node->safe_psql($db, $sql); + sleep(0.1); +} + +#------------------------------------------------------------------------------ +# Test 1: vacuum_statistics.enabled +#------------------------------------------------------------------------------ +subtest 'vacuum_statistics.enabled' => sub { + reset_and_vacuum($dbname); + + # Default: enabled - should have stats + my $count = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'guc_test'"); + ok($count > 0, 'stats collected when enabled'); + + # Disable, reset and vacuum in same session + reset_and_vacuum($dbname, 'guc_test', { gucs => ['vacuum_statistics.enabled = off'] }); + + $count = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'guc_test'"); + is($count, 0, 'no stats when disabled'); +}; + +#------------------------------------------------------------------------------ +# Test 2: vacuum_statistics.object_types (databases only, relations only) +#------------------------------------------------------------------------------ +subtest 'vacuum_statistics.object_types' => sub { + # track only db stats, no relation stats + reset_and_vacuum($dbname, 'guc_test', { + gucs => ["vacuum_statistics.object_types = 'databases'"], + modify => 1, + }); + my $db_has_dbs = $node->safe_psql($dbname, + "SELECT COALESCE(SUM(db_blks_hit), 0) FROM ext_vacuum_statistics.pg_stats_vacuum_database WHERE dboid = $dboid"); + my $rel_dbs = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'guc_test'"); + is($rel_dbs, 0, 'track=databases: no relation stats'); + ok($db_has_dbs > 0, 'track=databases: database stats collected'); + + # track only relation stats, no db stats + reset_and_vacuum($dbname, 'guc_test', { + gucs => ["vacuum_statistics.object_types = 'relations'"], + modify => 1, + }); + my $db_has_rels = $node->safe_psql($dbname, + "SELECT COALESCE(SUM(db_blks_hit), 0) > 0 FROM ext_vacuum_statistics.pg_stats_vacuum_database WHERE dboid = $dboid"); + my $rel_rels = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'guc_test'"); + ok($rel_rels > 0, 'track=relations: relation stats collected'); + is($db_has_rels, 'f', 'track=relations: no database stats'); +}; + +#------------------------------------------------------------------------------ +# Test 3: vacuum_statistics.track_relations (system, user) +#------------------------------------------------------------------------------ +subtest 'vacuum_statistics.track_relations' => sub { + # track_relations - only user tables + reset_and_vacuum($dbname, 'guc_test', { + gucs => [ + "vacuum_statistics.object_types = 'relations'", + "vacuum_statistics.track_relations = 'user'", + ], + extra_vacuum => ['pg_class'], + }); + + my $user_rel = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'guc_test'"); + my $sys_rel = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'pg_class'"); + ok($user_rel > 0, 'track_relations=user: user table stats collected'); + is($sys_rel, 0, 'track_relations=user: system table stats not collected'); + + # track_relations - only system tables + reset_and_vacuum($dbname, 'guc_test', { + gucs => [ + "vacuum_statistics.object_types = 'relations'", + "vacuum_statistics.track_relations = 'system'", + ], + extra_vacuum => ['pg_class'], + }); + + $user_rel = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'guc_test'"); + $sys_rel = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'pg_class'"); + is($user_rel, 0, 'track_relations=system: user table stats not collected'); + ok($sys_rel > 0, 'track_relations=system: system table stats collected'); +}; + +#------------------------------------------------------------------------------ +# Test 4: track_databases (via add/remove_track_database) +#------------------------------------------------------------------------------ +subtest 'track_databases (add/remove)' => sub { + $node->safe_psql($dbname, "SELECT ext_vacuum_statistics.remove_track_database($dboid)"); + $node->safe_psql($dbname, "SELECT ext_vacuum_statistics.add_track_database($dboid)"); + reset_and_vacuum($dbname, 'guc_test', { gucs => ["vacuum_statistics.track_databases_from_list = on"], modify => 1 }); + + my $rel_count = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'guc_test'"); + ok($rel_count > 0, 'db in list: stats collected'); + + $node->safe_psql($dbname, "SELECT ext_vacuum_statistics.remove_track_database($dboid)"); + reset_and_vacuum($dbname, 'guc_test', { gucs => ["vacuum_statistics.track_databases_from_list = on"], modify => 1 }); + + $rel_count = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'guc_test'"); + is($rel_count, 0, 'db removed from list: no stats'); +}; + +#------------------------------------------------------------------------------ +# Test 5: track_relations (via add/remove_track_relation) +#------------------------------------------------------------------------------ +subtest 'track_relations (add/remove)' => sub { + $node->safe_psql($dbname, "SELECT ext_vacuum_statistics.remove_track_relation($dboid, $reloid)"); + $node->safe_psql($dbname, "SELECT ext_vacuum_statistics.add_track_relation($dboid, $reloid)"); + reset_and_vacuum($dbname, 'guc_test', { gucs => ["vacuum_statistics.track_relations_from_list = on"], modify => 1 }); + + my $rel_count = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'guc_test'"); + ok($rel_count > 0, 'table in list: stats collected'); + + $node->safe_psql($dbname, "SELECT ext_vacuum_statistics.remove_track_relation($dboid, $reloid)"); + reset_and_vacuum($dbname, 'guc_test', { gucs => ["vacuum_statistics.track_relations_from_list = on"], modify => 1 }); + + $rel_count = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'guc_test'"); + is($rel_count, 0, 'table removed from list: no stats'); +}; + +$node->stop; + +done_testing(); diff --git a/contrib/ext_vacuum_statistics/vacuum_statistics.c b/contrib/ext_vacuum_statistics/vacuum_statistics.c new file mode 100644 index 00000000000..1f6f3e90614 --- /dev/null +++ b/contrib/ext_vacuum_statistics/vacuum_statistics.c @@ -0,0 +1,1000 @@ +/* + * ext_vacuum_statistics - Extended vacuum statistics for PostgreSQL + * + * This module collects detailed vacuum statistics (I/O, WAL, timing, etc.) + * at relation and database level by hooking into the vacuum reporting path. + * Statistics are stored via pgstat custom statistics. Management of statistics + * storage and output functions are implemented in this module. + */ +#include "postgres.h" + +#include "access/transam.h" +#include "catalog/catalog.h" +#include "catalog/objectaccess.h" +#include "catalog/pg_class.h" +#include "catalog/pg_database.h" +#include "fmgr.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "pgstat.h" +#include "storage/fd.h" +#include "utils/builtins.h" +#include "utils/fmgrprotos.h" +#include "utils/guc.h" +#include "utils/hsearch.h" +#include "utils/lsyscache.h" +#include "utils/pgstat_kind.h" +#include "utils/pgstat_internal.h" + +#ifdef PG_MODULE_MAGIC +PG_MODULE_MAGIC; +#endif + +/* Two kinds: relations (tables/indexes) and database aggregates */ +#define PGSTAT_KIND_EXTVAC_RELATION 24 +#define PGSTAT_KIND_EXTVAC_DB 25 + +#define SJ_NODENAME "vacuum_statistics" +#define EVS_TRACK_FILENAME "pg_stat/ext_vacuum_statistics_track.oid" + +/* Bit flags for evs_track (object_types): 'all', 'databases', 'relations' */ +#define EVS_TRACK_RELATIONS 0x01 +#define EVS_TRACK_DATABASES 0x02 + +/* Bit flags for evs_track_relations: 'all', 'system', 'user' */ +#define EVS_FILTER_SYSTEM 0x01 +#define EVS_FILTER_USER 0x02 + +/* GUCs */ +static bool evs_enabled = true; +static char *evs_track = "all"; /* 'all', 'databases', 'relations' */ +static char *evs_track_relations = "all"; /* 'all', 'system', 'user' */ +static int evs_track_bits = EVS_TRACK_RELATIONS | EVS_TRACK_DATABASES; +static int evs_track_relations_bits = EVS_FILTER_SYSTEM | EVS_FILTER_USER; +static bool evs_track_databases_from_list = false; /* if true, track only + * databases in list */ +static bool evs_track_relations_from_list = false; /* if true, track only + * relations in list */ + +/* Hook */ +static set_report_vacuum_hook_type prev_report_vacuum_hook = NULL; +static object_access_hook_type prev_object_access_hook = NULL; + +/* Forward declarations */ +static void pgstat_report_vacuum_extstats(Oid tableoid, bool shared, + PgStat_VacuumRelationCounts * params); +static bool evs_oid_in_list(HTAB *hash, Oid oid); +static void evs_track_hash_ensure_init(void); +static void evs_track_save_file(void); +static void evs_track_load_file(void); +static void evs_drop_access_hook(ObjectAccessType access, Oid classId, + Oid objectId, int subId, void *arg); + +/* Hash tables for track_databases and track_relations_list */ +static HTAB *evs_track_databases_hash = NULL; +static HTAB *evs_track_relations_hash = NULL; +static bool evs_track_hash_initialized = false; + +static void evs_track_load_file(void); + +/* + * objid encoding for relations: (relid << 2) | (type & 3) + */ +#define EXTVAC_OBJID(relid, type) (((uint64) (relid)) << 2 | ((type) & 3)) + +/* Key for relation tracking: (dboid, reloid). + * InvalidOid for dboid means it is a cluster object. + */ +typedef struct +{ + Oid dboid; + Oid reloid; +} EvsTrackRelKey; + +/* Shared memory entry for vacuum stats; one per relation or database. */ +typedef struct PgStatShared_ExtVacEntry +{ + PgStatShared_Common header; + PgStat_VacuumRelationCounts stats; +} PgStatShared_ExtVacEntry; + +/* PgStat kind for per-relation vacuum statistics (tables/indexes) */ +static const PgStat_KindInfo extvac_relation_kind_info = { + .name = "ext_vacuum_statistics_relation", + .fixed_amount = false, + .accessed_across_databases = true, + .write_to_file = true, + .track_entry_count = true, + .shared_size = sizeof(PgStatShared_ExtVacEntry), + .shared_data_off = offsetof(PgStatShared_ExtVacEntry, stats), + .shared_data_len = sizeof(PgStat_VacuumRelationCounts), + .pending_size = 0, + .flush_pending_cb = NULL, +}; + +/* PgStat kind for per-database aggregated vacuum statistics */ +static const PgStat_KindInfo extvac_db_kind_info = { + .name = "ext_vacuum_statistics_db", + .fixed_amount = false, + .accessed_across_databases = true, + .write_to_file = true, + .track_entry_count = true, + .shared_size = sizeof(PgStatShared_ExtVacEntry), + .shared_data_off = offsetof(PgStatShared_ExtVacEntry, stats), + .shared_data_len = sizeof(PgStat_VacuumRelationCounts), + .pending_size = 0, + .flush_pending_cb = NULL, +}; + +#define ACCUM_IF(dst, src, field) \ + do { (dst)->field += (src)->field; } while (0) + +static inline void +pgstat_accumulate_common(PgStat_CommonCounts * dst, const PgStat_CommonCounts * src) +{ + ACCUM_IF(dst, src, total_blks_read); + ACCUM_IF(dst, src, total_blks_hit); + ACCUM_IF(dst, src, total_blks_dirtied); + ACCUM_IF(dst, src, total_blks_written); + ACCUM_IF(dst, src, blks_fetched); + ACCUM_IF(dst, src, blks_hit); + ACCUM_IF(dst, src, blk_read_time); + ACCUM_IF(dst, src, blk_write_time); + ACCUM_IF(dst, src, delay_time); + ACCUM_IF(dst, src, total_time); + ACCUM_IF(dst, src, wal_records); + ACCUM_IF(dst, src, wal_fpi); + ACCUM_IF(dst, src, wal_bytes); + ACCUM_IF(dst, src, wraparound_failsafe_count); + ACCUM_IF(dst, src, interrupts_count); + ACCUM_IF(dst, src, tuples_deleted); +} + +static inline void +pgstat_accumulate_extvac_stats(PgStat_VacuumRelationCounts * dst, + const PgStat_VacuumRelationCounts * src) +{ + if (dst->type == PGSTAT_EXTVAC_INVALID) + dst->type = src->type; + + Assert(src->type != PGSTAT_EXTVAC_INVALID && src->type != PGSTAT_EXTVAC_DB); + Assert(src->type == dst->type); + + pgstat_accumulate_common(&dst->common, &src->common); + + if (dst->type == PGSTAT_EXTVAC_TABLE) + { + dst->table.pages_scanned += src->table.pages_scanned; + dst->table.pages_removed += src->table.pages_removed; + dst->table.tuples_frozen += src->table.tuples_frozen; + dst->table.recently_dead_tuples += src->table.recently_dead_tuples; + dst->table.vm_new_frozen_pages += src->table.vm_new_frozen_pages; + dst->table.vm_new_visible_pages += src->table.vm_new_visible_pages; + dst->table.vm_new_visible_frozen_pages += src->table.vm_new_visible_frozen_pages; + dst->table.missed_dead_pages += src->table.missed_dead_pages; + dst->table.missed_dead_tuples += src->table.missed_dead_tuples; + dst->table.index_vacuum_count += src->table.index_vacuum_count; + } + else if (dst->type == PGSTAT_EXTVAC_INDEX) + { + dst->index.pages_deleted += src->index.pages_deleted; + } +} + +/* GUC assign hooks: parse string and update bit flags */ +static void +evs_track_assign_hook(const char *newval, void *extra) +{ + if (strcmp(newval, "databases") == 0) + evs_track_bits = EVS_TRACK_DATABASES; + else if (strcmp(newval, "relations") == 0) + evs_track_bits = EVS_TRACK_RELATIONS; + else + evs_track_bits = EVS_TRACK_RELATIONS | EVS_TRACK_DATABASES; /* "all" or unknown */ +} + +static void +evs_track_relations_assign_hook(const char *newval, void *extra) +{ + if (strcmp(newval, "system") == 0) + evs_track_relations_bits = EVS_FILTER_SYSTEM; + else if (strcmp(newval, "user") == 0) + evs_track_relations_bits = EVS_FILTER_USER; + else + evs_track_relations_bits = EVS_FILTER_SYSTEM | EVS_FILTER_USER; /* "all" or unknown */ +} + +void +_PG_init(void) +{ + if (!process_shared_preload_libraries_in_progress) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ext_vacuum_statistics module could be loaded only on startup."), + errdetail("Add 'ext_vacuum_statistics' into the shared_preload_libraries list."))); + + DefineCustomBoolVariable("vacuum_statistics.enabled", + "Enable extended vacuum statistics collection.", + NULL, &evs_enabled, true, + PGC_SUSET, 0, NULL, NULL, NULL); + + DefineCustomStringVariable("vacuum_statistics.object_types", + "Object types for statistics: 'all', 'databases', 'relations'.", + NULL, &evs_track, "all", + PGC_SUSET, 0, NULL, evs_track_assign_hook, NULL); + + DefineCustomStringVariable("vacuum_statistics.track_relations", + "When tracking relations: 'all', 'system', 'user'.", + NULL, &evs_track_relations, "all", + PGC_SUSET, 0, NULL, evs_track_relations_assign_hook, NULL); + + DefineCustomBoolVariable("vacuum_statistics.track_databases_from_list", + "If true, track only databases added via add_track_database.", + NULL, &evs_track_databases_from_list, false, + PGC_SUSET, 0, NULL, NULL, NULL); + + DefineCustomBoolVariable("vacuum_statistics.track_relations_from_list", + "If true, track only relations added via add_track_relation.", + NULL, &evs_track_relations_from_list, false, + PGC_SUSET, 0, NULL, NULL, NULL); + + MarkGUCPrefixReserved(SJ_NODENAME); + + pgstat_register_kind(PGSTAT_KIND_EXTVAC_RELATION, &extvac_relation_kind_info); + pgstat_register_kind(PGSTAT_KIND_EXTVAC_DB, &extvac_db_kind_info); + + prev_report_vacuum_hook = set_report_vacuum_hook; + set_report_vacuum_hook = pgstat_report_vacuum_extstats; + + prev_object_access_hook = object_access_hook; + object_access_hook = evs_drop_access_hook; +} + +/* + * Object access hook: remove dropped objects from track lists. + */ +static void +evs_drop_access_hook(ObjectAccessType access, Oid classId, + Oid objectId, int subId, void *arg) +{ + if (prev_object_access_hook) + (*prev_object_access_hook) (access, classId, objectId, subId, arg); + + if (access == OAT_DROP) + { + if (classId == RelationRelationId && subId == 0) + { + char relkind = get_rel_relkind(objectId); + EvsTrackRelKey key; + bool found; + + if (relkind == RELKIND_RELATION || relkind == RELKIND_INDEX) + { + evs_track_hash_ensure_init(); + key.dboid = MyDatabaseId; + key.reloid = objectId; + hash_search(evs_track_relations_hash, &key, HASH_REMOVE, &found); + key.dboid = InvalidOid; + hash_search(evs_track_relations_hash, &key, HASH_REMOVE, &found); + evs_track_save_file(); + } + } + + if (classId == DatabaseRelationId && objectId != InvalidOid) + { + bool found; + + evs_track_hash_ensure_init(); + hash_search(evs_track_databases_hash, &objectId, HASH_REMOVE, &found); + evs_track_save_file(); + } + } +} + +/* + * Storage of track lists in a separate file. + * + * Stores the lists of database OIDs and (dboid, reloid) pairs used for + * selective tracking when track_databases_from_list or track_relations_from_list + * is enabled. + * Data stores in pg_stat/ext_vacuum_statistics_track.oid + */ +static void +evs_track_hash_ensure_init(void) +{ + HASHCTL ctl; + + if (evs_track_hash_initialized) + return; + + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(Oid); + ctl.entrysize = sizeof(Oid); + ctl.hcxt = TopMemoryContext; + /* Hash of database OIDs to track specific databases */ + evs_track_databases_hash = hash_create("ext_vacuum_statistics track databases", + 64, &ctl, HASH_ELEM | HASH_BLOBS); + + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(EvsTrackRelKey); + ctl.entrysize = sizeof(EvsTrackRelKey); + ctl.hcxt = TopMemoryContext; + /* Hash of (dboid, reloid) to track specific relations */ + evs_track_relations_hash = hash_create("ext_vacuum_statistics track relations", + 64, &ctl, HASH_ELEM | HASH_BLOBS); + + evs_track_load_file(); + evs_track_hash_initialized = true; +} + +static void +evs_track_load_file(void) +{ + char path[MAXPGPATH]; + FILE *fp; + char buf[256]; + bool in_relations = false; + Oid oid; + EvsTrackRelKey key; + bool found; + + if (!DataDir || DataDir[0] == '\0' || !evs_track_databases_hash || !evs_track_relations_hash) + return; + + snprintf(path, sizeof(path), "%s/%s", DataDir, EVS_TRACK_FILENAME); + fp = AllocateFile(path, "r"); + if (!fp) + return; + + while (fgets(buf, sizeof(buf), fp)) + { + if (strncmp(buf, "[databases]", 11) == 0) + { + in_relations = false; + continue; + } + if (strncmp(buf, "[relations]", 11) == 0) + { + in_relations = true; + continue; + } + if (in_relations) + { + if (sscanf(buf, "%u %u", &key.dboid, &key.reloid) == 2) + hash_search(evs_track_relations_hash, &key, HASH_ENTER, &found); + else if (sscanf(buf, "%u", &oid) == 1) + { + key.dboid = InvalidOid; + key.reloid = oid; + hash_search(evs_track_relations_hash, &key, HASH_ENTER, &found); + } + } + else + { + if (sscanf(buf, "%u", &oid) == 1) + hash_search(evs_track_databases_hash, &oid, HASH_ENTER, &found); + } + } + FreeFile(fp); +} + +static void +evs_track_save_file(void) +{ + char path[MAXPGPATH]; + char tmppath[MAXPGPATH]; + FILE *fp; + HASH_SEQ_STATUS status; + Oid *entry; + EvsTrackRelKey *rel_entry; + + if (!DataDir || DataDir[0] == '\0' || !evs_track_databases_hash || !evs_track_relations_hash) + return; + + snprintf(path, sizeof(path), "%s/%s", DataDir, EVS_TRACK_FILENAME); + snprintf(tmppath, sizeof(tmppath), "%s/%s.tmp", DataDir, EVS_TRACK_FILENAME); + fp = AllocateFile(tmppath, "w"); + if (!fp) + return; + + fprintf(fp, "[databases]\n"); + hash_seq_init(&status, evs_track_databases_hash); + while ((entry = (Oid *) hash_seq_search(&status)) != NULL) + fprintf(fp, "%u\n", *entry); + + fprintf(fp, "[relations]\n"); + hash_seq_init(&status, evs_track_relations_hash); + while ((rel_entry = (EvsTrackRelKey *) hash_seq_search(&status)) != NULL) + { + if (OidIsValid(rel_entry->dboid)) + fprintf(fp, "%u %u\n", rel_entry->dboid, rel_entry->reloid); + else + fprintf(fp, "0 %u\n", rel_entry->reloid); + } + + if (FreeFile(fp) != 0 || rename(tmppath, path) != 0) + unlink(tmppath); +} + +/* + * Check if OID is in the given hash + */ +static bool +evs_oid_in_list(HTAB *hash, Oid oid) +{ + if (!hash) + return false; + if (hash_get_num_entries(hash) == 0) + return false; + return hash_search(hash, &oid, HASH_FIND, NULL) != NULL; +} + +/* + * Check if (dboid, relid) is in track_relations list. + */ +static bool +evs_rel_in_list(Oid dboid, Oid relid) +{ + EvsTrackRelKey key; + + if (!evs_track_relations_hash) + return false; + if (hash_get_num_entries(evs_track_relations_hash) == 0) + return false; + key.dboid = dboid; + key.reloid = relid; + if (hash_search(evs_track_relations_hash, &key, HASH_FIND, NULL) != NULL) + return true; + key.dboid = InvalidOid; + return hash_search(evs_track_relations_hash, &key, HASH_FIND, NULL) != NULL; +} + +/* + * Decide whether to track statistics for relations. + * Relation is tracked if it is in the track list or a special filter is enabled. + */ +static bool +evs_should_track_relation_statistics(Oid dboid, Oid relid) +{ + evs_track_hash_ensure_init(); + + if (evs_track_databases_from_list && + !evs_oid_in_list(evs_track_databases_hash, dboid)) + return false; + if (evs_track_relations_from_list && + !(evs_rel_in_list(dboid, relid) || evs_rel_in_list(InvalidOid, relid))) + return false; + + if ((evs_track_bits & EVS_TRACK_RELATIONS) == 0) + return false; /* database-only mode */ + if (evs_track_relations_bits == EVS_FILTER_SYSTEM) + return IsCatalogRelationOid(relid); + if (evs_track_relations_bits == EVS_FILTER_USER) + return !IsCatalogRelationOid(relid); + return true; +} + +/* + * Decide whether to track statistics for databases. + * Database statistics is tracked if it is in the track list or a special filter is enabled. + */ +static bool +evs_should_track_database_statistics(Oid dboid) +{ + evs_track_hash_ensure_init(); + + if (evs_track_databases_from_list && + !evs_oid_in_list(evs_track_databases_hash, dboid)) + return false; + if ((evs_track_bits & EVS_TRACK_DATABASES) == 0) + return false; /* relations-only mode */ + if (evs_track_bits == EVS_TRACK_DATABASES) + return true; /* databases-only, accumulate to db */ + return true; +} + + +/* Accumulate common counts for database-level stats. */ +static inline void +pgstat_accumulate_common_for_db(PgStat_CommonCounts * dst, + const PgStat_CommonCounts * src) +{ + pgstat_accumulate_common(dst, src); +} + +/* + * Store incoming vacuum stats into pgstat custom statistics. + * store_relation: create/update per-relation entry + * store_db: accumulate into database-level entry (dboid, objid=0). + * Uses pgstat_get_entry_ref_locked and pgstat_accumulate_* for atomic updates. + */ +static void +extvac_store(Oid dboid, Oid relid, int type, + PgStat_VacuumRelationCounts * params, + bool store_relation, bool store_db) +{ + PgStat_EntryRef *entry_ref; + PgStatShared_ExtVacEntry *shared; + uint64 objid; + + if (!evs_enabled) + return; + + if (store_relation) + { + objid = EXTVAC_OBJID(relid, type); + entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_EXTVAC_RELATION, dboid, objid, false); + if (entry_ref) + { + shared = (PgStatShared_ExtVacEntry *) entry_ref->shared_stats; + if (shared->stats.type == PGSTAT_EXTVAC_INVALID) + { + memset(&shared->stats, 0, sizeof(shared->stats)); + shared->stats.type = params->type; + } + pgstat_accumulate_extvac_stats(&shared->stats, params); + pgstat_unlock_entry(entry_ref); + } + } + + if (store_db) + { + entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_EXTVAC_DB, dboid, InvalidOid, false); + if (entry_ref) + { + shared = (PgStatShared_ExtVacEntry *) entry_ref->shared_stats; + if (shared->stats.type == PGSTAT_EXTVAC_INVALID) + { + memset(&shared->stats, 0, sizeof(shared->stats)); + shared->stats.type = PGSTAT_EXTVAC_DB; + } + pgstat_accumulate_common_for_db(&shared->stats.common, ¶ms->common); + pgstat_unlock_entry(entry_ref); + } + } +} + +/* + * Vacuum report hook: called when vacuum finishes. Filters by track settings, + * stores stats per-relation and/or per-database, then chains to previous hook. + */ +static void +pgstat_report_vacuum_extstats(Oid tableoid, bool shared, + PgStat_VacuumRelationCounts * params) +{ + Oid dboid = shared ? InvalidOid : MyDatabaseId; + bool store_relation; + bool store_db; + + if (evs_enabled) + { + store_relation = evs_should_track_relation_statistics(dboid, tableoid); + store_db = evs_should_track_database_statistics(dboid); + + if (store_relation || store_db) + extvac_store(dboid, tableoid, params->type, params, store_relation, store_db); + } + if (prev_report_vacuum_hook) + prev_report_vacuum_hook(tableoid, shared, params); +} + +/* Reset statistics for a single relation entry. */ +static bool +extvac_reset_by_relid(Oid dboid, Oid relid, int type) +{ + uint64 objid = EXTVAC_OBJID(relid, type); + + pgstat_reset_entry(PGSTAT_KIND_EXTVAC_RELATION, dboid, objid, 0); + return true; +} + +/* Callback for pgstat_reset_matching_entries: match relation entries for given db */ +static bool +match_extvac_relations_for_db(PgStatShared_HashEntry *entry, Datum match_data) +{ + return entry->key.kind == PGSTAT_KIND_EXTVAC_RELATION && + entry->key.dboid == DatumGetObjectId(match_data); +} + +/* + * Reset statistics for a database (aggregate entry) and all its relations. + */ +static int64 +extvac_database_reset(Oid dboid) +{ + pgstat_reset_matching_entries(match_extvac_relations_for_db, + ObjectIdGetDatum(dboid), 0); + pgstat_reset_entry(PGSTAT_KIND_EXTVAC_DB, dboid, 0, 0); + return 1; +} + +/* Reset all vacuum statistics (both relation and database entries). */ +static int64 +extvac_stat_reset(void) +{ + pgstat_reset_of_kind(PGSTAT_KIND_EXTVAC_RELATION); + pgstat_reset_of_kind(PGSTAT_KIND_EXTVAC_DB); + return 0; /* count not available */ +} + +PG_FUNCTION_INFO_V1(vacuum_statistics_reset); +PG_FUNCTION_INFO_V1(extvac_shared_memory_size); +PG_FUNCTION_INFO_V1(extvac_reset_entry); +PG_FUNCTION_INFO_V1(extvac_reset_db_entry); + +Datum +vacuum_statistics_reset(PG_FUNCTION_ARGS) +{ + PG_RETURN_INT64(extvac_stat_reset()); +} + +Datum +extvac_reset_entry(PG_FUNCTION_ARGS) +{ + Oid dboid = PG_GETARG_OID(0); + Oid relid = PG_GETARG_OID(1); + int type = PG_GETARG_INT32(2); + + PG_RETURN_BOOL(extvac_reset_by_relid(dboid, relid, type)); +} + +Datum +extvac_reset_db_entry(PG_FUNCTION_ARGS) +{ + Oid dboid = PG_GETARG_OID(0); + + PG_RETURN_INT64(extvac_database_reset(dboid)); +} + +/* + * Return total shared memory in bytes used by the extension for vacuum stats. + * Used for monitoring and capacity planning: memory grows with the number of + * tracked relations and databases. + */ +Datum +extvac_shared_memory_size(PG_FUNCTION_ARGS) +{ + uint64 rel_count; + uint64 db_count; + uint64 total; + size_t entry_size = sizeof(PgStatShared_ExtVacEntry); + + rel_count = pgstat_get_entry_count(PGSTAT_KIND_EXTVAC_RELATION); + db_count = pgstat_get_entry_count(PGSTAT_KIND_EXTVAC_DB); + total = rel_count + db_count; + + PG_RETURN_INT64((int64) (total * entry_size)); +} + +/* + * Track list management: add/remove database or relation OIDs. + * Changes are persisted to pg_stat/ext_vacuum_statistics_track.oid. + */ + +PG_FUNCTION_INFO_V1(evs_add_track_database); +PG_FUNCTION_INFO_V1(evs_remove_track_database); +PG_FUNCTION_INFO_V1(evs_add_track_relation); +PG_FUNCTION_INFO_V1(evs_remove_track_relation); + +Datum +evs_add_track_database(PG_FUNCTION_ARGS) +{ + Oid oid = PG_GETARG_OID(0); + bool found; + + evs_track_hash_ensure_init(); + hash_search(evs_track_databases_hash, &oid, HASH_ENTER, &found); + evs_track_save_file(); + PG_RETURN_BOOL(!found); /* true if newly added */ +} + +Datum +evs_remove_track_database(PG_FUNCTION_ARGS) +{ + Oid oid = PG_GETARG_OID(0); + bool found; + + evs_track_hash_ensure_init(); + hash_search(evs_track_databases_hash, &oid, HASH_REMOVE, &found); + evs_track_save_file(); + PG_RETURN_BOOL(found); +} + +Datum +evs_add_track_relation(PG_FUNCTION_ARGS) +{ + EvsTrackRelKey key; + + key.dboid = PG_GETARG_OID(0); + key.reloid = PG_GETARG_OID(1); + { + bool found; + + evs_track_hash_ensure_init(); + hash_search(evs_track_relations_hash, &key, HASH_ENTER, &found); + evs_track_save_file(); + PG_RETURN_BOOL(!found); /* true if newly added */ + } +} + +Datum +evs_remove_track_relation(PG_FUNCTION_ARGS) +{ + EvsTrackRelKey key; + bool found; + + key.dboid = PG_GETARG_OID(0); + key.reloid = PG_GETARG_OID(1); + evs_track_hash_ensure_init(); + hash_search(evs_track_relations_hash, &key, HASH_REMOVE, &found); + evs_track_save_file(); + PG_RETURN_BOOL(found); +} + +/* + * Returns the list of database and relation OIDs for which statistics + * are collected. + */ +PG_FUNCTION_INFO_V1(evs_track_list); + +Datum +evs_track_list(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + TupleDesc tupdesc; + Tuplestorestate *tupstore; + MemoryContext per_query_ctx; + MemoryContext oldcontext; + Datum values[3]; + bool nulls[3] = {false, false, false}; + HASH_SEQ_STATUS status; + Oid *entry; + EvsTrackRelKey *rel_entry; + + if (!rsinfo || !IsA(rsinfo, ReturnSetInfo)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ext_vacuum_statistics: set-valued function called in context that cannot accept a set"))); + if (!(rsinfo->allowedModes & SFRM_Materialize)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ext_vacuum_statistics: materialize mode required"))); + + evs_track_hash_ensure_init(); + + per_query_ctx = rsinfo->econtext->ecxt_per_query_memory; + oldcontext = MemoryContextSwitchTo(per_query_ctx); + + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "ext_vacuum_statistics: return type must be a row type"); + + tupstore = tuplestore_begin_heap(true, false, work_mem); + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = tupdesc; + + /* Databases */ + if (hash_get_num_entries(evs_track_databases_hash) == 0) + { + values[0] = CStringGetTextDatum("database"); + nulls[1] = true; + nulls[2] = true; + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + nulls[1] = false; + nulls[2] = false; + } + else + { + hash_seq_init(&status, evs_track_databases_hash); + while ((entry = (Oid *) hash_seq_search(&status)) != NULL) + { + values[0] = CStringGetTextDatum("database"); + values[1] = ObjectIdGetDatum(*entry); + nulls[2] = true; + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + nulls[2] = false; + } + } + + /* Relations */ + if (hash_get_num_entries(evs_track_relations_hash) == 0) + { + values[0] = CStringGetTextDatum("relation"); + nulls[1] = true; + nulls[2] = true; + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + nulls[1] = false; + nulls[2] = false; + } + else + { + hash_seq_init(&status, evs_track_relations_hash); + while ((rel_entry = (EvsTrackRelKey *) hash_seq_search(&status)) != NULL) + { + values[0] = CStringGetTextDatum("relation"); + values[1] = ObjectIdGetDatum(rel_entry->dboid); + values[2] = ObjectIdGetDatum(rel_entry->reloid); + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + } + } + + MemoryContextSwitchTo(oldcontext); + + return (Datum) 0; +} + +/* + * Output vacuum statistics (tables, indexes, or per-database aggregates). + */ +#define EXTVAC_COMMON_STAT_COLS 12 + +static void +tuplestore_put_common(PgStat_CommonCounts * vacuum_ext, + Datum *values, bool *nulls, int *i) +{ + char buf[256]; + const int base = *i; + + values[(*i)++] = Int64GetDatum(vacuum_ext->total_blks_read); + values[(*i)++] = Int64GetDatum(vacuum_ext->total_blks_hit); + values[(*i)++] = Int64GetDatum(vacuum_ext->total_blks_dirtied); + values[(*i)++] = Int64GetDatum(vacuum_ext->total_blks_written); + values[(*i)++] = Int64GetDatum(vacuum_ext->wal_records); + values[(*i)++] = Int64GetDatum(vacuum_ext->wal_fpi); + snprintf(buf, sizeof buf, UINT64_FORMAT, vacuum_ext->wal_bytes); + values[(*i)++] = DirectFunctionCall3(numeric_in, + CStringGetDatum(buf), + ObjectIdGetDatum(0), + Int32GetDatum(-1)); + values[(*i)++] = Float8GetDatum(vacuum_ext->blk_read_time); + values[(*i)++] = Float8GetDatum(vacuum_ext->blk_write_time); + values[(*i)++] = Float8GetDatum(vacuum_ext->delay_time); + values[(*i)++] = Float8GetDatum(vacuum_ext->total_time); + values[(*i)++] = Int32GetDatum(vacuum_ext->wraparound_failsafe_count); + Assert((*i - base) == EXTVAC_COMMON_STAT_COLS); +} + +#define EXTVAC_HEAP_STAT_COLS 26 +#define EXTVAC_IDX_STAT_COLS 17 +#define EXTVAC_MAX_STAT_COLS Max(EXTVAC_HEAP_STAT_COLS, EXTVAC_IDX_STAT_COLS) + +static void +tuplestore_put_for_relation(Oid relid, Tuplestorestate *tupstore, + TupleDesc tupdesc, PgStat_VacuumRelationCounts * vacuum_ext) +{ + Datum values[EXTVAC_MAX_STAT_COLS]; + bool nulls[EXTVAC_MAX_STAT_COLS]; + int i = 0; + + memset(nulls, 0, sizeof(nulls)); + values[i++] = ObjectIdGetDatum(relid); + + tuplestore_put_common(&vacuum_ext->common, values, nulls, &i); + values[i++] = Int64GetDatum(vacuum_ext->common.blks_fetched - vacuum_ext->common.blks_hit); + values[i++] = Int64GetDatum(vacuum_ext->common.blks_hit); + + if (vacuum_ext->type == PGSTAT_EXTVAC_TABLE) + { + values[i++] = Int64GetDatum(vacuum_ext->common.tuples_deleted); + values[i++] = Int64GetDatum(vacuum_ext->table.pages_scanned); + values[i++] = Int64GetDatum(vacuum_ext->table.pages_removed); + values[i++] = Int64GetDatum(vacuum_ext->table.vm_new_frozen_pages); + values[i++] = Int64GetDatum(vacuum_ext->table.vm_new_visible_pages); + values[i++] = Int64GetDatum(vacuum_ext->table.vm_new_visible_frozen_pages); + values[i++] = Int64GetDatum(vacuum_ext->table.tuples_frozen); + values[i++] = Int64GetDatum(vacuum_ext->table.recently_dead_tuples); + values[i++] = Int64GetDatum(vacuum_ext->table.index_vacuum_count); + values[i++] = Int64GetDatum(vacuum_ext->table.missed_dead_pages); + values[i++] = Int64GetDatum(vacuum_ext->table.missed_dead_tuples); + } + else if (vacuum_ext->type == PGSTAT_EXTVAC_INDEX) + { + values[i++] = Int64GetDatum(vacuum_ext->common.tuples_deleted); + values[i++] = Int64GetDatum(vacuum_ext->index.pages_deleted); + } + + Assert(i == ((vacuum_ext->type == PGSTAT_EXTVAC_TABLE) ? EXTVAC_HEAP_STAT_COLS : EXTVAC_IDX_STAT_COLS)); + tuplestore_putvalues(tupstore, tupdesc, values, nulls); +} + +static Datum +pg_stats_vacuum(FunctionCallInfo fcinfo, int type) +{ + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + MemoryContext per_query_ctx; + MemoryContext oldcontext; + Tuplestorestate *tupstore; + TupleDesc tupdesc; + Oid dbid = PG_GETARG_OID(0); + + if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ext_vacuum_statistics: set-valued function called in context that cannot accept a set"))); + if (!(rsinfo->allowedModes & SFRM_Materialize)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ext_vacuum_statistics: materialize mode required"))); + + per_query_ctx = rsinfo->econtext->ecxt_per_query_memory; + oldcontext = MemoryContextSwitchTo(per_query_ctx); + + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "ext_vacuum_statistics: return type must be a row type"); + + tupstore = tuplestore_begin_heap(true, false, work_mem); + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = tupdesc; + + MemoryContextSwitchTo(oldcontext); + + if (type == PGSTAT_EXTVAC_INDEX || type == PGSTAT_EXTVAC_TABLE) + { + Oid relid = PG_GETARG_OID(1); + PgStat_VacuumRelationCounts *stats; + + if (!OidIsValid(relid)) + return (Datum) 0; + + stats = (PgStat_VacuumRelationCounts *) + pgstat_fetch_entry(PGSTAT_KIND_EXTVAC_RELATION, dbid, EXTVAC_OBJID(relid, type)); + + if (!stats) + stats = (PgStat_VacuumRelationCounts *) + pgstat_fetch_entry(PGSTAT_KIND_EXTVAC_RELATION, InvalidOid, EXTVAC_OBJID(relid, type)); + + if (stats && stats->type == type) + tuplestore_put_for_relation(relid, tupstore, tupdesc, stats); + } + else if (type == PGSTAT_EXTVAC_DB) + { + if (OidIsValid(dbid)) + { +#define EXTVAC_DB_STAT_COLS 14 + Datum values[EXTVAC_DB_STAT_COLS]; + bool nulls[EXTVAC_DB_STAT_COLS]; + int i = 0; + PgStat_VacuumRelationCounts *stats; + + stats = (PgStat_VacuumRelationCounts *) + pgstat_fetch_entry(PGSTAT_KIND_EXTVAC_DB, dbid, InvalidOid); + if (stats && stats->type == PGSTAT_EXTVAC_DB) + { + memset(nulls, 0, sizeof(nulls)); + values[i++] = ObjectIdGetDatum(dbid); + tuplestore_put_common(&stats->common, values, nulls, &i); + values[i++] = Int32GetDatum(stats->common.interrupts_count); + Assert(i == EXTVAC_DB_STAT_COLS); + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + } + } + /* invalid dbid: return empty set */ + } + else + elog(PANIC, "ext_vacuum_statistics: invalid type %d", type); + + return (Datum) 0; +} + +PG_FUNCTION_INFO_V1(pg_stats_get_vacuum_tables); +PG_FUNCTION_INFO_V1(pg_stats_get_vacuum_indexes); +PG_FUNCTION_INFO_V1(pg_stats_get_vacuum_database); + +Datum +pg_stats_get_vacuum_tables(PG_FUNCTION_ARGS) +{ + return pg_stats_vacuum(fcinfo, PGSTAT_EXTVAC_TABLE); +} + +Datum +pg_stats_get_vacuum_indexes(PG_FUNCTION_ARGS) +{ + return pg_stats_vacuum(fcinfo, PGSTAT_EXTVAC_INDEX); +} + +Datum +pg_stats_get_vacuum_database(PG_FUNCTION_ARGS) +{ + return pg_stats_vacuum(fcinfo, PGSTAT_EXTVAC_DB); +} diff --git a/contrib/meson.build b/contrib/meson.build index def13257cbe..b8cb62d22f1 100644 --- a/contrib/meson.build +++ b/contrib/meson.build @@ -26,6 +26,7 @@ subdir('cube') subdir('dblink') subdir('dict_int') subdir('dict_xsyn') +subdir('ext_vacuum_statistics') subdir('earthdistance') subdir('file_fdw') subdir('fuzzystrmatch') diff --git a/doc/src/sgml/contrib.sgml b/doc/src/sgml/contrib.sgml index 24b706b29ad..f8a5781bde3 100644 --- a/doc/src/sgml/contrib.sgml +++ b/doc/src/sgml/contrib.sgml @@ -141,6 +141,7 @@ CREATE EXTENSION extension_name; &dict-int; &dict-xsyn; &earthdistance; + &extvacuumstatistics; &file-fdw; &fuzzystrmatch; &hstore; diff --git a/doc/src/sgml/extvacuumstatistics.sgml b/doc/src/sgml/extvacuumstatistics.sgml new file mode 100644 index 00000000000..75eb4691c4d --- /dev/null +++ b/doc/src/sgml/extvacuumstatistics.sgml @@ -0,0 +1,502 @@ + + + + ext_vacuum_statistics — extended vacuum statistics + + + ext_vacuum_statistics + + + + The ext_vacuum_statistics module provides + extended per-table, per-index, and per-database vacuum statistics + (buffer I/O, WAL, general, timing) via views in the + ext_vacuum_statistics schema. + + + + The module must be loaded by adding ext_vacuum_statistics to + in + postgresql.conf, because it registers a vacuum hook at + server startup. This means that a server restart is needed to add or remove + the module. After installation, run + CREATE EXTENSION ext_vacuum_statistics in each database + where you want to use it. + + + + When active, the module provides views + ext_vacuum_statistics.pg_stats_vacuum_tables, + ext_vacuum_statistics.pg_stats_vacuum_indexes, and + ext_vacuum_statistics.pg_stats_vacuum_database, + plus functions to reset statistics and manage tracking. + + + + Each tracked object (one table, one index, or one database) uses + approximately 232 bytes of shared memory on Linux x86_64 (e.g. Ubuntu): + common stats (buffers, WAL, timing) plus header and LWLock ~144 bytes; + type + union ~88 bytes (the union holds table-specific or index-specific + fields; the allocated size is the same for both). The exact size depends on the platform. Call + ext_vacuum_statistics.shared_memory_size() to get + the total shared memory used by the extension. The extension's GUCs allow controlling memory by limiting + which objects are tracked: + vacuum_statistics.object_types, + vacuum_statistics.track_relations, and + track_*_from_list. + Example: a database with 1000 tables and 2000 indexes uses about 700 KB + on Ubuntu ((1000 + 2000 + 1) × 232 bytes). + + + + The <structname>ext_vacuum_statistics.pg_stats_vacuum_tables</structname> View + + + pg_stats_vacuum_tables + + + + The view ext_vacuum_statistics.pg_stats_vacuum_tables + contains one row for each table in the current database (including TOAST + tables), showing statistics about vacuuming that specific table. The columns + are shown in . + + + + <structname>ext_vacuum_statistics.pg_stats_vacuum_tables</structname> Columns + + + + + Column Type + + + Description + + + + + + + relid oid + + + OID of a table + + + + + schema name + + + Name of the schema this table is in + + + + + relname name + + + Name of this table + + + + + dbname name + + + Name of the database containing this table + + + + + total_blks_read int8 + + + Number of database blocks read by vacuum operations performed on this table + + + + + total_blks_hit int8 + + + Number of times database blocks were found in the buffer cache by vacuum operations + + + + + total_blks_dirtied int8 + + + Number of database blocks dirtied by vacuum operations + + + + + total_blks_written int8 + + + Number of database blocks written by vacuum operations + + + + + wal_records int8 + + + Total number of WAL records generated by vacuum operations performed on this table + + + + + wal_fpi int8 + + + Total number of WAL full page images generated by vacuum operations + + + + + wal_bytes numeric + + + Total amount of WAL bytes generated by vacuum operations + + + + + blk_read_time float8 + + + Time spent reading blocks by vacuum operations, in milliseconds + + + + + blk_write_time float8 + + + Time spent writing blocks by vacuum operations, in milliseconds + + + + + delay_time float8 + + + Time spent in vacuum delay points, in milliseconds + + + + + total_time float8 + + + Total time of vacuuming this table, in milliseconds + + + + + wraparound_failsafe_count int4 + + + Number of times vacuum was run to prevent a wraparound problem + + + + + rel_blks_read int8 + + + Number of blocks vacuum operations read from this table + + + + + rel_blks_hit int8 + + + Number of times blocks of this table were found in the buffer cache by vacuum + + + + + tuples_deleted int8 + + + Number of dead tuples vacuum operations deleted from this table + + + + + pages_scanned int8 + + + Number of pages examined by vacuum operations + + + + + pages_removed int8 + + + Number of pages removed from physical storage by vacuum operations + + + + + vm_new_frozen_pages int8 + + + Number of pages newly set all-frozen by vacuum in the visibility map + + + + + vm_new_visible_pages int8 + + + Number of pages newly set all-visible by vacuum in the visibility map + + + + + vm_new_visible_frozen_pages int8 + + + Number of pages newly set all-visible and all-frozen by vacuum in the visibility map + + + + + tuples_frozen int8 + + + Number of tuples that vacuum operations marked as frozen + + + + + recently_dead_tuples int8 + + + Number of dead tuples left due to visibility in transactions + + + + + index_vacuum_count int8 + + + Number of times indexes on this table were vacuumed + + + + + missed_dead_pages int8 + + + Number of pages that had at least one missed dead tuple + + + + + missed_dead_tuples int8 + + + Number of fully DEAD tuples that could not be pruned due to failure to acquire a cleanup lock + + + + +
+ + + + The <structname>ext_vacuum_statistics.pg_stats_vacuum_indexes</structname> View + + + pg_stats_vacuum_indexes + + + + The view ext_vacuum_statistics.pg_stats_vacuum_indexes + contains one row for each index in the current database, showing statistics + about vacuuming that specific index. Columns include + indexrelid, schema, + indexrelname, dbname, + buffer I/O (total_blks_read, + total_blks_hit, etc.), WAL + (wal_records, wal_fpi, + wal_bytes), timing + (blk_read_time, blk_write_time, + delay_time, total_time), + and tuples_deleted, pages_deleted. + + + + + The <structname>ext_vacuum_statistics.pg_stats_vacuum_database</structname> View + + + pg_stats_vacuum_database + + + + The view ext_vacuum_statistics.pg_stats_vacuum_database + contains one row for each database in the cluster, showing aggregate vacuum + statistics for that database. Columns include + dboid, dbname, + db_blks_read, db_blks_hit, + db_blks_dirtied, db_blks_written, + WAL stats (db_wal_records, + db_wal_fpi, db_wal_bytes), + timing (db_blk_read_time, + db_blk_write_time, db_delay_time, + db_total_time), + db_wraparound_failsafe_count, and + interrupts_count. + + + + + Functions + + + + + ext_vacuum_statistics.shared_memory_size() + bigint + + + + Returns the total shared memory in bytes used by the extension for + vacuum statistics (relations plus databases). + + + + + + ext_vacuum_statistics.vacuum_statistics_reset() + bigint + + + + Resets all vacuum statistics. Returns the number of entries reset. + + + + + + ext_vacuum_statistics.add_track_database(dboid oid) + boolean + + + + Adds a database OID to the tracking list (persisted to + pg_stat/ext_vacuum_statistics_track.oid). + Returns true if newly added. + + + + + + ext_vacuum_statistics.remove_track_database(dboid oid) + boolean + + + + Removes a database OID from the tracking list. Returns true if found and removed. + + + + + + ext_vacuum_statistics.add_track_relation(dboid oid, reloid oid) + boolean + + + + Adds a (database, relation) OID pair to the tracking list. Returns true if newly added. + + + + + + ext_vacuum_statistics.remove_track_relation(dboid oid, reloid oid) + boolean + + + + Removes a (database, relation) pair from the tracking list. Returns true if found and removed. + + + + + + ext_vacuum_statistics.track_list() + TABLE(track_kind text, dboid oid, reloid oid) + + + + Returns the list of database and relation OIDs for which vacuum statistics + are collected. When dboid or + reloid is NULL, statistics are collected for all. + + + + + + + + Configuration Parameters + + + + vacuum_statistics.enabled (boolean) + + + Enables extended vacuum statistics collection. Default: on. + + + + + vacuum_statistics.object_types (string) + + + Object types for statistics: all, databases, or + relations. Default: all. + + + + + vacuum_statistics.track_relations (string) + + + When tracking relations: all, system, or + user. Default: all. + + + + + vacuum_statistics.track_databases_from_list (boolean) + + + If on, track only databases added via add_track_database. + Default: off. + + + + + vacuum_statistics.track_relations_from_list (boolean) + + + If on, track only relations added via add_track_relation. + Default: off. + + + + + + diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index ac66fcbdb57..b03257c6973 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -133,6 +133,7 @@ + diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 04b087e2a5c..d20d5dddcc0 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -601,6 +601,65 @@ extvac_stats_end_idx(Relation rel, IndexBulkDeleteResult *stats, } } +/* + * Accumulate index stats into vacrel for later subtraction from heap stats. + * It needs to prevent double-counting of stats for heaps that + * include indexes because indexes are vacuumed before the heap. + * We need to be careful with buffer usage and wal usage during parallel vacuum + * because they are accumulated summarly for all indexes at once by leader after + * all workers have finished. + */ +static void +accumulate_idxs_vacuum_statistics(LVRelState *vacrel, + PgStat_VacuumRelationCounts * extVacIdxStats) +{ + vacrel->extVacReportIdx.common.blk_read_time += extVacIdxStats->common.blk_read_time; + vacrel->extVacReportIdx.common.blk_write_time += extVacIdxStats->common.blk_write_time; + vacrel->extVacReportIdx.common.total_blks_dirtied += extVacIdxStats->common.total_blks_dirtied; + vacrel->extVacReportIdx.common.total_blks_hit += extVacIdxStats->common.total_blks_hit; + vacrel->extVacReportIdx.common.total_blks_read += extVacIdxStats->common.total_blks_read; + vacrel->extVacReportIdx.common.total_blks_written += extVacIdxStats->common.total_blks_written; + vacrel->extVacReportIdx.common.wal_bytes += extVacIdxStats->common.wal_bytes; + vacrel->extVacReportIdx.common.wal_fpi += extVacIdxStats->common.wal_fpi; + vacrel->extVacReportIdx.common.wal_records += extVacIdxStats->common.wal_records; + vacrel->extVacReportIdx.common.delay_time += extVacIdxStats->common.delay_time; + vacrel->extVacReportIdx.common.total_time += extVacIdxStats->common.total_time; +} + +/* Build heap-specific extended stats */ +static void +accumulate_heap_vacuum_statistics(LVRelState *vacrel, PgStat_VacuumRelationCounts * extVacStats) +{ + extVacStats->type = PGSTAT_EXTVAC_TABLE; + extVacStats->table.pages_scanned = vacrel->scanned_pages; + extVacStats->table.pages_removed = vacrel->removed_pages; + extVacStats->table.vm_new_frozen_pages = vacrel->new_all_frozen_pages; + extVacStats->table.vm_new_visible_pages = vacrel->new_all_visible_pages; + extVacStats->table.vm_new_visible_frozen_pages = vacrel->new_all_visible_all_frozen_pages; + extVacStats->common.tuples_deleted = vacrel->tuples_deleted; + extVacStats->table.tuples_frozen = vacrel->tuples_frozen; + extVacStats->table.recently_dead_tuples = vacrel->recently_dead_tuples; + extVacStats->table.missed_dead_tuples = vacrel->missed_dead_tuples; + extVacStats->table.missed_dead_pages = vacrel->missed_dead_pages; + extVacStats->table.index_vacuum_count = vacrel->num_index_scans; + extVacStats->common.wraparound_failsafe_count = vacrel->wraparound_failsafe_count; + + /* Hook is invoked from pgstat_report_vacuum() when extstats is passed */ + + /* Subtract index stats from heap to avoid double-counting */ + extVacStats->common.blk_read_time -= vacrel->extVacReportIdx.common.blk_read_time; + extVacStats->common.blk_write_time -= vacrel->extVacReportIdx.common.blk_write_time; + extVacStats->common.total_blks_dirtied -= vacrel->extVacReportIdx.common.total_blks_dirtied; + extVacStats->common.total_blks_hit -= vacrel->extVacReportIdx.common.total_blks_hit; + extVacStats->common.total_blks_read -= vacrel->extVacReportIdx.common.total_blks_read; + extVacStats->common.total_blks_written -= vacrel->extVacReportIdx.common.total_blks_written; + extVacStats->common.wal_bytes -= vacrel->extVacReportIdx.common.wal_bytes; + extVacStats->common.wal_fpi -= vacrel->extVacReportIdx.common.wal_fpi; + extVacStats->common.wal_records -= vacrel->extVacReportIdx.common.wal_records; + extVacStats->common.total_time -= vacrel->extVacReportIdx.common.total_time; + extVacStats->common.delay_time -= vacrel->extVacReportIdx.common.delay_time; +} + /* * Helper to set up the eager scanning state for vacuuming a single relation. * Initializes the eager scan management related members of the LVRelState. @@ -778,7 +837,8 @@ heap_vacuum_rel(Relation rel, const VacuumParams params, /* Used for instrumentation and stats report */ starttime = GetCurrentTimestamp(); - extvac_stats_start(rel, &extVacCounters); + if (set_report_vacuum_hook) + extvac_stats_start(rel, &extVacCounters); pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); @@ -1094,11 +1154,25 @@ heap_vacuum_rel(Relation rel, const VacuumParams params, * soon in cases where the failsafe prevented significant amounts of heap * vacuuming. */ - pgstat_report_vacuum(rel, - Max(vacrel->new_live_tuples, 0), - vacrel->recently_dead_tuples + - vacrel->missed_dead_tuples, - starttime); + if (set_report_vacuum_hook) + { + extvac_stats_end(rel, &extVacCounters, &extVacReport.common); + accumulate_heap_vacuum_statistics(vacrel, &extVacReport); + + pgstat_report_vacuum_ext(rel, + Max(vacrel->new_live_tuples, 0), + vacrel->recently_dead_tuples + + vacrel->missed_dead_tuples, + starttime, + &extVacReport); + } + else + pgstat_report_vacuum_ext(rel, + Max(vacrel->new_live_tuples, 0), + vacrel->recently_dead_tuples + + vacrel->missed_dead_tuples, + starttime, + NULL); pgstat_progress_end_command(); @@ -3256,8 +3330,8 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, LVExtStatCountersIdx extVacCounters; PgStat_VacuumRelationCounts extVacReport; - extvac_stats_start_idx(indrel, istat, &extVacCounters); - + if (set_report_vacuum_hook) + extvac_stats_start_idx(indrel, istat, &extVacCounters); ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; ivinfo.analyze_only = false; @@ -3284,8 +3358,13 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, istat = vac_bulkdel_one_index(&ivinfo, istat, vacrel->dead_items, vacrel->dead_items_info); - memset(&extVacReport, 0, sizeof(extVacReport)); - extvac_stats_end_idx(indrel, istat, &extVacCounters, &extVacReport); + if (set_report_vacuum_hook) + { + memset(&extVacReport, 0, sizeof(extVacReport)); + extvac_stats_end_idx(indrel, istat, &extVacCounters, &extVacReport); + pgstat_report_vacuum_ext(indrel, -1, -1, 0, &extVacReport); + accumulate_idxs_vacuum_statistics(vacrel, &extVacReport); + } /* Revert to the previous phase information for error traceback */ restore_vacuum_error_info(vacrel, &saved_err_info); @@ -3314,8 +3393,8 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, LVExtStatCountersIdx extVacCounters; PgStat_VacuumRelationCounts extVacReport; - extvac_stats_start_idx(indrel, istat, &extVacCounters); - + if (set_report_vacuum_hook) + extvac_stats_start_idx(indrel, istat, &extVacCounters); ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; ivinfo.analyze_only = false; @@ -3341,8 +3420,13 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, istat = vac_cleanup_one_index(&ivinfo, istat); - memset(&extVacReport, 0, sizeof(extVacReport)); - extvac_stats_end_idx(indrel, istat, &extVacCounters, &extVacReport); + if (set_report_vacuum_hook) + { + memset(&extVacReport, 0, sizeof(extVacReport)); + extvac_stats_end_idx(indrel, istat, &extVacCounters, &extVacReport); + pgstat_report_vacuum_ext(indrel, -1, -1, 0, &extVacReport); + accumulate_idxs_vacuum_statistics(vacrel, &extVacReport); + } /* Revert to the previous phase information for error traceback */ restore_vacuum_error_info(vacrel, &saved_err_info); diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c index 7a85c644749..d0426e228b4 100644 --- a/src/backend/commands/vacuumparallel.c +++ b/src/backend/commands/vacuumparallel.c @@ -879,8 +879,8 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, if (indstats->istat_updated) istat = &(indstats->istat); - extvac_stats_start_idx(indrel, istat, &extVacCounters); - + if (set_report_vacuum_hook) + extvac_stats_start_idx(indrel, istat, &extVacCounters); ivinfo.index = indrel; ivinfo.heaprel = pvs->heaprel; ivinfo.analyze_only = false; @@ -909,8 +909,12 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, RelationGetRelationName(indrel)); } - memset(&extVacReport, 0, sizeof(extVacReport)); - extvac_stats_end_idx(indrel, istat_res, &extVacCounters, &extVacReport); + if (set_report_vacuum_hook) + { + memset(&extVacReport, 0, sizeof(extVacReport)); + extvac_stats_end_idx(indrel, istat_res, &extVacCounters, &extVacReport); + pgstat_report_vacuum_ext(indrel, -1, -1, 0, &extVacReport); + } /* * Copy the index bulk-deletion result returned from ambulkdelete and diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index 885d590d2b2..f0db10803d5 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -271,6 +271,30 @@ pgstat_report_vacuum(Relation rel, PgStat_Counter livetuples, (void) pgstat_flush_backend(false, PGSTAT_BACKEND_FLUSH_IO); } +/* + * Hook for extensions to receive extended vacuum statistics. + * NULL when no extension has registered. + */ +set_report_vacuum_hook_type set_report_vacuum_hook = NULL; + +/* + * Report extended vacuum statistics to extensions via set_report_vacuum_hook. + * When livetuples/deadtuples/starttime are provided (heap case), also calls + * pgstat_report_vacuum. For indexes, pass -1, -1, 0 to skip pgstat_report_vacuum. + */ +void +pgstat_report_vacuum_ext(Relation rel, PgStat_Counter livetuples, + PgStat_Counter deadtuples, TimestampTz starttime, + PgStat_VacuumRelationCounts * extstats) +{ + pgstat_report_vacuum(rel, livetuples, deadtuples, starttime); + + if (extstats != NULL && set_report_vacuum_hook) + (*set_report_vacuum_hook) (RelationGetRelid(rel), + rel->rd_rel->relisshared, + extstats); +} + /* * Report that the table was just analyzed and flush IO statistics. * diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index c50ce51e9da..09f7775b85e 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -23,6 +23,7 @@ #include "catalog/pg_type.h" #include "parser/parse_node.h" #include "storage/buf.h" +#include "executor/instrument.h" #include "storage/lock.h" #include "utils/relcache.h" #include "pgstat.h" diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 7fe8e5468b8..1013a52de6e 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -93,6 +93,7 @@ typedef struct PgStat_FunctionCounts * Working state needed to accumulate per-function-call timing statistics. */ /* + * Extended vacuum statistics - passed to extensions via set_report_vacuum_hook. * Type of entry: table (heap), index, or database aggregate. */ typedef enum ExtVacReportType @@ -738,6 +739,16 @@ extern void pgstat_report_vacuum(Relation rel, PgStat_Counter livetuples, PgStat_Counter deadtuples, TimestampTz starttime); +extern void pgstat_report_vacuum_ext(Relation rel, + PgStat_Counter livetuples, + PgStat_Counter deadtuples, + TimestampTz starttime, + PgStat_VacuumRelationCounts * extstats); + +/* Hook for extensions to receive extended vacuum statistics */ +typedef void (*set_report_vacuum_hook_type) (Oid tableoid, bool shared, + PgStat_VacuumRelationCounts * params); +extern PGDLLIMPORT set_report_vacuum_hook_type set_report_vacuum_hook; extern void pgstat_report_analyze(Relation rel, PgStat_Counter livetuples, PgStat_Counter deadtuples, bool resetcounter, TimestampTz starttime); -- 2.39.5 (Apple Git-154)