From cf8285d7557582d6995d58ca62599e7e47b20b1b Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Tue, 28 Apr 2026 03:43:29 +0300 Subject: [PATCH 3/3] ext_vacuum_statistics: extension for extended vacuum statistics Introduce a new extension that collects extended per-vacuum metrics via set_report_vacuum_hook and stores them through pgstat's custom statistics infrastructure. Tracking scope is controlled by GUCs: * vacuum_statistics.enabled -- master switch * vacuum_statistics.object_types -- databases / relations / all * vacuum_statistics.track_relations -- system / user / all * vacuum_statistics.track_{databases,relations}_from_list -- restrict tracking to objects registered via add_track_database() / add_track_relation(); removal via remove_track_*() and OAT_DROP hook * vacuum_statistics.collect -- buffers / wal / general / timing / all, consulted by ACCUM_IF() to skip unwanted categories at run time add_track_* / remove_track_* require superuser or pg_read_all_stats. --- contrib/Makefile | 1 + contrib/ext_vacuum_statistics/Makefile | 24 + contrib/ext_vacuum_statistics/README.md | 165 ++ .../expected/ext_vacuum_statistics.out | 52 + .../vacuum-extending-in-repetable-read.out | 52 + .../ext_vacuum_statistics--1.0.sql | 272 ++++ .../ext_vacuum_statistics.conf | 2 + .../ext_vacuum_statistics.control | 5 + contrib/ext_vacuum_statistics/meson.build | 41 + .../vacuum-extending-in-repetable-read.spec | 59 + .../t/052_vacuum_extending_basic_test.pl | 780 +++++++++ .../t/053_vacuum_extending_freeze_test.pl | 285 ++++ .../t/054_vacuum_extending_gucs_test.pl | 279 ++++ .../ext_vacuum_statistics/vacuum_statistics.c | 1387 +++++++++++++++++ contrib/meson.build | 1 + doc/src/sgml/contrib.sgml | 1 + doc/src/sgml/extvacuumstatistics.sgml | 502 ++++++ doc/src/sgml/filelist.sgml | 1 + 18 files changed, 3909 insertions(+) create mode 100644 contrib/ext_vacuum_statistics/Makefile create mode 100644 contrib/ext_vacuum_statistics/README.md create mode 100644 contrib/ext_vacuum_statistics/expected/ext_vacuum_statistics.out create mode 100644 contrib/ext_vacuum_statistics/expected/vacuum-extending-in-repetable-read.out create mode 100644 contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql create mode 100644 contrib/ext_vacuum_statistics/ext_vacuum_statistics.conf create mode 100644 contrib/ext_vacuum_statistics/ext_vacuum_statistics.control create mode 100644 contrib/ext_vacuum_statistics/meson.build create mode 100644 contrib/ext_vacuum_statistics/specs/vacuum-extending-in-repetable-read.spec create mode 100644 contrib/ext_vacuum_statistics/t/052_vacuum_extending_basic_test.pl create mode 100644 contrib/ext_vacuum_statistics/t/053_vacuum_extending_freeze_test.pl create mode 100644 contrib/ext_vacuum_statistics/t/054_vacuum_extending_gucs_test.pl create mode 100644 contrib/ext_vacuum_statistics/vacuum_statistics.c create mode 100644 doc/src/sgml/extvacuumstatistics.sgml diff --git a/contrib/Makefile b/contrib/Makefile index 7d91fe77db3..3140f2bf844 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -19,6 +19,7 @@ SUBDIRS = \ dict_int \ dict_xsyn \ earthdistance \ + ext_vacuum_statistics \ file_fdw \ fuzzystrmatch \ hstore \ diff --git a/contrib/ext_vacuum_statistics/Makefile b/contrib/ext_vacuum_statistics/Makefile new file mode 100644 index 00000000000..ed80bdf28d0 --- /dev/null +++ b/contrib/ext_vacuum_statistics/Makefile @@ -0,0 +1,24 @@ +# contrib/ext_vacuum_statistics/Makefile + +EXTENSION = ext_vacuum_statistics +MODULE_big = ext_vacuum_statistics +OBJS = vacuum_statistics.o +DATA = ext_vacuum_statistics--1.0.sql +PGFILEDESC = "ext_vacuum_statistics - convenience views for extended vacuum statistics" + +ISOLATION = vacuum-extending-in-repetable-read +ISOLATION_OPTS = --temp-config=$(top_srcdir)/contrib/ext_vacuum_statistics/ext_vacuum_statistics.conf +TAP_TESTS = 1 + +NO_INSTALLCHECK = 1 + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/ext_vacuum_statistics +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/contrib/ext_vacuum_statistics/README.md b/contrib/ext_vacuum_statistics/README.md new file mode 100644 index 00000000000..51697eab023 --- /dev/null +++ b/contrib/ext_vacuum_statistics/README.md @@ -0,0 +1,165 @@ +# ext_vacuum_statistics + +Extended vacuum statistics extension for PostgreSQL. It collects and exposes detailed per-table, per-index, and per-database vacuum statistics (buffer I/O, WAL, general, timing) via convenient views in the `ext_vacuum_statistics` schema. + +## Installation + +``` +./configure tmp_install="$(pwd)/my/inst" +make clean && make && make install +cd contrib/ext_vacuum_statistics +make && make install +``` + +It is essential that the extension is listed in `shared_preload_libraries` because it registers a vacuum hook at server startup. + +In your `postgresql.conf`: + +``` +shared_preload_libraries = 'ext_vacuum_statistics' +``` + +Restart PostgreSQL. + +In your database: + +```sql +CREATE EXTENSION ext_vacuum_statistics; +``` + +## Usage + +Query vacuum statistics via the provided views: + +```sql +-- Per-table heap vacuum statistics +SELECT * FROM ext_vacuum_statistics.pg_stats_vacuum_tables; + +-- Per-index vacuum statistics +SELECT * FROM ext_vacuum_statistics.pg_stats_vacuum_indexes; + +-- Per-database aggregate vacuum statistics +SELECT * FROM ext_vacuum_statistics.pg_stats_vacuum_database; +``` + +Example output: + +``` + relname | total_blks_read | total_blks_hit | wal_records | tuples_deleted | pages_removed +-----------+-----------------+----------------+-------------+----------------+--------------- + mytable | 120 | 340 | 15 | 500 | 10 +``` + +Reset statistics when needed: + +```sql +SELECT ext_vacuum_statistics.vacuum_statistics_reset(); +``` + +## Configuration (GUCs) + +| GUC | Default | Description | +|-----|---------|-------------| +| `vacuum_statistics.enabled` | on | Enable extended vacuum statistics collection | +| `vacuum_statistics.object_types` | all | Object types for statistics: `all`, `databases`, `relations` | +| `vacuum_statistics.track_relations` | all | When tracking relations: `all`, `system`, `user` | +| `vacuum_statistics.track_databases_from_list` | off | If on, track only databases added via add_track_database | +| `vacuum_statistics.track_relations_from_list` | off | If on, track only relations added via add_track_relation | + +## Memory usage + +Each tracked object (table, index, or database) uses approximately **232 bytes** of shared memory on Linux x86_64 (e.g. Ubuntu): common stats (buffers, WAL, timing) ~144 bytes; type + union ~88 bytes (union holds table-specific or index-specific fields, allocated size is the same for both). + +The exact size depends on the platform. Call `ext_vacuum_statistics.shared_memory_size()` to get the total shared memory used by the extension. The GUCs provided by the extension allow controlling the amount of memory used: `vacuum_statistics.object_types` to track only databases or relations, `vacuum_statistics.track_relations` to restrict to user or system tables/indexes, and `track_*_from_list` to track only selected databases and relations. + +Example: a database with 1000 tables and 2000 indexes, all tracked, uses about **700 KB** on Ubuntu (3001 entries × 232 bytes). Per-database entries add one entry per tracked database. + +## Advanced tuning + +### Track only database-level stats + +```sql +SET vacuum_statistics.object_types = 'databases'; +``` + +Statistics are accumulated per database; per-relation views remain empty. + +### Track only user or system tables + +```sql +SET vacuum_statistics.object_types = 'relations'; +SET vacuum_statistics.track_relations = 'user'; -- skip system catalogs +-- or +SET vacuum_statistics.track_relations = 'system'; -- only system catalogs +``` + +### Filter by database or relation OIDs + +Add OIDs via functions (persisted to `pg_stat/ext_vacuum_statistics_track.oid`) and enable filtering: + +```sql +-- Add databases and relations to track +SELECT ext_vacuum_statistics.add_track_database(16384); +SELECT ext_vacuum_statistics.add_track_relation(16384, 16385); -- dboid, reloid +SELECT ext_vacuum_statistics.add_track_relation(0, 16386); -- rel 16386 in any db + +-- Enable list-based filtering (off = track all) +SET vacuum_statistics.track_databases_from_list = on; +SET vacuum_statistics.track_relations_from_list = on; +``` + +Remove OIDs when no longer needed: + +```sql +SELECT ext_vacuum_statistics.remove_track_database(16384); +SELECT ext_vacuum_statistics.remove_track_relation(16384, 16385); +``` + +Inspect the current tracking configuration: + +```sql +SELECT * FROM ext_vacuum_statistics.track_list(); +``` + +Returns `track_kind`, `dboid`, `reloid`. When `dboid` or `reloid` is NULL, statistics are collected for all. + +## Recipes + +**Reduce overhead by tracking only databases:** + +```sql +SET vacuum_statistics.object_types = 'databases'; +``` + +**Track only a specific table in a specific database:** + +```sql +SELECT ext_vacuum_statistics.add_track_database( + (SELECT oid FROM pg_database WHERE datname = current_database()) +); +SELECT ext_vacuum_statistics.add_track_relation( + (SELECT oid FROM pg_database WHERE datname = current_database()), + 'mytable'::regclass +); +SET vacuum_statistics.track_databases_from_list = on; +SET vacuum_statistics.track_relations_from_list = on; +``` + +**Disable statistics collection temporarily:** + +```sql +SET vacuum_statistics.enabled = off; +``` + +## Views + +| View | Description | +|------|-------------| +| `ext_vacuum_statistics.pg_stats_vacuum_tables` | Per-table heap vacuum stats (pages scanned, tuples deleted, WAL, timing, etc.) | +| `ext_vacuum_statistics.pg_stats_vacuum_indexes` | Per-index vacuum stats | +| `ext_vacuum_statistics.pg_stats_vacuum_database` | Per-database aggregate vacuum stats | + +## Limitations + +- Must be loaded via `shared_preload_libraries`; it cannot be loaded on demand. +- Tracking configuration (`add_track_*`, `remove_track_*`) is stored in a file and shared across all databases in the cluster. diff --git a/contrib/ext_vacuum_statistics/expected/ext_vacuum_statistics.out b/contrib/ext_vacuum_statistics/expected/ext_vacuum_statistics.out new file mode 100644 index 00000000000..89c9594dea8 --- /dev/null +++ b/contrib/ext_vacuum_statistics/expected/ext_vacuum_statistics.out @@ -0,0 +1,52 @@ +-- ext_vacuum_statistics regression test + +-- Create extension +CREATE EXTENSION ext_vacuum_statistics; + +-- Verify schema and views exist +SELECT nspname FROM pg_namespace WHERE nspname = 'ext_vacuum_statistics'; + nspname +------------------ + ext_vacuum_statistics +(1 row) + +-- Views should be queryable (may return empty if no vacuum has run) +SELECT COUNT(*) >= 0 FROM ext_vacuum_statistics.pg_stats_vacuum_tables; + ?column? +---------- + t +(1 row) + +SELECT COUNT(*) >= 0 FROM ext_vacuum_statistics.pg_stats_vacuum_indexes; + ?column? +---------- + t +(1 row) + +SELECT COUNT(*) >= 0 FROM ext_vacuum_statistics.pg_stats_vacuum_database; + ?column? +---------- + t +(1 row) + +-- Verify views have expected columns +SELECT COUNT(*) AS tables_cols FROM information_schema.columns +WHERE table_schema = 'ext_vacuum_statistics' AND table_name = 'tables'; + tables_cols +------------- + 28 +(1 row) + +SELECT COUNT(*) AS indexes_cols FROM information_schema.columns +WHERE table_schema = 'ext_vacuum_statistics' AND table_name = 'indexes'; + indexes_cols +-------------- + 20 +(1 row) + +SELECT COUNT(*) AS database_cols FROM information_schema.columns +WHERE table_schema = 'ext_vacuum_statistics' AND table_name = 'database'; + database_cols +--------------- + 15 +(1 row) diff --git a/contrib/ext_vacuum_statistics/expected/vacuum-extending-in-repetable-read.out b/contrib/ext_vacuum_statistics/expected/vacuum-extending-in-repetable-read.out new file mode 100644 index 00000000000..6b381f9d232 --- /dev/null +++ b/contrib/ext_vacuum_statistics/expected/vacuum-extending-in-repetable-read.out @@ -0,0 +1,52 @@ +unused step name: s2_delete +Parsed test spec with 2 sessions + +starting permutation: s2_insert s2_print_vacuum_stats_table s1_begin_repeatable_read s2_update s2_insert_interrupt s2_vacuum s2_print_vacuum_stats_table s1_commit s2_checkpoint s2_vacuum s2_print_vacuum_stats_table +step s2_insert: INSERT INTO test_vacuum_stat_isolation(id, ival) SELECT ival, ival%10 FROM generate_series(1,1000) As ival; +step s2_print_vacuum_stats_table: + SELECT + vt.relname, vt.tuples_deleted, vt.recently_dead_tuples, vt.missed_dead_tuples, vt.missed_dead_pages, vt.tuples_frozen + FROM ext_vacuum_statistics.pg_stats_vacuum_tables vt, pg_class c + WHERE vt.relname = 'test_vacuum_stat_isolation' AND vt.relid = c.oid; + +relname|tuples_deleted|recently_dead_tuples|missed_dead_tuples|missed_dead_pages|tuples_frozen +-------+--------------+--------------------+------------------+-----------------+------------- +(0 rows) + +step s1_begin_repeatable_read: + BEGIN transaction ISOLATION LEVEL REPEATABLE READ; + select count(ival) from test_vacuum_stat_isolation where id>900; + +count +----- + 100 +(1 row) + +step s2_update: UPDATE test_vacuum_stat_isolation SET ival = ival + 2 where id > 900; +step s2_insert_interrupt: INSERT INTO test_vacuum_stat_isolation values (1,1); +step s2_vacuum: VACUUM test_vacuum_stat_isolation; +step s2_print_vacuum_stats_table: + SELECT + vt.relname, vt.tuples_deleted, vt.recently_dead_tuples, vt.missed_dead_tuples, vt.missed_dead_pages, vt.tuples_frozen + FROM ext_vacuum_statistics.pg_stats_vacuum_tables vt, pg_class c + WHERE vt.relname = 'test_vacuum_stat_isolation' AND vt.relid = c.oid; + +relname |tuples_deleted|recently_dead_tuples|missed_dead_tuples|missed_dead_pages|tuples_frozen +--------------------------+--------------+--------------------+------------------+-----------------+------------- +test_vacuum_stat_isolation| 0| 100| 0| 0| 0 +(1 row) + +step s1_commit: COMMIT; +step s2_checkpoint: CHECKPOINT; +step s2_vacuum: VACUUM test_vacuum_stat_isolation; +step s2_print_vacuum_stats_table: + SELECT + vt.relname, vt.tuples_deleted, vt.recently_dead_tuples, vt.missed_dead_tuples, vt.missed_dead_pages, vt.tuples_frozen + FROM ext_vacuum_statistics.pg_stats_vacuum_tables vt, pg_class c + WHERE vt.relname = 'test_vacuum_stat_isolation' AND vt.relid = c.oid; + +relname |tuples_deleted|recently_dead_tuples|missed_dead_tuples|missed_dead_pages|tuples_frozen +--------------------------+--------------+--------------------+------------------+-----------------+------------- +test_vacuum_stat_isolation| 100| 100| 0| 0| 101 +(1 row) + diff --git a/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql b/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql new file mode 100644 index 00000000000..aa3a9ec9699 --- /dev/null +++ b/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql @@ -0,0 +1,272 @@ +/*------------------------------------------------------------------------- + * + * ext_vacuum_statistics--1.0.sql + * Extended vacuum statistics via hook and custom storage + * + * This extension collects extended vacuum statistics via set_report_vacuum_hook + * and stores them in shared memory. + * + *------------------------------------------------------------------------- + */ + +\echo Use "CREATE EXTENSION ext_vacuum_statistics" to load this file. \quit + +CREATE SCHEMA IF NOT EXISTS ext_vacuum_statistics; + +COMMENT ON SCHEMA ext_vacuum_statistics IS + 'Extended vacuum statistics (heap, index, database)'; + +-- Reset functions +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.extvac_reset_entry( + dboid oid, + relid oid, + type int4 +) +RETURNS boolean +AS 'MODULE_PATHNAME', 'extvac_reset_entry' +LANGUAGE C STRICT PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.extvac_reset_db_entry(dboid oid) +RETURNS bigint +AS 'MODULE_PATHNAME', 'extvac_reset_db_entry' +LANGUAGE C STRICT PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.vacuum_statistics_reset() +RETURNS bigint +AS 'MODULE_PATHNAME', 'vacuum_statistics_reset' +LANGUAGE C STRICT PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.shared_memory_size() +RETURNS bigint +AS 'MODULE_PATHNAME', 'extvac_shared_memory_size' +LANGUAGE C STRICT PARALLEL SAFE; + +COMMENT ON FUNCTION ext_vacuum_statistics.shared_memory_size() IS + 'Total shared memory in bytes used by the extension for vacuum statistics.'; + +-- Add/remove OIDs for tracking +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.add_track_database(dboid oid) +RETURNS boolean +AS 'MODULE_PATHNAME', 'evs_add_track_database' +LANGUAGE C STRICT; + +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.remove_track_database(dboid oid) +RETURNS boolean +AS 'MODULE_PATHNAME', 'evs_remove_track_database' +LANGUAGE C STRICT; + +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.add_track_relation(dboid oid, reloid oid) +RETURNS boolean +AS 'MODULE_PATHNAME', 'evs_add_track_relation' +LANGUAGE C STRICT; + +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.remove_track_relation(dboid oid, reloid oid) +RETURNS boolean +AS 'MODULE_PATHNAME', 'evs_remove_track_relation' +LANGUAGE C STRICT; + +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.track_list() +RETURNS TABLE(track_kind text, dboid oid, reloid oid) +AS 'MODULE_PATHNAME', 'evs_track_list' +LANGUAGE C STRICT; + +COMMENT ON FUNCTION ext_vacuum_statistics.track_list() IS + 'List of database and relation OIDs for which vacuum statistics are collected.'; + +-- Track-list mutation requires superuser or pg_read_all_stats; hide the +-- functions from PUBLIC so the error is also produced for ordinary users +-- before the C-level privilege check runs. +REVOKE ALL ON FUNCTION ext_vacuum_statistics.add_track_database(oid) FROM PUBLIC; +REVOKE ALL ON FUNCTION ext_vacuum_statistics.remove_track_database(oid) FROM PUBLIC; +REVOKE ALL ON FUNCTION ext_vacuum_statistics.add_track_relation(oid, oid) FROM PUBLIC; +REVOKE ALL ON FUNCTION ext_vacuum_statistics.remove_track_relation(oid, oid) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION ext_vacuum_statistics.add_track_database(oid) TO pg_read_all_stats; +GRANT EXECUTE ON FUNCTION ext_vacuum_statistics.remove_track_database(oid) TO pg_read_all_stats; +GRANT EXECUTE ON FUNCTION ext_vacuum_statistics.add_track_relation(oid, oid) TO pg_read_all_stats; +GRANT EXECUTE ON FUNCTION ext_vacuum_statistics.remove_track_relation(oid, oid) TO pg_read_all_stats; + +-- Internal C function to fetch table vacuum stats +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.pg_stats_get_vacuum_tables( + IN dboid oid, + IN reloid oid, + OUT relid oid, + OUT total_blks_read bigint, + OUT total_blks_hit bigint, + OUT total_blks_dirtied bigint, + OUT total_blks_written bigint, + OUT wal_records bigint, + OUT wal_fpi bigint, + OUT wal_bytes numeric, + OUT blk_read_time double precision, + OUT blk_write_time double precision, + OUT delay_time double precision, + OUT total_time double precision, + OUT wraparound_failsafe_count integer, + OUT rel_blks_read bigint, + OUT rel_blks_hit bigint, + OUT tuples_deleted bigint, + OUT pages_scanned bigint, + OUT pages_removed bigint, + OUT vm_new_frozen_pages bigint, + OUT vm_new_visible_pages bigint, + OUT vm_new_visible_frozen_pages bigint, + OUT tuples_frozen bigint, + OUT recently_dead_tuples bigint, + OUT index_vacuum_count bigint, + OUT missed_dead_pages bigint, + OUT missed_dead_tuples bigint +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_stats_get_vacuum_tables' +LANGUAGE C STRICT STABLE; + +-- Internal C function to fetch index vacuum stats +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.pg_stats_get_vacuum_indexes( + IN dboid oid, + IN reloid oid, + OUT relid oid, + OUT total_blks_read bigint, + OUT total_blks_hit bigint, + OUT total_blks_dirtied bigint, + OUT total_blks_written bigint, + OUT wal_records bigint, + OUT wal_fpi bigint, + OUT wal_bytes numeric, + OUT blk_read_time double precision, + OUT blk_write_time double precision, + OUT delay_time double precision, + OUT total_time double precision, + OUT wraparound_failsafe_count integer, + OUT rel_blks_read bigint, + OUT rel_blks_hit bigint, + OUT tuples_deleted bigint, + OUT pages_deleted bigint +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_stats_get_vacuum_indexes' +LANGUAGE C STRICT STABLE; + +-- Internal C function to fetch database vacuum stats +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.pg_stats_get_vacuum_database( + IN dboid oid, + OUT dbid oid, + OUT total_blks_read bigint, + OUT total_blks_hit bigint, + OUT total_blks_dirtied bigint, + OUT total_blks_written bigint, + OUT wal_records bigint, + OUT wal_fpi bigint, + OUT wal_bytes numeric, + OUT blk_read_time double precision, + OUT blk_write_time double precision, + OUT delay_time double precision, + OUT total_time double precision, + OUT wraparound_failsafe_count integer, + OUT interrupts_count integer +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_stats_get_vacuum_database' +LANGUAGE C STRICT STABLE; + +-- View: vacuum statistics per table (heap) +CREATE VIEW ext_vacuum_statistics.pg_stats_vacuum_tables AS +SELECT + rel.oid AS relid, + ns.nspname AS schema, + rel.relname AS relname, + db.datname AS dbname, + stats.total_blks_read, + stats.total_blks_hit, + stats.total_blks_dirtied, + stats.total_blks_written, + stats.wal_records, + stats.wal_fpi, + stats.wal_bytes, + stats.blk_read_time, + stats.blk_write_time, + stats.delay_time, + stats.total_time, + stats.wraparound_failsafe_count, + stats.rel_blks_read, + stats.rel_blks_hit, + stats.tuples_deleted, + stats.pages_scanned, + stats.pages_removed, + stats.vm_new_frozen_pages, + stats.vm_new_visible_pages, + stats.vm_new_visible_frozen_pages, + stats.tuples_frozen, + stats.recently_dead_tuples, + stats.index_vacuum_count, + stats.missed_dead_pages, + stats.missed_dead_tuples +FROM pg_database db, + pg_class rel, + pg_namespace ns, + LATERAL ext_vacuum_statistics.pg_stats_get_vacuum_tables(db.oid, rel.oid) stats +WHERE db.datname = current_database() + AND rel.relkind = 'r' + AND rel.relnamespace = ns.oid + AND rel.oid = stats.relid; + +COMMENT ON VIEW ext_vacuum_statistics.pg_stats_vacuum_tables IS + 'Extended vacuum statistics per table (heap)'; + +-- View: vacuum statistics per index +CREATE VIEW ext_vacuum_statistics.pg_stats_vacuum_indexes AS +SELECT + rel.oid AS indexrelid, + ns.nspname AS schema, + rel.relname AS indexrelname, + db.datname AS dbname, + stats.total_blks_read, + stats.total_blks_hit, + stats.total_blks_dirtied, + stats.total_blks_written, + stats.wal_records, + stats.wal_fpi, + stats.wal_bytes, + stats.blk_read_time, + stats.blk_write_time, + stats.delay_time, + stats.total_time, + stats.wraparound_failsafe_count, + stats.rel_blks_read, + stats.rel_blks_hit, + stats.tuples_deleted, + stats.pages_deleted +FROM pg_database db, + pg_class rel, + pg_namespace ns, + LATERAL ext_vacuum_statistics.pg_stats_get_vacuum_indexes(db.oid, rel.oid) stats +WHERE db.datname = current_database() + AND rel.relkind = 'i' + AND rel.relnamespace = ns.oid + AND rel.oid = stats.relid; + +COMMENT ON VIEW ext_vacuum_statistics.pg_stats_vacuum_indexes IS + 'Extended vacuum statistics per index'; + +-- View: vacuum statistics per database (aggregate) +CREATE VIEW ext_vacuum_statistics.pg_stats_vacuum_database AS +SELECT + db.oid AS dboid, + db.datname AS dbname, + stats.total_blks_read AS db_blks_read, + stats.total_blks_hit AS db_blks_hit, + stats.total_blks_dirtied AS db_blks_dirtied, + stats.total_blks_written AS db_blks_written, + stats.wal_records AS db_wal_records, + stats.wal_fpi AS db_wal_fpi, + stats.wal_bytes AS db_wal_bytes, + stats.blk_read_time AS db_blk_read_time, + stats.blk_write_time AS db_blk_write_time, + stats.delay_time AS db_delay_time, + stats.total_time AS db_total_time, + stats.wraparound_failsafe_count AS db_wraparound_failsafe_count, + stats.interrupts_count +FROM pg_database db +LEFT JOIN LATERAL ext_vacuum_statistics.pg_stats_get_vacuum_database(db.oid) stats ON db.oid = stats.dbid; + +COMMENT ON VIEW ext_vacuum_statistics.pg_stats_vacuum_database IS + 'Extended vacuum statistics per database (aggregate)'; diff --git a/contrib/ext_vacuum_statistics/ext_vacuum_statistics.conf b/contrib/ext_vacuum_statistics/ext_vacuum_statistics.conf new file mode 100644 index 00000000000..9b711487623 --- /dev/null +++ b/contrib/ext_vacuum_statistics/ext_vacuum_statistics.conf @@ -0,0 +1,2 @@ +# Config for ext_vacuum_statistics regression tests +shared_preload_libraries = 'ext_vacuum_statistics' diff --git a/contrib/ext_vacuum_statistics/ext_vacuum_statistics.control b/contrib/ext_vacuum_statistics/ext_vacuum_statistics.control new file mode 100644 index 00000000000..518350a64b7 --- /dev/null +++ b/contrib/ext_vacuum_statistics/ext_vacuum_statistics.control @@ -0,0 +1,5 @@ +# ext_vacuum_statistics extension +comment = 'Extended vacuum statistics via hook (requires shared_preload_libraries)' +default_version = '1.0' +relocatable = true +module_pathname = '$libdir/ext_vacuum_statistics' diff --git a/contrib/ext_vacuum_statistics/meson.build b/contrib/ext_vacuum_statistics/meson.build new file mode 100644 index 00000000000..72338baa500 --- /dev/null +++ b/contrib/ext_vacuum_statistics/meson.build @@ -0,0 +1,41 @@ +# Copyright (c) 2022-2026, PostgreSQL Global Development Group +# +# ext_vacuum_statistics - extended vacuum statistics via hook +# Requires shared_preload_libraries = 'ext_vacuum_statistics' + +ext_vacuum_statistics_sources = files( + 'vacuum_statistics.c', +) + +ext_vacuum_statistics = shared_module('ext_vacuum_statistics', + ext_vacuum_statistics_sources, + kwargs: contrib_mod_args + { + 'dependencies': contrib_mod_args['dependencies'], + }, +) +contrib_targets += ext_vacuum_statistics + +install_data( + 'ext_vacuum_statistics.control', + 'ext_vacuum_statistics--1.0.sql', + kwargs: contrib_data_args, +) + +tests += { + 'name': 'ext_vacuum_statistics', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'isolation': { + 'specs': [ + 'vacuum-extending-in-repetable-read', + ], + 'regress_args': ['--temp-config', files('ext_vacuum_statistics.conf')], + 'runningcheck': false, + }, + 'tap': { + 'tests': [ + 't/052_vacuum_extending_basic_test.pl', + 't/053_vacuum_extending_freeze_test.pl', + ], + }, +} diff --git a/contrib/ext_vacuum_statistics/specs/vacuum-extending-in-repetable-read.spec b/contrib/ext_vacuum_statistics/specs/vacuum-extending-in-repetable-read.spec new file mode 100644 index 00000000000..4891e248cca --- /dev/null +++ b/contrib/ext_vacuum_statistics/specs/vacuum-extending-in-repetable-read.spec @@ -0,0 +1,59 @@ +# Test for checking recently_dead_tuples, tuples_deleted and frozen tuples in ext_vacuum_statistics.pg_stats_vacuum_tables. +# recently_dead_tuples values are counted when vacuum hasn't cleared tuples because they were deleted recently. +# recently_dead_tuples aren't increased after releasing lock compared with tuples_deleted, which increased +# by the value of the cleared tuples that the vacuum managed to clear. + +setup +{ + CREATE TABLE test_vacuum_stat_isolation(id int, ival int) WITH (autovacuum_enabled = off); + CREATE EXTENSION ext_vacuum_statistics; + SET track_io_timing = on; +} + +teardown +{ + DROP EXTENSION ext_vacuum_statistics CASCADE; + DROP TABLE test_vacuum_stat_isolation CASCADE; + RESET track_io_timing; +} + +session s1 +setup { + SET track_io_timing = on; +} +step s1_begin_repeatable_read { + BEGIN transaction ISOLATION LEVEL REPEATABLE READ; + select count(ival) from test_vacuum_stat_isolation where id>900; +} +step s1_commit { COMMIT; } + +session s2 +setup { + SET track_io_timing = on; +} +step s2_insert { INSERT INTO test_vacuum_stat_isolation(id, ival) SELECT ival, ival%10 FROM generate_series(1,1000) As ival; } +step s2_update { UPDATE test_vacuum_stat_isolation SET ival = ival + 2 where id > 900; } +step s2_delete { DELETE FROM test_vacuum_stat_isolation where id > 900; } +step s2_insert_interrupt { INSERT INTO test_vacuum_stat_isolation values (1,1); } +step s2_vacuum { VACUUM test_vacuum_stat_isolation; } +step s2_checkpoint { CHECKPOINT; } +step s2_print_vacuum_stats_table +{ + SELECT + vt.relname, vt.tuples_deleted, vt.recently_dead_tuples, vt.missed_dead_tuples, vt.missed_dead_pages, vt.tuples_frozen + FROM ext_vacuum_statistics.pg_stats_vacuum_tables vt, pg_class c + WHERE vt.relname = 'test_vacuum_stat_isolation' AND vt.relid = c.oid; +} + +permutation + s2_insert + s2_print_vacuum_stats_table + s1_begin_repeatable_read + s2_update + s2_insert_interrupt + s2_vacuum + s2_print_vacuum_stats_table + s1_commit + s2_checkpoint + s2_vacuum + s2_print_vacuum_stats_table diff --git a/contrib/ext_vacuum_statistics/t/052_vacuum_extending_basic_test.pl b/contrib/ext_vacuum_statistics/t/052_vacuum_extending_basic_test.pl new file mode 100644 index 00000000000..9463d5145f4 --- /dev/null +++ b/contrib/ext_vacuum_statistics/t/052_vacuum_extending_basic_test.pl @@ -0,0 +1,780 @@ +# Copyright (c) 2025 PostgreSQL Global Development Group +# Test cumulative vacuum stats system using TAP +# +# This test validates the accuracy and behavior of cumulative vacuum statistics +# across heap tables, indexes, and databases using: +# +# • ext_vacuum_statistics.pg_stats_vacuum_tables +# • ext_vacuum_statistics.pg_stats_vacuum_indexes +# • ext_vacuum_statistics.pg_stats_vacuum_database +# +# A polling helper function repeatedly checks the stats views until expected +# deltas appear or a configurable timeout expires. This guarantees that +# stats-collector propagation delays do not lead to flaky test behavior. + +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +#------------------------------------------------------------------------------ +# Test harness setup +#------------------------------------------------------------------------------ + +my $node = PostgreSQL::Test::Cluster->new('stat_vacuum'); +$node->init; + +# Configure the server: preload extension and logging level +$node->append_conf('postgresql.conf', q{ + shared_preload_libraries = 'ext_vacuum_statistics' + log_min_messages = notice +}); + +my $stderr; +my $base_stats; +my $wals; +my $ibase_stats; +my $iwals; + +$node->start( + '>' => \$base_stats, + '2>' => \$stderr +); + +#------------------------------------------------------------------------------ +# Database creation and initialization +#------------------------------------------------------------------------------ + +$node->safe_psql('postgres', q{ + CREATE DATABASE statistic_vacuum_database_regression; + CREATE EXTENSION ext_vacuum_statistics; +}); +# Main test database name and number of rows to insert +my $dbname = 'statistic_vacuum_database_regression'; +my $size_tab = 1000; + +# Enable required session settings and force the stats collector to flush next +$node->safe_psql($dbname, q{ + SET track_functions = 'all'; + SELECT pg_stat_force_next_flush(); +}); + +#------------------------------------------------------------------------------ +# Create test table and populate it +#------------------------------------------------------------------------------ + +$node->safe_psql( + $dbname, + "CREATE EXTENSION ext_vacuum_statistics; + CREATE TABLE vestat (x int PRIMARY KEY) + WITH (autovacuum_enabled = off, fillfactor = 10); + INSERT INTO vestat SELECT x FROM generate_series(1, $size_tab) AS g(x); + ANALYZE vestat;" +); + +#------------------------------------------------------------------------------ +# Timing parameters for polling loops +#------------------------------------------------------------------------------ + +my $timeout = 30; # overall wait timeout in seconds +my $interval = 0.015; # poll interval in seconds (15 ms) +my $start_time = time(); +my $updated = 0; + +#------------------------------------------------------------------------------ +# wait_for_vacuum_stats +# +# Polls ext_vacuum_statistics.pg_stats_vacuum_tables and ext_vacuum_statistics.pg_stats_vacuum_indexes until both the +# table-level and index-level counters exceed the provided baselines, or until +# the configured timeout elapses. +# +# Expected named args (baseline values): +# tab_tuples_deleted +# tab_wal_records +# idx_tuples_deleted +# idx_wal_records +# +# Returns: 1 if the condition is met before timeout, 0 otherwise. +#------------------------------------------------------------------------------ + +sub wait_for_vacuum_stats { + my (%args) = @_; + my $tab_tuples_deleted = ($args{tab_tuples_deleted} or 0); + my $tab_wal_records = ($args{tab_wal_records} or 0); + my $idx_tuples_deleted = ($args{idx_tuples_deleted} or 0); + my $idx_wal_records = ($args{idx_wal_records} or 0); + + my $start = time(); + while ((time() - $start) < $timeout) { + + my $result_query = $node->safe_psql( + $dbname, + "VACUUM vestat; + SELECT + (SELECT (tuples_deleted > $tab_tuples_deleted AND wal_records > $tab_wal_records) + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'vestat') + AND + (SELECT (tuples_deleted > $idx_tuples_deleted AND wal_records > $idx_wal_records) + FROM ext_vacuum_statistics.pg_stats_vacuum_indexes + WHERE indexrelname = 'vestat_pkey');" + ); + + return 1 if ($result_query eq 't'); + + sleep($interval); + } + + return 0; +} + +#------------------------------------------------------------------------------ +# Variables to hold vacuum-stat snapshots for later comparisons +#------------------------------------------------------------------------------ + +my $vm_new_visible_frozen_pages = 0; +my $tuples_deleted = 0; +my $pages_scanned = 0; +my $pages_removed = 0; +my $wal_records = 0; +my $wal_bytes = 0; +my $wal_fpi = 0; + +my $index_tuples_deleted = 0; +my $index_pages_deleted = 0; +my $index_wal_records = 0; +my $index_wal_bytes = 0; +my $index_wal_fpi = 0; + +my $vm_new_visible_frozen_pages_prev = 0; +my $tuples_deleted_prev = 0; +my $pages_scanned_prev = 0; +my $pages_removed_prev = 0; +my $wal_records_prev = 0; +my $wal_bytes_prev = 0; +my $wal_fpi_prev = 0; + +my $index_tuples_deleted_prev = 0; +my $index_pages_deleted_prev = 0; +my $index_wal_records_prev = 0; +my $index_wal_bytes_prev = 0; +my $index_wal_fpi_prev = 0; + +#------------------------------------------------------------------------------ +# fetch_vacuum_stats +# +# Reads current values of relevant vacuum counters for the test table and its +# primary index, storing them in package variables for subsequent comparisons. +#------------------------------------------------------------------------------ + +sub fetch_vacuum_stats { + # fetch actual base vacuum statistics + my $base_statistics = $node->safe_psql( + $dbname, + "SELECT vm_new_visible_frozen_pages, tuples_deleted, pages_scanned, pages_removed, wal_records, wal_bytes, wal_fpi + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'vestat';" + ); + + $base_statistics =~ s/\s*\|\s*/ /g; # transform " | " into space + ($vm_new_visible_frozen_pages, $tuples_deleted, $pages_scanned, $pages_removed, $wal_records, $wal_bytes, $wal_fpi) + = split /\s+/, $base_statistics; + + # --- index stats --- + my $index_base_statistics = $node->safe_psql( + $dbname, + "SELECT tuples_deleted, pages_deleted, wal_records, wal_bytes, wal_fpi + FROM ext_vacuum_statistics.pg_stats_vacuum_indexes + WHERE indexrelname = 'vestat_pkey';" + ); + + $index_base_statistics =~ s/\s*\|\s*/ /g; # transform " | " into space + ($index_tuples_deleted, $index_pages_deleted, $index_wal_records, $index_wal_bytes, $index_wal_fpi) + = split /\s+/, $index_base_statistics; +} + +#------------------------------------------------------------------------------ +# save_vacuum_stats +# +# Save current values (previously fetched by fetch_vacuum_stats) so that we +# later fetch new values and compare them. +#------------------------------------------------------------------------------ +sub save_vacuum_stats { + $vm_new_visible_frozen_pages_prev = $vm_new_visible_frozen_pages; + $tuples_deleted_prev = $tuples_deleted; + $pages_scanned_prev = $pages_scanned; + $pages_removed_prev = $pages_removed; + $wal_records_prev = $wal_records; + $wal_bytes_prev = $wal_bytes; + $wal_fpi_prev = $wal_fpi; + + $index_tuples_deleted_prev = $index_tuples_deleted; + $index_pages_deleted_prev = $index_pages_deleted; + $index_wal_records_prev = $index_wal_records; + $index_wal_bytes_prev = $index_wal_bytes; + $index_wal_fpi_prev = $index_wal_fpi; +} + +#------------------------------------------------------------------------------ +# print_vacuum_stats_on_error +# +# Print values in case of an error +#------------------------------------------------------------------------------ +sub print_vacuum_stats_on_error { + diag( + "Statistics in the failed test\n" . + "Table statistics:\n" . + " Before test:\n" . + " vm_new_visible_frozen_pages = $vm_new_visible_frozen_pages_prev\n" . + " tuples_deleted = $tuples_deleted_prev\n" . + " pages_scanned = $pages_scanned_prev\n" . + " pages_removed = $pages_removed_prev\n" . + " wal_records = $wal_records_prev\n" . + " wal_bytes = $wal_bytes_prev\n" . + " wal_fpi = $wal_fpi_prev\n" . + " After test:\n" . + " vm_new_visible_frozen_pages = $vm_new_visible_frozen_pages\n" . + " tuples_deleted = $tuples_deleted\n" . + " pages_scanned = $pages_scanned\n" . + " pages_removed = $pages_removed\n" . + " wal_records = $wal_records\n" . + " wal_bytes = $wal_bytes\n" . + " wal_fpi = $wal_fpi\n" . + "Index statistics:\n" . + " Before test:\n" . + " tuples_deleted = $index_tuples_deleted_prev\n" . + " pages_deleted = $index_pages_deleted_prev\n" . + " wal_records = $index_wal_records_prev\n" . + " wal_bytes = $index_wal_bytes_prev\n" . + " wal_fpi = $index_wal_fpi_prev\n" . + " After test:\n" . + " tuples_deleted = $index_tuples_deleted\n" . + " pages_deleted = $index_pages_deleted\n" . + " wal_records = $index_wal_records\n" . + " wal_bytes = $index_wal_bytes\n" . + " wal_fpi = $index_wal_fpi\n" + ); +}; + +sub fetch_error_base_db_vacuum_statistics { + my (%args) = @_; + + # Validate presence of required args (allow 0 as valid numeric baseline) + die "database name required" + unless exists $args{database_name} && defined $args{database_name}; + my $database_name = $args{database_name}; + + # fetch actual base database vacuum statistics + my $base_statistics = $node->safe_psql( + $database_name, + "SELECT db_blks_hit, db_blks_dirtied, + db_blks_written, db_wal_records, + db_wal_fpi, db_wal_bytes + FROM ext_vacuum_statistics.pg_stats_vacuum_database, pg_database + WHERE pg_database.datname = '$dbname' + AND pg_database.oid = ext_vacuum_statistics.pg_stats_vacuum_database.dboid;" + ); + $base_statistics =~ s/\s*\|\s*/ /g; # transform " | " in space + my ($db_blks_hit, $total_blks_dirtied, $total_blks_written, + $wal_records, $wal_fpi, $wal_bytes) = split /\s+/, $base_statistics; + + diag( + "BASE STATS MISMATCH FOR DATABASE $dbname:\n" . + " db_blks_hit = $db_blks_hit\n" . + " total_blks_dirtied = $total_blks_dirtied\n" . + " total_blks_written = $total_blks_written\n" . + " wal_records = $wal_records\n" . + " wal_fpi = $wal_fpi\n" . + " wal_bytes = $wal_bytes\n" + ); +} + + +#------------------------------------------------------------------------------ +# Test 1: Delete half the rows, run VACUUM, and wait for stats to advance +#------------------------------------------------------------------------------ +subtest 'Test 1: Delete half the rows, run VACUUM' => sub +{ + +$node->safe_psql($dbname, "DELETE FROM vestat WHERE x % 2 = 0;"); +$node->safe_psql($dbname, "VACUUM vestat;"); + +# Poll the stats view until expected deltas appear or timeout +$updated = wait_for_vacuum_stats( + tab_tuples_deleted => 0, + tab_wal_records => 0, + idx_tuples_deleted => 0, + idx_wal_records => 0, +); +ok($updated, 'vacuum stats updated after vacuuming half-deleted table (tuples_deleted and wal_fpi advanced)') + or diag "Timeout waiting for ext_vacuum_statistics update after $timeout seconds after vacuuming half-deleted table"; + +fetch_vacuum_stats(); + +ok($vm_new_visible_frozen_pages == $vm_new_visible_frozen_pages_prev, 'table vm_new_visible_frozen_pages stay the same'); +ok($tuples_deleted > $tuples_deleted_prev, 'table tuples_deleted has increased'); +ok($pages_scanned > $pages_scanned_prev, 'table pages_scanned has increased'); +ok($pages_removed == $pages_removed_prev, 'table pages_removed stay the same'); +ok($wal_records > $wal_records_prev, 'table wal_records has increased'); +ok($wal_bytes > $wal_bytes_prev, 'table wal_bytes has increased'); +ok($wal_fpi > $wal_fpi_prev, 'table wal_fpi has increased'); + +ok($index_pages_deleted == $index_pages_deleted_prev, 'index pages_deleted stay the same'); +ok($index_tuples_deleted > $index_tuples_deleted_prev, 'index tuples_deleted has increased'); +ok($index_wal_records > $index_wal_records_prev, 'index wal_records has increased'); +ok($index_wal_bytes > $index_wal_bytes_prev, 'index wal_bytes has increased'); +ok($index_wal_fpi == $index_wal_fpi_prev, 'index wal_fpi stay the same'); + +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 2: Delete all rows, run VACUUM, and wait for stats to advance +#------------------------------------------------------------------------------ +subtest 'Test 2: Delete all rows, run VACUUM' => sub +{ +save_vacuum_stats(); + +$node->safe_psql($dbname, "DELETE FROM vestat;"); +$node->safe_psql($dbname, "VACUUM vestat;"); + +$updated = wait_for_vacuum_stats( + tab_tuples_deleted => $tuples_deleted_prev, + tab_wal_records => $wal_records_prev, + idx_tuples_deleted => $index_tuples_deleted_prev, + idx_wal_records => $index_wal_records_prev, +); + +ok($updated, 'vacuum stats updated after vacuuming all-deleted table (tuples_deleted and wal_records advanced)') + or diag "Timeout waiting for ext_vacuum_statistics update after $timeout seconds after vacuuming all-deleted table"; + +fetch_vacuum_stats(); + +ok($vm_new_visible_frozen_pages > $vm_new_visible_frozen_pages_prev, 'table vm_new_visible_frozen_pages has increased'); +ok($tuples_deleted > $tuples_deleted_prev, 'table tuples_deleted has increased'); +ok($pages_scanned > $pages_scanned_prev, 'table pages_scanned has increased'); +ok($pages_removed > $pages_removed_prev, 'table pages_removed has increased'); +ok($wal_records > $wal_records_prev, 'table wal_records has increased'); +ok($wal_bytes > $wal_bytes_prev, 'table wal_bytes has increased'); +ok($wal_fpi > 0, 'table wal_fpi has increased'); + +ok($index_pages_deleted > $index_pages_deleted_prev, 'index pages_deleted has increased'); +ok($index_tuples_deleted > $index_tuples_deleted_prev, 'index tuples_deleted has increased'); +ok($index_wal_records > $index_wal_records_prev, 'index wal_records has increased'); +ok($index_wal_bytes > $index_wal_bytes_prev, 'index wal_bytes has increased'); +ok($index_wal_fpi == $index_wal_fpi_prev, 'index wal_fpi stay the same'); + +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 3: Test VACUUM FULL — it should not report to the stats collector +#------------------------------------------------------------------------------ +subtest 'Test 3: Test VACUUM FULL — it should not report to the stats collector' => sub +{ +save_vacuum_stats(); + +$node->safe_psql( + $dbname, + "INSERT INTO vestat SELECT x FROM generate_series(1, $size_tab) AS g(x); + CHECKPOINT; + DELETE FROM vestat; + VACUUM FULL vestat;" +); + +fetch_vacuum_stats(); + +ok($vm_new_visible_frozen_pages == $vm_new_visible_frozen_pages_prev, 'table vm_new_visible_frozen_pages stay the same'); +ok($tuples_deleted == $tuples_deleted_prev, 'table tuples_deleted stay the same'); +ok($pages_scanned == $pages_scanned_prev, 'table pages_scanned stay the same'); +ok($pages_removed == $pages_removed_prev, 'table pages_removed stay the same'); +ok($wal_records == $wal_records_prev, 'table wal_records stay the same'); +ok($wal_bytes == $wal_bytes_prev, 'table wal_bytes stay the same'); +ok($wal_fpi == $wal_fpi_prev, 'table wal_fpi stay the same'); + +ok($index_pages_deleted == $index_pages_deleted_prev, 'index pages_deleted stay the same'); +ok($index_tuples_deleted == $index_tuples_deleted_prev, 'index tuples_deleted stay the same'); +ok($index_wal_records == $index_wal_records_prev, 'index wal_records stay the same'); +ok($index_wal_bytes == $index_wal_bytes_prev, 'index wal_bytes stay the same'); +ok($index_wal_fpi == $index_wal_fpi_prev, 'index wal_fpi stay the same'); + +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 4: Update table, checkpoint, and VACUUM to provoke WAL/FPI accounting +#------------------------------------------------------------------------------ +subtest 'Test 4: Update table, checkpoint, and VACUUM to provoke WAL/FPI accounting' => sub +{ + +save_vacuum_stats(); + +$node->safe_psql( + $dbname, + "INSERT INTO vestat SELECT x FROM generate_series(1, $size_tab) AS g(x); + CHECKPOINT; + UPDATE vestat SET x = x + 1000; + VACUUM vestat;" +); + +$updated = wait_for_vacuum_stats( + tab_tuples_deleted => $tuples_deleted_prev, + tab_wal_records => $wal_records_prev, + idx_tuples_deleted => $index_tuples_deleted_prev, + idx_wal_records => $index_wal_records_prev, +); + +ok($updated, 'vacuum stats updated after updating tuples in the table (tuples_deleted and wal_records advanced)') + or diag "Timeout waiting for ext_vacuum_statistics update after $timeout seconds"; + +fetch_vacuum_stats(); + +ok($vm_new_visible_frozen_pages == $vm_new_visible_frozen_pages_prev, 'table vm_new_visible_frozen_pages stay the same'); +ok($tuples_deleted > $tuples_deleted_prev, 'table tuples_deleted has increased'); +ok($pages_scanned > $pages_scanned_prev, 'table pages_scanned has increased'); +ok($pages_removed == $pages_removed_prev, 'table pages_removed stay the same'); +ok($wal_records > $wal_records_prev, 'table wal_records has increased'); +ok($wal_bytes > $wal_bytes_prev, 'table wal_bytes has increased'); +ok($wal_fpi > $wal_fpi_prev, 'table wal_fpi has increased'); + +ok($index_pages_deleted > $index_pages_deleted_prev, 'index pages_deleted has increased'); +ok($index_tuples_deleted > $index_tuples_deleted_prev, 'index tuples_deleted has increased'); +ok($index_wal_records > $index_wal_records_prev, 'index wal_records has increased'); +ok($index_wal_bytes > $index_wal_bytes_prev, 'index wal_bytes has increased'); +ok($index_wal_fpi > $index_wal_fpi_prev, 'index wal_fpi has increased'); + +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 5: Update table, trancate and vacuuming +#------------------------------------------------------------------------------ +subtest 'Test 5: Update table, trancate and vacuuming' => sub +{ + +save_vacuum_stats(); + +$node->safe_psql( + $dbname, + "INSERT INTO vestat SELECT x FROM generate_series(1, $size_tab) AS g(x); + UPDATE vestat SET x = x + 1000;" +); +$node->safe_psql($dbname, "TRUNCATE vestat;"); +$node->safe_psql($dbname, "CHECKPOINT;"); +$node->safe_psql($dbname, "VACUUM vestat;"); + +$updated = wait_for_vacuum_stats( + tab_wal_records => $wal_records_prev, +); + +ok($updated, 'vacuum stats updated after updating tuples and trancation in the table (wal_records advanced)') + or diag "Timeout waiting for ext_vacuum_statistics update after $timeout seconds"; + +fetch_vacuum_stats(); + +ok($vm_new_visible_frozen_pages == $vm_new_visible_frozen_pages_prev, 'table vm_new_visible_frozen_pages stay the same'); +ok($tuples_deleted == $tuples_deleted_prev, 'table tuples_deleted stay the same'); +ok($pages_scanned == $pages_scanned_prev, 'table pages_scanned stay the same'); +ok($pages_removed == $pages_removed_prev, 'table pages_removed stay the same'); +ok($wal_records > $wal_records_prev, 'table wal_records has increased'); +ok($wal_bytes > $wal_bytes_prev, 'table wal_bytes has increased'); +ok($wal_fpi == $wal_fpi_prev, 'table wal_fpi stay the same'); + +ok($index_pages_deleted == $index_pages_deleted_prev, 'index pages_deleted stay the same'); +ok($index_tuples_deleted == $index_tuples_deleted_prev, 'index tuples_deleted stay the same'); +ok($index_wal_records == $index_wal_records_prev, 'index wal_records stay the same'); +ok($index_wal_bytes == $index_wal_bytes_prev, 'index wal_bytes stay the same'); +ok($index_wal_fpi == $index_wal_fpi_prev, 'index wal_fpi stay the same'); + +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 6: Delete all tuples from table, trancate, and vacuuming +#------------------------------------------------------------------------------ +subtest 'Test 6: Delete all tuples from table, trancate, and vacuuming' => sub +{ + +save_vacuum_stats(); + +$node->safe_psql( + $dbname, + "INSERT INTO vestat SELECT x FROM generate_series(1, $size_tab) AS g(x); + DELETE FROM vestat; + TRUNCATE vestat; + CHECKPOINT; + VACUUM vestat;" +); + +$updated = wait_for_vacuum_stats( + tab_wal_records => $wal_records, +); + +ok($updated, 'vacuum stats updated after deleting all tuples and trancation in the table (wal_records advanced)') + or diag "Timeout waiting for ext_vacuum_statistics update after $timeout seconds"; + +fetch_vacuum_stats(); + +ok($vm_new_visible_frozen_pages == $vm_new_visible_frozen_pages_prev, 'table vm_new_visible_frozen_pages stay the same'); +ok($tuples_deleted == $tuples_deleted_prev, 'table tuples_deleted stay the same'); +ok($pages_scanned == $pages_scanned_prev, 'table pages_scanned stay the same'); +ok($pages_removed == $pages_removed_prev, 'table pages_removed stay the same'); +ok($wal_records > $wal_records_prev, 'table wal_records has increased'); +ok($wal_bytes > $wal_bytes_prev, 'table wal_bytes has increased'); +ok($wal_fpi == $wal_fpi_prev, 'table wal_fpi stay the same'); + +ok($index_pages_deleted == $index_pages_deleted_prev, 'index pages_deleted stay the same'); +ok($index_tuples_deleted == $index_tuples_deleted_prev, 'index tuples_deleted stay the same'); +ok($index_wal_records == $index_wal_records_prev, 'index wal_records stay the same'); +ok($index_wal_bytes == $index_wal_bytes_prev, 'index wal_bytes stay the same'); +ok($index_wal_fpi == $index_wal_fpi_prev, 'index wal_fpi stay the same'); + +} or print_vacuum_stats_on_error(); + +my $dboid = $node->safe_psql( + $dbname, + "SELECT oid FROM pg_database WHERE datname = current_database();" +); + +#------------------------------------------------------------------------------------------------------- +# Test 7: Check if we return single vacuum statistics for particular relation from the current database +#------------------------------------------------------------------------------------------------------- +subtest 'Test 7: Check if we return vacuum statistics from the current database' => sub +{ +save_vacuum_stats(); + +my $reloid = $node->safe_psql( + $dbname, + q{ + SELECT oid FROM pg_class WHERE relname = 'vestat'; + } +); + +# Check if we can get vacuum statistics of particular heap relation in the current database +$base_stats = $node->safe_psql( + $dbname, + "SELECT count(*) FROM ext_vacuum_statistics.pg_stats_get_vacuum_tables((SELECT oid FROM pg_database WHERE datname = current_database()), $reloid);" +); +is($base_stats, 1, 'heap vacuum stats return from the current relation and database as expected'); + +$reloid = $node->safe_psql( + $dbname, + q{ + SELECT oid FROM pg_class WHERE relname = 'vestat_pkey'; + } +); + +# Check if we can get vacuum statistics of particular index relation in the current database +$base_stats = $node->safe_psql( + $dbname, + "SELECT count(*) FROM ext_vacuum_statistics.pg_stats_get_vacuum_indexes((SELECT oid FROM pg_database WHERE datname = current_database()), $reloid);" +); +is($base_stats, 1, 'index vacuum stats return from the current relation and database as expected'); + +# Check if we return empty results if vacuum statistics with particular oid doesn't exist +$base_stats = $node->safe_psql( + $dbname, + "SELECT count(*) FROM ext_vacuum_statistics.pg_stats_get_vacuum_tables((SELECT oid FROM pg_database WHERE datname = current_database()), 1);" +); +is($base_stats, 0, 'table vacuum stats return no rows, as expected'); + +$base_stats = $node->safe_psql( + $dbname, + "SELECT count(*) FROM ext_vacuum_statistics.pg_stats_get_vacuum_indexes((SELECT oid FROM pg_database WHERE datname = current_database()), 1);" +); +is($base_stats, 0, 'index vacuum stats return no rows, as expected'); + +# Check if we can get vacuum statistics of all relations in the current database +$base_stats = $node->safe_psql( + $dbname, + "SELECT count(*) > 0 FROM ext_vacuum_statistics.pg_stats_vacuum_tables;" +); +ok($base_stats eq 't', 'vacuum stats per all heap objects available'); + +$base_stats = $node->safe_psql( + $dbname, + "SELECT count(*) > 0 FROM ext_vacuum_statistics.pg_stats_vacuum_indexes;" +); +ok($base_stats eq 't', 'vacuum stats per all index objects available'); +}; + +#------------------------------------------------------------------------------ +# Test 8: Check relation-level vacuum statistics from another database +#------------------------------------------------------------------------------ +subtest 'Test 8: Check relation-level vacuum statistics from another database' => sub +{ +$base_stats = $node->safe_psql( + 'postgres', + "SELECT count(*) + FROM ext_vacuum_statistics.pg_stats_vacuum_indexes + WHERE indexrelname = 'vestat_pkey';" +); +is($base_stats, 0, 'check the printing index vacuum extended statistics from another database are not available'); + +$base_stats = $node->safe_psql( + 'postgres', + "SELECT count(*) + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'vestat';" +); +is($base_stats, 0, 'check the printing heap vacuum extended statistics from another database are not available'); + +# Check that relations from another database are not visible in the view when querying from postgres +$base_stats = $node->safe_psql( + 'postgres', + "SELECT count(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'vestat';" +); +is($base_stats, 0, 'vacuum stats per all tables objects from another database are not available as expected'); + +$base_stats = $node->safe_psql( + 'postgres', + "SELECT count(*) FROM ext_vacuum_statistics.pg_stats_vacuum_indexes WHERE indexrelname = 'vestat_pkey';" +); +is($base_stats, 0, 'vacuum stats per all index objects from another database are not available as expected'); +}; + +#-------------------------------------------------------------------------------------- +# Test 9: Check database-level vacuum statistics from the current and another database +#-------------------------------------------------------------------------------------- +subtest 'Test 9: Check database-level vacuum statistics from the current and another database' => sub +{ +my $db_blk_hit = 0; +my $total_blks_dirtied = 0; +my $total_blks_written = 0; +my $wal_records = 0; +my $wal_fpi = 0; +my $wal_bytes = 0; +$base_stats = $node->safe_psql( + $dbname, + "SELECT db_blks_hit, db_blks_dirtied, + db_blks_written, db_wal_records, + db_wal_fpi, db_wal_bytes + FROM ext_vacuum_statistics.pg_stats_vacuum_database, pg_database + WHERE pg_database.datname = '$dbname' + AND pg_database.oid = ext_vacuum_statistics.pg_stats_vacuum_database.dboid;" +); +$base_stats =~ s/\s*\|\s*/ /g; # transform " | " into space + ($db_blk_hit, $total_blks_dirtied, $total_blks_written, $wal_records, $wal_fpi, $wal_bytes) + = split /\s+/, $base_stats; + +ok($db_blk_hit > 0, 'db_blks_hit is more than 0'); +ok($total_blks_dirtied > 0, 'total_blks_dirtied is more than 0'); +ok($total_blks_written > 0, 'total_blks_written is more than 0'); +ok($wal_records > 0, 'wal_records is more than 0'); +ok($wal_fpi > 0, 'wal_fpi is more than 0'); +ok($wal_bytes > 0, 'wal_bytes is more than 0'); + +$base_stats = $node->safe_psql( + 'postgres', + "SELECT count(*) = 1 + FROM ext_vacuum_statistics.pg_stats_vacuum_database, pg_database + WHERE pg_database.datname = '$dbname' + AND pg_database.oid = ext_vacuum_statistics.pg_stats_vacuum_database.dboid;" +); +ok($base_stats eq 't', 'check database-level vacuum stats from another database are available'); +}; + +#------------------------------------------------------------------------------ +# Test 10: Cleanup checks: ensure functions return empty sets for OID = 0 +#------------------------------------------------------------------------------ +subtest 'Test 10: Cleanup checks: ensure functions return empty sets for OID = 0' => sub +{ +my $dboid = $node->safe_psql( + $dbname, + "SELECT oid FROM pg_database WHERE datname = current_database();" +); + +# Vacuum statistics for invalid relation OID return empty +$base_stats = $node->safe_psql( + $dbname, + q{ + SELECT COUNT(*) + FROM ext_vacuum_statistics.pg_stats_get_vacuum_tables((SELECT oid FROM pg_database WHERE datname = current_database()), 0); + } +); +is($base_stats, 0, 'vacuum stats per heap from invalid relation OID return empty as expected'); + +$base_stats = $node->safe_psql( + $dbname, + q{ + SELECT COUNT(*) + FROM ext_vacuum_statistics.pg_stats_get_vacuum_indexes((SELECT oid FROM pg_database WHERE datname = current_database()), 0); + } +); +is($base_stats, 0, 'vacuum stats per index from invalid relation OID return empty as expected'); + +$node->safe_psql($dbname, q{ + DROP TABLE vestat CASCADE; + VACUUM; +}); + +# Check that we don't print vacuum statistics for deleted objects +$base_stats = $node->safe_psql( + $dbname, + q{ + SELECT COUNT(*) + FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relid = 0; + } +); +is($base_stats, 0, 'ext_vacuum_statistics.pg_stats_vacuum_tables correctly returns no rows for OID = 0'); + +$base_stats = $node->safe_psql( + $dbname, + q{ + SELECT COUNT(*) + FROM ext_vacuum_statistics.pg_stats_vacuum_indexes WHERE indexrelid = 0; + } +); +is($base_stats, 0, 'ext_vacuum_statistics.pg_stats_vacuum_indexes correctly returns no rows for OID = 0'); + +my $reloid = $node->safe_psql( + $dbname, + q{ + SELECT oid FROM pg_class WHERE relname = 'pg_shdepend'; + } +); + +$node->safe_psql($dbname, "VACUUM pg_shdepend;"); + +# Check if we can get vacuum statistics for cluster relations (shared catalogs) +$base_stats = $node->safe_psql( + $dbname, + qq{ + SELECT count(*) > 0 + FROM ext_vacuum_statistics.pg_stats_get_vacuum_tables((SELECT oid FROM pg_database WHERE datname = current_database()), $reloid); + } +); + +is($base_stats, 't', 'vacuum stats for common heap objects available'); + +my $indoid = $node->safe_psql( + $dbname, + q{ + SELECT oid FROM pg_class WHERE relname = 'pg_shdepend_reference_index'; + } +); + +$base_stats = $node->safe_psql( + $dbname, + qq{ + SELECT count(*) > 0 + FROM ext_vacuum_statistics.pg_stats_get_vacuum_indexes((SELECT oid FROM pg_database WHERE datname = current_database()), $indoid); + } +); + +is($base_stats, 't', 'vacuum stats for common index objects available'); + +$node->safe_psql('postgres', + "DROP DATABASE $dbname; + VACUUM;" +); + +$base_stats = $node->safe_psql( + 'postgres', + q{ + SELECT count(*) = 0 + FROM ext_vacuum_statistics.pg_stats_get_vacuum_database(0); + } +); +is($base_stats, 't', 'vacuum stats from database with invalid database OID return empty, as expected'); +}; + +$node->stop; + +done_testing(); diff --git a/contrib/ext_vacuum_statistics/t/053_vacuum_extending_freeze_test.pl b/contrib/ext_vacuum_statistics/t/053_vacuum_extending_freeze_test.pl new file mode 100644 index 00000000000..4f8f025c63e --- /dev/null +++ b/contrib/ext_vacuum_statistics/t/053_vacuum_extending_freeze_test.pl @@ -0,0 +1,285 @@ +# Copyright (c) 2025 PostgreSQL Global Development Group +# +# Test cumulative vacuum stats using ext_vacuum_statistics extension (TAP) +# +# In short, this test validates the correctness and stability of cumulative +# vacuum statistics accounting around freezing, visibility, and revision +# tracking across multiple VACUUMs and backend operations. + +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +#------------------------------------------------------------------------------ +# Test cluster setup +#------------------------------------------------------------------------------ + +my $node = PostgreSQL::Test::Cluster->new('ext_stat_vacuum'); +$node->init; + +# Configure the server: preload extension and aggressive freezing behavior +$node->append_conf('postgresql.conf', q{ + shared_preload_libraries = 'ext_vacuum_statistics' + log_min_messages = notice + vacuum_freeze_min_age = 0 + vacuum_freeze_table_age = 0 + vacuum_multixact_freeze_min_age = 0 + vacuum_multixact_freeze_table_age = 0 + vacuum_max_eager_freeze_failure_rate = 1.0 + vacuum_failsafe_age = 0 + vacuum_multixact_failsafe_age = 0 + track_functions = 'all' +}); + +$node->start(); + +#------------------------------------------------------------------------------ +# Database creation and initialization +#------------------------------------------------------------------------------ + +$node->safe_psql('postgres', q{ + CREATE DATABASE statistic_vacuum_database_regression; +}); + +# Main test database name +my $dbname = 'statistic_vacuum_database_regression'; + +# Create extension +$node->safe_psql($dbname, q{ + CREATE EXTENSION ext_vacuum_statistics; +}); + +#------------------------------------------------------------------------------ +# Timing parameters for polling loops +#------------------------------------------------------------------------------ + +my $timeout = 30; # overall wait timeout in seconds +my $interval = 0.015; # poll interval in seconds (15 ms) +my $start_time = time(); +my $updated = 0; + +#------------------------------------------------------------------------------ +# wait_for_vacuum_stats +# +# Polls ext_vacuum_statistics.pg_stats_vacuum_tables until the named columns exceed the +# provided baseline values or until timeout. +# +# tab_all_frozen_pages_count => 0 # baseline numeric +# tab_all_visible_pages_count => 0 # baseline numeric +# run_vacuum => 0 # if true, run vacuum before polling +# +# Returns: 1 if the condition is met before timeout, 0 otherwise. +#------------------------------------------------------------------------------ +sub wait_for_vacuum_stats { + my (%args) = @_; + + my $tab_all_frozen_pages_count = $args{tab_all_frozen_pages_count} || 0; + my $tab_all_visible_pages_count = $args{tab_all_visible_pages_count} || 0; + my $run_vacuum = $args{run_vacuum} ? 1 : 0; + my $result_query; + + my $start = time(); + my $sql; + + # Run VACUUM once if requested, before polling + if ($run_vacuum) { + $node->safe_psql($dbname, 'VACUUM (FREEZE, VERBOSE) vestat'); + } + + while ((time() - $start) < $timeout) { + + if ($run_vacuum) { + $sql = " + SELECT (vm_new_visible_frozen_pages > $tab_all_frozen_pages_count) + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'vestat'"; + } + else { + $sql = " + SELECT (pg_stat_get_frozen_page_marks_cleared(c.oid) > $tab_all_frozen_pages_count AND + pg_stat_get_visible_page_marks_cleared(c.oid) > $tab_all_visible_pages_count) + FROM pg_class c + WHERE relname = 'vestat'"; + } + + $result_query = $node->safe_psql($dbname, $sql); + + return 1 if (defined $result_query && $result_query eq 't'); + + sleep($interval); + } + + return 0; +} + +#------------------------------------------------------------------------------ +# Variables to hold vacuum statistics snapshots for comparisons +#------------------------------------------------------------------------------ + +my $vm_new_visible_frozen_pages = 0; + +my $rev_all_frozen_pages = 0; +my $rev_all_visible_pages = 0; + +my $vm_new_visible_frozen_pages_prev = 0; + +my $rev_all_frozen_pages_prev = 0; +my $rev_all_visible_pages_prev = 0; + +my $res; + +#------------------------------------------------------------------------------ +# fetch_vacuum_stats +# +# Loads current values of the relevant vacuum counters for the test table +# into the package-level variables above so tests can compare later. +#------------------------------------------------------------------------------ + +sub fetch_vacuum_stats { + $vm_new_visible_frozen_pages = $node->safe_psql( + $dbname, + "SELECT vt.vm_new_visible_frozen_pages + FROM ext_vacuum_statistics.pg_stats_vacuum_tables vt + WHERE vt.relname = 'vestat';" + ); + + $rev_all_frozen_pages = $node->safe_psql( + $dbname, + "SELECT pg_stat_get_frozen_page_marks_cleared(c.oid) + FROM pg_class c + WHERE c.relname = 'vestat';" + ); + + $rev_all_visible_pages = $node->safe_psql( + $dbname, + "SELECT pg_stat_get_visible_page_marks_cleared(c.oid) + FROM pg_class c + WHERE c.relname = 'vestat';" + ); +} + +#------------------------------------------------------------------------------ +# save_vacuum_stats +#------------------------------------------------------------------------------ +sub save_vacuum_stats { + $vm_new_visible_frozen_pages_prev = $vm_new_visible_frozen_pages; + $rev_all_frozen_pages_prev = $rev_all_frozen_pages; + $rev_all_visible_pages_prev = $rev_all_visible_pages; +} + +#------------------------------------------------------------------------------ +# print_vacuum_stats_on_error +#------------------------------------------------------------------------------ +sub print_vacuum_stats_on_error { + diag( + "Statistics in the failed test\n" . + "Table statistics:\n" . + " Before test:\n" . + " vm_new_visible_frozen_pages = $vm_new_visible_frozen_pages_prev\n" . + " rev_all_frozen_pages = $rev_all_frozen_pages_prev\n" . + " rev_all_visible_pages = $rev_all_visible_pages_prev\n" . + " After test:\n" . + " vm_new_visible_frozen_pages = $vm_new_visible_frozen_pages\n" . + " rev_all_frozen_pages = $rev_all_frozen_pages\n" . + " rev_all_visible_pages = $rev_all_visible_pages\n" + ); +}; + +#------------------------------------------------------------------------------ +# Test 1: Create test table, populate it and run an initial vacuum to force freezing +#------------------------------------------------------------------------------ + +subtest 'Test 1: Create test table, populate it and run an initial vacuum to force freezing' => sub +{ +$node->safe_psql($dbname, q{ + CREATE TABLE vestat (x int) + WITH (autovacuum_enabled = off, fillfactor = 10); + INSERT INTO vestat SELECT x FROM generate_series(1, 1000) AS g(x); + ANALYZE vestat; + VACUUM (FREEZE, VERBOSE) vestat; +}); + +$updated = wait_for_vacuum_stats( + tab_all_frozen_pages_count => 0, + tab_all_visible_pages_count => 0, + run_vacuum => 1, +); + +ok($updated, + 'vacuum stats updated after vacuuming the table (vm_new_visible_frozen_pages advanced)') + or diag "Timeout waiting for ext_vacuum_statistics to update after $timeout seconds during vacuum"; + +fetch_vacuum_stats(); + +ok($vm_new_visible_frozen_pages > $vm_new_visible_frozen_pages_prev, 'table vm_new_visible_frozen_pages has increased'); +ok($rev_all_frozen_pages == $rev_all_frozen_pages_prev, 'table rev_all_frozen_pages stay the same'); +ok($rev_all_visible_pages == $rev_all_visible_pages_prev, 'table rev_all_visible_pages stay the same'); +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 2: Trigger backend updates +# Backend activity should reset per-page visibility/freeze marks and increment revision counters +#------------------------------------------------------------------------------ +subtest 'Test 2: Trigger backend updates' => sub +{ +save_vacuum_stats(); + +$node->safe_psql($dbname, q{ + UPDATE vestat SET x = x + 1001; +}); + +$updated = wait_for_vacuum_stats( + tab_all_frozen_pages_count => 0, + tab_all_visible_pages_count => 0, + run_vacuum => 0, +); + +ok($updated, + 'vacuum stats updated after backend tuple updates (rev_all_frozen_pages and rev_all_visible_pages advanced)') + or diag "Timeout waiting for vacuum stats update after $timeout seconds"; + +fetch_vacuum_stats(); + +ok($vm_new_visible_frozen_pages == $vm_new_visible_frozen_pages_prev, 'table vm_new_visible_frozen_pages stay the same'); +ok($rev_all_frozen_pages > $rev_all_frozen_pages_prev, 'table rev_all_frozen_pages has increased'); +ok($rev_all_visible_pages > $rev_all_visible_pages_prev, 'table rev_all_visible_pages has increased'); +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 3: Force another vacuum after backend modifications - vacuum should restore freeze/visibility +#------------------------------------------------------------------------------ +subtest 'Test 3: Force another vacuum after backend modifications - vacuum should restore freeze/visibility' => sub +{ +save_vacuum_stats(); + +$node->safe_psql($dbname, q{ VACUUM vestat; }); + +$updated = wait_for_vacuum_stats( + tab_all_frozen_pages_count => $vm_new_visible_frozen_pages, + tab_all_visible_pages_count => 0, + run_vacuum => 1, +); + +ok($updated, + 'vacuum stats updated after vacuuming the all-updated table (vm_new_visible_frozen_pages advanced)') + or diag "Timeout waiting for ext_vacuum_statistics to update after $timeout seconds during vacuum"; + +fetch_vacuum_stats(); + +ok($vm_new_visible_frozen_pages > $vm_new_visible_frozen_pages_prev, 'table vm_new_visible_frozen_pages has increased'); +ok($rev_all_frozen_pages == $rev_all_frozen_pages_prev, 'table rev_all_frozen_pages stay the same'); +ok($rev_all_visible_pages == $rev_all_visible_pages_prev, 'table rev_all_visible_pages stay the same'); +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Cleanup +#------------------------------------------------------------------------------ + +$node->safe_psql('postgres', q{ + DROP DATABASE statistic_vacuum_database_regression; +}); + +$node->stop; +done_testing(); diff --git a/contrib/ext_vacuum_statistics/t/054_vacuum_extending_gucs_test.pl b/contrib/ext_vacuum_statistics/t/054_vacuum_extending_gucs_test.pl new file mode 100644 index 00000000000..a195249842b --- /dev/null +++ b/contrib/ext_vacuum_statistics/t/054_vacuum_extending_gucs_test.pl @@ -0,0 +1,279 @@ +# Copyright (c) 2025 PostgreSQL Global Development Group +# +# Test GUC parameters for ext_vacuum_statistics extension: +# vacuum_statistics.enabled +# vacuum_statistics.object_types (all, databases, relations) +# vacuum_statistics.track_relations (all, system, user) +# vacuum_statistics.track_databases_from_list, add/remove_track_database +# add/remove_track_database, add/remove_track_relation, track_*_from_list + +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; + +use Test::More; + +#------------------------------------------------------------------------------ +# Test cluster setup +#------------------------------------------------------------------------------ + +my $node = PostgreSQL::Test::Cluster->new('ext_stat_vacuum_gucs'); +$node->init; + +$node->append_conf('postgresql.conf', q{ + shared_preload_libraries = 'ext_vacuum_statistics' + log_min_messages = notice +}); + +$node->start; + +#------------------------------------------------------------------------------ +# Database creation and initialization +#------------------------------------------------------------------------------ + +$node->safe_psql('postgres', q{ + CREATE DATABASE statistic_vacuum_gucs; +}); + +my $dbname = 'statistic_vacuum_gucs'; + +$node->safe_psql($dbname, q{ + CREATE EXTENSION ext_vacuum_statistics; + CREATE TABLE guc_test (x int PRIMARY KEY) + WITH (autovacuum_enabled = off); + INSERT INTO guc_test SELECT x FROM generate_series(1, 100) AS g(x); + ANALYZE guc_test; +}); + +# Get OIDs for filtering tests +my $dboid = $node->safe_psql($dbname, q{SELECT oid FROM pg_database WHERE datname = current_database()}); +my $reloid = $node->safe_psql($dbname, q{SELECT oid FROM pg_class WHERE relname = 'guc_test'}); + +#------------------------------------------------------------------------------ +# Reset stats and run vacuum (all in one session so GUCs persist) +#------------------------------------------------------------------------------ + +sub reset_and_vacuum { + my ($db, $table, $opts) = @_; + $table ||= 'guc_test'; + my $gucs = $opts && $opts->{gucs} ? $opts->{gucs} : []; + my $modify = $opts && $opts->{modify}; + my $extra = $opts && $opts->{extra_vacuum} ? $opts->{extra_vacuum} : []; + $extra = [$extra] unless ref $extra eq 'ARRAY'; + my $sql = join("\n", (map { "SET $_;" } @$gucs), + "SELECT ext_vacuum_statistics.vacuum_statistics_reset();", + $modify ? ( + "TRUNCATE $table;", + "INSERT INTO $table SELECT x FROM generate_series(1, 100) AS g(x);", + "DELETE FROM $table;", + ) : (), + "VACUUM $table;", + (map { "VACUUM $_;" } @$extra), + # Make pending stats visible to subsequent sessions without sleeping. + "SELECT pg_stat_force_next_flush();"); + $node->safe_psql($db, $sql); +} + +#------------------------------------------------------------------------------ +# Test 1: vacuum_statistics.enabled +#------------------------------------------------------------------------------ +subtest 'vacuum_statistics.enabled' => sub { + reset_and_vacuum($dbname); + + # Default: enabled - should have stats + my $count = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'guc_test'"); + ok($count > 0, 'stats collected when enabled'); + + # Disable, reset and vacuum in same session. Assert not only that the + # row count is zero, but that the specific counters remain zero: a stray + # row with zero counters would otherwise pass a bare COUNT(*)=0 check. + reset_and_vacuum($dbname, 'guc_test', { gucs => ['vacuum_statistics.enabled = off'] }); + + $count = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'guc_test'"); + is($count, 0, 'no rows when disabled'); + + my $sums = $node->safe_psql($dbname, q{ + SELECT COALESCE(SUM(total_blks_read), 0) + + COALESCE(SUM(total_blks_dirtied), 0) + + COALESCE(SUM(pages_scanned), 0) + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'guc_test' + }); + is($sums, '0', 'no counters accumulated when disabled'); +}; + +#------------------------------------------------------------------------------ +# Test 2: vacuum_statistics.object_types (databases only, relations only) +#------------------------------------------------------------------------------ +subtest 'vacuum_statistics.object_types' => sub { + # track only db stats, no relation stats + reset_and_vacuum($dbname, 'guc_test', { + gucs => ["vacuum_statistics.object_types = 'databases'"], + modify => 1, + }); + my $db_has_dbs = $node->safe_psql($dbname, + "SELECT COALESCE(SUM(db_blks_hit), 0) FROM ext_vacuum_statistics.pg_stats_vacuum_database WHERE dboid = $dboid"); + my $rel_dbs = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'guc_test'"); + is($rel_dbs, 0, 'track=databases: no relation stats'); + ok($db_has_dbs > 0, 'track=databases: database stats collected'); + + # track only relation stats, no db stats + reset_and_vacuum($dbname, 'guc_test', { + gucs => ["vacuum_statistics.object_types = 'relations'"], + modify => 1, + }); + my $db_has_rels = $node->safe_psql($dbname, + "SELECT COALESCE(SUM(db_blks_hit), 0) > 0 FROM ext_vacuum_statistics.pg_stats_vacuum_database WHERE dboid = $dboid"); + my $rel_rels = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'guc_test'"); + ok($rel_rels > 0, 'track=relations: relation stats collected'); + is($db_has_rels, 'f', 'track=relations: no database stats'); +}; + +#------------------------------------------------------------------------------ +# Test 3: vacuum_statistics.track_relations (system, user) +#------------------------------------------------------------------------------ +subtest 'vacuum_statistics.track_relations' => sub { + # track_relations - only user tables + reset_and_vacuum($dbname, 'guc_test', { + gucs => [ + "vacuum_statistics.object_types = 'relations'", + "vacuum_statistics.track_relations = 'user'", + ], + extra_vacuum => ['pg_class'], + }); + + my $user_rel = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'guc_test'"); + my $sys_rel = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'pg_class'"); + ok($user_rel > 0, 'track_relations=user: user table stats collected'); + is($sys_rel, 0, 'track_relations=user: system table stats not collected'); + + # track_relations - only system tables + reset_and_vacuum($dbname, 'guc_test', { + gucs => [ + "vacuum_statistics.object_types = 'relations'", + "vacuum_statistics.track_relations = 'system'", + ], + extra_vacuum => ['pg_class'], + }); + + $user_rel = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'guc_test'"); + $sys_rel = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'pg_class'"); + is($user_rel, 0, 'track_relations=system: user table stats not collected'); + ok($sys_rel > 0, 'track_relations=system: system table stats collected'); +}; + +#------------------------------------------------------------------------------ +# Test 4: track_databases (via add/remove_track_database) +#------------------------------------------------------------------------------ +subtest 'track_databases (add/remove)' => sub { + $node->safe_psql($dbname, "SELECT ext_vacuum_statistics.remove_track_database($dboid)"); + $node->safe_psql($dbname, "SELECT ext_vacuum_statistics.add_track_database($dboid)"); + reset_and_vacuum($dbname, 'guc_test', { gucs => ["vacuum_statistics.track_databases_from_list = on"], modify => 1 }); + + my $rel_count = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'guc_test'"); + ok($rel_count > 0, 'db in list: stats collected'); + + $node->safe_psql($dbname, "SELECT ext_vacuum_statistics.remove_track_database($dboid)"); + reset_and_vacuum($dbname, 'guc_test', { gucs => ["vacuum_statistics.track_databases_from_list = on"], modify => 1 }); + + $rel_count = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'guc_test'"); + is($rel_count, 0, 'db removed from list: no stats'); +}; + +#------------------------------------------------------------------------------ +# Test 5: track_relations (via add/remove_track_relation) +#------------------------------------------------------------------------------ +subtest 'track_relations (add/remove)' => sub { + $node->safe_psql($dbname, "SELECT ext_vacuum_statistics.remove_track_relation($dboid, $reloid)"); + $node->safe_psql($dbname, "SELECT ext_vacuum_statistics.add_track_relation($dboid, $reloid)"); + reset_and_vacuum($dbname, 'guc_test', { gucs => ["vacuum_statistics.track_relations_from_list = on"], modify => 1 }); + + my $rel_count = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'guc_test'"); + ok($rel_count > 0, 'table in list: stats collected'); + + $node->safe_psql($dbname, "SELECT ext_vacuum_statistics.remove_track_relation($dboid, $reloid)"); + reset_and_vacuum($dbname, 'guc_test', { gucs => ["vacuum_statistics.track_relations_from_list = on"], modify => 1 }); + + $rel_count = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'guc_test'"); + is($rel_count, 0, 'table removed from list: no stats'); +}; + +#------------------------------------------------------------------------------ +# Test 6: vacuum_statistics.collect - per-category gating +# +# With collect='wal' only wal_* counters must advance; buffer, timing, and +# general categories must stay at zero. With collect='buffers' the inverse +# holds. Unknown tokens must be rejected by the check-hook. +#------------------------------------------------------------------------------ +subtest 'vacuum_statistics.collect' => sub { + # wal-only: WAL counters should accumulate, buffers/timing/general should not. + reset_and_vacuum($dbname, 'guc_test', { + gucs => ["vacuum_statistics.collect = 'wal'"], + modify => 1, + }); + + my $wal = $node->safe_psql($dbname, q{ + SELECT COALESCE(SUM(wal_records), 0) > 0 + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'guc_test' + }); + is($wal, 't', "collect='wal': wal_records accumulated"); + + my $other = $node->safe_psql($dbname, q{ + SELECT COALESCE(SUM(total_blks_read), 0) + + COALESCE(SUM(total_blks_hit), 0) + + COALESCE(SUM(total_time), 0) + + COALESCE(SUM(tuples_deleted), 0) + + COALESCE(SUM(pages_scanned), 0) + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'guc_test' + }); + is($other, '0', + "collect='wal': buffer/timing/general counters not accumulated"); + + # buffers-only: buffer counters should advance, WAL should not. + reset_and_vacuum($dbname, 'guc_test', { + gucs => ["vacuum_statistics.collect = 'buffers'"], + modify => 1, + }); + + my $buf = $node->safe_psql($dbname, q{ + SELECT COALESCE(SUM(total_blks_read), 0) + + COALESCE(SUM(total_blks_hit), 0) > 0 + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'guc_test' + }); + is($buf, 't', "collect='buffers': buffer counters accumulated"); + + my $wal_off = $node->safe_psql($dbname, q{ + SELECT COALESCE(SUM(wal_records), 0) + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'guc_test' + }); + is($wal_off, '0', + "collect='buffers': WAL counters not accumulated"); + + # Unknown category must be rejected by the check-hook. + my ($ret, $stdout, $stderr) = $node->psql($dbname, + "SET vacuum_statistics.collect = 'nope'"); + isnt($ret, 0, "collect='nope': rejected by check-hook"); + like($stderr, qr/Unrecognized category "nope"/, + "collect='nope': errdetail names the offending token"); +}; + +$node->stop; + +done_testing(); diff --git a/contrib/ext_vacuum_statistics/vacuum_statistics.c b/contrib/ext_vacuum_statistics/vacuum_statistics.c new file mode 100644 index 00000000000..75d1bd2cf06 --- /dev/null +++ b/contrib/ext_vacuum_statistics/vacuum_statistics.c @@ -0,0 +1,1387 @@ +/* + * ext_vacuum_statistics - Extended vacuum statistics for PostgreSQL + * + * This module collects detailed vacuum statistics (I/O, WAL, timing, etc.) + * at relation and database level by hooking into the vacuum reporting path. + * Statistics are stored via pgstat custom statistics. Management of statistics + * storage and output functions are implemented in this module. + */ +#include "postgres.h" + +#include "access/transam.h" +#include "catalog/catalog.h" +#include "catalog/objectaccess.h" +#include "catalog/pg_authid.h" +#include "catalog/pg_class.h" +#include "catalog/pg_database.h" +#include "fmgr.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "pgstat.h" +#include "storage/fd.h" +#include "storage/ipc.h" +#include "storage/lwlock.h" +#include "utils/acl.h" +#include "utils/builtins.h" +#include "utils/fmgrprotos.h" +#include "utils/guc.h" +#include "utils/hsearch.h" +#include "utils/lsyscache.h" +#include "utils/pgstat_kind.h" +#include "utils/pgstat_internal.h" +#include "utils/tuplestore.h" + +#ifdef PG_MODULE_MAGIC +PG_MODULE_MAGIC; +#endif + +/* Two kinds: relations (tables/indexes) and database aggregates */ +#define PGSTAT_KIND_EXTVAC_RELATION 24 +#define PGSTAT_KIND_EXTVAC_DB 25 + +#define SJ_NODENAME "vacuum_statistics" +#define EVS_TRACK_FILENAME "pg_stat/ext_vacuum_statistics_track.oid" + +/* Bit flags for evs_track (object_types): 'all', 'databases', 'relations' */ +#define EVS_TRACK_RELATIONS 0x01 +#define EVS_TRACK_DATABASES 0x02 + +/* Bit flags for evs_track_relations: 'all', 'system', 'user' */ +#define EVS_FILTER_SYSTEM 0x01 +#define EVS_FILTER_USER 0x02 + +/* + * Bit flags for evs_collect_mask. Each category groups counters that can be + * accumulated (or skipped) together, letting users reduce overhead at run + * time by turning off categories they don't need. + */ +#define EVS_COLLECT_BUFFERS 0x1 /* blks_*, blk_*_time */ +#define EVS_COLLECT_WAL 0x2 /* wal_records, wal_fpi, wal_bytes */ +#define EVS_COLLECT_GENERAL 0x4 /* tuples_deleted, pages_*, vm_*, + * wraparound_failsafe_count, + * interrupts_count */ +#define EVS_COLLECT_TIMING 0x8 /* delay_time, total_time */ +#define EVS_COLLECT_ALL (EVS_COLLECT_BUFFERS | EVS_COLLECT_WAL | \ + EVS_COLLECT_GENERAL | EVS_COLLECT_TIMING) + +/* GUCs */ +static bool evs_enabled = true; +static char *evs_track = "all"; /* 'all', 'databases', 'relations' */ +static char *evs_track_relations = "all"; /* 'all', 'system', 'user' */ +static int evs_track_bits = EVS_TRACK_RELATIONS | EVS_TRACK_DATABASES; +static int evs_track_relations_bits = EVS_FILTER_SYSTEM | EVS_FILTER_USER; +static bool evs_track_databases_from_list = false; /* if true, track only + * databases in list */ +static bool evs_track_relations_from_list = false; /* if true, track only + * relations in list */ +static char *evs_collect = "all"; /* categories to collect */ +static int evs_collect_mask = EVS_COLLECT_ALL; + +/* Hook */ +static set_report_vacuum_hook_type prev_report_vacuum_hook = NULL; +static object_access_hook_type prev_object_access_hook = NULL; +static shmem_request_hook_type prev_shmem_request_hook = NULL; + +/* Forward declarations */ +static void pgstat_report_vacuum_extstats(Oid tableoid, bool shared, + PgStat_VacuumRelationCounts * params); +static bool evs_oid_in_list(HTAB *hash, Oid oid); +static void evs_track_hash_ensure_init(void); +static void evs_track_save_file(void); +static void evs_track_load_file(void); +static void evs_drop_access_hook(ObjectAccessType access, Oid classId, + Oid objectId, int subId, void *arg); +static void evs_shmem_request(void); + +/* Hash tables for track_databases and track_relations_list (backend-local) */ +static HTAB *evs_track_databases_hash = NULL; +static HTAB *evs_track_relations_hash = NULL; +static bool evs_track_hash_initialized = false; + +/* + * Named LWLock tranche protecting the on-disk track file and serializing + * backend-local reloads/saves across concurrent backends. + */ +#define EVS_TRACK_TRANCHE_NAME "ext_vacuum_statistics_track" +static LWLock *evs_track_lock = NULL; + +static inline LWLock * +evs_get_track_lock(void) +{ + if (evs_track_lock == NULL) + evs_track_lock = &GetNamedLWLockTranche(EVS_TRACK_TRANCHE_NAME)->lock; + return evs_track_lock; +} + +/* + * objid encoding for relations: (relid << 2) | (type & 3) + */ +#define EXTVAC_OBJID(relid, type) (((uint64) (relid)) << 2 | ((type) & 3)) + +/* Key for relation tracking: (dboid, reloid). + * InvalidOid for dboid means it is a cluster object. + */ +typedef struct +{ + Oid dboid; + Oid reloid; +} EvsTrackRelKey; + +/* Shared memory entry for vacuum stats; one per relation or database. */ +typedef struct PgStatShared_ExtVacEntry +{ + PgStatShared_Common header; + PgStat_VacuumRelationCounts stats; +} PgStatShared_ExtVacEntry; + +/* PgStat kind for per-relation vacuum statistics (tables/indexes) */ +static const PgStat_KindInfo extvac_relation_kind_info = { + .name = "ext_vacuum_statistics_relation", + .fixed_amount = false, + .accessed_across_databases = true, + .write_to_file = true, + .track_entry_count = true, + .shared_size = sizeof(PgStatShared_ExtVacEntry), + .shared_data_off = offsetof(PgStatShared_ExtVacEntry, stats), + .shared_data_len = sizeof(PgStat_VacuumRelationCounts), + .pending_size = 0, + .flush_pending_cb = NULL, +}; + +/* PgStat kind for per-database aggregated vacuum statistics */ +static const PgStat_KindInfo extvac_db_kind_info = { + .name = "ext_vacuum_statistics_db", + .fixed_amount = false, + .accessed_across_databases = true, + .write_to_file = true, + .track_entry_count = true, + .shared_size = sizeof(PgStatShared_ExtVacEntry), + .shared_data_off = offsetof(PgStatShared_ExtVacEntry, stats), + .shared_data_len = sizeof(PgStat_VacuumRelationCounts), + .pending_size = 0, + .flush_pending_cb = NULL, +}; + +/* + * Accumulate a single counter only if its category is enabled in + * evs_collect_mask. Parentheses around every argument: the macro is invoked + * from expression contexts and with expressions as the destination pointer. + */ +#define ACCUM_IF(dst, src, field, cat) \ + do { \ + if ((evs_collect_mask) & (cat)) \ + ((dst))->field += ((src))->field; \ + } while (0) + +static inline void +pgstat_accumulate_common(PgStat_CommonCounts * dst, const PgStat_CommonCounts * src) +{ + ACCUM_IF(dst, src, total_blks_read, EVS_COLLECT_BUFFERS); + ACCUM_IF(dst, src, total_blks_hit, EVS_COLLECT_BUFFERS); + ACCUM_IF(dst, src, total_blks_dirtied, EVS_COLLECT_BUFFERS); + ACCUM_IF(dst, src, total_blks_written, EVS_COLLECT_BUFFERS); + ACCUM_IF(dst, src, blks_fetched, EVS_COLLECT_BUFFERS); + ACCUM_IF(dst, src, blks_hit, EVS_COLLECT_BUFFERS); + ACCUM_IF(dst, src, blk_read_time, EVS_COLLECT_BUFFERS); + ACCUM_IF(dst, src, blk_write_time, EVS_COLLECT_BUFFERS); + ACCUM_IF(dst, src, delay_time, EVS_COLLECT_TIMING); + ACCUM_IF(dst, src, total_time, EVS_COLLECT_TIMING); + ACCUM_IF(dst, src, wal_records, EVS_COLLECT_WAL); + ACCUM_IF(dst, src, wal_fpi, EVS_COLLECT_WAL); + ACCUM_IF(dst, src, wal_bytes, EVS_COLLECT_WAL); + ACCUM_IF(dst, src, wraparound_failsafe_count, EVS_COLLECT_GENERAL); + ACCUM_IF(dst, src, interrupts_count, EVS_COLLECT_GENERAL); + ACCUM_IF(dst, src, tuples_deleted, EVS_COLLECT_GENERAL); +} + +static inline void +pgstat_accumulate_extvac_stats(PgStat_VacuumRelationCounts * dst, + const PgStat_VacuumRelationCounts * src) +{ + if (dst->type == PGSTAT_EXTVAC_INVALID) + dst->type = src->type; + + Assert(src->type != PGSTAT_EXTVAC_INVALID && src->type != PGSTAT_EXTVAC_DB); + Assert(src->type == dst->type); + + pgstat_accumulate_common(&dst->common, &src->common); + + if (dst->type == PGSTAT_EXTVAC_TABLE && + (evs_collect_mask & EVS_COLLECT_GENERAL) != 0) + { + dst->table.pages_scanned += src->table.pages_scanned; + dst->table.pages_removed += src->table.pages_removed; + dst->table.tuples_frozen += src->table.tuples_frozen; + dst->table.recently_dead_tuples += src->table.recently_dead_tuples; + dst->table.vm_new_frozen_pages += src->table.vm_new_frozen_pages; + dst->table.vm_new_visible_pages += src->table.vm_new_visible_pages; + dst->table.vm_new_visible_frozen_pages += src->table.vm_new_visible_frozen_pages; + dst->table.missed_dead_pages += src->table.missed_dead_pages; + dst->table.missed_dead_tuples += src->table.missed_dead_tuples; + dst->table.index_vacuum_count += src->table.index_vacuum_count; + } + else if (dst->type == PGSTAT_EXTVAC_INDEX && + (evs_collect_mask & EVS_COLLECT_GENERAL) != 0) + { + dst->index.pages_deleted += src->index.pages_deleted; + } +} + +/* + * GUC check hooks: validate the string and compute the bitmask into *extra. + * Rejecting unknown values here prevents silent fall-through to "all". + */ +static bool +evs_track_check_hook(char **newval, void **extra, GucSource source) +{ + int *bits; + + if (*newval == NULL) + return false; + + bits = (int *) guc_malloc(LOG, sizeof(int)); + if (!bits) + return false; + + if (strcmp(*newval, "all") == 0) + *bits = EVS_TRACK_RELATIONS | EVS_TRACK_DATABASES; + else if (strcmp(*newval, "databases") == 0) + *bits = EVS_TRACK_DATABASES; + else if (strcmp(*newval, "relations") == 0) + *bits = EVS_TRACK_RELATIONS; + else + { + guc_free(bits); + GUC_check_errdetail("Allowed values are \"all\", \"databases\", \"relations\"."); + return false; + } + *extra = bits; + return true; +} + +static void +evs_track_assign_hook(const char *newval, void *extra) +{ + evs_track_bits = *((int *) extra); +} + +static bool +evs_track_relations_check_hook(char **newval, void **extra, GucSource source) +{ + int *bits; + + if (*newval == NULL) + return false; + + bits = (int *) guc_malloc(LOG, sizeof(int)); + if (!bits) + return false; + + if (strcmp(*newval, "all") == 0) + *bits = EVS_FILTER_SYSTEM | EVS_FILTER_USER; + else if (strcmp(*newval, "system") == 0) + *bits = EVS_FILTER_SYSTEM; + else if (strcmp(*newval, "user") == 0) + *bits = EVS_FILTER_USER; + else + { + guc_free(bits); + GUC_check_errdetail("Allowed values are \"all\", \"system\", \"user\"."); + return false; + } + *extra = bits; + return true; +} + +static void +evs_track_relations_assign_hook(const char *newval, void *extra) +{ + evs_track_relations_bits = *((int *) extra); +} + +/* + * Check hook for vacuum_statistics.collect. + * + * Accepts a comma- or whitespace-separated list of category names + * (buffers, wal, general, timing) or the shorthand "all". Computes the + * matching bitmask once and stashes it in *extra; the assign hook just + * copies it into evs_collect_mask. Unknown tokens are rejected so the + * setting cannot silently collapse to the "all" default. + */ +static bool +evs_collect_check_hook(char **newval, void **extra, GucSource source) +{ + int *mask; + char *copy; + char *p; + char *tok; + int accum = 0; + bool saw_all = false; + + if (*newval == NULL) + return false; + + mask = (int *) guc_malloc(LOG, sizeof(int)); + if (!mask) + return false; + + /* Empty string means "all", matching the default behavior. */ + if ((*newval)[0] == '\0') + { + *mask = EVS_COLLECT_ALL; + *extra = mask; + return true; + } + + copy = pstrdup(*newval); + for (p = copy; (tok = strtok(p, " \t,")) != NULL; p = NULL) + { + if (pg_strcasecmp(tok, "all") == 0) + saw_all = true; + else if (pg_strcasecmp(tok, "buffers") == 0) + accum |= EVS_COLLECT_BUFFERS; + else if (pg_strcasecmp(tok, "wal") == 0) + accum |= EVS_COLLECT_WAL; + else if (pg_strcasecmp(tok, "general") == 0) + accum |= EVS_COLLECT_GENERAL; + else if (pg_strcasecmp(tok, "timing") == 0) + accum |= EVS_COLLECT_TIMING; + else + { + /* + * GUC_check_errdetail formats the message immediately, but tok + * points into copy; emit the detail first, then free the + * scratch buffer so the formatted string is already stashed in + * GUC_check_errdetail_string. + */ + GUC_check_errdetail("Unrecognized category \"%s\" in vacuum_statistics.collect; " + "allowed values are \"all\", \"buffers\", \"wal\", \"general\", \"timing\".", + tok); + pfree(copy); + guc_free(mask); + return false; + } + } + pfree(copy); + + *mask = saw_all ? EVS_COLLECT_ALL : accum; + if (*mask == 0) + *mask = EVS_COLLECT_ALL; + *extra = mask; + return true; +} + +static void +evs_collect_assign_hook(const char *newval, void *extra) +{ + evs_collect_mask = *((int *) extra); +} + +void +_PG_init(void) +{ + if (!process_shared_preload_libraries_in_progress) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ext_vacuum_statistics module could be loaded only on startup."), + errdetail("Add 'ext_vacuum_statistics' into the shared_preload_libraries list."))); + + DefineCustomBoolVariable("vacuum_statistics.enabled", + "Enable extended vacuum statistics collection.", + NULL, &evs_enabled, true, + PGC_SUSET, 0, NULL, NULL, NULL); + + DefineCustomStringVariable("vacuum_statistics.object_types", + "Object types for statistics: 'all', 'databases', 'relations'.", + NULL, &evs_track, "all", + PGC_SUSET, 0, + evs_track_check_hook, + evs_track_assign_hook, NULL); + + DefineCustomStringVariable("vacuum_statistics.track_relations", + "When tracking relations: 'all', 'system', 'user'.", + NULL, &evs_track_relations, "all", + PGC_SUSET, 0, + evs_track_relations_check_hook, + evs_track_relations_assign_hook, NULL); + + DefineCustomBoolVariable("vacuum_statistics.track_databases_from_list", + "If true, track only databases added via add_track_database.", + NULL, &evs_track_databases_from_list, false, + PGC_SUSET, 0, NULL, NULL, NULL); + + DefineCustomBoolVariable("vacuum_statistics.track_relations_from_list", + "If true, track only relations added via add_track_relation.", + NULL, &evs_track_relations_from_list, false, + PGC_SUSET, 0, NULL, NULL, NULL); + + DefineCustomStringVariable("vacuum_statistics.collect", + "Statistics categories to collect.", + "Comma- or whitespace-separated list of: " + "\"buffers\", \"wal\", \"general\", \"timing\"; " + "or \"all\" for every category (default).", + &evs_collect, "all", + PGC_SUSET, 0, + evs_collect_check_hook, + evs_collect_assign_hook, NULL); + + MarkGUCPrefixReserved(SJ_NODENAME); + + pgstat_register_kind(PGSTAT_KIND_EXTVAC_RELATION, &extvac_relation_kind_info); + pgstat_register_kind(PGSTAT_KIND_EXTVAC_DB, &extvac_db_kind_info); + + prev_shmem_request_hook = shmem_request_hook; + shmem_request_hook = evs_shmem_request; + + prev_report_vacuum_hook = set_report_vacuum_hook; + set_report_vacuum_hook = pgstat_report_vacuum_extstats; + + prev_object_access_hook = object_access_hook; + object_access_hook = evs_drop_access_hook; +} + +static void +evs_shmem_request(void) +{ + if (prev_shmem_request_hook) + prev_shmem_request_hook(); + + RequestNamedLWLockTranche(EVS_TRACK_TRANCHE_NAME, 1); +} + +/* + * Object access hook: remove dropped objects from track lists. + */ +static void +evs_drop_access_hook(ObjectAccessType access, Oid classId, + Oid objectId, int subId, void *arg) +{ + if (prev_object_access_hook) + (*prev_object_access_hook) (access, classId, objectId, subId, arg); + + if (access == OAT_DROP) + { + if (classId == RelationRelationId && subId == 0) + { + char relkind = get_rel_relkind(objectId); + EvsTrackRelKey key; + bool found; + + if (relkind == RELKIND_RELATION || relkind == RELKIND_INDEX) + { + LWLock *lock = evs_get_track_lock(); + + LWLockAcquire(lock, LW_EXCLUSIVE); + evs_track_hash_ensure_init(); + key.dboid = MyDatabaseId; + key.reloid = objectId; + hash_search(evs_track_relations_hash, &key, HASH_REMOVE, &found); + key.dboid = InvalidOid; + hash_search(evs_track_relations_hash, &key, HASH_REMOVE, &found); + evs_track_save_file(); + LWLockRelease(lock); + } + } + + if (classId == DatabaseRelationId && objectId != InvalidOid) + { + LWLock *lock = evs_get_track_lock(); + bool found; + + LWLockAcquire(lock, LW_EXCLUSIVE); + evs_track_hash_ensure_init(); + hash_search(evs_track_databases_hash, &objectId, HASH_REMOVE, &found); + evs_track_save_file(); + LWLockRelease(lock); + } + } +} + +/* + * Storage of track lists in a separate file. + * + * Stores the lists of database OIDs and (dboid, reloid) pairs used for + * selective tracking when track_databases_from_list or track_relations_from_list + * is enabled. + * Data stores in pg_stat/ext_vacuum_statistics_track.oid + */ +/* + * Initialize the backend-local tracking hashes and load their contents + * from the on-disk file. + * + * The hashes are per-backend, so no lock is needed to protect them from + * other processes; however, another backend may be concurrently rewriting + * the track file, so we take a shared lock for the file read. + */ +static void +evs_track_hash_ensure_init(void) +{ + HASHCTL ctl; + LWLock *lock; + bool need_load; + + if (evs_track_hash_initialized) + return; + + lock = evs_get_track_lock(); + + if (evs_track_databases_hash == NULL) + { + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(Oid); + ctl.entrysize = sizeof(Oid); + ctl.hcxt = TopMemoryContext; + evs_track_databases_hash = + hash_create("ext_vacuum_statistics track databases", + 64, &ctl, HASH_ELEM | HASH_BLOBS); + } + + if (evs_track_relations_hash == NULL) + { + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(EvsTrackRelKey); + ctl.entrysize = sizeof(EvsTrackRelKey); + ctl.hcxt = TopMemoryContext; + evs_track_relations_hash = + hash_create("ext_vacuum_statistics track relations", + 64, &ctl, HASH_ELEM | HASH_BLOBS); + } + + need_load = !LWLockHeldByMe(lock); + if (need_load) + LWLockAcquire(lock, LW_SHARED); + PG_TRY(); + { + evs_track_load_file(); + evs_track_hash_initialized = true; + } + PG_FINALLY(); + { + if (need_load) + LWLockRelease(lock); + } + PG_END_TRY(); +} + +/* + * Load track lists from disk into the backend-local hashes. + * + * Caller must hold evs_track_lock at least in shared mode, since the file + * may be concurrently rewritten by another backend. + */ +static void +evs_track_load_file(void) +{ + char path[MAXPGPATH]; + FILE *fp; + char buf[MAXPGPATH]; + bool in_relations = false; + Oid oid; + EvsTrackRelKey key; + bool found; + + if (!DataDir || DataDir[0] == '\0' || + !evs_track_databases_hash || !evs_track_relations_hash) + return; + + snprintf(path, sizeof(path), "%s/%s", DataDir, EVS_TRACK_FILENAME); + fp = AllocateFile(path, "r"); + if (!fp) + { + if (errno != ENOENT) + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not open track file \"%s\": %m", path))); + return; + } + + PG_TRY(); + { + while (fgets(buf, sizeof(buf), fp)) + { + size_t len = strlen(buf); + + /* Reject unterminated lines (longer than buffer) as corruption. */ + if (len > 0 && buf[len - 1] != '\n' && !feof(fp)) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("line too long in track file \"%s\"", path))); + + if (strncmp(buf, "[databases]", 11) == 0) + { + in_relations = false; + continue; + } + if (strncmp(buf, "[relations]", 11) == 0) + { + in_relations = true; + continue; + } + if (in_relations) + { + if (sscanf(buf, "%u %u", &key.dboid, &key.reloid) == 2) + hash_search(evs_track_relations_hash, &key, HASH_ENTER, &found); + else if (sscanf(buf, "%u", &oid) == 1) + { + key.dboid = InvalidOid; + key.reloid = oid; + hash_search(evs_track_relations_hash, &key, HASH_ENTER, &found); + } + } + else if (sscanf(buf, "%u", &oid) == 1) + hash_search(evs_track_databases_hash, &oid, HASH_ENTER, &found); + } + + if (ferror(fp)) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read track file \"%s\": %m", path))); + } + PG_FINALLY(); + { + FreeFile(fp); + } + PG_END_TRY(); +} + +/* + * Atomically rewrite the track file. Caller must hold evs_track_lock + * in exclusive mode. + */ +static void +evs_track_save_file(void) +{ + char path[MAXPGPATH]; + char tmppath[MAXPGPATH]; + FILE *fp; + HASH_SEQ_STATUS status; + Oid *entry; + EvsTrackRelKey *rel_entry; + bool failed = false; + + if (!DataDir || DataDir[0] == '\0' || + !evs_track_databases_hash || !evs_track_relations_hash) + return; + + snprintf(path, sizeof(path), "%s/%s", DataDir, EVS_TRACK_FILENAME); + snprintf(tmppath, sizeof(tmppath), "%s.tmp", path); + + fp = AllocateFile(tmppath, PG_BINARY_W); + if (!fp) + { + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not create track file \"%s\": %m", tmppath))); + return; + } + + PG_TRY(); + { + if (fputs("[databases]\n", fp) == EOF) + failed = true; + + if (!failed) + { + hash_seq_init(&status, evs_track_databases_hash); + while ((entry = (Oid *) hash_seq_search(&status)) != NULL) + { + if (fprintf(fp, "%u\n", *entry) < 0) + { + hash_seq_term(&status); + failed = true; + break; + } + } + } + + if (!failed && fputs("[relations]\n", fp) == EOF) + failed = true; + + if (!failed) + { + hash_seq_init(&status, evs_track_relations_hash); + while ((rel_entry = (EvsTrackRelKey *) hash_seq_search(&status)) != NULL) + { + int rc; + + if (OidIsValid(rel_entry->dboid)) + rc = fprintf(fp, "%u %u\n", rel_entry->dboid, rel_entry->reloid); + else + rc = fprintf(fp, "0 %u\n", rel_entry->reloid); + if (rc < 0) + { + hash_seq_term(&status); + failed = true; + break; + } + } + } + + if (!failed && fflush(fp) != 0) + failed = true; + + if (!failed) + { + int fd = fileno(fp); + + if (fd >= 0 && pg_fsync(fd) != 0) + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not fsync track file \"%s\": %m", + tmppath))); + } + } + PG_CATCH(); + { + FreeFile(fp); + (void) unlink(tmppath); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (FreeFile(fp) != 0) + { + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not close track file \"%s\": %m", tmppath))); + failed = true; + } + + if (failed) + { + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not write track file \"%s\": %m", tmppath))); + if (unlink(tmppath) != 0 && errno != ENOENT) + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not unlink \"%s\": %m", tmppath))); + return; + } + + if (durable_rename(tmppath, path, LOG) != 0) + { + if (unlink(tmppath) != 0 && errno != ENOENT) + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not unlink \"%s\": %m", tmppath))); + } +} + +/* + * Check if OID is in the given hash + */ +static bool +evs_oid_in_list(HTAB *hash, Oid oid) +{ + if (!hash) + return false; + if (hash_get_num_entries(hash) == 0) + return false; + return hash_search(hash, &oid, HASH_FIND, NULL) != NULL; +} + +/* + * Check if (dboid, relid) is in track_relations list. + */ +static bool +evs_rel_in_list(Oid dboid, Oid relid) +{ + EvsTrackRelKey key; + + if (!evs_track_relations_hash) + return false; + if (hash_get_num_entries(evs_track_relations_hash) == 0) + return false; + key.dboid = dboid; + key.reloid = relid; + if (hash_search(evs_track_relations_hash, &key, HASH_FIND, NULL) != NULL) + return true; + key.dboid = InvalidOid; + return hash_search(evs_track_relations_hash, &key, HASH_FIND, NULL) != NULL; +} + +/* + * Decide whether to track statistics for relations. + * Relation is tracked if it is in the track list or a special filter is enabled. + */ +static bool +evs_should_track_relation_statistics(Oid dboid, Oid relid) +{ + evs_track_hash_ensure_init(); + + if (evs_track_databases_from_list && + !evs_oid_in_list(evs_track_databases_hash, dboid)) + return false; + if (evs_track_relations_from_list && + !(evs_rel_in_list(dboid, relid) || evs_rel_in_list(InvalidOid, relid))) + return false; + + if ((evs_track_bits & EVS_TRACK_RELATIONS) == 0) + return false; /* database-only mode */ + if (evs_track_relations_bits == EVS_FILTER_SYSTEM) + return IsCatalogRelationOid(relid); + if (evs_track_relations_bits == EVS_FILTER_USER) + return !IsCatalogRelationOid(relid); + return true; +} + +/* + * Decide whether to track statistics for databases. + * Database statistics is tracked if it is in the track list or a special filter is enabled. + */ +static bool +evs_should_track_database_statistics(Oid dboid) +{ + evs_track_hash_ensure_init(); + + if (evs_track_databases_from_list && + !evs_oid_in_list(evs_track_databases_hash, dboid)) + return false; + if ((evs_track_bits & EVS_TRACK_DATABASES) == 0) + return false; /* relations-only mode */ + if (evs_track_bits == EVS_TRACK_DATABASES) + return true; /* databases-only, accumulate to db */ + return true; +} + + +/* Accumulate common counts for database-level stats. */ +static inline void +pgstat_accumulate_common_for_db(PgStat_CommonCounts * dst, + const PgStat_CommonCounts * src) +{ + pgstat_accumulate_common(dst, src); +} + +/* + * Store incoming vacuum stats into pgstat custom statistics. + * store_relation: create/update per-relation entry + * store_db: accumulate into database-level entry (dboid, objid=0). + * Uses pgstat_get_entry_ref_locked and pgstat_accumulate_* for atomic updates. + */ +static void +extvac_store(Oid dboid, Oid relid, int type, + PgStat_VacuumRelationCounts * params, + bool store_relation, bool store_db) +{ + PgStat_EntryRef *entry_ref; + PgStatShared_ExtVacEntry *shared; + uint64 objid; + + if (!evs_enabled) + return; + + if (store_relation) + { + objid = EXTVAC_OBJID(relid, type); + entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_EXTVAC_RELATION, dboid, objid, false); + if (entry_ref) + { + shared = (PgStatShared_ExtVacEntry *) entry_ref->shared_stats; + if (shared->stats.type == PGSTAT_EXTVAC_INVALID) + { + memset(&shared->stats, 0, sizeof(shared->stats)); + shared->stats.type = params->type; + } + pgstat_accumulate_extvac_stats(&shared->stats, params); + pgstat_unlock_entry(entry_ref); + } + } + + if (store_db) + { + entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_EXTVAC_DB, dboid, InvalidOid, false); + if (entry_ref) + { + shared = (PgStatShared_ExtVacEntry *) entry_ref->shared_stats; + if (shared->stats.type == PGSTAT_EXTVAC_INVALID) + { + memset(&shared->stats, 0, sizeof(shared->stats)); + shared->stats.type = PGSTAT_EXTVAC_DB; + } + pgstat_accumulate_common_for_db(&shared->stats.common, ¶ms->common); + pgstat_unlock_entry(entry_ref); + } + } +} + +/* + * Vacuum report hook: called when vacuum finishes. Filters by track settings, + * stores stats per-relation and/or per-database, then chains to previous hook. + */ +static void +pgstat_report_vacuum_extstats(Oid tableoid, bool shared, + PgStat_VacuumRelationCounts * params) +{ + Oid dboid = shared ? InvalidOid : MyDatabaseId; + bool store_relation; + bool store_db; + + if (evs_enabled) + { + store_relation = evs_should_track_relation_statistics(dboid, tableoid); + store_db = evs_should_track_database_statistics(dboid); + + if (store_relation || store_db) + extvac_store(dboid, tableoid, params->type, params, store_relation, store_db); + } + if (prev_report_vacuum_hook) + prev_report_vacuum_hook(tableoid, shared, params); +} + +/* Reset statistics for a single relation entry. */ +static bool +extvac_reset_by_relid(Oid dboid, Oid relid, int type) +{ + uint64 objid = EXTVAC_OBJID(relid, type); + + pgstat_reset_entry(PGSTAT_KIND_EXTVAC_RELATION, dboid, objid, 0); + return true; +} + +/* Callback for pgstat_reset_matching_entries: match relation entries for given db */ +static bool +match_extvac_relations_for_db(PgStatShared_HashEntry *entry, Datum match_data) +{ + return entry->key.kind == PGSTAT_KIND_EXTVAC_RELATION && + entry->key.dboid == DatumGetObjectId(match_data); +} + +/* + * Reset statistics for a database (aggregate entry) and all its relations. + */ +static int64 +extvac_database_reset(Oid dboid) +{ + pgstat_reset_matching_entries(match_extvac_relations_for_db, + ObjectIdGetDatum(dboid), 0); + pgstat_reset_entry(PGSTAT_KIND_EXTVAC_DB, dboid, 0, 0); + return 1; +} + +/* Reset all vacuum statistics (both relation and database entries). */ +static int64 +extvac_stat_reset(void) +{ + pgstat_reset_of_kind(PGSTAT_KIND_EXTVAC_RELATION); + pgstat_reset_of_kind(PGSTAT_KIND_EXTVAC_DB); + return 0; /* count not available */ +} + +PG_FUNCTION_INFO_V1(vacuum_statistics_reset); +PG_FUNCTION_INFO_V1(extvac_shared_memory_size); +PG_FUNCTION_INFO_V1(extvac_reset_entry); +PG_FUNCTION_INFO_V1(extvac_reset_db_entry); + +Datum +vacuum_statistics_reset(PG_FUNCTION_ARGS) +{ + PG_RETURN_INT64(extvac_stat_reset()); +} + +Datum +extvac_reset_entry(PG_FUNCTION_ARGS) +{ + Oid dboid = PG_GETARG_OID(0); + Oid relid = PG_GETARG_OID(1); + int type = PG_GETARG_INT32(2); + + PG_RETURN_BOOL(extvac_reset_by_relid(dboid, relid, type)); +} + +Datum +extvac_reset_db_entry(PG_FUNCTION_ARGS) +{ + Oid dboid = PG_GETARG_OID(0); + + PG_RETURN_INT64(extvac_database_reset(dboid)); +} + +/* + * Return total shared memory in bytes used by the extension for vacuum stats. + * Used for monitoring and capacity planning: memory grows with the number of + * tracked relations and databases. + */ +Datum +extvac_shared_memory_size(PG_FUNCTION_ARGS) +{ + uint64 rel_count; + uint64 db_count; + uint64 total; + size_t entry_size = sizeof(PgStatShared_ExtVacEntry); + + rel_count = pgstat_get_entry_count(PGSTAT_KIND_EXTVAC_RELATION); + db_count = pgstat_get_entry_count(PGSTAT_KIND_EXTVAC_DB); + total = rel_count + db_count; + + PG_RETURN_INT64((int64) (total * entry_size)); +} + +/* + * Track list management: add/remove database or relation OIDs. + * Changes are persisted to pg_stat/ext_vacuum_statistics_track.oid. + */ + +PG_FUNCTION_INFO_V1(evs_add_track_database); +PG_FUNCTION_INFO_V1(evs_remove_track_database); +PG_FUNCTION_INFO_V1(evs_add_track_relation); +PG_FUNCTION_INFO_V1(evs_remove_track_relation); + +/* + * Mutating track-list entry points: require server-wide privilege, since + * the underlying lists steer tracking for every backend. + */ +static void +evs_require_track_privilege(const char *funcname) +{ + if (!superuser() && !has_privs_of_role(GetUserId(), ROLE_PG_READ_ALL_STATS)) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("permission denied for function %s", funcname), + errhint("Only superusers and members of pg_read_all_stats " + "may change the vacuum statistics track list."))); +} + +Datum +evs_add_track_database(PG_FUNCTION_ARGS) +{ + Oid oid = PG_GETARG_OID(0); + bool found; + LWLock *lock; + + evs_require_track_privilege("add_track_database"); + lock = evs_get_track_lock(); + LWLockAcquire(lock, LW_EXCLUSIVE); + evs_track_hash_ensure_init(); + hash_search(evs_track_databases_hash, &oid, HASH_ENTER, &found); + evs_track_save_file(); + LWLockRelease(lock); + PG_RETURN_BOOL(!found); /* true if newly added */ +} + +Datum +evs_remove_track_database(PG_FUNCTION_ARGS) +{ + Oid oid = PG_GETARG_OID(0); + bool found; + LWLock *lock; + + evs_require_track_privilege("remove_track_database"); + lock = evs_get_track_lock(); + LWLockAcquire(lock, LW_EXCLUSIVE); + evs_track_hash_ensure_init(); + hash_search(evs_track_databases_hash, &oid, HASH_REMOVE, &found); + evs_track_save_file(); + LWLockRelease(lock); + PG_RETURN_BOOL(found); +} + +Datum +evs_add_track_relation(PG_FUNCTION_ARGS) +{ + EvsTrackRelKey key; + bool found; + LWLock *lock; + + evs_require_track_privilege("add_track_relation"); + key.dboid = PG_GETARG_OID(0); + key.reloid = PG_GETARG_OID(1); + lock = evs_get_track_lock(); + LWLockAcquire(lock, LW_EXCLUSIVE); + evs_track_hash_ensure_init(); + hash_search(evs_track_relations_hash, &key, HASH_ENTER, &found); + evs_track_save_file(); + LWLockRelease(lock); + PG_RETURN_BOOL(!found); /* true if newly added */ +} + +Datum +evs_remove_track_relation(PG_FUNCTION_ARGS) +{ + EvsTrackRelKey key; + bool found; + LWLock *lock; + + evs_require_track_privilege("remove_track_relation"); + key.dboid = PG_GETARG_OID(0); + key.reloid = PG_GETARG_OID(1); + lock = evs_get_track_lock(); + LWLockAcquire(lock, LW_EXCLUSIVE); + evs_track_hash_ensure_init(); + hash_search(evs_track_relations_hash, &key, HASH_REMOVE, &found); + evs_track_save_file(); + LWLockRelease(lock); + PG_RETURN_BOOL(found); +} + +/* + * Returns the list of database and relation OIDs for which statistics + * are collected. + */ +PG_FUNCTION_INFO_V1(evs_track_list); + +Datum +evs_track_list(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + TupleDesc tupdesc; + Tuplestorestate *tupstore; + MemoryContext per_query_ctx; + MemoryContext oldcontext; + Datum values[3]; + bool nulls[3] = {false, false, false}; + HASH_SEQ_STATUS status; + Oid *entry; + EvsTrackRelKey *rel_entry; + + if (!rsinfo || !IsA(rsinfo, ReturnSetInfo)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ext_vacuum_statistics: set-valued function called in context that cannot accept a set"))); + if (!(rsinfo->allowedModes & SFRM_Materialize)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ext_vacuum_statistics: materialize mode required"))); + + evs_track_hash_ensure_init(); + + per_query_ctx = rsinfo->econtext->ecxt_per_query_memory; + oldcontext = MemoryContextSwitchTo(per_query_ctx); + + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "ext_vacuum_statistics: return type must be a row type"); + + tupstore = tuplestore_begin_heap(true, false, work_mem); + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = tupdesc; + + /* Databases */ + if (hash_get_num_entries(evs_track_databases_hash) == 0) + { + values[0] = CStringGetTextDatum("database"); + nulls[1] = true; + nulls[2] = true; + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + nulls[1] = false; + nulls[2] = false; + } + else + { + hash_seq_init(&status, evs_track_databases_hash); + while ((entry = (Oid *) hash_seq_search(&status)) != NULL) + { + values[0] = CStringGetTextDatum("database"); + values[1] = ObjectIdGetDatum(*entry); + nulls[2] = true; + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + nulls[2] = false; + } + } + + /* Relations */ + if (hash_get_num_entries(evs_track_relations_hash) == 0) + { + values[0] = CStringGetTextDatum("relation"); + nulls[1] = true; + nulls[2] = true; + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + nulls[1] = false; + nulls[2] = false; + } + else + { + hash_seq_init(&status, evs_track_relations_hash); + while ((rel_entry = (EvsTrackRelKey *) hash_seq_search(&status)) != NULL) + { + values[0] = CStringGetTextDatum("relation"); + values[1] = ObjectIdGetDatum(rel_entry->dboid); + values[2] = ObjectIdGetDatum(rel_entry->reloid); + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + } + } + + MemoryContextSwitchTo(oldcontext); + + return (Datum) 0; +} + +/* + * Output vacuum statistics (tables, indexes, or per-database aggregates). + */ +#define EXTVAC_COMMON_STAT_COLS 12 + +static void +tuplestore_put_common(PgStat_CommonCounts * vacuum_ext, + Datum *values, bool *nulls, int *i) +{ + char buf[256]; + const int base = *i; + + values[(*i)++] = Int64GetDatum(vacuum_ext->total_blks_read); + values[(*i)++] = Int64GetDatum(vacuum_ext->total_blks_hit); + values[(*i)++] = Int64GetDatum(vacuum_ext->total_blks_dirtied); + values[(*i)++] = Int64GetDatum(vacuum_ext->total_blks_written); + values[(*i)++] = Int64GetDatum(vacuum_ext->wal_records); + values[(*i)++] = Int64GetDatum(vacuum_ext->wal_fpi); + snprintf(buf, sizeof buf, UINT64_FORMAT, vacuum_ext->wal_bytes); + values[(*i)++] = DirectFunctionCall3(numeric_in, + CStringGetDatum(buf), + ObjectIdGetDatum(0), + Int32GetDatum(-1)); + values[(*i)++] = Float8GetDatum(vacuum_ext->blk_read_time); + values[(*i)++] = Float8GetDatum(vacuum_ext->blk_write_time); + values[(*i)++] = Float8GetDatum(vacuum_ext->delay_time); + values[(*i)++] = Float8GetDatum(vacuum_ext->total_time); + values[(*i)++] = Int32GetDatum(vacuum_ext->wraparound_failsafe_count); + Assert((*i - base) == EXTVAC_COMMON_STAT_COLS); +} + +#define EXTVAC_HEAP_STAT_COLS 26 +#define EXTVAC_IDX_STAT_COLS 17 +#define EXTVAC_MAX_STAT_COLS Max(EXTVAC_HEAP_STAT_COLS, EXTVAC_IDX_STAT_COLS) + +static void +tuplestore_put_for_relation(Oid relid, Tuplestorestate *tupstore, + TupleDesc tupdesc, PgStat_VacuumRelationCounts * vacuum_ext) +{ + Datum values[EXTVAC_MAX_STAT_COLS]; + bool nulls[EXTVAC_MAX_STAT_COLS]; + int i = 0; + + memset(nulls, 0, sizeof(nulls)); + values[i++] = ObjectIdGetDatum(relid); + + tuplestore_put_common(&vacuum_ext->common, values, nulls, &i); + values[i++] = Int64GetDatum(vacuum_ext->common.blks_fetched - vacuum_ext->common.blks_hit); + values[i++] = Int64GetDatum(vacuum_ext->common.blks_hit); + + if (vacuum_ext->type == PGSTAT_EXTVAC_TABLE) + { + values[i++] = Int64GetDatum(vacuum_ext->common.tuples_deleted); + values[i++] = Int64GetDatum(vacuum_ext->table.pages_scanned); + values[i++] = Int64GetDatum(vacuum_ext->table.pages_removed); + values[i++] = Int64GetDatum(vacuum_ext->table.vm_new_frozen_pages); + values[i++] = Int64GetDatum(vacuum_ext->table.vm_new_visible_pages); + values[i++] = Int64GetDatum(vacuum_ext->table.vm_new_visible_frozen_pages); + values[i++] = Int64GetDatum(vacuum_ext->table.tuples_frozen); + values[i++] = Int64GetDatum(vacuum_ext->table.recently_dead_tuples); + values[i++] = Int64GetDatum(vacuum_ext->table.index_vacuum_count); + values[i++] = Int64GetDatum(vacuum_ext->table.missed_dead_pages); + values[i++] = Int64GetDatum(vacuum_ext->table.missed_dead_tuples); + } + else if (vacuum_ext->type == PGSTAT_EXTVAC_INDEX) + { + values[i++] = Int64GetDatum(vacuum_ext->common.tuples_deleted); + values[i++] = Int64GetDatum(vacuum_ext->index.pages_deleted); + } + + Assert(i == ((vacuum_ext->type == PGSTAT_EXTVAC_TABLE) ? EXTVAC_HEAP_STAT_COLS : EXTVAC_IDX_STAT_COLS)); + tuplestore_putvalues(tupstore, tupdesc, values, nulls); +} + +static Datum +pg_stats_vacuum(FunctionCallInfo fcinfo, int type) +{ + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + MemoryContext per_query_ctx; + MemoryContext oldcontext; + Tuplestorestate *tupstore; + TupleDesc tupdesc; + Oid dbid = PG_GETARG_OID(0); + + if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ext_vacuum_statistics: set-valued function called in context that cannot accept a set"))); + if (!(rsinfo->allowedModes & SFRM_Materialize)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ext_vacuum_statistics: materialize mode required"))); + + per_query_ctx = rsinfo->econtext->ecxt_per_query_memory; + oldcontext = MemoryContextSwitchTo(per_query_ctx); + + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "ext_vacuum_statistics: return type must be a row type"); + + tupstore = tuplestore_begin_heap(true, false, work_mem); + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = tupdesc; + + MemoryContextSwitchTo(oldcontext); + + if (type == PGSTAT_EXTVAC_INDEX || type == PGSTAT_EXTVAC_TABLE) + { + Oid relid = PG_GETARG_OID(1); + PgStat_VacuumRelationCounts *stats; + + if (!OidIsValid(relid)) + return (Datum) 0; + + stats = (PgStat_VacuumRelationCounts *) + pgstat_fetch_entry(PGSTAT_KIND_EXTVAC_RELATION, dbid, + EXTVAC_OBJID(relid, type), NULL); + + if (!stats) + stats = (PgStat_VacuumRelationCounts *) + pgstat_fetch_entry(PGSTAT_KIND_EXTVAC_RELATION, InvalidOid, + EXTVAC_OBJID(relid, type), NULL); + + if (stats && stats->type == type) + tuplestore_put_for_relation(relid, tupstore, tupdesc, stats); + } + else if (type == PGSTAT_EXTVAC_DB) + { + if (OidIsValid(dbid)) + { +#define EXTVAC_DB_STAT_COLS 14 + Datum values[EXTVAC_DB_STAT_COLS]; + bool nulls[EXTVAC_DB_STAT_COLS]; + int i = 0; + PgStat_VacuumRelationCounts *stats; + + stats = (PgStat_VacuumRelationCounts *) + pgstat_fetch_entry(PGSTAT_KIND_EXTVAC_DB, dbid, + InvalidOid, NULL); + if (stats && stats->type == PGSTAT_EXTVAC_DB) + { + memset(nulls, 0, sizeof(nulls)); + values[i++] = ObjectIdGetDatum(dbid); + tuplestore_put_common(&stats->common, values, nulls, &i); + values[i++] = Int32GetDatum(stats->common.interrupts_count); + Assert(i == EXTVAC_DB_STAT_COLS); + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + } + } + /* invalid dbid: return empty set */ + } + else + elog(PANIC, "ext_vacuum_statistics: invalid type %d", type); + + return (Datum) 0; +} + +PG_FUNCTION_INFO_V1(pg_stats_get_vacuum_tables); +PG_FUNCTION_INFO_V1(pg_stats_get_vacuum_indexes); +PG_FUNCTION_INFO_V1(pg_stats_get_vacuum_database); + +Datum +pg_stats_get_vacuum_tables(PG_FUNCTION_ARGS) +{ + return pg_stats_vacuum(fcinfo, PGSTAT_EXTVAC_TABLE); +} + +Datum +pg_stats_get_vacuum_indexes(PG_FUNCTION_ARGS) +{ + return pg_stats_vacuum(fcinfo, PGSTAT_EXTVAC_INDEX); +} + +Datum +pg_stats_get_vacuum_database(PG_FUNCTION_ARGS) +{ + return pg_stats_vacuum(fcinfo, PGSTAT_EXTVAC_DB); +} diff --git a/contrib/meson.build b/contrib/meson.build index ebb7f83d8c5..d7dc0fd07f0 100644 --- a/contrib/meson.build +++ b/contrib/meson.build @@ -26,6 +26,7 @@ subdir('cube') subdir('dblink') subdir('dict_int') subdir('dict_xsyn') +subdir('ext_vacuum_statistics') subdir('earthdistance') subdir('file_fdw') subdir('fuzzystrmatch') diff --git a/doc/src/sgml/contrib.sgml b/doc/src/sgml/contrib.sgml index b9b03654aad..2a38f9042bb 100644 --- a/doc/src/sgml/contrib.sgml +++ b/doc/src/sgml/contrib.sgml @@ -141,6 +141,7 @@ CREATE EXTENSION extension_name; &dict-int; &dict-xsyn; &earthdistance; + &extvacuumstatistics; &file-fdw; &fuzzystrmatch; &hstore; diff --git a/doc/src/sgml/extvacuumstatistics.sgml b/doc/src/sgml/extvacuumstatistics.sgml new file mode 100644 index 00000000000..75eb4691c4d --- /dev/null +++ b/doc/src/sgml/extvacuumstatistics.sgml @@ -0,0 +1,502 @@ + + + + ext_vacuum_statistics — extended vacuum statistics + + + ext_vacuum_statistics + + + + The ext_vacuum_statistics module provides + extended per-table, per-index, and per-database vacuum statistics + (buffer I/O, WAL, general, timing) via views in the + ext_vacuum_statistics schema. + + + + The module must be loaded by adding ext_vacuum_statistics to + in + postgresql.conf, because it registers a vacuum hook at + server startup. This means that a server restart is needed to add or remove + the module. After installation, run + CREATE EXTENSION ext_vacuum_statistics in each database + where you want to use it. + + + + When active, the module provides views + ext_vacuum_statistics.pg_stats_vacuum_tables, + ext_vacuum_statistics.pg_stats_vacuum_indexes, and + ext_vacuum_statistics.pg_stats_vacuum_database, + plus functions to reset statistics and manage tracking. + + + + Each tracked object (one table, one index, or one database) uses + approximately 232 bytes of shared memory on Linux x86_64 (e.g. Ubuntu): + common stats (buffers, WAL, timing) plus header and LWLock ~144 bytes; + type + union ~88 bytes (the union holds table-specific or index-specific + fields; the allocated size is the same for both). The exact size depends on the platform. Call + ext_vacuum_statistics.shared_memory_size() to get + the total shared memory used by the extension. The extension's GUCs allow controlling memory by limiting + which objects are tracked: + vacuum_statistics.object_types, + vacuum_statistics.track_relations, and + track_*_from_list. + Example: a database with 1000 tables and 2000 indexes uses about 700 KB + on Ubuntu ((1000 + 2000 + 1) × 232 bytes). + + + + The <structname>ext_vacuum_statistics.pg_stats_vacuum_tables</structname> View + + + pg_stats_vacuum_tables + + + + The view ext_vacuum_statistics.pg_stats_vacuum_tables + contains one row for each table in the current database (including TOAST + tables), showing statistics about vacuuming that specific table. The columns + are shown in . + + + + <structname>ext_vacuum_statistics.pg_stats_vacuum_tables</structname> Columns + + + + + Column Type + + + Description + + + + + + + relid oid + + + OID of a table + + + + + schema name + + + Name of the schema this table is in + + + + + relname name + + + Name of this table + + + + + dbname name + + + Name of the database containing this table + + + + + total_blks_read int8 + + + Number of database blocks read by vacuum operations performed on this table + + + + + total_blks_hit int8 + + + Number of times database blocks were found in the buffer cache by vacuum operations + + + + + total_blks_dirtied int8 + + + Number of database blocks dirtied by vacuum operations + + + + + total_blks_written int8 + + + Number of database blocks written by vacuum operations + + + + + wal_records int8 + + + Total number of WAL records generated by vacuum operations performed on this table + + + + + wal_fpi int8 + + + Total number of WAL full page images generated by vacuum operations + + + + + wal_bytes numeric + + + Total amount of WAL bytes generated by vacuum operations + + + + + blk_read_time float8 + + + Time spent reading blocks by vacuum operations, in milliseconds + + + + + blk_write_time float8 + + + Time spent writing blocks by vacuum operations, in milliseconds + + + + + delay_time float8 + + + Time spent in vacuum delay points, in milliseconds + + + + + total_time float8 + + + Total time of vacuuming this table, in milliseconds + + + + + wraparound_failsafe_count int4 + + + Number of times vacuum was run to prevent a wraparound problem + + + + + rel_blks_read int8 + + + Number of blocks vacuum operations read from this table + + + + + rel_blks_hit int8 + + + Number of times blocks of this table were found in the buffer cache by vacuum + + + + + tuples_deleted int8 + + + Number of dead tuples vacuum operations deleted from this table + + + + + pages_scanned int8 + + + Number of pages examined by vacuum operations + + + + + pages_removed int8 + + + Number of pages removed from physical storage by vacuum operations + + + + + vm_new_frozen_pages int8 + + + Number of pages newly set all-frozen by vacuum in the visibility map + + + + + vm_new_visible_pages int8 + + + Number of pages newly set all-visible by vacuum in the visibility map + + + + + vm_new_visible_frozen_pages int8 + + + Number of pages newly set all-visible and all-frozen by vacuum in the visibility map + + + + + tuples_frozen int8 + + + Number of tuples that vacuum operations marked as frozen + + + + + recently_dead_tuples int8 + + + Number of dead tuples left due to visibility in transactions + + + + + index_vacuum_count int8 + + + Number of times indexes on this table were vacuumed + + + + + missed_dead_pages int8 + + + Number of pages that had at least one missed dead tuple + + + + + missed_dead_tuples int8 + + + Number of fully DEAD tuples that could not be pruned due to failure to acquire a cleanup lock + + + + +
+ + + + The <structname>ext_vacuum_statistics.pg_stats_vacuum_indexes</structname> View + + + pg_stats_vacuum_indexes + + + + The view ext_vacuum_statistics.pg_stats_vacuum_indexes + contains one row for each index in the current database, showing statistics + about vacuuming that specific index. Columns include + indexrelid, schema, + indexrelname, dbname, + buffer I/O (total_blks_read, + total_blks_hit, etc.), WAL + (wal_records, wal_fpi, + wal_bytes), timing + (blk_read_time, blk_write_time, + delay_time, total_time), + and tuples_deleted, pages_deleted. + + + + + The <structname>ext_vacuum_statistics.pg_stats_vacuum_database</structname> View + + + pg_stats_vacuum_database + + + + The view ext_vacuum_statistics.pg_stats_vacuum_database + contains one row for each database in the cluster, showing aggregate vacuum + statistics for that database. Columns include + dboid, dbname, + db_blks_read, db_blks_hit, + db_blks_dirtied, db_blks_written, + WAL stats (db_wal_records, + db_wal_fpi, db_wal_bytes), + timing (db_blk_read_time, + db_blk_write_time, db_delay_time, + db_total_time), + db_wraparound_failsafe_count, and + interrupts_count. + + + + + Functions + + + + + ext_vacuum_statistics.shared_memory_size() + bigint + + + + Returns the total shared memory in bytes used by the extension for + vacuum statistics (relations plus databases). + + + + + + ext_vacuum_statistics.vacuum_statistics_reset() + bigint + + + + Resets all vacuum statistics. Returns the number of entries reset. + + + + + + ext_vacuum_statistics.add_track_database(dboid oid) + boolean + + + + Adds a database OID to the tracking list (persisted to + pg_stat/ext_vacuum_statistics_track.oid). + Returns true if newly added. + + + + + + ext_vacuum_statistics.remove_track_database(dboid oid) + boolean + + + + Removes a database OID from the tracking list. Returns true if found and removed. + + + + + + ext_vacuum_statistics.add_track_relation(dboid oid, reloid oid) + boolean + + + + Adds a (database, relation) OID pair to the tracking list. Returns true if newly added. + + + + + + ext_vacuum_statistics.remove_track_relation(dboid oid, reloid oid) + boolean + + + + Removes a (database, relation) pair from the tracking list. Returns true if found and removed. + + + + + + ext_vacuum_statistics.track_list() + TABLE(track_kind text, dboid oid, reloid oid) + + + + Returns the list of database and relation OIDs for which vacuum statistics + are collected. When dboid or + reloid is NULL, statistics are collected for all. + + + + + + + + Configuration Parameters + + + + vacuum_statistics.enabled (boolean) + + + Enables extended vacuum statistics collection. Default: on. + + + + + vacuum_statistics.object_types (string) + + + Object types for statistics: all, databases, or + relations. Default: all. + + + + + vacuum_statistics.track_relations (string) + + + When tracking relations: all, system, or + user. Default: all. + + + + + vacuum_statistics.track_databases_from_list (boolean) + + + If on, track only databases added via add_track_database. + Default: off. + + + + + vacuum_statistics.track_relations_from_list (boolean) + + + If on, track only relations added via add_track_relation. + Default: off. + + + + + + diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index 25a85082759..85d721467c0 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -133,6 +133,7 @@ + -- 2.39.5 (Apple Git-154)