From c10bf672375d1fde10adf88c5df3fdd38824936f Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Mon, 20 Jul 2026 17:06:11 +0300 Subject: [PATCH v43 5/9] Vacuum report hook and the ext_vacuum_statistics extension. Add set_report_vacuum_hook, which gives extensions a report after a vacuum of a table and of each of its indexes. Add the ext_vacuum_statistics extension, which stores these reports as custom cumulative statistics and shows them in the pg_stats_vacuum_tables and pg_stats_vacuum_indexes views. This commit adds the tuple counters: tuples_deleted, tuples_frozen, recently_dead_tuples and missed_dead_tuples. Page, WAL and buffer counters come in the following commits. When a relation is dropped, its statistics are dropped too. When a relation is created, any old statistics for its OID are reset. The statistics are keyed by the relation OID alone, so the object access hook does not need to read pg_class. If the server is started without the module, all cumulative statistics are discarded, because the statistics file has entries of an unknown kind. The documentation mentions this. Collection is controlled by vacuum_statistics.enabled, which can be set per database. Tests cover the tuple counters, the GUC and the removal of statistics of dropped relations. Authors: Alena Rybakina , Andrei Lepikhov , Andrei Zubkov --- contrib/Makefile | 1 + contrib/ext_vacuum_statistics/.gitignore | 5 + contrib/ext_vacuum_statistics/Makefile | 23 ++ contrib/ext_vacuum_statistics/README.md | 85 +++++ .../expected/ext_vacuum_statistics.out | 52 +++ .../vacuum-extending-in-repetable-read.out | 52 +++ .../ext_vacuum_statistics--1.0.sql | 99 +++++ .../ext_vacuum_statistics.conf | 2 + .../ext_vacuum_statistics.control | 5 + contrib/ext_vacuum_statistics/meson.build | 41 +++ .../vacuum-extending-in-repetable-read.spec | 59 +++ .../t/054_vacuum_extending_gucs_test.pl | 100 ++++++ .../t/056_vacuum_stats_gc.pl | 221 ++++++++++++ .../ext_vacuum_statistics/vacuum_statistics.c | 340 ++++++++++++++++++ contrib/meson.build | 1 + doc/src/sgml/contrib.sgml | 1 + doc/src/sgml/extvacuumstatistics.sgml | 254 +++++++++++++ doc/src/sgml/filelist.sgml | 1 + src/backend/access/heap/vacuumlazy.c | 102 +++++- src/backend/commands/vacuumparallel.c | 18 + src/backend/utils/activity/pgstat_relation.c | 29 ++ src/include/commands/vacuum.h | 1 + src/include/pgstat.h | 46 +++ 23 files changed, 1531 insertions(+), 7 deletions(-) create mode 100644 contrib/ext_vacuum_statistics/.gitignore create mode 100644 contrib/ext_vacuum_statistics/Makefile create mode 100644 contrib/ext_vacuum_statistics/README.md create mode 100644 contrib/ext_vacuum_statistics/expected/ext_vacuum_statistics.out create mode 100644 contrib/ext_vacuum_statistics/expected/vacuum-extending-in-repetable-read.out create mode 100644 contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql create mode 100644 contrib/ext_vacuum_statistics/ext_vacuum_statistics.conf create mode 100644 contrib/ext_vacuum_statistics/ext_vacuum_statistics.control create mode 100644 contrib/ext_vacuum_statistics/meson.build create mode 100644 contrib/ext_vacuum_statistics/specs/vacuum-extending-in-repetable-read.spec create mode 100644 contrib/ext_vacuum_statistics/t/054_vacuum_extending_gucs_test.pl create mode 100644 contrib/ext_vacuum_statistics/t/056_vacuum_stats_gc.pl create mode 100644 contrib/ext_vacuum_statistics/vacuum_statistics.c create mode 100644 doc/src/sgml/extvacuumstatistics.sgml diff --git a/contrib/Makefile b/contrib/Makefile index 7d91fe77db3..3140f2bf844 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -19,6 +19,7 @@ SUBDIRS = \ dict_int \ dict_xsyn \ earthdistance \ + ext_vacuum_statistics \ file_fdw \ fuzzystrmatch \ hstore \ diff --git a/contrib/ext_vacuum_statistics/.gitignore b/contrib/ext_vacuum_statistics/.gitignore new file mode 100644 index 00000000000..4d88e8bdc41 --- /dev/null +++ b/contrib/ext_vacuum_statistics/.gitignore @@ -0,0 +1,5 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ +/output_iso/ diff --git a/contrib/ext_vacuum_statistics/Makefile b/contrib/ext_vacuum_statistics/Makefile new file mode 100644 index 00000000000..0c30e2856a6 --- /dev/null +++ b/contrib/ext_vacuum_statistics/Makefile @@ -0,0 +1,23 @@ +# contrib/ext_vacuum_statistics/Makefile + +EXTENSION = ext_vacuum_statistics +MODULE_big = ext_vacuum_statistics +OBJS = vacuum_statistics.o +DATA = ext_vacuum_statistics--1.0.sql +PGFILEDESC = "ext_vacuum_statistics - convenience views for extended vacuum statistics" + +ISOLATION = vacuum-extending-in-repetable-read +ISOLATION_OPTS = --temp-config=$(top_srcdir)/contrib/ext_vacuum_statistics/ext_vacuum_statistics.conf +TAP_TESTS = 1 +NO_INSTALLCHECK = 1 + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/ext_vacuum_statistics +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/contrib/ext_vacuum_statistics/README.md b/contrib/ext_vacuum_statistics/README.md new file mode 100644 index 00000000000..ac80bda4a66 --- /dev/null +++ b/contrib/ext_vacuum_statistics/README.md @@ -0,0 +1,85 @@ +# ext_vacuum_statistics + +Extended vacuum statistics extension for PostgreSQL. It collects and exposes detailed per-table and per-index vacuum statistics via convenient views in the `ext_vacuum_statistics` schema. + +## Installation + +``` +./configure tmp_install="$(pwd)/my/inst" +make clean && make && make install +cd contrib/ext_vacuum_statistics +make && make install +``` + +It is essential that the extension is listed in `shared_preload_libraries` because it registers a vacuum hook at server startup. + +In your `postgresql.conf`: + +``` +shared_preload_libraries = 'ext_vacuum_statistics' +``` + +Restart PostgreSQL. + +In your database: + +```sql +CREATE EXTENSION ext_vacuum_statistics; +``` + +## Usage + +Query vacuum statistics via the provided views: + +```sql +-- Per-table heap vacuum statistics +SELECT * FROM ext_vacuum_statistics.pg_stats_vacuum_tables; + +-- Per-index vacuum statistics +SELECT * FROM ext_vacuum_statistics.pg_stats_vacuum_indexes; +``` + +Example output: + +``` + relname | tuples_deleted | pages_removed +-----------+----------------+--------------- + mytable | 500 | 10 +``` + +Reset statistics when needed: + +```sql +SELECT ext_vacuum_statistics.vacuum_statistics_reset(); +``` + +## Configuration (GUCs) + +| GUC | Default | Description | +|-----|---------|-------------| +| `vacuum_statistics.enabled` | on | Enable extended vacuum statistics collection | + +## Memory usage + +Each tracked object (table or index) uses a fixed-size shared memory entry; the exact size depends on the platform. + +Example: a database with 1000 tables and 2000 indexes, all tracked, uses about **700 KB** on Ubuntu (3001 entries × 232 bytes). Per-database entries add one entry per tracked database. + +## Recipes + +**Disable statistics collection temporarily:** + +```sql +SET vacuum_statistics.enabled = off; +``` + +## Views + +| View | Description | +|------|-------------| +| `ext_vacuum_statistics.pg_stats_vacuum_tables` | Per-table heap vacuum stats (pages scanned, tuples deleted, dead tuples, etc.) | +| `ext_vacuum_statistics.pg_stats_vacuum_indexes` | Per-index vacuum stats | + +## Limitations + +- Must be loaded via `shared_preload_libraries`; it cannot be loaded on demand. diff --git a/contrib/ext_vacuum_statistics/expected/ext_vacuum_statistics.out b/contrib/ext_vacuum_statistics/expected/ext_vacuum_statistics.out new file mode 100644 index 00000000000..89c9594dea8 --- /dev/null +++ b/contrib/ext_vacuum_statistics/expected/ext_vacuum_statistics.out @@ -0,0 +1,52 @@ +-- ext_vacuum_statistics regression test + +-- Create extension +CREATE EXTENSION ext_vacuum_statistics; + +-- Verify schema and views exist +SELECT nspname FROM pg_namespace WHERE nspname = 'ext_vacuum_statistics'; + nspname +------------------ + ext_vacuum_statistics +(1 row) + +-- Views should be queryable (may return empty if no vacuum has run) +SELECT COUNT(*) >= 0 FROM ext_vacuum_statistics.pg_stats_vacuum_tables; + ?column? +---------- + t +(1 row) + +SELECT COUNT(*) >= 0 FROM ext_vacuum_statistics.pg_stats_vacuum_indexes; + ?column? +---------- + t +(1 row) + +SELECT COUNT(*) >= 0 FROM ext_vacuum_statistics.pg_stats_vacuum_database; + ?column? +---------- + t +(1 row) + +-- Verify views have expected columns +SELECT COUNT(*) AS tables_cols FROM information_schema.columns +WHERE table_schema = 'ext_vacuum_statistics' AND table_name = 'tables'; + tables_cols +------------- + 28 +(1 row) + +SELECT COUNT(*) AS indexes_cols FROM information_schema.columns +WHERE table_schema = 'ext_vacuum_statistics' AND table_name = 'indexes'; + indexes_cols +-------------- + 20 +(1 row) + +SELECT COUNT(*) AS database_cols FROM information_schema.columns +WHERE table_schema = 'ext_vacuum_statistics' AND table_name = 'database'; + database_cols +--------------- + 15 +(1 row) diff --git a/contrib/ext_vacuum_statistics/expected/vacuum-extending-in-repetable-read.out b/contrib/ext_vacuum_statistics/expected/vacuum-extending-in-repetable-read.out new file mode 100644 index 00000000000..f8962fad325 --- /dev/null +++ b/contrib/ext_vacuum_statistics/expected/vacuum-extending-in-repetable-read.out @@ -0,0 +1,52 @@ +unused step name: s2_delete +Parsed test spec with 2 sessions + +starting permutation: s2_insert s2_print_vacuum_stats_table s1_begin_repeatable_read s2_update s2_insert_interrupt s2_vacuum s2_print_vacuum_stats_table s1_commit s2_checkpoint s2_vacuum s2_print_vacuum_stats_table +step s2_insert: INSERT INTO test_vacuum_stat_isolation(id, ival) SELECT ival, ival%10 FROM generate_series(1,1000) As ival; +step s2_print_vacuum_stats_table: + SELECT + vt.relname, vt.tuples_deleted, vt.recently_dead_tuples, vt.missed_dead_tuples, vt.tuples_frozen + FROM ext_vacuum_statistics.pg_stats_vacuum_tables vt, pg_class c + WHERE vt.relname = 'test_vacuum_stat_isolation' AND vt.relid = c.oid; + +relname|tuples_deleted|recently_dead_tuples|missed_dead_tuples|tuples_frozen +-------+--------------+--------------------+------------------+------------- +(0 rows) + +step s1_begin_repeatable_read: + BEGIN transaction ISOLATION LEVEL REPEATABLE READ; + select count(ival) from test_vacuum_stat_isolation where id>900; + +count +----- + 100 +(1 row) + +step s2_update: UPDATE test_vacuum_stat_isolation SET ival = ival + 2 where id > 900; +step s2_insert_interrupt: INSERT INTO test_vacuum_stat_isolation values (1,1); +step s2_vacuum: VACUUM test_vacuum_stat_isolation; +step s2_print_vacuum_stats_table: + SELECT + vt.relname, vt.tuples_deleted, vt.recently_dead_tuples, vt.missed_dead_tuples, vt.tuples_frozen + FROM ext_vacuum_statistics.pg_stats_vacuum_tables vt, pg_class c + WHERE vt.relname = 'test_vacuum_stat_isolation' AND vt.relid = c.oid; + +relname |tuples_deleted|recently_dead_tuples|missed_dead_tuples|tuples_frozen +--------------------------+--------------+--------------------+------------------+------------- +test_vacuum_stat_isolation| 0| 100| 0| 0 +(1 row) + +step s1_commit: COMMIT; +step s2_checkpoint: CHECKPOINT; +step s2_vacuum: VACUUM test_vacuum_stat_isolation; +step s2_print_vacuum_stats_table: + SELECT + vt.relname, vt.tuples_deleted, vt.recently_dead_tuples, vt.missed_dead_tuples, vt.tuples_frozen + FROM ext_vacuum_statistics.pg_stats_vacuum_tables vt, pg_class c + WHERE vt.relname = 'test_vacuum_stat_isolation' AND vt.relid = c.oid; + +relname |tuples_deleted|recently_dead_tuples|missed_dead_tuples|tuples_frozen +--------------------------+--------------+--------------------+------------------+------------- +test_vacuum_stat_isolation| 100| 100| 0| 101 +(1 row) + diff --git a/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql b/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql new file mode 100644 index 00000000000..c3a34d6e103 --- /dev/null +++ b/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql @@ -0,0 +1,99 @@ +/*------------------------------------------------------------------------- + * + * ext_vacuum_statistics--1.0.sql + * Extended vacuum statistics via hook and custom storage + * + * This extension collects extended vacuum statistics via set_report_vacuum_hook + * and stores them in shared memory. + * + *------------------------------------------------------------------------- + */ + +\echo Use "CREATE EXTENSION ext_vacuum_statistics" to load this file. \quit + +CREATE SCHEMA IF NOT EXISTS ext_vacuum_statistics; + +COMMENT ON SCHEMA ext_vacuum_statistics IS + 'Extended vacuum statistics (heap, index, database)'; + +-- Reset functions +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.extvac_reset_entry( + dboid oid, + relid oid +) +RETURNS boolean +AS 'MODULE_PATHNAME', 'extvac_reset_entry' +LANGUAGE C STRICT PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.vacuum_statistics_reset() +RETURNS bigint +AS 'MODULE_PATHNAME', 'vacuum_statistics_reset' +LANGUAGE C STRICT PARALLEL SAFE; + +-- Internal C function to fetch table vacuum stats +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.pg_stats_get_vacuum_tables( + IN dboid oid, + IN reloid oid, + OUT relid oid, + OUT tuples_deleted bigint, + OUT tuples_frozen bigint, + OUT recently_dead_tuples bigint, + OUT missed_dead_tuples bigint +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_stats_get_vacuum_tables' +LANGUAGE C STRICT STABLE; + +-- Internal C function to fetch index vacuum stats +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.pg_stats_get_vacuum_indexes( + IN dboid oid, + IN reloid oid, + OUT relid oid, + OUT tuples_deleted bigint +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_stats_get_vacuum_indexes' +LANGUAGE C STRICT STABLE; + +-- View: vacuum statistics per table (heap) +CREATE VIEW ext_vacuum_statistics.pg_stats_vacuum_tables AS +SELECT + rel.oid AS relid, + ns.nspname AS schema, + rel.relname AS relname, + db.datname AS dbname, + stats.tuples_deleted, + stats.tuples_frozen, + stats.recently_dead_tuples, + stats.missed_dead_tuples +FROM pg_database db, + pg_class rel, + pg_namespace ns, + LATERAL ext_vacuum_statistics.pg_stats_get_vacuum_tables(db.oid, rel.oid) stats +WHERE db.datname = current_database() + AND rel.relkind = 'r' + AND rel.relnamespace = ns.oid + AND rel.oid = stats.relid; + +COMMENT ON VIEW ext_vacuum_statistics.pg_stats_vacuum_tables IS + 'Extended vacuum statistics per table (heap)'; + +-- View: vacuum statistics per index +CREATE VIEW ext_vacuum_statistics.pg_stats_vacuum_indexes AS +SELECT + rel.oid AS indexrelid, + ns.nspname AS schema, + rel.relname AS indexrelname, + db.datname AS dbname, + stats.tuples_deleted +FROM pg_database db, + pg_class rel, + pg_namespace ns, + LATERAL ext_vacuum_statistics.pg_stats_get_vacuum_indexes(db.oid, rel.oid) stats +WHERE db.datname = current_database() + AND rel.relkind = 'i' + AND rel.relnamespace = ns.oid + AND rel.oid = stats.relid; + +COMMENT ON VIEW ext_vacuum_statistics.pg_stats_vacuum_indexes IS + 'Extended vacuum statistics per index'; diff --git a/contrib/ext_vacuum_statistics/ext_vacuum_statistics.conf b/contrib/ext_vacuum_statistics/ext_vacuum_statistics.conf new file mode 100644 index 00000000000..9b711487623 --- /dev/null +++ b/contrib/ext_vacuum_statistics/ext_vacuum_statistics.conf @@ -0,0 +1,2 @@ +# Config for ext_vacuum_statistics regression tests +shared_preload_libraries = 'ext_vacuum_statistics' diff --git a/contrib/ext_vacuum_statistics/ext_vacuum_statistics.control b/contrib/ext_vacuum_statistics/ext_vacuum_statistics.control new file mode 100644 index 00000000000..518350a64b7 --- /dev/null +++ b/contrib/ext_vacuum_statistics/ext_vacuum_statistics.control @@ -0,0 +1,5 @@ +# ext_vacuum_statistics extension +comment = 'Extended vacuum statistics via hook (requires shared_preload_libraries)' +default_version = '1.0' +relocatable = true +module_pathname = '$libdir/ext_vacuum_statistics' diff --git a/contrib/ext_vacuum_statistics/meson.build b/contrib/ext_vacuum_statistics/meson.build new file mode 100644 index 00000000000..85eb2d6b85d --- /dev/null +++ b/contrib/ext_vacuum_statistics/meson.build @@ -0,0 +1,41 @@ +# Copyright (c) 2022-2026, PostgreSQL Global Development Group +# +# ext_vacuum_statistics - extended vacuum statistics via hook +# Requires shared_preload_libraries = 'ext_vacuum_statistics' + +ext_vacuum_statistics_sources = files( + 'vacuum_statistics.c', +) + +ext_vacuum_statistics = shared_module('ext_vacuum_statistics', + ext_vacuum_statistics_sources, + kwargs: contrib_mod_args + { + 'dependencies': contrib_mod_args['dependencies'], + }, +) +contrib_targets += ext_vacuum_statistics + +install_data( + 'ext_vacuum_statistics.control', + 'ext_vacuum_statistics--1.0.sql', + kwargs: contrib_data_args, +) + +tests += { + 'name': 'ext_vacuum_statistics', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'isolation': { + 'specs': [ + 'vacuum-extending-in-repetable-read', + ], + 'regress_args': ['--temp-config', files('ext_vacuum_statistics.conf')], + 'runningcheck': false, + }, + 'tap': { + 'tests': [ + 't/054_vacuum_extending_gucs_test.pl', + 't/056_vacuum_stats_gc.pl', + ], + }, +} diff --git a/contrib/ext_vacuum_statistics/specs/vacuum-extending-in-repetable-read.spec b/contrib/ext_vacuum_statistics/specs/vacuum-extending-in-repetable-read.spec new file mode 100644 index 00000000000..3001b56ac2b --- /dev/null +++ b/contrib/ext_vacuum_statistics/specs/vacuum-extending-in-repetable-read.spec @@ -0,0 +1,59 @@ +# Test for checking recently_dead_tuples, tuples_deleted and frozen tuples in ext_vacuum_statistics.pg_stats_vacuum_tables. +# recently_dead_tuples values are counted when vacuum hasn't cleared tuples because they were deleted recently. +# recently_dead_tuples aren't increased after releasing lock compared with tuples_deleted, which increased +# by the value of the cleared tuples that the vacuum managed to clear. + +setup +{ + CREATE TABLE test_vacuum_stat_isolation(id int, ival int) WITH (autovacuum_enabled = off); + CREATE EXTENSION ext_vacuum_statistics; + SET track_io_timing = on; +} + +teardown +{ + DROP EXTENSION ext_vacuum_statistics CASCADE; + DROP TABLE test_vacuum_stat_isolation CASCADE; + RESET track_io_timing; +} + +session s1 +setup { + SET track_io_timing = on; +} +step s1_begin_repeatable_read { + BEGIN transaction ISOLATION LEVEL REPEATABLE READ; + select count(ival) from test_vacuum_stat_isolation where id>900; +} +step s1_commit { COMMIT; } + +session s2 +setup { + SET track_io_timing = on; +} +step s2_insert { INSERT INTO test_vacuum_stat_isolation(id, ival) SELECT ival, ival%10 FROM generate_series(1,1000) As ival; } +step s2_update { UPDATE test_vacuum_stat_isolation SET ival = ival + 2 where id > 900; } +step s2_delete { DELETE FROM test_vacuum_stat_isolation where id > 900; } +step s2_insert_interrupt { INSERT INTO test_vacuum_stat_isolation values (1,1); } +step s2_vacuum { VACUUM test_vacuum_stat_isolation; } +step s2_checkpoint { CHECKPOINT; } +step s2_print_vacuum_stats_table +{ + SELECT + vt.relname, vt.tuples_deleted, vt.recently_dead_tuples, vt.missed_dead_tuples, vt.tuples_frozen + FROM ext_vacuum_statistics.pg_stats_vacuum_tables vt, pg_class c + WHERE vt.relname = 'test_vacuum_stat_isolation' AND vt.relid = c.oid; +} + +permutation + s2_insert + s2_print_vacuum_stats_table + s1_begin_repeatable_read + s2_update + s2_insert_interrupt + s2_vacuum + s2_print_vacuum_stats_table + s1_commit + s2_checkpoint + s2_vacuum + s2_print_vacuum_stats_table diff --git a/contrib/ext_vacuum_statistics/t/054_vacuum_extending_gucs_test.pl b/contrib/ext_vacuum_statistics/t/054_vacuum_extending_gucs_test.pl new file mode 100644 index 00000000000..4dc43364330 --- /dev/null +++ b/contrib/ext_vacuum_statistics/t/054_vacuum_extending_gucs_test.pl @@ -0,0 +1,100 @@ +# Copyright (c) 2025 PostgreSQL Global Development Group +# +# Test GUC parameters for ext_vacuum_statistics extension: +# vacuum_statistics.enabled +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; + +use Test::More; + +#------------------------------------------------------------------------------ +# Test cluster setup +#------------------------------------------------------------------------------ + +my $node = PostgreSQL::Test::Cluster->new('ext_stat_vacuum_gucs'); +$node->init; + +$node->append_conf('postgresql.conf', q{ + shared_preload_libraries = 'ext_vacuum_statistics' + log_min_messages = notice +}); + +$node->start; + +#------------------------------------------------------------------------------ +# Database creation and initialization +#------------------------------------------------------------------------------ + +$node->safe_psql('postgres', q{ + CREATE DATABASE statistic_vacuum_gucs; +}); + +my $dbname = 'statistic_vacuum_gucs'; + +$node->safe_psql($dbname, q{ + CREATE EXTENSION ext_vacuum_statistics; + CREATE TABLE guc_test (x int PRIMARY KEY) + WITH (autovacuum_enabled = off); + INSERT INTO guc_test SELECT x FROM generate_series(1, 100) AS g(x); + ANALYZE guc_test; +}); + +#------------------------------------------------------------------------------ +# Reset stats and run vacuum (all in one session so GUCs persist) +#------------------------------------------------------------------------------ + +sub reset_and_vacuum { + my ($db, $table, $opts) = @_; + $table ||= 'guc_test'; + my $gucs = $opts && $opts->{gucs} ? $opts->{gucs} : []; + my $modify = $opts && $opts->{modify}; + my $extra = $opts && $opts->{extra_vacuum} ? $opts->{extra_vacuum} : []; + $extra = [$extra] unless ref $extra eq 'ARRAY'; + my $sql = join("\n", (map { "SET $_;" } @$gucs), + "SELECT ext_vacuum_statistics.vacuum_statistics_reset();", + $modify ? ( + "TRUNCATE $table;", + "INSERT INTO $table SELECT x FROM generate_series(1, 100) AS g(x);", + "DELETE FROM $table;", + ) : (), + "VACUUM $table;", + (map { "VACUUM $_;" } @$extra), + # Make pending stats visible to subsequent sessions without sleeping. + "SELECT pg_stat_force_next_flush();"); + $node->safe_psql($db, $sql); +} + +#------------------------------------------------------------------------------ +# Test 1: vacuum_statistics.enabled +#------------------------------------------------------------------------------ +subtest 'vacuum_statistics.enabled' => sub { + reset_and_vacuum($dbname); + + # Default: enabled - should have stats + my $count = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'guc_test'"); + ok($count > 0, 'stats collected when enabled'); + + # Disable, reset and vacuum in same session. Assert not only that the + # row count is zero, but that the specific counters remain zero: a stray + # row with zero counters would otherwise pass a bare COUNT(*)=0 check. + reset_and_vacuum($dbname, 'guc_test', { gucs => ['vacuum_statistics.enabled = off'] }); + + $count = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'guc_test'"); + is($count, 0, 'no rows when disabled'); + + my $sums = $node->safe_psql($dbname, q{ + SELECT COALESCE(SUM(tuples_deleted), 0) + + COALESCE(SUM(tuples_frozen), 0) + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'guc_test' + }); + is($sums, '0', 'no counters accumulated when disabled'); +}; + +$node->stop; + +done_testing(); diff --git a/contrib/ext_vacuum_statistics/t/056_vacuum_stats_gc.pl b/contrib/ext_vacuum_statistics/t/056_vacuum_stats_gc.pl new file mode 100644 index 00000000000..7f96943c0a2 --- /dev/null +++ b/contrib/ext_vacuum_statistics/t/056_vacuum_stats_gc.pl @@ -0,0 +1,221 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test that vacuum statistics entries do not outlive the relations they +# describe: DROP TABLE, a rolled back DROP, a restart, DROP DATABASE, and a +# new relation that gets the OID of an old one. +# +# Entries are looked up with pg_stats_get_vacuum_tables() and +# pg_stats_get_vacuum_indexes(), which find an entry by OID even when the +# relation is no longer in pg_class. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('vacstat_gc'); +$node->init; +$node->append_conf( + 'postgresql.conf', q{ +shared_preload_libraries = 'ext_vacuum_statistics' +autovacuum = off +}); +$node->start; + +$node->safe_psql('postgres', 'CREATE EXTENSION ext_vacuum_statistics'); + +# SQL that counts the statistics entries of the given tables and indexes. +sub entries_sql +{ + my ($dboid, $tables, $indexes) = @_; + my $t = join(',', @$tables); + my $i = join(',', @$indexes); + + return qq{ +SELECT (SELECT count(*) + FROM unnest('{$t}'::oid[]) AS r, + ext_vacuum_statistics.pg_stats_get_vacuum_tables($dboid, r)) + + (SELECT count(*) + FROM unnest('{$i}'::oid[]) AS r, + ext_vacuum_statistics.pg_stats_get_vacuum_indexes($dboid, r))}; +} + +sub count_entries +{ + return $node->safe_psql('postgres', entries_sql(@_)); +} + +# Wait until the entries are gone. +sub wait_for_no_entries +{ + my ($what, @args) = @_; + + $node->poll_query_until('postgres', + 'SELECT (' . entries_sql(@args) . ') = 0') + or die "timed out waiting for $what"; + return; +} + +# OIDs of the relations of the given kind whose names match a pattern. +sub relids +{ + my ($dbname, $relkind, $pattern) = @_; + + return split /\n/, + $node->safe_psql($dbname, + "SELECT oid FROM pg_class WHERE relkind = '$relkind' AND relname LIKE '$pattern' ORDER BY oid" + ); +} + +my $dboid = $node->safe_psql('postgres', + "SELECT oid FROM pg_database WHERE datname = 'postgres'"); + +# Vacuum a handful of tables, each with an index. +my @tables = map { "gc_tab$_" } (1 .. 20); + +$node->safe_psql('postgres', + join('', map { qq[ +CREATE TABLE $_ (id int); +CREATE INDEX ${_}_idx ON $_ (id); +INSERT INTO $_ SELECT generate_series(1, 100); +DELETE FROM $_;] } @tables)); +$node->safe_psql('postgres', 'VACUUM ' . join(', ', @tables)); + +my @tabids = relids('postgres', 'r', 'gc_tab%'); +my @idxids = relids('postgres', 'i', 'gc_tab%'); +is(scalar(@tabids) + scalar(@idxids), 40, 'found the tables and indexes'); +is(count_entries($dboid, \@tabids, \@idxids), + 40, 'vacuum created an entry for each table and index'); + +# A rolled back DROP must keep the statistics. +$node->safe_psql('postgres', 'BEGIN; DROP TABLE gc_tab1; ROLLBACK;'); +is(count_entries($dboid, \@tabids, \@idxids), + 40, 'rolled back DROP TABLE kept the statistics'); + +# Dropping the tables drops their entries and those of their indexes. +$node->safe_psql('postgres', 'DROP TABLE ' . join(', ', @tables)); +wait_for_no_entries('DROP TABLE to drop the statistics', + $dboid, \@tabids, \@idxids); +is(count_entries($dboid, \@tabids, \@idxids), + 0, 'DROP TABLE dropped the statistics entries'); + +# Nothing may come back from the statistics file either. +$node->restart; +is(count_entries($dboid, \@tabids, \@idxids), + 0, 'no dropped entries came back after a restart'); + +# The same for a whole database. +$node->safe_psql('postgres', 'CREATE DATABASE vacstat_gc_db'); +$node->safe_psql( + 'vacstat_gc_db', q{ +CREATE TABLE gc_dbtab (id int); +CREATE INDEX gc_dbtab_idx ON gc_dbtab (id); +INSERT INTO gc_dbtab SELECT generate_series(1, 100); +DELETE FROM gc_dbtab; +VACUUM gc_dbtab; +}); +my $otherdb = $node->safe_psql('postgres', + "SELECT oid FROM pg_database WHERE datname = 'vacstat_gc_db'"); +my @dbtabids = relids('vacstat_gc_db', 'r', 'gc_dbtab'); +my @dbidxids = relids('vacstat_gc_db', 'i', 'gc_dbtab_idx'); +is(count_entries($otherdb, \@dbtabids, \@dbidxids), + 2, 'vacuum in another database created statistics entries'); + +$node->safe_psql('postgres', 'DROP DATABASE vacstat_gc_db'); +wait_for_no_entries('DROP DATABASE to drop the statistics', + $otherdb, \@dbtabids, \@dbidxids); +is(count_entries($otherdb, \@dbtabids, \@dbidxids), + 0, 'DROP DATABASE dropped the statistics entries'); + +# A relation created on the OID of an earlier one must not inherit its +# statistics. DROP never leaves an entry behind, and OIDs are only handed +# out again after a wraparound, so fake the situation: take a vacuumed table +# out of the catalogs behind the back of DROP, which leaves its entry +# orphaned, then create a new table with the very same OIDs in binary upgrade +# mode. +$node->safe_psql( + 'postgres', q{ +CREATE TABLE gc_reuse (id int); +INSERT INTO gc_reuse SELECT generate_series(1, 100); +DELETE FROM gc_reuse; +VACUUM gc_reuse; +}); +my ($relid, $reltype, $typarray) = split /\|/, + $node->safe_psql( + 'postgres', q{ +SELECT c.oid, c.reltype, t.typarray + FROM pg_class c JOIN pg_type t ON t.oid = c.reltype + WHERE c.relname = 'gc_reuse'}); +is( $node->safe_psql( + 'postgres', + "SELECT tuples_deleted FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relid = $relid" + ), + '100', + 'the table whose OID gets recycled has statistics'); + +$node->safe_psql( + 'postgres', qq{ +SET allow_system_table_mods = on; +DELETE FROM pg_depend + WHERE (classid = 'pg_class'::regclass AND objid = $relid) + OR (refclassid = 'pg_class'::regclass AND refobjid = $relid) + OR (classid = 'pg_type'::regclass AND objid IN ($reltype, $typarray)) + OR (refclassid = 'pg_type'::regclass AND refobjid IN ($reltype, $typarray)); +DELETE FROM pg_attribute WHERE attrelid = $relid; +DELETE FROM pg_type WHERE oid IN ($reltype, $typarray); +DELETE FROM pg_class WHERE oid = $relid; +}); + +# Binary upgrade mode is a postmaster switch, so start the node by hand. +$node->stop; +command_ok( + [ + 'pg_ctl', + '--pgdata' => $node->data_dir, + '--log' => $node->logfile, + '--options' => '-b', + '--wait', 'start' + ], + 'started in binary upgrade mode'); + +# The relfilenumber only has to be unused; the old file is still on disk. +my ($ret, $stdout, $stderr) = $node->psql( + 'postgres', qq{ +SELECT binary_upgrade_set_next_heap_pg_class_oid('$relid'::oid); +SELECT binary_upgrade_set_next_heap_relfilenode('3000000000'::oid); +SELECT binary_upgrade_set_next_pg_type_oid('$reltype'::oid); +SELECT binary_upgrade_set_next_array_pg_type_oid('$typarray'::oid); +CREATE TABLE gc_reuse_new (id int); +}); +is($ret, 0, 'created a table on the recycled OID'); +like( + $stderr, + qr/resetting existing statistics for kind ext_vacuum_statistics_relation/, + 'creating the table reset the orphaned statistics entry'); + +command_ok( + [ + 'pg_ctl', + '--pgdata' => $node->data_dir, + '--mode' => 'fast', + '--wait', 'stop' + ], + 'stopped binary upgrade mode'); +$node->start; + +is( $node->safe_psql( + 'postgres', "SELECT oid FROM pg_class WHERE relname = 'gc_reuse_new'"), + $relid, + 'the new table got the recycled OID'); +is( $node->safe_psql( + 'postgres', q{ +SELECT coalesce((SELECT tuples_deleted + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'gc_reuse_new'), 0)}), + '0', + 'the new table did not inherit the statistics of the old one'); + +$node->stop; + +done_testing(); diff --git a/contrib/ext_vacuum_statistics/vacuum_statistics.c b/contrib/ext_vacuum_statistics/vacuum_statistics.c new file mode 100644 index 00000000000..2aa3c054266 --- /dev/null +++ b/contrib/ext_vacuum_statistics/vacuum_statistics.c @@ -0,0 +1,340 @@ +/* + * ext_vacuum_statistics - Extended vacuum statistics for PostgreSQL + * + * This module collects detailed vacuum statistics (I/O, WAL, timing, etc.) + * at relation and database level by hooking into the vacuum reporting path. + * Statistics are stored via pgstat custom statistics. Management of statistics + * storage and output functions are implemented in this module. + */ +#include "postgres.h" + +#include "catalog/objectaccess.h" +#include "catalog/pg_class.h" +#include "fmgr.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "pgstat.h" +#include "utils/builtins.h" +#include "utils/fmgrprotos.h" +#include "utils/guc.h" +#include "utils/pgstat_internal.h" +#include "utils/pgstat_kind.h" +#include "utils/tuplestore.h" + +#ifdef PG_MODULE_MAGIC +PG_MODULE_MAGIC; +#endif + +/* Custom stats kind for per-relation vacuum statistics */ +#define PGSTAT_KIND_EXTVAC_RELATION 24 + +#define SJ_NODENAME "vacuum_statistics" + +/* GUCs */ +static bool evs_enabled = true; + +/* Hooks */ +static set_report_vacuum_hook_type prev_report_vacuum_hook = NULL; +static object_access_hook_type prev_object_access_hook = NULL; + +/* Forward declarations */ +static void pgstat_report_vacuum_extstats(Oid tableoid, bool shared, + PgStat_VacuumRelationCounts * params); +static void extvac_object_access(ObjectAccessType access, Oid classId, + Oid objectId, int subId, void *arg); + +/* Shared memory entry for vacuum stats; one per relation or database. */ +typedef struct PgStatShared_ExtVacEntry +{ + PgStatShared_Common header; + PgStat_VacuumRelationCounts stats; +} PgStatShared_ExtVacEntry; + +/* PgStat kind for per-relation vacuum statistics (tables/indexes) */ +static const PgStat_KindInfo extvac_relation_kind_info = { + .name = "ext_vacuum_statistics_relation", + .fixed_amount = false, + .accessed_across_databases = true, + .write_to_file = true, + .shared_size = sizeof(PgStatShared_ExtVacEntry), + .shared_data_off = offsetof(PgStatShared_ExtVacEntry, stats), + .shared_data_len = sizeof(PgStat_VacuumRelationCounts), + .pending_size = 0, + .flush_pending_cb = NULL, +}; + +static inline void +pgstat_accumulate_common(PgStat_CommonCounts * dst, const PgStat_CommonCounts * src) +{ + dst->tuples_deleted += src->tuples_deleted; +} + +static inline void +pgstat_accumulate_extvac_stats(PgStat_VacuumRelationCounts * dst, + const PgStat_VacuumRelationCounts * src) +{ + if (dst->type == PGSTAT_EXTVAC_INVALID) + dst->type = src->type; + + Assert(src->type != PGSTAT_EXTVAC_INVALID && src->type != PGSTAT_EXTVAC_DB); + Assert(src->type == dst->type); + + pgstat_accumulate_common(&dst->common, &src->common); + + if (dst->type == PGSTAT_EXTVAC_TABLE) + { + dst->table.tuples_frozen += src->table.tuples_frozen; + dst->table.recently_dead_tuples += src->table.recently_dead_tuples; + dst->table.missed_dead_tuples += src->table.missed_dead_tuples; + } +} + +void +_PG_init(void) +{ + if (!process_shared_preload_libraries_in_progress) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ext_vacuum_statistics module could be loaded only on startup."), + errdetail("Add 'ext_vacuum_statistics' into the shared_preload_libraries list."))); + + DefineCustomBoolVariable("vacuum_statistics.enabled", + "Enable extended vacuum statistics collection.", + NULL, &evs_enabled, true, + PGC_SUSET, 0, NULL, NULL, NULL); + + MarkGUCPrefixReserved(SJ_NODENAME); + + pgstat_register_kind(PGSTAT_KIND_EXTVAC_RELATION, &extvac_relation_kind_info); + + prev_report_vacuum_hook = set_report_vacuum_hook; + set_report_vacuum_hook = pgstat_report_vacuum_extstats; + + prev_object_access_hook = object_access_hook; + object_access_hook = extvac_object_access; +} + +/* + * Object access hook: drop the statistics of a dropped relation, and reset + * old statistics when a new relation is created. + * + * Otherwise the statistics of a dropped relation stay in memory and in the + * statistics file, and a new relation with the same OID gets them. Both + * actions are transactional, as for the built-in relation statistics. + * + * Statistics are keyed by the relation OID, and shared catalogs are never + * created or dropped after initdb, so we do not need to read pg_class. + * + * Dropped databases need no handling: dropping a database drops all its + * statistics entries, ours included. + */ +static void +extvac_object_access(ObjectAccessType access, Oid classId, Oid objectId, + int subId, void *arg) +{ + if (prev_object_access_hook) + prev_object_access_hook(access, classId, objectId, subId, arg); + + /* only whole relations are of interest, not their columns */ + if (classId != RelationRelationId || subId != 0) + return; + + if (access == OAT_DROP) + pgstat_drop_transactional(PGSTAT_KIND_EXTVAC_RELATION, MyDatabaseId, + objectId); + else if (access == OAT_POST_CREATE) + { + /* + * Discard whatever an earlier owner of this OID left behind, and + * arrange for the entry to be dropped again should the creating + * transaction roll back. This mirrors what pgstat_create_relation() + * does for the built-in relation statistics. + */ + pgstat_create_transactional(PGSTAT_KIND_EXTVAC_RELATION, MyDatabaseId, + objectId); + } +} + +/* + * Store incoming vacuum stats into a per-relation pgstat custom statistics + * entry. Uses pgstat_get_entry_ref_locked and pgstat_accumulate_* for + * atomic updates. + */ +static void +extvac_store(Oid dboid, Oid relid, PgStat_VacuumRelationCounts * params) +{ + PgStat_EntryRef *entry_ref; + PgStatShared_ExtVacEntry *shared; + + if (!evs_enabled) + return; + + entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_EXTVAC_RELATION, dboid, relid, false); + if (entry_ref) + { + shared = (PgStatShared_ExtVacEntry *) entry_ref->shared_stats; + if (shared->stats.type == PGSTAT_EXTVAC_INVALID) + { + memset(&shared->stats, 0, sizeof(shared->stats)); + shared->stats.type = params->type; + } + pgstat_accumulate_extvac_stats(&shared->stats, params); + pgstat_unlock_entry(entry_ref); + } +} + +/* + * Vacuum report hook: called when vacuum finishes. Stores stats per-relation, + * then chains to previous hook. + */ +static void +pgstat_report_vacuum_extstats(Oid tableoid, bool shared, + PgStat_VacuumRelationCounts * params) +{ + Oid dboid = shared ? InvalidOid : MyDatabaseId; + + if (evs_enabled) + extvac_store(dboid, tableoid, params); + if (prev_report_vacuum_hook) + prev_report_vacuum_hook(tableoid, shared, params); +} + +/* Reset statistics for a single relation entry. */ +static bool +extvac_reset_by_relid(Oid dboid, Oid relid) +{ + pgstat_reset_entry(PGSTAT_KIND_EXTVAC_RELATION, dboid, relid, 0); + return true; +} + +/* Reset all vacuum statistics. */ +static int64 +extvac_stat_reset(void) +{ + pgstat_reset_of_kind(PGSTAT_KIND_EXTVAC_RELATION); + return 0; /* count not available */ +} + +PG_FUNCTION_INFO_V1(vacuum_statistics_reset); +PG_FUNCTION_INFO_V1(extvac_reset_entry); + +Datum +vacuum_statistics_reset(PG_FUNCTION_ARGS) +{ + PG_RETURN_INT64(extvac_stat_reset()); +} + +Datum +extvac_reset_entry(PG_FUNCTION_ARGS) +{ + Oid dboid = PG_GETARG_OID(0); + Oid relid = PG_GETARG_OID(1); + + PG_RETURN_BOOL(extvac_reset_by_relid(dboid, relid)); +} + +/* + * Output vacuum statistics (tables or indexes). + */ +#define EXTVAC_HEAP_STAT_COLS 5 +#define EXTVAC_IDX_STAT_COLS 2 +#define EXTVAC_MAX_STAT_COLS Max(EXTVAC_HEAP_STAT_COLS, EXTVAC_IDX_STAT_COLS) + +static void +tuplestore_put_for_relation(Oid relid, Tuplestorestate *tupstore, + TupleDesc tupdesc, PgStat_VacuumRelationCounts * vacuum_ext) +{ + Datum values[EXTVAC_MAX_STAT_COLS]; + bool nulls[EXTVAC_MAX_STAT_COLS]; + int i = 0; + + memset(nulls, 0, sizeof(nulls)); + values[i++] = ObjectIdGetDatum(relid); + + if (vacuum_ext->type == PGSTAT_EXTVAC_TABLE) + { + values[i++] = Int64GetDatum(vacuum_ext->common.tuples_deleted); + values[i++] = Int64GetDatum(vacuum_ext->table.tuples_frozen); + values[i++] = Int64GetDatum(vacuum_ext->table.recently_dead_tuples); + values[i++] = Int64GetDatum(vacuum_ext->table.missed_dead_tuples); + } + else if (vacuum_ext->type == PGSTAT_EXTVAC_INDEX) + { + values[i++] = Int64GetDatum(vacuum_ext->common.tuples_deleted); + } + + Assert(i == ((vacuum_ext->type == PGSTAT_EXTVAC_TABLE) ? EXTVAC_HEAP_STAT_COLS : EXTVAC_IDX_STAT_COLS)); + tuplestore_putvalues(tupstore, tupdesc, values, nulls); +} + +static Datum +pg_stats_vacuum(FunctionCallInfo fcinfo, int type) +{ + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + MemoryContext per_query_ctx; + MemoryContext oldcontext; + Tuplestorestate *tupstore; + TupleDesc tupdesc; + Oid dbid = PG_GETARG_OID(0); + + if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ext_vacuum_statistics: set-valued function called in context that cannot accept a set"))); + if (!(rsinfo->allowedModes & SFRM_Materialize)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ext_vacuum_statistics: materialize mode required"))); + + per_query_ctx = rsinfo->econtext->ecxt_per_query_memory; + oldcontext = MemoryContextSwitchTo(per_query_ctx); + + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "ext_vacuum_statistics: return type must be a row type"); + + tupstore = tuplestore_begin_heap(true, false, work_mem); + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = tupdesc; + + MemoryContextSwitchTo(oldcontext); + + if (type == PGSTAT_EXTVAC_INDEX || type == PGSTAT_EXTVAC_TABLE) + { + Oid relid = PG_GETARG_OID(1); + PgStat_VacuumRelationCounts *stats; + + if (!OidIsValid(relid)) + return (Datum) 0; + + stats = (PgStat_VacuumRelationCounts *) + pgstat_fetch_entry(PGSTAT_KIND_EXTVAC_RELATION, dbid, relid, NULL); + + if (!stats) + stats = (PgStat_VacuumRelationCounts *) + pgstat_fetch_entry(PGSTAT_KIND_EXTVAC_RELATION, InvalidOid, + relid, NULL); + + if (stats && stats->type == type) + tuplestore_put_for_relation(relid, tupstore, tupdesc, stats); + } + else + elog(PANIC, "ext_vacuum_statistics: invalid type %d", type); + + return (Datum) 0; +} + +PG_FUNCTION_INFO_V1(pg_stats_get_vacuum_tables); +PG_FUNCTION_INFO_V1(pg_stats_get_vacuum_indexes); + +Datum +pg_stats_get_vacuum_tables(PG_FUNCTION_ARGS) +{ + return pg_stats_vacuum(fcinfo, PGSTAT_EXTVAC_TABLE); +} + +Datum +pg_stats_get_vacuum_indexes(PG_FUNCTION_ARGS) +{ + return pg_stats_vacuum(fcinfo, PGSTAT_EXTVAC_INDEX); +} diff --git a/contrib/meson.build b/contrib/meson.build index ebb7f83d8c5..d7dc0fd07f0 100644 --- a/contrib/meson.build +++ b/contrib/meson.build @@ -26,6 +26,7 @@ subdir('cube') subdir('dblink') subdir('dict_int') subdir('dict_xsyn') +subdir('ext_vacuum_statistics') subdir('earthdistance') subdir('file_fdw') subdir('fuzzystrmatch') diff --git a/doc/src/sgml/contrib.sgml b/doc/src/sgml/contrib.sgml index b9b03654aad..2a38f9042bb 100644 --- a/doc/src/sgml/contrib.sgml +++ b/doc/src/sgml/contrib.sgml @@ -141,6 +141,7 @@ CREATE EXTENSION extension_name; &dict-int; &dict-xsyn; &earthdistance; + &extvacuumstatistics; &file-fdw; &fuzzystrmatch; &hstore; diff --git a/doc/src/sgml/extvacuumstatistics.sgml b/doc/src/sgml/extvacuumstatistics.sgml new file mode 100644 index 00000000000..b637744ed79 --- /dev/null +++ b/doc/src/sgml/extvacuumstatistics.sgml @@ -0,0 +1,254 @@ + + + + ext_vacuum_statistics — extended vacuum statistics + + + ext_vacuum_statistics + + + + The ext_vacuum_statistics module provides + extended per-table and per-index vacuum statistics via views in the + ext_vacuum_statistics schema. + + + + The module must be loaded by adding ext_vacuum_statistics to + in + postgresql.conf, because it registers a vacuum hook at + server startup. This means that a server restart is needed to add or remove + the module. After installation, run + CREATE EXTENSION ext_vacuum_statistics in each database + where you want to use it. + + + + Note that starting the server without the module in + , even once, discards + all cumulative statistics, not only those of this + module. The statistics file then holds entries of a statistics kind that + is not registered, so the server reports the file as corrupted and resets + every statistics kind, built-in ones such as those shown in + pg_stat_user_tables included. To stop collecting + vacuum statistics without that side effect, keep the module loaded and + set vacuum_statistics.enabled to off + instead. + + + + When active, the module provides views + ext_vacuum_statistics.pg_stats_vacuum_tables and + ext_vacuum_statistics.pg_stats_vacuum_indexes, + plus functions to reset statistics. + + + + Each tracked object (one table or one index) uses a fixed-size shared + memory entry; the exact size depends on the platform. + + + + The <structname>ext_vacuum_statistics.pg_stats_vacuum_tables</structname> View + + + pg_stats_vacuum_tables + + + + The view ext_vacuum_statistics.pg_stats_vacuum_tables + contains one row for each table in the current database (including TOAST + tables), showing statistics about vacuuming that specific table. The columns + are shown in . + + + + <structname>ext_vacuum_statistics.pg_stats_vacuum_tables</structname> Columns + + + + + Column Type + + + Description + + + + + + + relid oid + + + OID of a table + + + + + schema name + + + Name of the schema this table is in + + + + + relname name + + + Name of this table + + + + + dbname name + + + Name of the database containing this table + + + + + tuples_deleted int8 + + + Number of dead tuples vacuum operations deleted from this table + + + + + tuples_frozen int8 + + + Number of tuples that vacuum operations marked as frozen + + + + + recently_dead_tuples int8 + + + Number of dead tuples left due to visibility in transactions + + + + + missed_dead_tuples int8 + + + Number of fully DEAD tuples that could not be pruned due to failure to acquire a cleanup lock + + + + +
+ + + + The <structname>ext_vacuum_statistics.pg_stats_vacuum_indexes</structname> View + + + pg_stats_vacuum_indexes + + + + The view ext_vacuum_statistics.pg_stats_vacuum_indexes + contains one row for each index in the current database, showing statistics + about vacuuming that specific index. The columns are shown in + . + + + + <structname>ext_vacuum_statistics.pg_stats_vacuum_indexes</structname> Columns + + + + + Column Type + + + Description + + + + + + + indexrelid oid + + + OID of this index + + + + + schema name + + + Name of the schema this index is in + + + + + indexrelname name + + + Name of this index + + + + + dbname name + + + Name of the database containing this index + + + + + tuples_deleted int8 + + + Number of index entries removed by the bulkdelete passes of vacuums + of the parent table + + + + +
+ + + + Functions + + + + + ext_vacuum_statistics.vacuum_statistics_reset() + bigint + + + + Resets all vacuum statistics. Returns the number of entries reset. + + + + + + + + Configuration Parameters + + + + vacuum_statistics.enabled (boolean) + + + Enables extended vacuum statistics collection. Default: on. + + + + + + diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index 0797dcf96da..935cfa1d2a6 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -133,6 +133,7 @@ + diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 241a10c0805..c04def12120 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -285,6 +285,8 @@ typedef struct LVRelState /* Error reporting state */ char *dbname; char *relnamespace; + Oid reloid; + Oid indoid; char *relname; char *indname; /* Current index name */ BlockNumber blkno; /* used only for heap operations */ @@ -412,6 +414,11 @@ typedef struct LVRelState * been permanently disabled. */ BlockNumber eager_scan_remaining_fails; + + /* + * We need to accumulate index statistics for later subtraction from heap + * stats. + */ } LVRelState; @@ -487,6 +494,29 @@ static void restore_vacuum_error_info(LVRelState *vacrel, const LVSavedErrInfo *saved_vacrel); +/* Extended vacuum statistics functions */ + +/* + * Build the heap-specific part of the extended vacuum report from the + * counters gathered in vacrel. + */ +static void +accumulate_heap_vacuum_statistics(LVRelState *vacrel, PgStat_VacuumRelationCounts * extVacStats) +{ + extVacStats->type = PGSTAT_EXTVAC_TABLE; + extVacStats->common.tuples_deleted = vacrel->tuples_deleted; + extVacStats->table.tuples_frozen = vacrel->tuples_frozen; + extVacStats->table.recently_dead_tuples = vacrel->recently_dead_tuples; + extVacStats->table.missed_dead_tuples = vacrel->missed_dead_tuples; + + /* + * The wraparound failsafe is reported as a per-relation flag (0/1): did + * the latest vacuum of this relation engage it? Consumers can sum the + * flags to build a per-database total. + */ + + /* Hook is invoked from pgstat_report_vacuum() when extstats is passed */ +} /* * Helper to set up the eager scanning state for vacuuming a single relation. @@ -646,7 +676,9 @@ heap_vacuum_rel(Relation rel, const VacuumParams *params, ErrorContextCallback errcallback; char **indnames = NULL; Size dead_items_max_bytes = 0; + PgStat_VacuumRelationCounts extVacReport; + memset(&extVacReport, 0, sizeof(extVacReport)); verbose = (params->options & VACOPT_VERBOSE) != 0; instrument = (verbose || (AmAutoVacuumWorkerProcess() && params->log_vacuum_min_duration >= 0)); @@ -691,6 +723,7 @@ heap_vacuum_rel(Relation rel, const VacuumParams *params, vacrel->dbname = get_database_name(MyDatabaseId); vacrel->relnamespace = get_namespace_name(RelationGetNamespace(rel)); vacrel->relname = pstrdup(RelationGetRelationName(rel)); + vacrel->reloid = RelationGetRelid(rel); vacrel->indname = NULL; vacrel->phase = VACUUM_ERRCB_PHASE_UNKNOWN; vacrel->verbose = verbose; @@ -994,13 +1027,30 @@ heap_vacuum_rel(Relation rel, const VacuumParams *params, * leader's sleeps only; parallel workers account their own sleeps to the * indexes they process. */ - pgstat_report_vacuum(rel, - Max(vacrel->new_live_tuples, 0), - vacrel->recently_dead_tuples + - vacrel->missed_dead_tuples, - starttime, - (PgStat_Counter) rint(VacuumDelayTime - startdelaytime), - VacuumFailsafeActive); + if (set_report_vacuum_hook) + { + accumulate_heap_vacuum_statistics(vacrel, &extVacReport); + + pgstat_report_vacuum_ext(rel, + Max(vacrel->new_live_tuples, 0), + vacrel->recently_dead_tuples + + vacrel->missed_dead_tuples, + starttime, + (PgStat_Counter) rint(VacuumDelayTime - + startdelaytime), + VacuumFailsafeActive, + &extVacReport); + } + else + pgstat_report_vacuum_ext(rel, + Max(vacrel->new_live_tuples, 0), + vacrel->recently_dead_tuples + + vacrel->missed_dead_tuples, + starttime, + (PgStat_Counter) rint(VacuumDelayTime - + startdelaytime), + VacuumFailsafeActive, + NULL); pgstat_progress_end_command(); if (instrument) @@ -3053,7 +3103,15 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, LVSavedErrInfo saved_err_info; TimestampTz istarttime = GetCurrentTimestamp(); double startdelaytime = VacuumDelayTime; + double prev_tuples_removed = 0; + PgStat_VacuumRelationCounts extVacReport; + /* + * Snapshot the running bulkdelete totals: an index may be processed + * several times per vacuum, and the report below covers this pass only. + */ + if (istat != NULL) + prev_tuples_removed = istat->tuples_removed; ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; ivinfo.analyze_only = false; @@ -3071,6 +3129,7 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, */ Assert(vacrel->indname == NULL); vacrel->indname = pstrdup(RelationGetRelationName(indrel)); + vacrel->indoid = RelationGetRelid(indrel); update_vacuum_error_info(vacrel, &saved_err_info, VACUUM_ERRCB_PHASE_VACUUM_INDEX, InvalidBlockNumber, InvalidOffsetNumber); @@ -3087,6 +3146,16 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, startdelaytime), AmAutoVacuumWorkerProcess()); + if (set_report_vacuum_hook) + { + memset(&extVacReport, 0, sizeof(extVacReport)); + extVacReport.type = PGSTAT_EXTVAC_INDEX; + if (istat != NULL) + extVacReport.common.tuples_deleted = + istat->tuples_removed - prev_tuples_removed; + pgstat_report_vacuum_ext(indrel, -1, -1, 0, 0, false, &extVacReport); + } + /* Revert to the previous phase information for error traceback */ restore_vacuum_error_info(vacrel, &saved_err_info); pfree(vacrel->indname); @@ -3113,7 +3182,15 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, LVSavedErrInfo saved_err_info; TimestampTz istarttime = GetCurrentTimestamp(); double startdelaytime = VacuumDelayTime; + double prev_tuples_removed = 0; + PgStat_VacuumRelationCounts extVacReport; + /* + * Snapshot the running bulkdelete totals: an index may be processed + * several times per vacuum, and the report below covers this pass only. + */ + if (istat != NULL) + prev_tuples_removed = istat->tuples_removed; ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; ivinfo.analyze_only = false; @@ -3132,6 +3209,7 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, */ Assert(vacrel->indname == NULL); vacrel->indname = pstrdup(RelationGetRelationName(indrel)); + vacrel->indoid = RelationGetRelid(indrel); update_vacuum_error_info(vacrel, &saved_err_info, VACUUM_ERRCB_PHASE_INDEX_CLEANUP, InvalidBlockNumber, InvalidOffsetNumber); @@ -3146,6 +3224,16 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, startdelaytime), AmAutoVacuumWorkerProcess()); + if (set_report_vacuum_hook) + { + memset(&extVacReport, 0, sizeof(extVacReport)); + extVacReport.type = PGSTAT_EXTVAC_INDEX; + if (istat != NULL) + extVacReport.common.tuples_deleted = + istat->tuples_removed - prev_tuples_removed; + pgstat_report_vacuum_ext(indrel, -1, -1, 0, 0, false, &extVacReport); + } + /* Revert to the previous phase information for error traceback */ restore_vacuum_error_info(vacrel, &saved_err_info); pfree(vacrel->indname); diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c index a3c1ae3cc29..2cb0d985fdf 100644 --- a/src/backend/commands/vacuumparallel.c +++ b/src/backend/commands/vacuumparallel.c @@ -1080,6 +1080,8 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, IndexVacuumInfo ivinfo; TimestampTz istarttime = GetCurrentTimestamp(); double startdelaytime = VacuumDelayTime; + double prev_tuples_removed = 0; + PgStat_VacuumRelationCounts extVacReport; /* * Update the pointer to the corresponding bulk-deletion result if someone @@ -1088,6 +1090,12 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, if (indstats->istat_updated) istat = &(indstats->istat); + /* + * Snapshot the running bulkdelete totals: an index may be processed + * several times per vacuum, and the report below covers this pass only. + */ + if (istat != NULL) + prev_tuples_removed = istat->tuples_removed; ivinfo.index = indrel; ivinfo.heaprel = pvs->heaprel; ivinfo.analyze_only = false; @@ -1116,6 +1124,16 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, RelationGetRelationName(indrel)); } + if (set_report_vacuum_hook) + { + memset(&extVacReport, 0, sizeof(extVacReport)); + extVacReport.type = PGSTAT_EXTVAC_INDEX; + if (istat_res != NULL) + extVacReport.common.tuples_deleted = + istat_res->tuples_removed - prev_tuples_removed; + pgstat_report_vacuum_ext(indrel, -1, -1, 0, 0, false, &extVacReport); + } + /* * Accumulate this pass into the index's cumulative vacuum times. Use the * leader's DSM flag to classify autovacuum: a parallel worker of an diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index a169199b21b..ea8776ad6f7 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -354,6 +354,35 @@ pgstat_report_vacuum(Relation rel, PgStat_Counter livetuples, (void) pgstat_flush_backend(false, PGSTAT_BACKEND_FLUSH_IO); } +/* + * Hook for extensions to receive extended vacuum statistics. + * NULL when no extension has registered. + */ +set_report_vacuum_hook_type set_report_vacuum_hook = NULL; + +/* + * Report extended vacuum statistics to extensions via set_report_vacuum_hook. + * When livetuples/deadtuples/starttime are provided (heap case), also calls + * pgstat_report_vacuum. For indexes, pass -1, -1, 0, 0 to skip pgstat_report_vacuum. + */ +void +pgstat_report_vacuum_ext(Relation rel, PgStat_Counter livetuples, + PgStat_Counter deadtuples, TimestampTz starttime, + PgStat_Counter delaytime, bool failsafe, + PgStat_VacuumRelationCounts * extstats) +{ + /* Index reports pass starttime = 0: no per-vacuum pgstat side-effects */ + if (starttime != 0) + pgstat_report_vacuum(rel, livetuples, deadtuples, starttime, delaytime, + failsafe); + + if (extstats != NULL && set_report_vacuum_hook) + (*set_report_vacuum_hook) (RelationGetRelid(rel), + rel->rd_rel->relisshared, + extstats); +} + + /* * Report that the table was just analyzed and flush IO statistics. * diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 21cc777fa1f..d0f92ed6625 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -24,6 +24,7 @@ #include "parser/parse_node.h" #include "storage/buf.h" #include "utils/relcache.h" +#include "pgstat.h" /* * Flags for amparallelvacuumoptions to control the participation of bulkdelete diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 218a9bfd93b..1b529c43368 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -93,6 +93,38 @@ typedef struct PgStat_FunctionCounts /* * Working state needed to accumulate per-function-call timing statistics. */ +/* + * Extended vacuum statistics - passed to extensions via set_report_vacuum_hook. + * Type of entry: table (heap), index, or database aggregate. + */ +typedef enum ExtVacReportType +{ + PGSTAT_EXTVAC_INVALID = 0, + PGSTAT_EXTVAC_TABLE = 1, + PGSTAT_EXTVAC_INDEX = 2, + PGSTAT_EXTVAC_DB = 3, +} ExtVacReportType; + +typedef struct PgStat_CommonCounts +{ + int64 tuples_deleted; +} PgStat_CommonCounts; + +typedef struct PgStat_VacuumRelationCounts +{ + PgStat_CommonCounts common; + ExtVacReportType type; + union + { + struct + { + int64 tuples_frozen; + int64 recently_dead_tuples; + int64 missed_dead_tuples; + } table; + }; +} PgStat_VacuumRelationCounts; + typedef struct PgStat_FunctionCallUsage { /* Link to function's hashtable entry (must still be there at exit!) */ @@ -839,6 +871,20 @@ extern void pgstat_report_index_vacuum_time(Relation rel, PgStat_Counter delaytime, bool is_autovacuum); extern void pgstat_count_vacuum_error(bool shared); + +extern void pgstat_report_vacuum_ext(Relation rel, + PgStat_Counter livetuples, + PgStat_Counter deadtuples, + TimestampTz starttime, + PgStat_Counter delaytime, + bool failsafe, + PgStat_VacuumRelationCounts * extstats); + +/* Hook for extensions to receive extended vacuum statistics */ +typedef void (*set_report_vacuum_hook_type) (Oid tableoid, bool shared, + PgStat_VacuumRelationCounts * params); +extern PGDLLIMPORT set_report_vacuum_hook_type set_report_vacuum_hook; + extern void pgstat_report_analyze(Relation rel, PgStat_Counter livetuples, PgStat_Counter deadtuples, bool resetcounter, TimestampTz starttime); -- 2.50.1 (Apple Git-155)