From b6f5be8518ca705e16f2f0134e65621327331f55 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Fri, 17 Jul 2026 04:32:15 +0300 Subject: [PATCH v43 7/9] ext_vacuum_statistics: WAL metrics and the per-database aggregate. Add the WAL category (wal_records, wal_fpi, wal_bytes) to the per-relation views, sampled by the core machinery around the heap run and each index pass from pgWalUsage; the resource usage of index passes is subtracted from the heap report to avoid double-counting in aggregates. Parallel index passes sample inside the worker. Also introduce the per-database aggregate: a second custom stats kind accumulating the counters of every vacuum in the database, exposed through the new pg_stats_vacuum_database view and resettable per database with extvac_reset_db_entry(). Authors: Alena Rybakina , Andrei Lepikhov , Andrei Zubkov --- contrib/ext_vacuum_statistics/README.md | 12 +- .../ext_vacuum_statistics--1.0.sql | 43 + contrib/ext_vacuum_statistics/meson.build | 1 + .../t/052_vacuum_extending_basic_test.pl | 754 ++++++++++++++++++ .../ext_vacuum_statistics/vacuum_statistics.c | 169 +++- doc/src/sgml/extvacuumstatistics.sgml | 145 +++- src/backend/access/heap/vacuumlazy.c | 72 +- src/backend/commands/vacuumparallel.c | 4 + src/include/commands/vacuum.h | 11 + src/include/pgstat.h | 3 + 10 files changed, 1182 insertions(+), 32 deletions(-) create mode 100644 contrib/ext_vacuum_statistics/t/052_vacuum_extending_basic_test.pl diff --git a/contrib/ext_vacuum_statistics/README.md b/contrib/ext_vacuum_statistics/README.md index ac80bda4a66..8c8b42f014d 100644 --- a/contrib/ext_vacuum_statistics/README.md +++ b/contrib/ext_vacuum_statistics/README.md @@ -1,6 +1,6 @@ # ext_vacuum_statistics -Extended vacuum statistics extension for PostgreSQL. It collects and exposes detailed per-table and per-index vacuum statistics via convenient views in the `ext_vacuum_statistics` schema. +Extended vacuum statistics extension for PostgreSQL. It collects and exposes detailed per-table, per-index, and per-database vacuum statistics (buffer I/O, WAL, general, timing) via convenient views in the `ext_vacuum_statistics` schema. ## Installation @@ -37,14 +37,17 @@ SELECT * FROM ext_vacuum_statistics.pg_stats_vacuum_tables; -- Per-index vacuum statistics SELECT * FROM ext_vacuum_statistics.pg_stats_vacuum_indexes; + +-- Per-database aggregate vacuum statistics +SELECT * FROM ext_vacuum_statistics.pg_stats_vacuum_database; ``` Example output: ``` - relname | tuples_deleted | pages_removed ------------+----------------+--------------- - mytable | 500 | 10 + relname | wal_records | tuples_deleted | pages_removed +-----------+-------------+----------------+--------------- + mytable | 15 | 500 | 10 ``` Reset statistics when needed: @@ -79,6 +82,7 @@ SET vacuum_statistics.enabled = off; |------|-------------| | `ext_vacuum_statistics.pg_stats_vacuum_tables` | Per-table heap vacuum stats (pages scanned, tuples deleted, dead tuples, etc.) | | `ext_vacuum_statistics.pg_stats_vacuum_indexes` | Per-index vacuum stats | +| `ext_vacuum_statistics.pg_stats_vacuum_database` | Per-database aggregate vacuum stats | ## Limitations diff --git a/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql b/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql index fa6da7dcc40..d8d327741f8 100644 --- a/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql +++ b/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql @@ -25,6 +25,11 @@ RETURNS boolean AS 'MODULE_PATHNAME', 'extvac_reset_entry' LANGUAGE C STRICT PARALLEL SAFE; +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.extvac_reset_db_entry(dboid oid) +RETURNS bigint +AS 'MODULE_PATHNAME', 'extvac_reset_db_entry' +LANGUAGE C STRICT PARALLEL SAFE; + CREATE OR REPLACE FUNCTION ext_vacuum_statistics.vacuum_statistics_reset() RETURNS bigint AS 'MODULE_PATHNAME', 'vacuum_statistics_reset' @@ -35,6 +40,9 @@ CREATE OR REPLACE FUNCTION ext_vacuum_statistics.pg_stats_get_vacuum_tables( IN dboid oid, IN reloid oid, OUT relid oid, + OUT wal_records bigint, + OUT wal_fpi bigint, + OUT wal_bytes numeric, OUT tuples_deleted bigint, OUT pages_scanned bigint, OUT pages_removed bigint, @@ -52,6 +60,9 @@ CREATE OR REPLACE FUNCTION ext_vacuum_statistics.pg_stats_get_vacuum_indexes( IN dboid oid, IN reloid oid, OUT relid oid, + OUT wal_records bigint, + OUT wal_fpi bigint, + OUT wal_bytes numeric, OUT tuples_deleted bigint, OUT pages_deleted bigint ) @@ -59,6 +70,18 @@ RETURNS SETOF record AS 'MODULE_PATHNAME', 'pg_stats_get_vacuum_indexes' LANGUAGE C STRICT STABLE; +-- Internal C function to fetch database vacuum stats +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.pg_stats_get_vacuum_database( + IN dboid oid, + OUT dbid oid, + OUT wal_records bigint, + OUT wal_fpi bigint, + OUT wal_bytes numeric +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_stats_get_vacuum_database' +LANGUAGE C STRICT STABLE; + -- View: vacuum statistics per table (heap) CREATE VIEW ext_vacuum_statistics.pg_stats_vacuum_tables AS SELECT @@ -66,6 +89,9 @@ SELECT ns.nspname AS schema, rel.relname AS relname, db.datname AS dbname, + stats.wal_records, + stats.wal_fpi, + stats.wal_bytes, stats.tuples_deleted, stats.pages_scanned, stats.pages_removed, @@ -92,6 +118,9 @@ SELECT ns.nspname AS schema, rel.relname AS indexrelname, db.datname AS dbname, + stats.wal_records, + stats.wal_fpi, + stats.wal_bytes, stats.tuples_deleted, stats.pages_deleted FROM pg_database db, @@ -105,3 +134,17 @@ WHERE db.datname = current_database() COMMENT ON VIEW ext_vacuum_statistics.pg_stats_vacuum_indexes IS 'Extended vacuum statistics per index'; + +-- View: vacuum statistics per database (aggregate) +CREATE VIEW ext_vacuum_statistics.pg_stats_vacuum_database AS +SELECT + db.oid AS dboid, + db.datname AS dbname, + stats.wal_records AS db_wal_records, + stats.wal_fpi AS db_wal_fpi, + stats.wal_bytes AS db_wal_bytes +FROM pg_database db +LEFT JOIN LATERAL ext_vacuum_statistics.pg_stats_get_vacuum_database(db.oid) stats ON db.oid = stats.dbid; + +COMMENT ON VIEW ext_vacuum_statistics.pg_stats_vacuum_database IS + 'Extended vacuum statistics per database (aggregate)'; diff --git a/contrib/ext_vacuum_statistics/meson.build b/contrib/ext_vacuum_statistics/meson.build index 85eb2d6b85d..3e6858493ef 100644 --- a/contrib/ext_vacuum_statistics/meson.build +++ b/contrib/ext_vacuum_statistics/meson.build @@ -34,6 +34,7 @@ tests += { }, 'tap': { 'tests': [ + 't/052_vacuum_extending_basic_test.pl', 't/054_vacuum_extending_gucs_test.pl', 't/056_vacuum_stats_gc.pl', ], diff --git a/contrib/ext_vacuum_statistics/t/052_vacuum_extending_basic_test.pl b/contrib/ext_vacuum_statistics/t/052_vacuum_extending_basic_test.pl new file mode 100644 index 00000000000..494c6cc8259 --- /dev/null +++ b/contrib/ext_vacuum_statistics/t/052_vacuum_extending_basic_test.pl @@ -0,0 +1,754 @@ +# Copyright (c) 2025 PostgreSQL Global Development Group +# Test cumulative vacuum stats system using TAP +# +# This test validates the accuracy and behavior of cumulative vacuum statistics +# across heap tables, indexes, and databases using: +# +# • ext_vacuum_statistics.pg_stats_vacuum_tables +# • ext_vacuum_statistics.pg_stats_vacuum_indexes +# • ext_vacuum_statistics.pg_stats_vacuum_database +# +# A polling helper function repeatedly checks the stats views until expected +# deltas appear or a configurable timeout expires. This guarantees that +# stats-collector propagation delays do not lead to flaky test behavior. + +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +#------------------------------------------------------------------------------ +# Test harness setup +#------------------------------------------------------------------------------ + +my $node = PostgreSQL::Test::Cluster->new('stat_vacuum'); +$node->init; + +# Configure the server: preload extension and logging level +$node->append_conf('postgresql.conf', q{ + shared_preload_libraries = 'ext_vacuum_statistics' + log_min_messages = notice +}); + +my $stderr; +my $base_stats; +my $wals; +my $ibase_stats; +my $iwals; + +$node->start( + '>' => \$base_stats, + '2>' => \$stderr +); + +#------------------------------------------------------------------------------ +# Database creation and initialization +#------------------------------------------------------------------------------ + +$node->safe_psql('postgres', q{ + CREATE DATABASE statistic_vacuum_database_regression; + CREATE EXTENSION ext_vacuum_statistics; +}); +# Main test database name and number of rows to insert +my $dbname = 'statistic_vacuum_database_regression'; +my $size_tab = 1000; + +# Enable required session settings and force the stats collector to flush next +$node->safe_psql($dbname, q{ + SET track_functions = 'all'; + SELECT pg_stat_force_next_flush(); +}); + +#------------------------------------------------------------------------------ +# Create test table and populate it +#------------------------------------------------------------------------------ + +$node->safe_psql( + $dbname, + "CREATE EXTENSION ext_vacuum_statistics; + CREATE TABLE vestat (x int PRIMARY KEY) + WITH (autovacuum_enabled = off, fillfactor = 10); + INSERT INTO vestat SELECT x FROM generate_series(1, $size_tab) AS g(x); + ANALYZE vestat;" +); + +#------------------------------------------------------------------------------ +# Timing parameters for polling loops +#------------------------------------------------------------------------------ + +my $timeout = 30; # overall wait timeout in seconds +my $interval = 0.015; # poll interval in seconds (15 ms) +my $start_time = time(); +my $updated = 0; + +#------------------------------------------------------------------------------ +# wait_for_vacuum_stats +# +# Polls ext_vacuum_statistics.pg_stats_vacuum_tables and ext_vacuum_statistics.pg_stats_vacuum_indexes until both the +# table-level and index-level counters exceed the provided baselines, or until +# the configured timeout elapses. +# +# Expected named args (baseline values): +# tab_tuples_deleted +# tab_wal_records +# idx_tuples_deleted +# idx_wal_records +# +# Returns: 1 if the condition is met before timeout, 0 otherwise. +#------------------------------------------------------------------------------ + +sub wait_for_vacuum_stats { + my (%args) = @_; + my $tab_tuples_deleted = ($args{tab_tuples_deleted} or 0); + my $tab_wal_records = ($args{tab_wal_records} or 0); + my $idx_tuples_deleted = ($args{idx_tuples_deleted} or 0); + my $idx_wal_records = ($args{idx_wal_records} or 0); + + my $start = time(); + while ((time() - $start) < $timeout) { + + my $result_query = $node->safe_psql( + $dbname, + "VACUUM vestat; + SELECT + (SELECT (tuples_deleted > $tab_tuples_deleted AND wal_records > $tab_wal_records) + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'vestat') + AND + (SELECT (tuples_deleted > $idx_tuples_deleted AND wal_records > $idx_wal_records) + FROM ext_vacuum_statistics.pg_stats_vacuum_indexes + WHERE indexrelname = 'vestat_pkey');" + ); + + return 1 if ($result_query eq 't'); + + sleep($interval); + } + + return 0; +} + +#------------------------------------------------------------------------------ +# Variables to hold vacuum-stat snapshots for later comparisons +#------------------------------------------------------------------------------ + +my $tuples_deleted = 0; +my $pages_scanned = 0; +my $pages_removed = 0; +my $wal_records = 0; +my $wal_bytes = 0; +my $wal_fpi = 0; + +my $index_tuples_deleted = 0; +my $index_pages_deleted = 0; +my $index_wal_records = 0; +my $index_wal_bytes = 0; +my $index_wal_fpi = 0; + +my $tuples_deleted_prev = 0; +my $pages_scanned_prev = 0; +my $pages_removed_prev = 0; +my $wal_records_prev = 0; +my $wal_bytes_prev = 0; +my $wal_fpi_prev = 0; + +my $index_tuples_deleted_prev = 0; +my $index_pages_deleted_prev = 0; +my $index_wal_records_prev = 0; +my $index_wal_bytes_prev = 0; +my $index_wal_fpi_prev = 0; + +#------------------------------------------------------------------------------ +# fetch_vacuum_stats +# +# Reads current values of relevant vacuum counters for the test table and its +# primary index, storing them in package variables for subsequent comparisons. +#------------------------------------------------------------------------------ + +sub fetch_vacuum_stats { + # fetch actual base vacuum statistics + my $base_statistics = $node->safe_psql( + $dbname, + "SELECT tuples_deleted, pages_scanned, pages_removed, wal_records, wal_bytes, wal_fpi + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'vestat';" + ); + + $base_statistics =~ s/\s*\|\s*/ /g; # transform " | " into space + ($tuples_deleted, $pages_scanned, $pages_removed, $wal_records, $wal_bytes, $wal_fpi) + = split /\s+/, $base_statistics; + + # --- index stats --- + my $index_base_statistics = $node->safe_psql( + $dbname, + "SELECT tuples_deleted, pages_deleted, wal_records, wal_bytes, wal_fpi + FROM ext_vacuum_statistics.pg_stats_vacuum_indexes + WHERE indexrelname = 'vestat_pkey';" + ); + + $index_base_statistics =~ s/\s*\|\s*/ /g; # transform " | " into space + ($index_tuples_deleted, $index_pages_deleted, $index_wal_records, $index_wal_bytes, $index_wal_fpi) + = split /\s+/, $index_base_statistics; +} + +#------------------------------------------------------------------------------ +# save_vacuum_stats +# +# Save current values (previously fetched by fetch_vacuum_stats) so that we +# later fetch new values and compare them. +#------------------------------------------------------------------------------ +sub save_vacuum_stats { + $tuples_deleted_prev = $tuples_deleted; + $pages_scanned_prev = $pages_scanned; + $pages_removed_prev = $pages_removed; + $wal_records_prev = $wal_records; + $wal_bytes_prev = $wal_bytes; + $wal_fpi_prev = $wal_fpi; + + $index_tuples_deleted_prev = $index_tuples_deleted; + $index_pages_deleted_prev = $index_pages_deleted; + $index_wal_records_prev = $index_wal_records; + $index_wal_bytes_prev = $index_wal_bytes; + $index_wal_fpi_prev = $index_wal_fpi; +} + +#------------------------------------------------------------------------------ +# print_vacuum_stats_on_error +# +# Print values in case of an error +#------------------------------------------------------------------------------ +sub print_vacuum_stats_on_error { + diag( + "Statistics in the failed test\n" . + "Table statistics:\n" . + " Before test:\n" . + " tuples_deleted = $tuples_deleted_prev\n" . + " pages_scanned = $pages_scanned_prev\n" . + " pages_removed = $pages_removed_prev\n" . + " wal_records = $wal_records_prev\n" . + " wal_bytes = $wal_bytes_prev\n" . + " wal_fpi = $wal_fpi_prev\n" . + " After test:\n" . + " tuples_deleted = $tuples_deleted\n" . + " pages_scanned = $pages_scanned\n" . + " pages_removed = $pages_removed\n" . + " wal_records = $wal_records\n" . + " wal_bytes = $wal_bytes\n" . + " wal_fpi = $wal_fpi\n" . + "Index statistics:\n" . + " Before test:\n" . + " tuples_deleted = $index_tuples_deleted_prev\n" . + " pages_deleted = $index_pages_deleted_prev\n" . + " wal_records = $index_wal_records_prev\n" . + " wal_bytes = $index_wal_bytes_prev\n" . + " wal_fpi = $index_wal_fpi_prev\n" . + " After test:\n" . + " tuples_deleted = $index_tuples_deleted\n" . + " pages_deleted = $index_pages_deleted\n" . + " wal_records = $index_wal_records\n" . + " wal_bytes = $index_wal_bytes\n" . + " wal_fpi = $index_wal_fpi\n" + ); +}; + +sub fetch_error_base_db_vacuum_statistics { + my (%args) = @_; + + # Validate presence of required args (allow 0 as valid numeric baseline) + die "database name required" + unless exists $args{database_name} && defined $args{database_name}; + my $database_name = $args{database_name}; + + # fetch actual base database vacuum statistics + my $base_statistics = $node->safe_psql( + $database_name, + "SELECT db_wal_records, db_wal_fpi, db_wal_bytes + FROM ext_vacuum_statistics.pg_stats_vacuum_database, pg_database + WHERE pg_database.datname = '$dbname' + AND pg_database.oid = ext_vacuum_statistics.pg_stats_vacuum_database.dboid;" + ); + $base_statistics =~ s/\s*\|\s*/ /g; # transform " | " in space + my ($wal_records, $wal_fpi, $wal_bytes) = split /\s+/, $base_statistics; + + diag( + "BASE STATS MISMATCH FOR DATABASE $dbname:\n" . + " wal_records = $wal_records\n" . + " wal_fpi = $wal_fpi\n" . + " wal_bytes = $wal_bytes\n" + ); +} + + +#------------------------------------------------------------------------------ +# Test 1: Delete half the rows, run VACUUM, and wait for stats to advance +#------------------------------------------------------------------------------ +subtest 'Test 1: Delete half the rows, run VACUUM' => sub +{ + +$node->safe_psql($dbname, "DELETE FROM vestat WHERE x % 2 = 0;"); +$node->safe_psql($dbname, "VACUUM vestat;"); + +# Poll the stats view until expected deltas appear or timeout +$updated = wait_for_vacuum_stats( + tab_tuples_deleted => 0, + tab_wal_records => 0, + idx_tuples_deleted => 0, + idx_wal_records => 0, +); +ok($updated, 'vacuum stats updated after vacuuming half-deleted table (tuples_deleted and wal_fpi advanced)') + or diag "Timeout waiting for ext_vacuum_statistics update after $timeout seconds after vacuuming half-deleted table"; + +fetch_vacuum_stats(); + +ok($tuples_deleted > $tuples_deleted_prev, 'table tuples_deleted has increased'); +ok($pages_scanned > $pages_scanned_prev, 'table pages_scanned has increased'); +ok($pages_removed == $pages_removed_prev, 'table pages_removed stay the same'); +ok($wal_records > $wal_records_prev, 'table wal_records has increased'); +ok($wal_bytes > $wal_bytes_prev, 'table wal_bytes has increased'); +ok($wal_fpi > $wal_fpi_prev, 'table wal_fpi has increased'); + +ok($index_pages_deleted == $index_pages_deleted_prev, 'index pages_deleted stay the same'); +ok($index_tuples_deleted > $index_tuples_deleted_prev, 'index tuples_deleted has increased'); +ok($index_wal_records > $index_wal_records_prev, 'index wal_records has increased'); +ok($index_wal_bytes > $index_wal_bytes_prev, 'index wal_bytes has increased'); +ok($index_wal_fpi == $index_wal_fpi_prev, 'index wal_fpi stay the same'); + +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 2: Delete all rows, run VACUUM, and wait for stats to advance +#------------------------------------------------------------------------------ +subtest 'Test 2: Delete all rows, run VACUUM' => sub +{ +save_vacuum_stats(); + +$node->safe_psql($dbname, "DELETE FROM vestat;"); +$node->safe_psql($dbname, "VACUUM vestat;"); + +$updated = wait_for_vacuum_stats( + tab_tuples_deleted => $tuples_deleted_prev, + tab_wal_records => $wal_records_prev, + idx_tuples_deleted => $index_tuples_deleted_prev, + idx_wal_records => $index_wal_records_prev, +); + +ok($updated, 'vacuum stats updated after vacuuming all-deleted table (tuples_deleted and wal_records advanced)') + or diag "Timeout waiting for ext_vacuum_statistics update after $timeout seconds after vacuuming all-deleted table"; + +fetch_vacuum_stats(); + +ok($tuples_deleted > $tuples_deleted_prev, 'table tuples_deleted has increased'); +ok($pages_scanned > $pages_scanned_prev, 'table pages_scanned has increased'); +ok($pages_removed > $pages_removed_prev, 'table pages_removed has increased'); +ok($wal_records > $wal_records_prev, 'table wal_records has increased'); +ok($wal_bytes > $wal_bytes_prev, 'table wal_bytes has increased'); +ok($wal_fpi > 0, 'table wal_fpi has increased'); + +ok($index_pages_deleted > $index_pages_deleted_prev, 'index pages_deleted has increased'); +ok($index_tuples_deleted > $index_tuples_deleted_prev, 'index tuples_deleted has increased'); +ok($index_wal_records > $index_wal_records_prev, 'index wal_records has increased'); +ok($index_wal_bytes > $index_wal_bytes_prev, 'index wal_bytes has increased'); +ok($index_wal_fpi == $index_wal_fpi_prev, 'index wal_fpi stay the same'); + +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 3: Test VACUUM FULL — it should not report to the stats collector +#------------------------------------------------------------------------------ +subtest 'Test 3: Test VACUUM FULL — it should not report to the stats collector' => sub +{ +save_vacuum_stats(); + +$node->safe_psql( + $dbname, + "INSERT INTO vestat SELECT x FROM generate_series(1, $size_tab) AS g(x); + CHECKPOINT; + DELETE FROM vestat; + VACUUM FULL vestat;" +); + +fetch_vacuum_stats(); + +ok($tuples_deleted == $tuples_deleted_prev, 'table tuples_deleted stay the same'); +ok($pages_scanned == $pages_scanned_prev, 'table pages_scanned stay the same'); +ok($pages_removed == $pages_removed_prev, 'table pages_removed stay the same'); +ok($wal_records == $wal_records_prev, 'table wal_records stay the same'); +ok($wal_bytes == $wal_bytes_prev, 'table wal_bytes stay the same'); +ok($wal_fpi == $wal_fpi_prev, 'table wal_fpi stay the same'); + +ok($index_pages_deleted == $index_pages_deleted_prev, 'index pages_deleted stay the same'); +ok($index_tuples_deleted == $index_tuples_deleted_prev, 'index tuples_deleted stay the same'); +ok($index_wal_records == $index_wal_records_prev, 'index wal_records stay the same'); +ok($index_wal_bytes == $index_wal_bytes_prev, 'index wal_bytes stay the same'); +ok($index_wal_fpi == $index_wal_fpi_prev, 'index wal_fpi stay the same'); + +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 4: Update table, checkpoint, and VACUUM to provoke WAL/FPI accounting +#------------------------------------------------------------------------------ +subtest 'Test 4: Update table, checkpoint, and VACUUM to provoke WAL/FPI accounting' => sub +{ + +save_vacuum_stats(); + +$node->safe_psql( + $dbname, + "INSERT INTO vestat SELECT x FROM generate_series(1, $size_tab) AS g(x); + CHECKPOINT; + UPDATE vestat SET x = x + 1000; + VACUUM vestat;" +); + +$updated = wait_for_vacuum_stats( + tab_tuples_deleted => $tuples_deleted_prev, + tab_wal_records => $wal_records_prev, + idx_tuples_deleted => $index_tuples_deleted_prev, + idx_wal_records => $index_wal_records_prev, +); + +ok($updated, 'vacuum stats updated after updating tuples in the table (tuples_deleted and wal_records advanced)') + or diag "Timeout waiting for ext_vacuum_statistics update after $timeout seconds"; + +fetch_vacuum_stats(); + +ok($tuples_deleted > $tuples_deleted_prev, 'table tuples_deleted has increased'); +ok($pages_scanned > $pages_scanned_prev, 'table pages_scanned has increased'); +ok($pages_removed == $pages_removed_prev, 'table pages_removed stay the same'); +ok($wal_records > $wal_records_prev, 'table wal_records has increased'); +ok($wal_bytes > $wal_bytes_prev, 'table wal_bytes has increased'); +ok($wal_fpi > $wal_fpi_prev, 'table wal_fpi has increased'); + +ok($index_pages_deleted > $index_pages_deleted_prev, 'index pages_deleted has increased'); +ok($index_tuples_deleted > $index_tuples_deleted_prev, 'index tuples_deleted has increased'); +ok($index_wal_records > $index_wal_records_prev, 'index wal_records has increased'); +ok($index_wal_bytes > $index_wal_bytes_prev, 'index wal_bytes has increased'); +ok($index_wal_fpi > $index_wal_fpi_prev, 'index wal_fpi has increased'); + +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 5: Update table, trancate and vacuuming +#------------------------------------------------------------------------------ +subtest 'Test 5: Update table, trancate and vacuuming' => sub +{ + +save_vacuum_stats(); + +$node->safe_psql( + $dbname, + "INSERT INTO vestat SELECT x FROM generate_series(1, $size_tab) AS g(x); + UPDATE vestat SET x = x + 1000;" +); +$node->safe_psql($dbname, "TRUNCATE vestat;"); +$node->safe_psql($dbname, "CHECKPOINT;"); +$node->safe_psql($dbname, "VACUUM vestat;"); + +$updated = wait_for_vacuum_stats( + tab_wal_records => $wal_records_prev, +); + +ok($updated, 'vacuum stats updated after updating tuples and trancation in the table (wal_records advanced)') + or diag "Timeout waiting for ext_vacuum_statistics update after $timeout seconds"; + +fetch_vacuum_stats(); + +ok($tuples_deleted == $tuples_deleted_prev, 'table tuples_deleted stay the same'); +ok($pages_scanned == $pages_scanned_prev, 'table pages_scanned stay the same'); +ok($pages_removed == $pages_removed_prev, 'table pages_removed stay the same'); +ok($wal_records > $wal_records_prev, 'table wal_records has increased'); +ok($wal_bytes > $wal_bytes_prev, 'table wal_bytes has increased'); +ok($wal_fpi == $wal_fpi_prev, 'table wal_fpi stay the same'); + +ok($index_pages_deleted == $index_pages_deleted_prev, 'index pages_deleted stay the same'); +ok($index_tuples_deleted == $index_tuples_deleted_prev, 'index tuples_deleted stay the same'); +ok($index_wal_records == $index_wal_records_prev, 'index wal_records stay the same'); +ok($index_wal_bytes == $index_wal_bytes_prev, 'index wal_bytes stay the same'); +ok($index_wal_fpi == $index_wal_fpi_prev, 'index wal_fpi stay the same'); + +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 6: Delete all tuples from table, trancate, and vacuuming +#------------------------------------------------------------------------------ +subtest 'Test 6: Delete all tuples from table, trancate, and vacuuming' => sub +{ + +save_vacuum_stats(); + +$node->safe_psql( + $dbname, + "INSERT INTO vestat SELECT x FROM generate_series(1, $size_tab) AS g(x); + DELETE FROM vestat; + TRUNCATE vestat; + CHECKPOINT; + VACUUM vestat;" +); + +$updated = wait_for_vacuum_stats( + tab_wal_records => $wal_records, +); + +ok($updated, 'vacuum stats updated after deleting all tuples and trancation in the table (wal_records advanced)') + or diag "Timeout waiting for ext_vacuum_statistics update after $timeout seconds"; + +fetch_vacuum_stats(); + +ok($tuples_deleted == $tuples_deleted_prev, 'table tuples_deleted stay the same'); +ok($pages_scanned == $pages_scanned_prev, 'table pages_scanned stay the same'); +ok($pages_removed == $pages_removed_prev, 'table pages_removed stay the same'); +ok($wal_records > $wal_records_prev, 'table wal_records has increased'); +ok($wal_bytes > $wal_bytes_prev, 'table wal_bytes has increased'); +ok($wal_fpi == $wal_fpi_prev, 'table wal_fpi stay the same'); + +ok($index_pages_deleted == $index_pages_deleted_prev, 'index pages_deleted stay the same'); +ok($index_tuples_deleted == $index_tuples_deleted_prev, 'index tuples_deleted stay the same'); +ok($index_wal_records == $index_wal_records_prev, 'index wal_records stay the same'); +ok($index_wal_bytes == $index_wal_bytes_prev, 'index wal_bytes stay the same'); +ok($index_wal_fpi == $index_wal_fpi_prev, 'index wal_fpi stay the same'); + +} or print_vacuum_stats_on_error(); + +my $dboid = $node->safe_psql( + $dbname, + "SELECT oid FROM pg_database WHERE datname = current_database();" +); + +#------------------------------------------------------------------------------------------------------- +# Test 7: Check if we return single vacuum statistics for particular relation from the current database +#------------------------------------------------------------------------------------------------------- +subtest 'Test 7: Check if we return vacuum statistics from the current database' => sub +{ +save_vacuum_stats(); + +my $reloid = $node->safe_psql( + $dbname, + q{ + SELECT oid FROM pg_class WHERE relname = 'vestat'; + } +); + +# Check if we can get vacuum statistics of particular heap relation in the current database +$base_stats = $node->safe_psql( + $dbname, + "SELECT count(*) FROM ext_vacuum_statistics.pg_stats_get_vacuum_tables((SELECT oid FROM pg_database WHERE datname = current_database()), $reloid);" +); +is($base_stats, 1, 'heap vacuum stats return from the current relation and database as expected'); + +$reloid = $node->safe_psql( + $dbname, + q{ + SELECT oid FROM pg_class WHERE relname = 'vestat_pkey'; + } +); + +# Check if we can get vacuum statistics of particular index relation in the current database +$base_stats = $node->safe_psql( + $dbname, + "SELECT count(*) FROM ext_vacuum_statistics.pg_stats_get_vacuum_indexes((SELECT oid FROM pg_database WHERE datname = current_database()), $reloid);" +); +is($base_stats, 1, 'index vacuum stats return from the current relation and database as expected'); + +# Check if we return empty results if vacuum statistics with particular oid doesn't exist +$base_stats = $node->safe_psql( + $dbname, + "SELECT count(*) FROM ext_vacuum_statistics.pg_stats_get_vacuum_tables((SELECT oid FROM pg_database WHERE datname = current_database()), 1);" +); +is($base_stats, 0, 'table vacuum stats return no rows, as expected'); + +$base_stats = $node->safe_psql( + $dbname, + "SELECT count(*) FROM ext_vacuum_statistics.pg_stats_get_vacuum_indexes((SELECT oid FROM pg_database WHERE datname = current_database()), 1);" +); +is($base_stats, 0, 'index vacuum stats return no rows, as expected'); + +# Check if we can get vacuum statistics of all relations in the current database +$base_stats = $node->safe_psql( + $dbname, + "SELECT count(*) > 0 FROM ext_vacuum_statistics.pg_stats_vacuum_tables;" +); +ok($base_stats eq 't', 'vacuum stats per all heap objects available'); + +$base_stats = $node->safe_psql( + $dbname, + "SELECT count(*) > 0 FROM ext_vacuum_statistics.pg_stats_vacuum_indexes;" +); +ok($base_stats eq 't', 'vacuum stats per all index objects available'); +}; + +#------------------------------------------------------------------------------ +# Test 8: Check relation-level vacuum statistics from another database +#------------------------------------------------------------------------------ +subtest 'Test 8: Check relation-level vacuum statistics from another database' => sub +{ +$base_stats = $node->safe_psql( + 'postgres', + "SELECT count(*) + FROM ext_vacuum_statistics.pg_stats_vacuum_indexes + WHERE indexrelname = 'vestat_pkey';" +); +is($base_stats, 0, 'check the printing index vacuum extended statistics from another database are not available'); + +$base_stats = $node->safe_psql( + 'postgres', + "SELECT count(*) + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'vestat';" +); +is($base_stats, 0, 'check the printing heap vacuum extended statistics from another database are not available'); + +# Check that relations from another database are not visible in the view when querying from postgres +$base_stats = $node->safe_psql( + 'postgres', + "SELECT count(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'vestat';" +); +is($base_stats, 0, 'vacuum stats per all tables objects from another database are not available as expected'); + +$base_stats = $node->safe_psql( + 'postgres', + "SELECT count(*) FROM ext_vacuum_statistics.pg_stats_vacuum_indexes WHERE indexrelname = 'vestat_pkey';" +); +is($base_stats, 0, 'vacuum stats per all index objects from another database are not available as expected'); +}; + +#-------------------------------------------------------------------------------------- +# Test 9: Check database-level vacuum statistics from the current and another database +#-------------------------------------------------------------------------------------- +subtest 'Test 9: Check database-level vacuum statistics from the current and another database' => sub +{ +my $wal_records = 0; +my $wal_fpi = 0; +my $wal_bytes = 0; +$base_stats = $node->safe_psql( + $dbname, + "SELECT db_wal_records, db_wal_fpi, db_wal_bytes + FROM ext_vacuum_statistics.pg_stats_vacuum_database, pg_database + WHERE pg_database.datname = '$dbname' + AND pg_database.oid = ext_vacuum_statistics.pg_stats_vacuum_database.dboid;" +); +$base_stats =~ s/\s*\|\s*/ /g; # transform " | " into space + ($wal_records, $wal_fpi, $wal_bytes) = split /\s+/, $base_stats; + +ok($wal_records > 0, 'wal_records is more than 0'); +ok($wal_fpi > 0, 'wal_fpi is more than 0'); +ok($wal_bytes > 0, 'wal_bytes is more than 0'); + +$base_stats = $node->safe_psql( + 'postgres', + "SELECT count(*) = 1 + FROM ext_vacuum_statistics.pg_stats_vacuum_database, pg_database + WHERE pg_database.datname = '$dbname' + AND pg_database.oid = ext_vacuum_statistics.pg_stats_vacuum_database.dboid;" +); +ok($base_stats eq 't', 'check database-level vacuum stats from another database are available'); +}; + +#------------------------------------------------------------------------------ +# Test 10: Cleanup checks: ensure functions return empty sets for OID = 0 +#------------------------------------------------------------------------------ +subtest 'Test 10: Cleanup checks: ensure functions return empty sets for OID = 0' => sub +{ +my $dboid = $node->safe_psql( + $dbname, + "SELECT oid FROM pg_database WHERE datname = current_database();" +); + +# Vacuum statistics for invalid relation OID return empty +$base_stats = $node->safe_psql( + $dbname, + q{ + SELECT COUNT(*) + FROM ext_vacuum_statistics.pg_stats_get_vacuum_tables((SELECT oid FROM pg_database WHERE datname = current_database()), 0); + } +); +is($base_stats, 0, 'vacuum stats per heap from invalid relation OID return empty as expected'); + +$base_stats = $node->safe_psql( + $dbname, + q{ + SELECT COUNT(*) + FROM ext_vacuum_statistics.pg_stats_get_vacuum_indexes((SELECT oid FROM pg_database WHERE datname = current_database()), 0); + } +); +is($base_stats, 0, 'vacuum stats per index from invalid relation OID return empty as expected'); + +$node->safe_psql($dbname, q{ + DROP TABLE vestat CASCADE; + VACUUM; +}); + +# Check that we don't print vacuum statistics for deleted objects +$base_stats = $node->safe_psql( + $dbname, + q{ + SELECT COUNT(*) + FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relid = 0; + } +); +is($base_stats, 0, 'ext_vacuum_statistics.pg_stats_vacuum_tables correctly returns no rows for OID = 0'); + +$base_stats = $node->safe_psql( + $dbname, + q{ + SELECT COUNT(*) + FROM ext_vacuum_statistics.pg_stats_vacuum_indexes WHERE indexrelid = 0; + } +); +is($base_stats, 0, 'ext_vacuum_statistics.pg_stats_vacuum_indexes correctly returns no rows for OID = 0'); + +my $reloid = $node->safe_psql( + $dbname, + q{ + SELECT oid FROM pg_class WHERE relname = 'pg_shdepend'; + } +); + +$node->safe_psql($dbname, "VACUUM pg_shdepend;"); + +# Check if we can get vacuum statistics for cluster relations (shared catalogs) +$base_stats = $node->safe_psql( + $dbname, + qq{ + SELECT count(*) > 0 + FROM ext_vacuum_statistics.pg_stats_get_vacuum_tables((SELECT oid FROM pg_database WHERE datname = current_database()), $reloid); + } +); + +is($base_stats, 't', 'vacuum stats for common heap objects available'); + +my $indoid = $node->safe_psql( + $dbname, + q{ + SELECT oid FROM pg_class WHERE relname = 'pg_shdepend_reference_index'; + } +); + +$base_stats = $node->safe_psql( + $dbname, + qq{ + SELECT count(*) > 0 + FROM ext_vacuum_statistics.pg_stats_get_vacuum_indexes((SELECT oid FROM pg_database WHERE datname = current_database()), $indoid); + } +); + +is($base_stats, 't', 'vacuum stats for common index objects available'); + +$node->safe_psql('postgres', + "DROP DATABASE $dbname; + VACUUM;" +); + +$base_stats = $node->safe_psql( + 'postgres', + q{ + SELECT count(*) = 0 + FROM ext_vacuum_statistics.pg_stats_get_vacuum_database(0); + } +); +is($base_stats, 't', 'vacuum stats from database with invalid database OID return empty, as expected'); +}; + +$node->stop; + +done_testing(); diff --git a/contrib/ext_vacuum_statistics/vacuum_statistics.c b/contrib/ext_vacuum_statistics/vacuum_statistics.c index 7b4cc9b5331..f8f991e6460 100644 --- a/contrib/ext_vacuum_statistics/vacuum_statistics.c +++ b/contrib/ext_vacuum_statistics/vacuum_statistics.c @@ -25,8 +25,9 @@ PG_MODULE_MAGIC; #endif -/* Custom stats kind for per-relation vacuum statistics */ +/* Two kinds: relations (tables/indexes) and database aggregates */ #define PGSTAT_KIND_EXTVAC_RELATION 24 +#define PGSTAT_KIND_EXTVAC_DB 25 #define SJ_NODENAME "vacuum_statistics" @@ -63,9 +64,25 @@ static const PgStat_KindInfo extvac_relation_kind_info = { .flush_pending_cb = NULL, }; +/* PgStat kind for per-database aggregated vacuum statistics */ +static const PgStat_KindInfo extvac_db_kind_info = { + .name = "ext_vacuum_statistics_db", + .fixed_amount = false, + .accessed_across_databases = true, + .write_to_file = true, + .shared_size = sizeof(PgStatShared_ExtVacEntry), + .shared_data_off = offsetof(PgStatShared_ExtVacEntry, stats), + .shared_data_len = sizeof(PgStat_VacuumRelationCounts), + .pending_size = 0, + .flush_pending_cb = NULL, +}; + static inline void pgstat_accumulate_common(PgStat_CommonCounts * dst, const PgStat_CommonCounts * src) { + dst->wal_records += src->wal_records; + dst->wal_fpi += src->wal_fpi; + dst->wal_bytes += src->wal_bytes; dst->tuples_deleted += src->tuples_deleted; } @@ -113,6 +130,7 @@ _PG_init(void) MarkGUCPrefixReserved(SJ_NODENAME); pgstat_register_kind(PGSTAT_KIND_EXTVAC_RELATION, &extvac_relation_kind_info); + pgstat_register_kind(PGSTAT_KIND_EXTVAC_DB, &extvac_db_kind_info); prev_report_vacuum_hook = set_report_vacuum_hook; set_report_vacuum_hook = pgstat_report_vacuum_extstats; @@ -162,13 +180,23 @@ extvac_object_access(ObjectAccessType access, Oid classId, Oid objectId, } } +/* Accumulate common counts for database-level stats. */ +static inline void +pgstat_accumulate_common_for_db(PgStat_CommonCounts * dst, + const PgStat_CommonCounts * src) +{ + pgstat_accumulate_common(dst, src); +} + /* - * Store incoming vacuum stats into a per-relation pgstat custom statistics - * entry. Uses pgstat_get_entry_ref_locked and pgstat_accumulate_* for - * atomic updates. + * Store incoming vacuum stats into pgstat custom statistics. + * store_relation: create/update per-relation entry + * store_db: accumulate into database-level entry (dboid, objid=0). + * Uses pgstat_get_entry_ref_locked and pgstat_accumulate_* for atomic updates. */ static void -extvac_store(Oid dboid, Oid relid, PgStat_VacuumRelationCounts * params) +extvac_store(Oid dboid, Oid relid, PgStat_VacuumRelationCounts * params, + bool store_relation, bool store_db) { PgStat_EntryRef *entry_ref; PgStatShared_ExtVacEntry *shared; @@ -176,23 +204,42 @@ extvac_store(Oid dboid, Oid relid, PgStat_VacuumRelationCounts * params) if (!evs_enabled) return; - entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_EXTVAC_RELATION, dboid, relid, false); - if (entry_ref) + if (store_relation) { - shared = (PgStatShared_ExtVacEntry *) entry_ref->shared_stats; - if (shared->stats.type == PGSTAT_EXTVAC_INVALID) + entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_EXTVAC_RELATION, dboid, relid, false); + if (entry_ref) { - memset(&shared->stats, 0, sizeof(shared->stats)); - shared->stats.type = params->type; + shared = (PgStatShared_ExtVacEntry *) entry_ref->shared_stats; + if (shared->stats.type == PGSTAT_EXTVAC_INVALID) + { + memset(&shared->stats, 0, sizeof(shared->stats)); + shared->stats.type = params->type; + } + pgstat_accumulate_extvac_stats(&shared->stats, params); + pgstat_unlock_entry(entry_ref); + } + } + + if (store_db) + { + entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_EXTVAC_DB, dboid, InvalidOid, false); + if (entry_ref) + { + shared = (PgStatShared_ExtVacEntry *) entry_ref->shared_stats; + if (shared->stats.type == PGSTAT_EXTVAC_INVALID) + { + memset(&shared->stats, 0, sizeof(shared->stats)); + shared->stats.type = PGSTAT_EXTVAC_DB; + } + pgstat_accumulate_common_for_db(&shared->stats.common, ¶ms->common); + pgstat_unlock_entry(entry_ref); } - pgstat_accumulate_extvac_stats(&shared->stats, params); - pgstat_unlock_entry(entry_ref); } } /* - * Vacuum report hook: called when vacuum finishes. Stores stats per-relation, - * then chains to previous hook. + * Vacuum report hook: called when vacuum finishes. Stores stats per-relation + * and per-database, then chains to previous hook. */ static void pgstat_report_vacuum_extstats(Oid tableoid, bool shared, @@ -201,7 +248,7 @@ pgstat_report_vacuum_extstats(Oid tableoid, bool shared, Oid dboid = shared ? InvalidOid : MyDatabaseId; if (evs_enabled) - extvac_store(dboid, tableoid, params); + extvac_store(dboid, tableoid, params, true, true); if (prev_report_vacuum_hook) prev_report_vacuum_hook(tableoid, shared, params); } @@ -214,16 +261,38 @@ extvac_reset_by_relid(Oid dboid, Oid relid) return true; } -/* Reset all vacuum statistics. */ +/* Callback for pgstat_reset_matching_entries: match relation entries for given db */ +static bool +match_extvac_relations_for_db(PgStatShared_HashEntry *entry, Datum match_data) +{ + return entry->key.kind == PGSTAT_KIND_EXTVAC_RELATION && + entry->key.dboid == DatumGetObjectId(match_data); +} + +/* + * Reset statistics for a database (aggregate entry) and all its relations. + */ +static int64 +extvac_database_reset(Oid dboid) +{ + pgstat_reset_matching_entries(match_extvac_relations_for_db, + ObjectIdGetDatum(dboid), 0); + pgstat_reset_entry(PGSTAT_KIND_EXTVAC_DB, dboid, 0, 0); + return 1; +} + +/* Reset all vacuum statistics (both relation and database entries). */ static int64 extvac_stat_reset(void) { pgstat_reset_of_kind(PGSTAT_KIND_EXTVAC_RELATION); + pgstat_reset_of_kind(PGSTAT_KIND_EXTVAC_DB); return 0; /* count not available */ } PG_FUNCTION_INFO_V1(vacuum_statistics_reset); PG_FUNCTION_INFO_V1(extvac_reset_entry); +PG_FUNCTION_INFO_V1(extvac_reset_db_entry); Datum vacuum_statistics_reset(PG_FUNCTION_ARGS) @@ -240,11 +309,38 @@ extvac_reset_entry(PG_FUNCTION_ARGS) PG_RETURN_BOOL(extvac_reset_by_relid(dboid, relid)); } +Datum +extvac_reset_db_entry(PG_FUNCTION_ARGS) +{ + Oid dboid = PG_GETARG_OID(0); + + PG_RETURN_INT64(extvac_database_reset(dboid)); +} + /* - * Output vacuum statistics (tables or indexes). + * Output vacuum statistics (tables, indexes, or per-database aggregates). */ -#define EXTVAC_HEAP_STAT_COLS 8 -#define EXTVAC_IDX_STAT_COLS 3 +#define EXTVAC_COMMON_STAT_COLS 3 + +static void +tuplestore_put_common(PgStat_CommonCounts * vacuum_ext, + Datum *values, bool *nulls, int *i) +{ + char buf[256]; + const int base PG_USED_FOR_ASSERTS_ONLY = *i; + + values[(*i)++] = Int64GetDatum(vacuum_ext->wal_records); + values[(*i)++] = Int64GetDatum(vacuum_ext->wal_fpi); + snprintf(buf, sizeof buf, UINT64_FORMAT, vacuum_ext->wal_bytes); + values[(*i)++] = DirectFunctionCall3(numeric_in, + CStringGetDatum(buf), + ObjectIdGetDatum(0), + Int32GetDatum(-1)); + Assert((*i - base) == EXTVAC_COMMON_STAT_COLS); +} + +#define EXTVAC_HEAP_STAT_COLS 11 +#define EXTVAC_IDX_STAT_COLS 6 #define EXTVAC_MAX_STAT_COLS Max(EXTVAC_HEAP_STAT_COLS, EXTVAC_IDX_STAT_COLS) static void @@ -258,6 +354,8 @@ tuplestore_put_for_relation(Oid relid, Tuplestorestate *tupstore, memset(nulls, 0, sizeof(nulls)); values[i++] = ObjectIdGetDatum(relid); + tuplestore_put_common(&vacuum_ext->common, values, nulls, &i); + if (vacuum_ext->type == PGSTAT_EXTVAC_TABLE) { values[i++] = Int64GetDatum(vacuum_ext->common.tuples_deleted); @@ -329,6 +427,30 @@ pg_stats_vacuum(FunctionCallInfo fcinfo, int type) if (stats && stats->type == type) tuplestore_put_for_relation(relid, tupstore, tupdesc, stats); } + else if (type == PGSTAT_EXTVAC_DB) + { + if (OidIsValid(dbid)) + { +#define EXTVAC_DB_STAT_COLS 4 + Datum values[EXTVAC_DB_STAT_COLS]; + bool nulls[EXTVAC_DB_STAT_COLS]; + int i = 0; + PgStat_VacuumRelationCounts *stats; + + stats = (PgStat_VacuumRelationCounts *) + pgstat_fetch_entry(PGSTAT_KIND_EXTVAC_DB, dbid, + InvalidOid, NULL); + if (stats && stats->type == PGSTAT_EXTVAC_DB) + { + memset(nulls, 0, sizeof(nulls)); + values[i++] = ObjectIdGetDatum(dbid); + tuplestore_put_common(&stats->common, values, nulls, &i); + Assert(i == EXTVAC_DB_STAT_COLS); + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + } + } + /* invalid dbid: return empty set */ + } else elog(PANIC, "ext_vacuum_statistics: invalid type %d", type); @@ -337,6 +459,7 @@ pg_stats_vacuum(FunctionCallInfo fcinfo, int type) PG_FUNCTION_INFO_V1(pg_stats_get_vacuum_tables); PG_FUNCTION_INFO_V1(pg_stats_get_vacuum_indexes); +PG_FUNCTION_INFO_V1(pg_stats_get_vacuum_database); Datum pg_stats_get_vacuum_tables(PG_FUNCTION_ARGS) @@ -349,3 +472,9 @@ pg_stats_get_vacuum_indexes(PG_FUNCTION_ARGS) { return pg_stats_vacuum(fcinfo, PGSTAT_EXTVAC_INDEX); } + +Datum +pg_stats_get_vacuum_database(PG_FUNCTION_ARGS) +{ + return pg_stats_vacuum(fcinfo, PGSTAT_EXTVAC_DB); +} diff --git a/doc/src/sgml/extvacuumstatistics.sgml b/doc/src/sgml/extvacuumstatistics.sgml index 8308f05fdf6..23471350c5a 100644 --- a/doc/src/sgml/extvacuumstatistics.sgml +++ b/doc/src/sgml/extvacuumstatistics.sgml @@ -9,7 +9,8 @@ The ext_vacuum_statistics module provides - extended per-table and per-index vacuum statistics via views in the + extended per-table, per-index, and per-database vacuum statistics + (buffer I/O, WAL, general, timing) via views in the ext_vacuum_statistics schema. @@ -38,9 +39,10 @@ When active, the module provides views - ext_vacuum_statistics.pg_stats_vacuum_tables and - ext_vacuum_statistics.pg_stats_vacuum_indexes, - plus functions to reset statistics. + ext_vacuum_statistics.pg_stats_vacuum_tables, + ext_vacuum_statistics.pg_stats_vacuum_indexes, and + ext_vacuum_statistics.pg_stats_vacuum_database, + plus functions to reset statistics and manage tracking. @@ -108,6 +110,30 @@ Name of the database containing this table + + + wal_records int8 + + + Total number of WAL records generated by vacuum operations performed on this table + + + + + wal_fpi int8 + + + Total number of WAL full page images generated by vacuum operations + + + + + wal_bytes numeric + + + Total amount of WAL bytes generated by vacuum operations + + tuples_deleted int8 @@ -183,6 +209,117 @@ . + + <structname>ext_vacuum_statistics.pg_stats_vacuum_indexes</structname> Columns + + + + + Column Type + + + Description + + + + + + + indexrelid oid + + + OID of this index + + + + + schema name + + + Name of the schema this index is in + + + + + indexrelname name + + + Name of this index + + + + + dbname name + + + Name of the database containing this index + + + + + wal_records int8 + + + Total number of WAL records generated by vacuum operations while + processing this index + + + + + wal_fpi int8 + + + Total number of WAL full page images generated by vacuum operations + while processing this index + + + + + wal_bytes numeric + + + Total amount of WAL bytes generated by vacuum operations while + processing this index + + + + + tuples_deleted int8 + + + Number of index entries removed by the bulkdelete passes of vacuums + of the parent table + + + + + pages_deleted int8 + + + Number of index pages marked deleted by vacuum operations + + + + +
+ + + + The <structname>ext_vacuum_statistics.pg_stats_vacuum_database</structname> View + + + pg_stats_vacuum_database + + + + The view ext_vacuum_statistics.pg_stats_vacuum_database + contains one row for each database in the cluster, showing aggregate vacuum + statistics for that database. Columns include + dboid, dbname, + and WAL stats (db_wal_records, + db_wal_fpi, db_wal_bytes). + + <structname>ext_vacuum_statistics.pg_stats_vacuum_indexes</structname> Columns diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 00a68f7f8d3..3626f92b652 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -415,6 +415,13 @@ typedef struct LVRelState */ BlockNumber eager_scan_remaining_fails; + /* + * Resource usage of the index passes this process ran, subtracted from + * the heap report to avoid double-counting (see + * accumulate_heap_vacuum_statistics). + */ + PgStat_VacuumRelationCounts extVacReportIdx; + /* * We need to accumulate index statistics for later subtraction from heap * stats. @@ -496,6 +503,33 @@ static void restore_vacuum_error_info(LVRelState *vacrel, /* Extended vacuum statistics functions */ +/* + * extvac_stats_start - Snapshot resource usage before a vacuum phase. + */ +void +extvac_stats_start(Relation rel, LVExtStatCounters * counters) +{ + memset(counters, 0, sizeof(LVExtStatCounters)); + counters->walusage = pgWalUsage; +} + +/* + * extvac_stats_end - Diff resource usage since extvac_stats_start into report. + */ +void +extvac_stats_end(Relation rel, LVExtStatCounters * counters, + PgStat_CommonCounts * report) +{ + WalUsage walusage; + + memset(&walusage, 0, sizeof(WalUsage)); + WalUsageAccumDiff(&walusage, &pgWalUsage, &counters->walusage); + + report->wal_records = walusage.wal_records; + report->wal_fpi = walusage.wal_fpi; + report->wal_bytes = walusage.wal_bytes; +} + /* * Build the heap-specific part of the extended vacuum report from the * counters gathered in vacrel. @@ -513,12 +547,27 @@ accumulate_heap_vacuum_statistics(LVRelState *vacrel, PgStat_VacuumRelationCount extVacStats->table.missed_dead_pages = vacrel->missed_dead_pages; /* - * The wraparound failsafe is reported as a per-relation flag (0/1): did - * the latest vacuum of this relation engage it? Consumers can sum the - * flags to build a per-database total. + * Subtract the resource usage of the index passes this process ran: they + * are reported per index, and the database-wide aggregate would count + * them twice otherwise. Parallel workers report their own usage with + * their index passes, so it never enters the leader's counters. */ + extVacStats->common.wal_records -= vacrel->extVacReportIdx.common.wal_records; + extVacStats->common.wal_fpi -= vacrel->extVacReportIdx.common.wal_fpi; + extVacStats->common.wal_bytes -= vacrel->extVacReportIdx.common.wal_bytes; +} - /* Hook is invoked from pgstat_report_vacuum() when extstats is passed */ +/* + * Accumulate an index pass report into the running total that is later + * subtracted from the heap report. + */ +static void +accumulate_idxs_vacuum_statistics(LVRelState *vacrel, + PgStat_VacuumRelationCounts * extVacIdxStats) +{ + vacrel->extVacReportIdx.common.wal_records += extVacIdxStats->common.wal_records; + vacrel->extVacReportIdx.common.wal_fpi += extVacIdxStats->common.wal_fpi; + vacrel->extVacReportIdx.common.wal_bytes += extVacIdxStats->common.wal_bytes; } /* @@ -674,6 +723,7 @@ heap_vacuum_rel(Relation rel, const VacuumParams *params, PgStat_Counter startreadtime = 0, startwritetime = 0; double startdelaytime; + LVExtStatCounters extVacCounters; WalUsage startwalusage = pgWalUsage; BufferUsage startbufferusage = pgBufferUsage; ErrorContextCallback errcallback; @@ -699,6 +749,9 @@ heap_vacuum_rel(Relation rel, const VacuumParams *params, starttime = GetCurrentTimestamp(); startdelaytime = VacuumDelayTime; + if (set_report_vacuum_hook) + extvac_stats_start(rel, &extVacCounters); + pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); if (AmAutoVacuumWorkerProcess()) @@ -1032,6 +1085,7 @@ heap_vacuum_rel(Relation rel, const VacuumParams *params, */ if (set_report_vacuum_hook) { + extvac_stats_end(rel, &extVacCounters, &extVacReport.common); accumulate_heap_vacuum_statistics(vacrel, &extVacReport); pgstat_report_vacuum_ext(rel, @@ -3108,6 +3162,7 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, double startdelaytime = VacuumDelayTime; double prev_tuples_removed = 0; BlockNumber prev_pages_deleted = 0; + LVExtStatCounters extVacCounters; PgStat_VacuumRelationCounts extVacReport; /* @@ -3119,6 +3174,8 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, prev_tuples_removed = istat->tuples_removed; prev_pages_deleted = istat->pages_deleted; } + if (set_report_vacuum_hook) + extvac_stats_start(indrel, &extVacCounters); ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; ivinfo.analyze_only = false; @@ -3156,6 +3213,7 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, if (set_report_vacuum_hook) { memset(&extVacReport, 0, sizeof(extVacReport)); + extvac_stats_end(indrel, &extVacCounters, &extVacReport.common); extVacReport.type = PGSTAT_EXTVAC_INDEX; if (istat != NULL) { @@ -3165,6 +3223,7 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, istat->pages_deleted - prev_pages_deleted; } pgstat_report_vacuum_ext(indrel, -1, -1, 0, 0, false, &extVacReport); + accumulate_idxs_vacuum_statistics(vacrel, &extVacReport); } /* Revert to the previous phase information for error traceback */ @@ -3195,6 +3254,7 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, double startdelaytime = VacuumDelayTime; double prev_tuples_removed = 0; BlockNumber prev_pages_deleted = 0; + LVExtStatCounters extVacCounters; PgStat_VacuumRelationCounts extVacReport; /* @@ -3206,6 +3266,8 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, prev_tuples_removed = istat->tuples_removed; prev_pages_deleted = istat->pages_deleted; } + if (set_report_vacuum_hook) + extvac_stats_start(indrel, &extVacCounters); ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; ivinfo.analyze_only = false; @@ -3242,6 +3304,7 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, if (set_report_vacuum_hook) { memset(&extVacReport, 0, sizeof(extVacReport)); + extvac_stats_end(indrel, &extVacCounters, &extVacReport.common); extVacReport.type = PGSTAT_EXTVAC_INDEX; if (istat != NULL) { @@ -3251,6 +3314,7 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, istat->pages_deleted - prev_pages_deleted; } pgstat_report_vacuum_ext(indrel, -1, -1, 0, 0, false, &extVacReport); + accumulate_idxs_vacuum_statistics(vacrel, &extVacReport); } /* Revert to the previous phase information for error traceback */ diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c index c22a7d824a2..bbed8e34331 100644 --- a/src/backend/commands/vacuumparallel.c +++ b/src/backend/commands/vacuumparallel.c @@ -1082,6 +1082,7 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, double startdelaytime = VacuumDelayTime; double prev_tuples_removed = 0; BlockNumber prev_pages_deleted = 0; + LVExtStatCounters extVacCounters; PgStat_VacuumRelationCounts extVacReport; /* @@ -1100,6 +1101,8 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, prev_tuples_removed = istat->tuples_removed; prev_pages_deleted = istat->pages_deleted; } + if (set_report_vacuum_hook) + extvac_stats_start(indrel, &extVacCounters); ivinfo.index = indrel; ivinfo.heaprel = pvs->heaprel; ivinfo.analyze_only = false; @@ -1131,6 +1134,7 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, if (set_report_vacuum_hook) { memset(&extVacReport, 0, sizeof(extVacReport)); + extvac_stats_end(indrel, &extVacCounters, &extVacReport.common); extVacReport.type = PGSTAT_EXTVAC_INDEX; if (istat_res != NULL) { diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index d0f92ed6625..758b5ebad38 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -21,6 +21,7 @@ #include "catalog/pg_class.h" #include "catalog/pg_statistic.h" #include "catalog/pg_type.h" +#include "executor/instrument.h" #include "parser/parse_node.h" #include "storage/buf.h" #include "utils/relcache.h" @@ -363,6 +364,16 @@ extern PGDLLIMPORT pg_atomic_uint32 *VacuumSharedCostBalance; extern PGDLLIMPORT pg_atomic_uint32 *VacuumActiveNWorkers; extern PGDLLIMPORT int VacuumCostBalanceLocal; +/* Snapshots of resource usage taken before a vacuum phase */ +typedef struct LVExtStatCounters +{ + WalUsage walusage; +} LVExtStatCounters; + +extern void extvac_stats_start(Relation rel, LVExtStatCounters *counters); +extern void extvac_stats_end(Relation rel, LVExtStatCounters *counters, + PgStat_CommonCounts *report); + extern PGDLLIMPORT bool VacuumFailsafeActive; extern PGDLLIMPORT double vacuum_cost_delay; extern PGDLLIMPORT int vacuum_cost_limit; diff --git a/src/include/pgstat.h b/src/include/pgstat.h index b60567aa573..3cb24981192 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -107,6 +107,9 @@ typedef enum ExtVacReportType typedef struct PgStat_CommonCounts { + int64 wal_records; + int64 wal_fpi; + uint64 wal_bytes; int64 tuples_deleted; } PgStat_CommonCounts; -- 2.50.1 (Apple Git-155)