From 53f35af28dafc5663d2149aa97c1a5b9aa50f1ce Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Fri, 17 Jul 2026 04:32:15 +0300 Subject: [PATCH v42 7/9] ext_vacuum_statistics: WAL metrics and the per-database aggregate. Add the WAL category (wal_records, wal_fpi, wal_bytes) to the per-relation views, sampled by the core machinery around the heap run and each index pass from pgWalUsage; the resource usage of index passes is subtracted from the heap report to avoid double-counting in aggregates. Parallel index passes sample inside the worker. Also introduce the per-database aggregate: a second custom stats kind accumulating the counters of every vacuum in the database, exposed through the new pg_stats_vacuum_database view and resettable per database with extvac_reset_db_entry(). WAL usage is one of the primary answers to "which relations or databases were the most expensive to vacuum, and why", so it is tracked at both the relation and the database level. A TAP test covers WAL and tuple counters advancing across vacuums for tables and indexes. --- contrib/ext_vacuum_statistics/Makefile | 1 + contrib/ext_vacuum_statistics/README.md | 12 +- .../ext_vacuum_statistics--1.0.sql | 42 + contrib/ext_vacuum_statistics/meson.build | 1 + .../t/052_vacuum_extending_basic_test.pl | 754 ++++++++++++++++++ .../ext_vacuum_statistics/vacuum_statistics.c | 176 +++- doc/src/sgml/extvacuumstatistics.sgml | 145 +++- src/backend/access/heap/vacuumlazy.c | 72 +- src/backend/commands/vacuumparallel.c | 4 + src/include/commands/vacuum.h | 11 + src/include/pgstat.h | 3 + 11 files changed, 1187 insertions(+), 34 deletions(-) create mode 100644 contrib/ext_vacuum_statistics/t/052_vacuum_extending_basic_test.pl diff --git a/contrib/ext_vacuum_statistics/Makefile b/contrib/ext_vacuum_statistics/Makefile index 0c30e2856a6..ed80bdf28d0 100644 --- a/contrib/ext_vacuum_statistics/Makefile +++ b/contrib/ext_vacuum_statistics/Makefile @@ -9,6 +9,7 @@ PGFILEDESC = "ext_vacuum_statistics - convenience views for extended vacuum stat ISOLATION = vacuum-extending-in-repetable-read ISOLATION_OPTS = --temp-config=$(top_srcdir)/contrib/ext_vacuum_statistics/ext_vacuum_statistics.conf TAP_TESTS = 1 + NO_INSTALLCHECK = 1 ifdef USE_PGXS diff --git a/contrib/ext_vacuum_statistics/README.md b/contrib/ext_vacuum_statistics/README.md index 3e77bfffb3a..2a8477535c3 100644 --- a/contrib/ext_vacuum_statistics/README.md +++ b/contrib/ext_vacuum_statistics/README.md @@ -1,6 +1,6 @@ # ext_vacuum_statistics -Extended vacuum statistics extension for PostgreSQL. It collects and exposes detailed per-table and per-index vacuum statistics via convenient views in the `ext_vacuum_statistics` schema. +Extended vacuum statistics extension for PostgreSQL. It collects and exposes detailed per-table, per-index, and per-database vacuum statistics (buffer I/O, WAL, general, timing) via convenient views in the `ext_vacuum_statistics` schema. ## Installation @@ -37,14 +37,17 @@ SELECT * FROM ext_vacuum_statistics.pg_stats_vacuum_tables; -- Per-index vacuum statistics SELECT * FROM ext_vacuum_statistics.pg_stats_vacuum_indexes; + +-- Per-database aggregate vacuum statistics +SELECT * FROM ext_vacuum_statistics.pg_stats_vacuum_database; ``` Example output: ``` - relname | tuples_deleted | pages_removed ------------+----------------+--------------- - mytable | 500 | 10 + relname | wal_records | tuples_deleted | pages_removed +-----------+-------------+----------------+--------------- + mytable | 15 | 500 | 10 ``` Reset statistics when needed: @@ -81,6 +84,7 @@ SET vacuum_statistics.enabled = off; |------|-------------| | `ext_vacuum_statistics.pg_stats_vacuum_tables` | Per-table heap vacuum stats (pages scanned, tuples deleted, dead tuples, etc.) | | `ext_vacuum_statistics.pg_stats_vacuum_indexes` | Per-index vacuum stats | +| `ext_vacuum_statistics.pg_stats_vacuum_database` | Per-database aggregate vacuum stats | ## Limitations diff --git a/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql b/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql index 0faca18a463..279f673f6be 100644 --- a/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql +++ b/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql @@ -26,6 +26,11 @@ RETURNS boolean AS 'MODULE_PATHNAME', 'extvac_reset_entry' LANGUAGE C STRICT PARALLEL SAFE; +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.extvac_reset_db_entry(dboid oid) +RETURNS bigint +AS 'MODULE_PATHNAME', 'extvac_reset_db_entry' +LANGUAGE C STRICT PARALLEL SAFE; + CREATE OR REPLACE FUNCTION ext_vacuum_statistics.vacuum_statistics_reset() RETURNS bigint AS 'MODULE_PATHNAME', 'vacuum_statistics_reset' @@ -44,6 +49,9 @@ CREATE OR REPLACE FUNCTION ext_vacuum_statistics.pg_stats_get_vacuum_tables( IN dboid oid, IN reloid oid, OUT relid oid, + OUT wal_records bigint, + OUT wal_fpi bigint, + OUT wal_bytes numeric, OUT tuples_deleted bigint, OUT pages_scanned bigint, OUT pages_removed bigint, @@ -61,6 +69,9 @@ CREATE OR REPLACE FUNCTION ext_vacuum_statistics.pg_stats_get_vacuum_indexes( IN dboid oid, IN reloid oid, OUT relid oid, + OUT wal_records bigint, + OUT wal_fpi bigint, + OUT wal_bytes numeric, OUT tuples_deleted bigint, OUT pages_deleted bigint ) @@ -68,6 +79,18 @@ RETURNS SETOF record AS 'MODULE_PATHNAME', 'pg_stats_get_vacuum_indexes' LANGUAGE C STRICT STABLE; +-- Internal C function to fetch database vacuum stats +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.pg_stats_get_vacuum_database( + IN dboid oid, + OUT dbid oid, + OUT wal_records bigint, + OUT wal_fpi bigint, + OUT wal_bytes numeric +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_stats_get_vacuum_database' +LANGUAGE C STRICT STABLE; + -- View: vacuum statistics per table (heap) CREATE VIEW ext_vacuum_statistics.pg_stats_vacuum_tables AS SELECT @@ -75,6 +98,9 @@ SELECT ns.nspname AS schema, rel.relname AS relname, db.datname AS dbname, + stats.wal_records, + stats.wal_fpi, + stats.wal_bytes, stats.tuples_deleted, stats.pages_scanned, stats.pages_removed, @@ -101,6 +127,9 @@ SELECT ns.nspname AS schema, rel.relname AS indexrelname, db.datname AS dbname, + stats.wal_records, + stats.wal_fpi, + stats.wal_bytes, stats.tuples_deleted, stats.pages_deleted FROM pg_database db, @@ -115,3 +144,16 @@ WHERE db.datname = current_database() COMMENT ON VIEW ext_vacuum_statistics.pg_stats_vacuum_indexes IS 'Extended vacuum statistics per index'; +-- View: vacuum statistics per database (aggregate) +CREATE VIEW ext_vacuum_statistics.pg_stats_vacuum_database AS +SELECT + db.oid AS dboid, + db.datname AS dbname, + stats.wal_records AS db_wal_records, + stats.wal_fpi AS db_wal_fpi, + stats.wal_bytes AS db_wal_bytes +FROM pg_database db +LEFT JOIN LATERAL ext_vacuum_statistics.pg_stats_get_vacuum_database(db.oid) stats ON db.oid = stats.dbid; + +COMMENT ON VIEW ext_vacuum_statistics.pg_stats_vacuum_database IS + 'Extended vacuum statistics per database (aggregate)'; diff --git a/contrib/ext_vacuum_statistics/meson.build b/contrib/ext_vacuum_statistics/meson.build index 1cf0dc5da9d..6f4b4833460 100644 --- a/contrib/ext_vacuum_statistics/meson.build +++ b/contrib/ext_vacuum_statistics/meson.build @@ -34,6 +34,7 @@ tests += { }, 'tap': { 'tests': [ + 't/052_vacuum_extending_basic_test.pl', 't/054_vacuum_extending_gucs_test.pl', ], }, diff --git a/contrib/ext_vacuum_statistics/t/052_vacuum_extending_basic_test.pl b/contrib/ext_vacuum_statistics/t/052_vacuum_extending_basic_test.pl new file mode 100644 index 00000000000..494c6cc8259 --- /dev/null +++ b/contrib/ext_vacuum_statistics/t/052_vacuum_extending_basic_test.pl @@ -0,0 +1,754 @@ +# Copyright (c) 2025 PostgreSQL Global Development Group +# Test cumulative vacuum stats system using TAP +# +# This test validates the accuracy and behavior of cumulative vacuum statistics +# across heap tables, indexes, and databases using: +# +# • ext_vacuum_statistics.pg_stats_vacuum_tables +# • ext_vacuum_statistics.pg_stats_vacuum_indexes +# • ext_vacuum_statistics.pg_stats_vacuum_database +# +# A polling helper function repeatedly checks the stats views until expected +# deltas appear or a configurable timeout expires. This guarantees that +# stats-collector propagation delays do not lead to flaky test behavior. + +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +#------------------------------------------------------------------------------ +# Test harness setup +#------------------------------------------------------------------------------ + +my $node = PostgreSQL::Test::Cluster->new('stat_vacuum'); +$node->init; + +# Configure the server: preload extension and logging level +$node->append_conf('postgresql.conf', q{ + shared_preload_libraries = 'ext_vacuum_statistics' + log_min_messages = notice +}); + +my $stderr; +my $base_stats; +my $wals; +my $ibase_stats; +my $iwals; + +$node->start( + '>' => \$base_stats, + '2>' => \$stderr +); + +#------------------------------------------------------------------------------ +# Database creation and initialization +#------------------------------------------------------------------------------ + +$node->safe_psql('postgres', q{ + CREATE DATABASE statistic_vacuum_database_regression; + CREATE EXTENSION ext_vacuum_statistics; +}); +# Main test database name and number of rows to insert +my $dbname = 'statistic_vacuum_database_regression'; +my $size_tab = 1000; + +# Enable required session settings and force the stats collector to flush next +$node->safe_psql($dbname, q{ + SET track_functions = 'all'; + SELECT pg_stat_force_next_flush(); +}); + +#------------------------------------------------------------------------------ +# Create test table and populate it +#------------------------------------------------------------------------------ + +$node->safe_psql( + $dbname, + "CREATE EXTENSION ext_vacuum_statistics; + CREATE TABLE vestat (x int PRIMARY KEY) + WITH (autovacuum_enabled = off, fillfactor = 10); + INSERT INTO vestat SELECT x FROM generate_series(1, $size_tab) AS g(x); + ANALYZE vestat;" +); + +#------------------------------------------------------------------------------ +# Timing parameters for polling loops +#------------------------------------------------------------------------------ + +my $timeout = 30; # overall wait timeout in seconds +my $interval = 0.015; # poll interval in seconds (15 ms) +my $start_time = time(); +my $updated = 0; + +#------------------------------------------------------------------------------ +# wait_for_vacuum_stats +# +# Polls ext_vacuum_statistics.pg_stats_vacuum_tables and ext_vacuum_statistics.pg_stats_vacuum_indexes until both the +# table-level and index-level counters exceed the provided baselines, or until +# the configured timeout elapses. +# +# Expected named args (baseline values): +# tab_tuples_deleted +# tab_wal_records +# idx_tuples_deleted +# idx_wal_records +# +# Returns: 1 if the condition is met before timeout, 0 otherwise. +#------------------------------------------------------------------------------ + +sub wait_for_vacuum_stats { + my (%args) = @_; + my $tab_tuples_deleted = ($args{tab_tuples_deleted} or 0); + my $tab_wal_records = ($args{tab_wal_records} or 0); + my $idx_tuples_deleted = ($args{idx_tuples_deleted} or 0); + my $idx_wal_records = ($args{idx_wal_records} or 0); + + my $start = time(); + while ((time() - $start) < $timeout) { + + my $result_query = $node->safe_psql( + $dbname, + "VACUUM vestat; + SELECT + (SELECT (tuples_deleted > $tab_tuples_deleted AND wal_records > $tab_wal_records) + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'vestat') + AND + (SELECT (tuples_deleted > $idx_tuples_deleted AND wal_records > $idx_wal_records) + FROM ext_vacuum_statistics.pg_stats_vacuum_indexes + WHERE indexrelname = 'vestat_pkey');" + ); + + return 1 if ($result_query eq 't'); + + sleep($interval); + } + + return 0; +} + +#------------------------------------------------------------------------------ +# Variables to hold vacuum-stat snapshots for later comparisons +#------------------------------------------------------------------------------ + +my $tuples_deleted = 0; +my $pages_scanned = 0; +my $pages_removed = 0; +my $wal_records = 0; +my $wal_bytes = 0; +my $wal_fpi = 0; + +my $index_tuples_deleted = 0; +my $index_pages_deleted = 0; +my $index_wal_records = 0; +my $index_wal_bytes = 0; +my $index_wal_fpi = 0; + +my $tuples_deleted_prev = 0; +my $pages_scanned_prev = 0; +my $pages_removed_prev = 0; +my $wal_records_prev = 0; +my $wal_bytes_prev = 0; +my $wal_fpi_prev = 0; + +my $index_tuples_deleted_prev = 0; +my $index_pages_deleted_prev = 0; +my $index_wal_records_prev = 0; +my $index_wal_bytes_prev = 0; +my $index_wal_fpi_prev = 0; + +#------------------------------------------------------------------------------ +# fetch_vacuum_stats +# +# Reads current values of relevant vacuum counters for the test table and its +# primary index, storing them in package variables for subsequent comparisons. +#------------------------------------------------------------------------------ + +sub fetch_vacuum_stats { + # fetch actual base vacuum statistics + my $base_statistics = $node->safe_psql( + $dbname, + "SELECT tuples_deleted, pages_scanned, pages_removed, wal_records, wal_bytes, wal_fpi + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'vestat';" + ); + + $base_statistics =~ s/\s*\|\s*/ /g; # transform " | " into space + ($tuples_deleted, $pages_scanned, $pages_removed, $wal_records, $wal_bytes, $wal_fpi) + = split /\s+/, $base_statistics; + + # --- index stats --- + my $index_base_statistics = $node->safe_psql( + $dbname, + "SELECT tuples_deleted, pages_deleted, wal_records, wal_bytes, wal_fpi + FROM ext_vacuum_statistics.pg_stats_vacuum_indexes + WHERE indexrelname = 'vestat_pkey';" + ); + + $index_base_statistics =~ s/\s*\|\s*/ /g; # transform " | " into space + ($index_tuples_deleted, $index_pages_deleted, $index_wal_records, $index_wal_bytes, $index_wal_fpi) + = split /\s+/, $index_base_statistics; +} + +#------------------------------------------------------------------------------ +# save_vacuum_stats +# +# Save current values (previously fetched by fetch_vacuum_stats) so that we +# later fetch new values and compare them. +#------------------------------------------------------------------------------ +sub save_vacuum_stats { + $tuples_deleted_prev = $tuples_deleted; + $pages_scanned_prev = $pages_scanned; + $pages_removed_prev = $pages_removed; + $wal_records_prev = $wal_records; + $wal_bytes_prev = $wal_bytes; + $wal_fpi_prev = $wal_fpi; + + $index_tuples_deleted_prev = $index_tuples_deleted; + $index_pages_deleted_prev = $index_pages_deleted; + $index_wal_records_prev = $index_wal_records; + $index_wal_bytes_prev = $index_wal_bytes; + $index_wal_fpi_prev = $index_wal_fpi; +} + +#------------------------------------------------------------------------------ +# print_vacuum_stats_on_error +# +# Print values in case of an error +#------------------------------------------------------------------------------ +sub print_vacuum_stats_on_error { + diag( + "Statistics in the failed test\n" . + "Table statistics:\n" . + " Before test:\n" . + " tuples_deleted = $tuples_deleted_prev\n" . + " pages_scanned = $pages_scanned_prev\n" . + " pages_removed = $pages_removed_prev\n" . + " wal_records = $wal_records_prev\n" . + " wal_bytes = $wal_bytes_prev\n" . + " wal_fpi = $wal_fpi_prev\n" . + " After test:\n" . + " tuples_deleted = $tuples_deleted\n" . + " pages_scanned = $pages_scanned\n" . + " pages_removed = $pages_removed\n" . + " wal_records = $wal_records\n" . + " wal_bytes = $wal_bytes\n" . + " wal_fpi = $wal_fpi\n" . + "Index statistics:\n" . + " Before test:\n" . + " tuples_deleted = $index_tuples_deleted_prev\n" . + " pages_deleted = $index_pages_deleted_prev\n" . + " wal_records = $index_wal_records_prev\n" . + " wal_bytes = $index_wal_bytes_prev\n" . + " wal_fpi = $index_wal_fpi_prev\n" . + " After test:\n" . + " tuples_deleted = $index_tuples_deleted\n" . + " pages_deleted = $index_pages_deleted\n" . + " wal_records = $index_wal_records\n" . + " wal_bytes = $index_wal_bytes\n" . + " wal_fpi = $index_wal_fpi\n" + ); +}; + +sub fetch_error_base_db_vacuum_statistics { + my (%args) = @_; + + # Validate presence of required args (allow 0 as valid numeric baseline) + die "database name required" + unless exists $args{database_name} && defined $args{database_name}; + my $database_name = $args{database_name}; + + # fetch actual base database vacuum statistics + my $base_statistics = $node->safe_psql( + $database_name, + "SELECT db_wal_records, db_wal_fpi, db_wal_bytes + FROM ext_vacuum_statistics.pg_stats_vacuum_database, pg_database + WHERE pg_database.datname = '$dbname' + AND pg_database.oid = ext_vacuum_statistics.pg_stats_vacuum_database.dboid;" + ); + $base_statistics =~ s/\s*\|\s*/ /g; # transform " | " in space + my ($wal_records, $wal_fpi, $wal_bytes) = split /\s+/, $base_statistics; + + diag( + "BASE STATS MISMATCH FOR DATABASE $dbname:\n" . + " wal_records = $wal_records\n" . + " wal_fpi = $wal_fpi\n" . + " wal_bytes = $wal_bytes\n" + ); +} + + +#------------------------------------------------------------------------------ +# Test 1: Delete half the rows, run VACUUM, and wait for stats to advance +#------------------------------------------------------------------------------ +subtest 'Test 1: Delete half the rows, run VACUUM' => sub +{ + +$node->safe_psql($dbname, "DELETE FROM vestat WHERE x % 2 = 0;"); +$node->safe_psql($dbname, "VACUUM vestat;"); + +# Poll the stats view until expected deltas appear or timeout +$updated = wait_for_vacuum_stats( + tab_tuples_deleted => 0, + tab_wal_records => 0, + idx_tuples_deleted => 0, + idx_wal_records => 0, +); +ok($updated, 'vacuum stats updated after vacuuming half-deleted table (tuples_deleted and wal_fpi advanced)') + or diag "Timeout waiting for ext_vacuum_statistics update after $timeout seconds after vacuuming half-deleted table"; + +fetch_vacuum_stats(); + +ok($tuples_deleted > $tuples_deleted_prev, 'table tuples_deleted has increased'); +ok($pages_scanned > $pages_scanned_prev, 'table pages_scanned has increased'); +ok($pages_removed == $pages_removed_prev, 'table pages_removed stay the same'); +ok($wal_records > $wal_records_prev, 'table wal_records has increased'); +ok($wal_bytes > $wal_bytes_prev, 'table wal_bytes has increased'); +ok($wal_fpi > $wal_fpi_prev, 'table wal_fpi has increased'); + +ok($index_pages_deleted == $index_pages_deleted_prev, 'index pages_deleted stay the same'); +ok($index_tuples_deleted > $index_tuples_deleted_prev, 'index tuples_deleted has increased'); +ok($index_wal_records > $index_wal_records_prev, 'index wal_records has increased'); +ok($index_wal_bytes > $index_wal_bytes_prev, 'index wal_bytes has increased'); +ok($index_wal_fpi == $index_wal_fpi_prev, 'index wal_fpi stay the same'); + +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 2: Delete all rows, run VACUUM, and wait for stats to advance +#------------------------------------------------------------------------------ +subtest 'Test 2: Delete all rows, run VACUUM' => sub +{ +save_vacuum_stats(); + +$node->safe_psql($dbname, "DELETE FROM vestat;"); +$node->safe_psql($dbname, "VACUUM vestat;"); + +$updated = wait_for_vacuum_stats( + tab_tuples_deleted => $tuples_deleted_prev, + tab_wal_records => $wal_records_prev, + idx_tuples_deleted => $index_tuples_deleted_prev, + idx_wal_records => $index_wal_records_prev, +); + +ok($updated, 'vacuum stats updated after vacuuming all-deleted table (tuples_deleted and wal_records advanced)') + or diag "Timeout waiting for ext_vacuum_statistics update after $timeout seconds after vacuuming all-deleted table"; + +fetch_vacuum_stats(); + +ok($tuples_deleted > $tuples_deleted_prev, 'table tuples_deleted has increased'); +ok($pages_scanned > $pages_scanned_prev, 'table pages_scanned has increased'); +ok($pages_removed > $pages_removed_prev, 'table pages_removed has increased'); +ok($wal_records > $wal_records_prev, 'table wal_records has increased'); +ok($wal_bytes > $wal_bytes_prev, 'table wal_bytes has increased'); +ok($wal_fpi > 0, 'table wal_fpi has increased'); + +ok($index_pages_deleted > $index_pages_deleted_prev, 'index pages_deleted has increased'); +ok($index_tuples_deleted > $index_tuples_deleted_prev, 'index tuples_deleted has increased'); +ok($index_wal_records > $index_wal_records_prev, 'index wal_records has increased'); +ok($index_wal_bytes > $index_wal_bytes_prev, 'index wal_bytes has increased'); +ok($index_wal_fpi == $index_wal_fpi_prev, 'index wal_fpi stay the same'); + +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 3: Test VACUUM FULL — it should not report to the stats collector +#------------------------------------------------------------------------------ +subtest 'Test 3: Test VACUUM FULL — it should not report to the stats collector' => sub +{ +save_vacuum_stats(); + +$node->safe_psql( + $dbname, + "INSERT INTO vestat SELECT x FROM generate_series(1, $size_tab) AS g(x); + CHECKPOINT; + DELETE FROM vestat; + VACUUM FULL vestat;" +); + +fetch_vacuum_stats(); + +ok($tuples_deleted == $tuples_deleted_prev, 'table tuples_deleted stay the same'); +ok($pages_scanned == $pages_scanned_prev, 'table pages_scanned stay the same'); +ok($pages_removed == $pages_removed_prev, 'table pages_removed stay the same'); +ok($wal_records == $wal_records_prev, 'table wal_records stay the same'); +ok($wal_bytes == $wal_bytes_prev, 'table wal_bytes stay the same'); +ok($wal_fpi == $wal_fpi_prev, 'table wal_fpi stay the same'); + +ok($index_pages_deleted == $index_pages_deleted_prev, 'index pages_deleted stay the same'); +ok($index_tuples_deleted == $index_tuples_deleted_prev, 'index tuples_deleted stay the same'); +ok($index_wal_records == $index_wal_records_prev, 'index wal_records stay the same'); +ok($index_wal_bytes == $index_wal_bytes_prev, 'index wal_bytes stay the same'); +ok($index_wal_fpi == $index_wal_fpi_prev, 'index wal_fpi stay the same'); + +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 4: Update table, checkpoint, and VACUUM to provoke WAL/FPI accounting +#------------------------------------------------------------------------------ +subtest 'Test 4: Update table, checkpoint, and VACUUM to provoke WAL/FPI accounting' => sub +{ + +save_vacuum_stats(); + +$node->safe_psql( + $dbname, + "INSERT INTO vestat SELECT x FROM generate_series(1, $size_tab) AS g(x); + CHECKPOINT; + UPDATE vestat SET x = x + 1000; + VACUUM vestat;" +); + +$updated = wait_for_vacuum_stats( + tab_tuples_deleted => $tuples_deleted_prev, + tab_wal_records => $wal_records_prev, + idx_tuples_deleted => $index_tuples_deleted_prev, + idx_wal_records => $index_wal_records_prev, +); + +ok($updated, 'vacuum stats updated after updating tuples in the table (tuples_deleted and wal_records advanced)') + or diag "Timeout waiting for ext_vacuum_statistics update after $timeout seconds"; + +fetch_vacuum_stats(); + +ok($tuples_deleted > $tuples_deleted_prev, 'table tuples_deleted has increased'); +ok($pages_scanned > $pages_scanned_prev, 'table pages_scanned has increased'); +ok($pages_removed == $pages_removed_prev, 'table pages_removed stay the same'); +ok($wal_records > $wal_records_prev, 'table wal_records has increased'); +ok($wal_bytes > $wal_bytes_prev, 'table wal_bytes has increased'); +ok($wal_fpi > $wal_fpi_prev, 'table wal_fpi has increased'); + +ok($index_pages_deleted > $index_pages_deleted_prev, 'index pages_deleted has increased'); +ok($index_tuples_deleted > $index_tuples_deleted_prev, 'index tuples_deleted has increased'); +ok($index_wal_records > $index_wal_records_prev, 'index wal_records has increased'); +ok($index_wal_bytes > $index_wal_bytes_prev, 'index wal_bytes has increased'); +ok($index_wal_fpi > $index_wal_fpi_prev, 'index wal_fpi has increased'); + +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 5: Update table, trancate and vacuuming +#------------------------------------------------------------------------------ +subtest 'Test 5: Update table, trancate and vacuuming' => sub +{ + +save_vacuum_stats(); + +$node->safe_psql( + $dbname, + "INSERT INTO vestat SELECT x FROM generate_series(1, $size_tab) AS g(x); + UPDATE vestat SET x = x + 1000;" +); +$node->safe_psql($dbname, "TRUNCATE vestat;"); +$node->safe_psql($dbname, "CHECKPOINT;"); +$node->safe_psql($dbname, "VACUUM vestat;"); + +$updated = wait_for_vacuum_stats( + tab_wal_records => $wal_records_prev, +); + +ok($updated, 'vacuum stats updated after updating tuples and trancation in the table (wal_records advanced)') + or diag "Timeout waiting for ext_vacuum_statistics update after $timeout seconds"; + +fetch_vacuum_stats(); + +ok($tuples_deleted == $tuples_deleted_prev, 'table tuples_deleted stay the same'); +ok($pages_scanned == $pages_scanned_prev, 'table pages_scanned stay the same'); +ok($pages_removed == $pages_removed_prev, 'table pages_removed stay the same'); +ok($wal_records > $wal_records_prev, 'table wal_records has increased'); +ok($wal_bytes > $wal_bytes_prev, 'table wal_bytes has increased'); +ok($wal_fpi == $wal_fpi_prev, 'table wal_fpi stay the same'); + +ok($index_pages_deleted == $index_pages_deleted_prev, 'index pages_deleted stay the same'); +ok($index_tuples_deleted == $index_tuples_deleted_prev, 'index tuples_deleted stay the same'); +ok($index_wal_records == $index_wal_records_prev, 'index wal_records stay the same'); +ok($index_wal_bytes == $index_wal_bytes_prev, 'index wal_bytes stay the same'); +ok($index_wal_fpi == $index_wal_fpi_prev, 'index wal_fpi stay the same'); + +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 6: Delete all tuples from table, trancate, and vacuuming +#------------------------------------------------------------------------------ +subtest 'Test 6: Delete all tuples from table, trancate, and vacuuming' => sub +{ + +save_vacuum_stats(); + +$node->safe_psql( + $dbname, + "INSERT INTO vestat SELECT x FROM generate_series(1, $size_tab) AS g(x); + DELETE FROM vestat; + TRUNCATE vestat; + CHECKPOINT; + VACUUM vestat;" +); + +$updated = wait_for_vacuum_stats( + tab_wal_records => $wal_records, +); + +ok($updated, 'vacuum stats updated after deleting all tuples and trancation in the table (wal_records advanced)') + or diag "Timeout waiting for ext_vacuum_statistics update after $timeout seconds"; + +fetch_vacuum_stats(); + +ok($tuples_deleted == $tuples_deleted_prev, 'table tuples_deleted stay the same'); +ok($pages_scanned == $pages_scanned_prev, 'table pages_scanned stay the same'); +ok($pages_removed == $pages_removed_prev, 'table pages_removed stay the same'); +ok($wal_records > $wal_records_prev, 'table wal_records has increased'); +ok($wal_bytes > $wal_bytes_prev, 'table wal_bytes has increased'); +ok($wal_fpi == $wal_fpi_prev, 'table wal_fpi stay the same'); + +ok($index_pages_deleted == $index_pages_deleted_prev, 'index pages_deleted stay the same'); +ok($index_tuples_deleted == $index_tuples_deleted_prev, 'index tuples_deleted stay the same'); +ok($index_wal_records == $index_wal_records_prev, 'index wal_records stay the same'); +ok($index_wal_bytes == $index_wal_bytes_prev, 'index wal_bytes stay the same'); +ok($index_wal_fpi == $index_wal_fpi_prev, 'index wal_fpi stay the same'); + +} or print_vacuum_stats_on_error(); + +my $dboid = $node->safe_psql( + $dbname, + "SELECT oid FROM pg_database WHERE datname = current_database();" +); + +#------------------------------------------------------------------------------------------------------- +# Test 7: Check if we return single vacuum statistics for particular relation from the current database +#------------------------------------------------------------------------------------------------------- +subtest 'Test 7: Check if we return vacuum statistics from the current database' => sub +{ +save_vacuum_stats(); + +my $reloid = $node->safe_psql( + $dbname, + q{ + SELECT oid FROM pg_class WHERE relname = 'vestat'; + } +); + +# Check if we can get vacuum statistics of particular heap relation in the current database +$base_stats = $node->safe_psql( + $dbname, + "SELECT count(*) FROM ext_vacuum_statistics.pg_stats_get_vacuum_tables((SELECT oid FROM pg_database WHERE datname = current_database()), $reloid);" +); +is($base_stats, 1, 'heap vacuum stats return from the current relation and database as expected'); + +$reloid = $node->safe_psql( + $dbname, + q{ + SELECT oid FROM pg_class WHERE relname = 'vestat_pkey'; + } +); + +# Check if we can get vacuum statistics of particular index relation in the current database +$base_stats = $node->safe_psql( + $dbname, + "SELECT count(*) FROM ext_vacuum_statistics.pg_stats_get_vacuum_indexes((SELECT oid FROM pg_database WHERE datname = current_database()), $reloid);" +); +is($base_stats, 1, 'index vacuum stats return from the current relation and database as expected'); + +# Check if we return empty results if vacuum statistics with particular oid doesn't exist +$base_stats = $node->safe_psql( + $dbname, + "SELECT count(*) FROM ext_vacuum_statistics.pg_stats_get_vacuum_tables((SELECT oid FROM pg_database WHERE datname = current_database()), 1);" +); +is($base_stats, 0, 'table vacuum stats return no rows, as expected'); + +$base_stats = $node->safe_psql( + $dbname, + "SELECT count(*) FROM ext_vacuum_statistics.pg_stats_get_vacuum_indexes((SELECT oid FROM pg_database WHERE datname = current_database()), 1);" +); +is($base_stats, 0, 'index vacuum stats return no rows, as expected'); + +# Check if we can get vacuum statistics of all relations in the current database +$base_stats = $node->safe_psql( + $dbname, + "SELECT count(*) > 0 FROM ext_vacuum_statistics.pg_stats_vacuum_tables;" +); +ok($base_stats eq 't', 'vacuum stats per all heap objects available'); + +$base_stats = $node->safe_psql( + $dbname, + "SELECT count(*) > 0 FROM ext_vacuum_statistics.pg_stats_vacuum_indexes;" +); +ok($base_stats eq 't', 'vacuum stats per all index objects available'); +}; + +#------------------------------------------------------------------------------ +# Test 8: Check relation-level vacuum statistics from another database +#------------------------------------------------------------------------------ +subtest 'Test 8: Check relation-level vacuum statistics from another database' => sub +{ +$base_stats = $node->safe_psql( + 'postgres', + "SELECT count(*) + FROM ext_vacuum_statistics.pg_stats_vacuum_indexes + WHERE indexrelname = 'vestat_pkey';" +); +is($base_stats, 0, 'check the printing index vacuum extended statistics from another database are not available'); + +$base_stats = $node->safe_psql( + 'postgres', + "SELECT count(*) + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'vestat';" +); +is($base_stats, 0, 'check the printing heap vacuum extended statistics from another database are not available'); + +# Check that relations from another database are not visible in the view when querying from postgres +$base_stats = $node->safe_psql( + 'postgres', + "SELECT count(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'vestat';" +); +is($base_stats, 0, 'vacuum stats per all tables objects from another database are not available as expected'); + +$base_stats = $node->safe_psql( + 'postgres', + "SELECT count(*) FROM ext_vacuum_statistics.pg_stats_vacuum_indexes WHERE indexrelname = 'vestat_pkey';" +); +is($base_stats, 0, 'vacuum stats per all index objects from another database are not available as expected'); +}; + +#-------------------------------------------------------------------------------------- +# Test 9: Check database-level vacuum statistics from the current and another database +#-------------------------------------------------------------------------------------- +subtest 'Test 9: Check database-level vacuum statistics from the current and another database' => sub +{ +my $wal_records = 0; +my $wal_fpi = 0; +my $wal_bytes = 0; +$base_stats = $node->safe_psql( + $dbname, + "SELECT db_wal_records, db_wal_fpi, db_wal_bytes + FROM ext_vacuum_statistics.pg_stats_vacuum_database, pg_database + WHERE pg_database.datname = '$dbname' + AND pg_database.oid = ext_vacuum_statistics.pg_stats_vacuum_database.dboid;" +); +$base_stats =~ s/\s*\|\s*/ /g; # transform " | " into space + ($wal_records, $wal_fpi, $wal_bytes) = split /\s+/, $base_stats; + +ok($wal_records > 0, 'wal_records is more than 0'); +ok($wal_fpi > 0, 'wal_fpi is more than 0'); +ok($wal_bytes > 0, 'wal_bytes is more than 0'); + +$base_stats = $node->safe_psql( + 'postgres', + "SELECT count(*) = 1 + FROM ext_vacuum_statistics.pg_stats_vacuum_database, pg_database + WHERE pg_database.datname = '$dbname' + AND pg_database.oid = ext_vacuum_statistics.pg_stats_vacuum_database.dboid;" +); +ok($base_stats eq 't', 'check database-level vacuum stats from another database are available'); +}; + +#------------------------------------------------------------------------------ +# Test 10: Cleanup checks: ensure functions return empty sets for OID = 0 +#------------------------------------------------------------------------------ +subtest 'Test 10: Cleanup checks: ensure functions return empty sets for OID = 0' => sub +{ +my $dboid = $node->safe_psql( + $dbname, + "SELECT oid FROM pg_database WHERE datname = current_database();" +); + +# Vacuum statistics for invalid relation OID return empty +$base_stats = $node->safe_psql( + $dbname, + q{ + SELECT COUNT(*) + FROM ext_vacuum_statistics.pg_stats_get_vacuum_tables((SELECT oid FROM pg_database WHERE datname = current_database()), 0); + } +); +is($base_stats, 0, 'vacuum stats per heap from invalid relation OID return empty as expected'); + +$base_stats = $node->safe_psql( + $dbname, + q{ + SELECT COUNT(*) + FROM ext_vacuum_statistics.pg_stats_get_vacuum_indexes((SELECT oid FROM pg_database WHERE datname = current_database()), 0); + } +); +is($base_stats, 0, 'vacuum stats per index from invalid relation OID return empty as expected'); + +$node->safe_psql($dbname, q{ + DROP TABLE vestat CASCADE; + VACUUM; +}); + +# Check that we don't print vacuum statistics for deleted objects +$base_stats = $node->safe_psql( + $dbname, + q{ + SELECT COUNT(*) + FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relid = 0; + } +); +is($base_stats, 0, 'ext_vacuum_statistics.pg_stats_vacuum_tables correctly returns no rows for OID = 0'); + +$base_stats = $node->safe_psql( + $dbname, + q{ + SELECT COUNT(*) + FROM ext_vacuum_statistics.pg_stats_vacuum_indexes WHERE indexrelid = 0; + } +); +is($base_stats, 0, 'ext_vacuum_statistics.pg_stats_vacuum_indexes correctly returns no rows for OID = 0'); + +my $reloid = $node->safe_psql( + $dbname, + q{ + SELECT oid FROM pg_class WHERE relname = 'pg_shdepend'; + } +); + +$node->safe_psql($dbname, "VACUUM pg_shdepend;"); + +# Check if we can get vacuum statistics for cluster relations (shared catalogs) +$base_stats = $node->safe_psql( + $dbname, + qq{ + SELECT count(*) > 0 + FROM ext_vacuum_statistics.pg_stats_get_vacuum_tables((SELECT oid FROM pg_database WHERE datname = current_database()), $reloid); + } +); + +is($base_stats, 't', 'vacuum stats for common heap objects available'); + +my $indoid = $node->safe_psql( + $dbname, + q{ + SELECT oid FROM pg_class WHERE relname = 'pg_shdepend_reference_index'; + } +); + +$base_stats = $node->safe_psql( + $dbname, + qq{ + SELECT count(*) > 0 + FROM ext_vacuum_statistics.pg_stats_get_vacuum_indexes((SELECT oid FROM pg_database WHERE datname = current_database()), $indoid); + } +); + +is($base_stats, 't', 'vacuum stats for common index objects available'); + +$node->safe_psql('postgres', + "DROP DATABASE $dbname; + VACUUM;" +); + +$base_stats = $node->safe_psql( + 'postgres', + q{ + SELECT count(*) = 0 + FROM ext_vacuum_statistics.pg_stats_get_vacuum_database(0); + } +); +is($base_stats, 't', 'vacuum stats from database with invalid database OID return empty, as expected'); +}; + +$node->stop; + +done_testing(); diff --git a/contrib/ext_vacuum_statistics/vacuum_statistics.c b/contrib/ext_vacuum_statistics/vacuum_statistics.c index 0570fc5c071..da8f5f56c26 100644 --- a/contrib/ext_vacuum_statistics/vacuum_statistics.c +++ b/contrib/ext_vacuum_statistics/vacuum_statistics.c @@ -23,8 +23,9 @@ PG_MODULE_MAGIC; #endif -/* Custom stats kind for per-relation vacuum statistics */ +/* Two kinds: relations (tables/indexes) and database aggregates */ #define PGSTAT_KIND_EXTVAC_RELATION 24 +#define PGSTAT_KIND_EXTVAC_DB 25 #define SJ_NODENAME "vacuum_statistics" @@ -64,9 +65,26 @@ static const PgStat_KindInfo extvac_relation_kind_info = { .flush_pending_cb = NULL, }; +/* PgStat kind for per-database aggregated vacuum statistics */ +static const PgStat_KindInfo extvac_db_kind_info = { + .name = "ext_vacuum_statistics_db", + .fixed_amount = false, + .accessed_across_databases = true, + .write_to_file = true, + .track_entry_count = true, + .shared_size = sizeof(PgStatShared_ExtVacEntry), + .shared_data_off = offsetof(PgStatShared_ExtVacEntry, stats), + .shared_data_len = sizeof(PgStat_VacuumRelationCounts), + .pending_size = 0, + .flush_pending_cb = NULL, +}; + static inline void pgstat_accumulate_common(PgStat_CommonCounts * dst, const PgStat_CommonCounts * src) { + dst->wal_records += src->wal_records; + dst->wal_fpi += src->wal_fpi; + dst->wal_bytes += src->wal_bytes; dst->tuples_deleted += src->tuples_deleted; } @@ -114,19 +132,30 @@ _PG_init(void) MarkGUCPrefixReserved(SJ_NODENAME); pgstat_register_kind(PGSTAT_KIND_EXTVAC_RELATION, &extvac_relation_kind_info); + pgstat_register_kind(PGSTAT_KIND_EXTVAC_DB, &extvac_db_kind_info); prev_report_vacuum_hook = set_report_vacuum_hook; set_report_vacuum_hook = pgstat_report_vacuum_extstats; } +/* Accumulate common counts for database-level stats. */ +static inline void +pgstat_accumulate_common_for_db(PgStat_CommonCounts * dst, + const PgStat_CommonCounts * src) +{ + pgstat_accumulate_common(dst, src); +} + /* - * Store incoming vacuum stats into a per-relation pgstat custom statistics - * entry. Uses pgstat_get_entry_ref_locked and pgstat_accumulate_* for - * atomic updates. + * Store incoming vacuum stats into pgstat custom statistics. + * store_relation: create/update per-relation entry + * store_db: accumulate into database-level entry (dboid, objid=0). + * Uses pgstat_get_entry_ref_locked and pgstat_accumulate_* for atomic updates. */ static void extvac_store(Oid dboid, Oid relid, int type, - PgStat_VacuumRelationCounts * params) + PgStat_VacuumRelationCounts * params, + bool store_relation, bool store_db) { PgStat_EntryRef *entry_ref; PgStatShared_ExtVacEntry *shared; @@ -135,24 +164,43 @@ extvac_store(Oid dboid, Oid relid, int type, if (!evs_enabled) return; - objid = EXTVAC_OBJID(relid, type); - entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_EXTVAC_RELATION, dboid, objid, false); - if (entry_ref) + if (store_relation) + { + objid = EXTVAC_OBJID(relid, type); + entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_EXTVAC_RELATION, dboid, objid, false); + if (entry_ref) + { + shared = (PgStatShared_ExtVacEntry *) entry_ref->shared_stats; + if (shared->stats.type == PGSTAT_EXTVAC_INVALID) + { + memset(&shared->stats, 0, sizeof(shared->stats)); + shared->stats.type = params->type; + } + pgstat_accumulate_extvac_stats(&shared->stats, params); + pgstat_unlock_entry(entry_ref); + } + } + + if (store_db) { - shared = (PgStatShared_ExtVacEntry *) entry_ref->shared_stats; - if (shared->stats.type == PGSTAT_EXTVAC_INVALID) + entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_EXTVAC_DB, dboid, InvalidOid, false); + if (entry_ref) { - memset(&shared->stats, 0, sizeof(shared->stats)); - shared->stats.type = params->type; + shared = (PgStatShared_ExtVacEntry *) entry_ref->shared_stats; + if (shared->stats.type == PGSTAT_EXTVAC_INVALID) + { + memset(&shared->stats, 0, sizeof(shared->stats)); + shared->stats.type = PGSTAT_EXTVAC_DB; + } + pgstat_accumulate_common_for_db(&shared->stats.common, ¶ms->common); + pgstat_unlock_entry(entry_ref); } - pgstat_accumulate_extvac_stats(&shared->stats, params); - pgstat_unlock_entry(entry_ref); } } /* - * Vacuum report hook: called when vacuum finishes. Stores stats per-relation, - * then chains to previous hook. + * Vacuum report hook: called when vacuum finishes. Stores stats per-relation + * and per-database, then chains to previous hook. */ static void pgstat_report_vacuum_extstats(Oid tableoid, bool shared, @@ -161,7 +209,7 @@ pgstat_report_vacuum_extstats(Oid tableoid, bool shared, Oid dboid = shared ? InvalidOid : MyDatabaseId; if (evs_enabled) - extvac_store(dboid, tableoid, params->type, params); + extvac_store(dboid, tableoid, params->type, params, true, true); if (prev_report_vacuum_hook) prev_report_vacuum_hook(tableoid, shared, params); } @@ -176,17 +224,39 @@ extvac_reset_by_relid(Oid dboid, Oid relid, int type) return true; } -/* Reset all vacuum statistics. */ +/* Callback for pgstat_reset_matching_entries: match relation entries for given db */ +static bool +match_extvac_relations_for_db(PgStatShared_HashEntry *entry, Datum match_data) +{ + return entry->key.kind == PGSTAT_KIND_EXTVAC_RELATION && + entry->key.dboid == DatumGetObjectId(match_data); +} + +/* + * Reset statistics for a database (aggregate entry) and all its relations. + */ +static int64 +extvac_database_reset(Oid dboid) +{ + pgstat_reset_matching_entries(match_extvac_relations_for_db, + ObjectIdGetDatum(dboid), 0); + pgstat_reset_entry(PGSTAT_KIND_EXTVAC_DB, dboid, 0, 0); + return 1; +} + +/* Reset all vacuum statistics (both relation and database entries). */ static int64 extvac_stat_reset(void) { pgstat_reset_of_kind(PGSTAT_KIND_EXTVAC_RELATION); + pgstat_reset_of_kind(PGSTAT_KIND_EXTVAC_DB); return 0; /* count not available */ } PG_FUNCTION_INFO_V1(vacuum_statistics_reset); PG_FUNCTION_INFO_V1(extvac_shared_memory_size); PG_FUNCTION_INFO_V1(extvac_reset_entry); +PG_FUNCTION_INFO_V1(extvac_reset_db_entry); Datum vacuum_statistics_reset(PG_FUNCTION_ARGS) @@ -204,6 +274,14 @@ extvac_reset_entry(PG_FUNCTION_ARGS) PG_RETURN_BOOL(extvac_reset_by_relid(dboid, relid, type)); } +Datum +extvac_reset_db_entry(PG_FUNCTION_ARGS) +{ + Oid dboid = PG_GETARG_OID(0); + + PG_RETURN_INT64(extvac_database_reset(dboid)); +} + /* * Return total shared memory in bytes used by the extension for vacuum stats. * Used for monitoring and capacity planning: memory grows with the number of @@ -213,20 +291,41 @@ Datum extvac_shared_memory_size(PG_FUNCTION_ARGS) { uint64 rel_count; + uint64 db_count; uint64 total; size_t entry_size = sizeof(PgStatShared_ExtVacEntry); rel_count = pgstat_get_entry_count(PGSTAT_KIND_EXTVAC_RELATION); - total = rel_count; + db_count = pgstat_get_entry_count(PGSTAT_KIND_EXTVAC_DB); + total = rel_count + db_count; PG_RETURN_INT64((int64) (total * entry_size)); } /* - * Output vacuum statistics (tables or indexes). + * Output vacuum statistics (tables, indexes, or per-database aggregates). */ -#define EXTVAC_HEAP_STAT_COLS 8 -#define EXTVAC_IDX_STAT_COLS 3 +#define EXTVAC_COMMON_STAT_COLS 3 + +static void +tuplestore_put_common(PgStat_CommonCounts * vacuum_ext, + Datum *values, bool *nulls, int *i) +{ + char buf[256]; + const int base PG_USED_FOR_ASSERTS_ONLY = *i; + + values[(*i)++] = Int64GetDatum(vacuum_ext->wal_records); + values[(*i)++] = Int64GetDatum(vacuum_ext->wal_fpi); + snprintf(buf, sizeof buf, UINT64_FORMAT, vacuum_ext->wal_bytes); + values[(*i)++] = DirectFunctionCall3(numeric_in, + CStringGetDatum(buf), + ObjectIdGetDatum(0), + Int32GetDatum(-1)); + Assert((*i - base) == EXTVAC_COMMON_STAT_COLS); +} + +#define EXTVAC_HEAP_STAT_COLS 11 +#define EXTVAC_IDX_STAT_COLS 6 #define EXTVAC_MAX_STAT_COLS Max(EXTVAC_HEAP_STAT_COLS, EXTVAC_IDX_STAT_COLS) static void @@ -240,6 +339,8 @@ tuplestore_put_for_relation(Oid relid, Tuplestorestate *tupstore, memset(nulls, 0, sizeof(nulls)); values[i++] = ObjectIdGetDatum(relid); + tuplestore_put_common(&vacuum_ext->common, values, nulls, &i); + if (vacuum_ext->type == PGSTAT_EXTVAC_TABLE) { values[i++] = Int64GetDatum(vacuum_ext->common.tuples_deleted); @@ -312,6 +413,30 @@ pg_stats_vacuum(FunctionCallInfo fcinfo, int type) if (stats && stats->type == type) tuplestore_put_for_relation(relid, tupstore, tupdesc, stats); } + else if (type == PGSTAT_EXTVAC_DB) + { + if (OidIsValid(dbid)) + { +#define EXTVAC_DB_STAT_COLS 4 + Datum values[EXTVAC_DB_STAT_COLS]; + bool nulls[EXTVAC_DB_STAT_COLS]; + int i = 0; + PgStat_VacuumRelationCounts *stats; + + stats = (PgStat_VacuumRelationCounts *) + pgstat_fetch_entry(PGSTAT_KIND_EXTVAC_DB, dbid, + InvalidOid, NULL); + if (stats && stats->type == PGSTAT_EXTVAC_DB) + { + memset(nulls, 0, sizeof(nulls)); + values[i++] = ObjectIdGetDatum(dbid); + tuplestore_put_common(&stats->common, values, nulls, &i); + Assert(i == EXTVAC_DB_STAT_COLS); + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + } + } + /* invalid dbid: return empty set */ + } else elog(PANIC, "ext_vacuum_statistics: invalid type %d", type); @@ -320,6 +445,7 @@ pg_stats_vacuum(FunctionCallInfo fcinfo, int type) PG_FUNCTION_INFO_V1(pg_stats_get_vacuum_tables); PG_FUNCTION_INFO_V1(pg_stats_get_vacuum_indexes); +PG_FUNCTION_INFO_V1(pg_stats_get_vacuum_database); Datum pg_stats_get_vacuum_tables(PG_FUNCTION_ARGS) @@ -332,3 +458,9 @@ pg_stats_get_vacuum_indexes(PG_FUNCTION_ARGS) { return pg_stats_vacuum(fcinfo, PGSTAT_EXTVAC_INDEX); } + +Datum +pg_stats_get_vacuum_database(PG_FUNCTION_ARGS) +{ + return pg_stats_vacuum(fcinfo, PGSTAT_EXTVAC_DB); +} diff --git a/doc/src/sgml/extvacuumstatistics.sgml b/doc/src/sgml/extvacuumstatistics.sgml index 5645b58b640..9e601f0951d 100644 --- a/doc/src/sgml/extvacuumstatistics.sgml +++ b/doc/src/sgml/extvacuumstatistics.sgml @@ -9,7 +9,8 @@ The ext_vacuum_statistics module provides - extended per-table and per-index vacuum statistics via views in the + extended per-table, per-index, and per-database vacuum statistics + (buffer I/O, WAL, general, timing) via views in the ext_vacuum_statistics schema. @@ -25,9 +26,10 @@ When active, the module provides views - ext_vacuum_statistics.pg_stats_vacuum_tables and - ext_vacuum_statistics.pg_stats_vacuum_indexes, - plus functions to reset statistics. + ext_vacuum_statistics.pg_stats_vacuum_tables, + ext_vacuum_statistics.pg_stats_vacuum_indexes, and + ext_vacuum_statistics.pg_stats_vacuum_database, + plus functions to reset statistics and manage tracking. @@ -97,6 +99,30 @@ Name of the database containing this table + + + wal_records int8 + + + Total number of WAL records generated by vacuum operations performed on this table + + + + + wal_fpi int8 + + + Total number of WAL full page images generated by vacuum operations + + + + + wal_bytes numeric + + + Total amount of WAL bytes generated by vacuum operations + + tuples_deleted int8 @@ -172,6 +198,117 @@ . + + <structname>ext_vacuum_statistics.pg_stats_vacuum_indexes</structname> Columns + + + + + Column Type + + + Description + + + + + + + indexrelid oid + + + OID of this index + + + + + schema name + + + Name of the schema this index is in + + + + + indexrelname name + + + Name of this index + + + + + dbname name + + + Name of the database containing this index + + + + + wal_records int8 + + + Total number of WAL records generated by vacuum operations while + processing this index + + + + + wal_fpi int8 + + + Total number of WAL full page images generated by vacuum operations + while processing this index + + + + + wal_bytes numeric + + + Total amount of WAL bytes generated by vacuum operations while + processing this index + + + + + tuples_deleted int8 + + + Number of index entries removed by the bulkdelete passes of vacuums + of the parent table + + + + + pages_deleted int8 + + + Number of index pages marked deleted by vacuum operations + + + + +
+ + + + The <structname>ext_vacuum_statistics.pg_stats_vacuum_database</structname> View + + + pg_stats_vacuum_database + + + + The view ext_vacuum_statistics.pg_stats_vacuum_database + contains one row for each database in the cluster, showing aggregate vacuum + statistics for that database. Columns include + dboid, dbname, + and WAL stats (db_wal_records, + db_wal_fpi, db_wal_bytes). + + <structname>ext_vacuum_statistics.pg_stats_vacuum_indexes</structname> Columns diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index e27df6a6d0b..6adbcd31224 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -415,6 +415,13 @@ typedef struct LVRelState */ BlockNumber eager_scan_remaining_fails; + /* + * Resource usage of the index passes this process ran, subtracted from + * the heap report to avoid double-counting (see + * accumulate_heap_vacuum_statistics). + */ + PgStat_VacuumRelationCounts extVacReportIdx; + /* * We need to accumulate index statistics for later subtraction from heap * stats. @@ -496,6 +503,33 @@ static void restore_vacuum_error_info(LVRelState *vacrel, /* Extended vacuum statistics functions */ +/* + * extvac_stats_start - Snapshot resource usage before a vacuum phase. + */ +void +extvac_stats_start(Relation rel, LVExtStatCounters * counters) +{ + memset(counters, 0, sizeof(LVExtStatCounters)); + counters->walusage = pgWalUsage; +} + +/* + * extvac_stats_end - Diff resource usage since extvac_stats_start into report. + */ +void +extvac_stats_end(Relation rel, LVExtStatCounters * counters, + PgStat_CommonCounts * report) +{ + WalUsage walusage; + + memset(&walusage, 0, sizeof(WalUsage)); + WalUsageAccumDiff(&walusage, &pgWalUsage, &counters->walusage); + + report->wal_records = walusage.wal_records; + report->wal_fpi = walusage.wal_fpi; + report->wal_bytes = walusage.wal_bytes; +} + /* * Build the heap-specific part of the extended vacuum report from the * counters gathered in vacrel. @@ -513,12 +547,27 @@ accumulate_heap_vacuum_statistics(LVRelState *vacrel, PgStat_VacuumRelationCount extVacStats->table.missed_dead_pages = vacrel->missed_dead_pages; /* - * The wraparound failsafe is reported as a per-relation flag (0/1): did - * the latest vacuum of this relation engage it? Consumers can sum the - * flags to build a per-database total. + * Subtract the resource usage of the index passes this process ran: they + * are reported per index, and the database-wide aggregate would count + * them twice otherwise. Parallel workers report their own usage with + * their index passes, so it never enters the leader's counters. */ + extVacStats->common.wal_records -= vacrel->extVacReportIdx.common.wal_records; + extVacStats->common.wal_fpi -= vacrel->extVacReportIdx.common.wal_fpi; + extVacStats->common.wal_bytes -= vacrel->extVacReportIdx.common.wal_bytes; +} - /* Hook is invoked from pgstat_report_vacuum() when extstats is passed */ +/* + * Accumulate an index pass report into the running total that is later + * subtracted from the heap report. + */ +static void +accumulate_idxs_vacuum_statistics(LVRelState *vacrel, + PgStat_VacuumRelationCounts * extVacIdxStats) +{ + vacrel->extVacReportIdx.common.wal_records += extVacIdxStats->common.wal_records; + vacrel->extVacReportIdx.common.wal_fpi += extVacIdxStats->common.wal_fpi; + vacrel->extVacReportIdx.common.wal_bytes += extVacIdxStats->common.wal_bytes; } /* @@ -674,6 +723,7 @@ heap_vacuum_rel(Relation rel, const VacuumParams *params, PgStat_Counter startreadtime = 0, startwritetime = 0; double startdelaytime; + LVExtStatCounters extVacCounters; WalUsage startwalusage = pgWalUsage; BufferUsage startbufferusage = pgBufferUsage; ErrorContextCallback errcallback; @@ -699,6 +749,9 @@ heap_vacuum_rel(Relation rel, const VacuumParams *params, starttime = GetCurrentTimestamp(); startdelaytime = VacuumDelayTime; + if (set_report_vacuum_hook) + extvac_stats_start(rel, &extVacCounters); + pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); if (AmAutoVacuumWorkerProcess()) @@ -1033,6 +1086,7 @@ heap_vacuum_rel(Relation rel, const VacuumParams *params, */ if (set_report_vacuum_hook) { + extvac_stats_end(rel, &extVacCounters, &extVacReport.common); accumulate_heap_vacuum_statistics(vacrel, &extVacReport); pgstat_report_vacuum_ext(rel, @@ -3085,6 +3139,7 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, double startdelaytime = VacuumDelayTime; double prev_tuples_removed = 0; BlockNumber prev_pages_deleted = 0; + LVExtStatCounters extVacCounters; PgStat_VacuumRelationCounts extVacReport; /* @@ -3096,6 +3151,8 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, prev_tuples_removed = istat->tuples_removed; prev_pages_deleted = istat->pages_deleted; } + if (set_report_vacuum_hook) + extvac_stats_start(indrel, &extVacCounters); ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; ivinfo.analyze_only = false; @@ -3133,6 +3190,7 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, if (set_report_vacuum_hook) { memset(&extVacReport, 0, sizeof(extVacReport)); + extvac_stats_end(indrel, &extVacCounters, &extVacReport.common); extVacReport.type = PGSTAT_EXTVAC_INDEX; if (istat != NULL) { @@ -3142,6 +3200,7 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, istat->pages_deleted - prev_pages_deleted; } pgstat_report_vacuum_ext(indrel, -1, -1, 0, 0, false, &extVacReport); + accumulate_idxs_vacuum_statistics(vacrel, &extVacReport); } /* Revert to the previous phase information for error traceback */ @@ -3172,6 +3231,7 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, double startdelaytime = VacuumDelayTime; double prev_tuples_removed = 0; BlockNumber prev_pages_deleted = 0; + LVExtStatCounters extVacCounters; PgStat_VacuumRelationCounts extVacReport; /* @@ -3183,6 +3243,8 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, prev_tuples_removed = istat->tuples_removed; prev_pages_deleted = istat->pages_deleted; } + if (set_report_vacuum_hook) + extvac_stats_start(indrel, &extVacCounters); ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; ivinfo.analyze_only = false; @@ -3219,6 +3281,7 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, if (set_report_vacuum_hook) { memset(&extVacReport, 0, sizeof(extVacReport)); + extvac_stats_end(indrel, &extVacCounters, &extVacReport.common); extVacReport.type = PGSTAT_EXTVAC_INDEX; if (istat != NULL) { @@ -3228,6 +3291,7 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, istat->pages_deleted - prev_pages_deleted; } pgstat_report_vacuum_ext(indrel, -1, -1, 0, 0, false, &extVacReport); + accumulate_idxs_vacuum_statistics(vacrel, &extVacReport); } /* Revert to the previous phase information for error traceback */ diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c index 2f3415fbfcb..c48ef437389 100644 --- a/src/backend/commands/vacuumparallel.c +++ b/src/backend/commands/vacuumparallel.c @@ -1082,6 +1082,7 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, double startdelaytime = VacuumDelayTime; double prev_tuples_removed = 0; BlockNumber prev_pages_deleted = 0; + LVExtStatCounters extVacCounters; PgStat_VacuumRelationCounts extVacReport; /* @@ -1100,6 +1101,8 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, prev_tuples_removed = istat->tuples_removed; prev_pages_deleted = istat->pages_deleted; } + if (set_report_vacuum_hook) + extvac_stats_start(indrel, &extVacCounters); ivinfo.index = indrel; ivinfo.heaprel = pvs->heaprel; ivinfo.analyze_only = false; @@ -1131,6 +1134,7 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, if (set_report_vacuum_hook) { memset(&extVacReport, 0, sizeof(extVacReport)); + extvac_stats_end(indrel, &extVacCounters, &extVacReport.common); extVacReport.type = PGSTAT_EXTVAC_INDEX; if (istat_res != NULL) { diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index ffd2918b0f3..856c6fefd17 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -21,6 +21,7 @@ #include "catalog/pg_class.h" #include "catalog/pg_statistic.h" #include "catalog/pg_type.h" +#include "executor/instrument.h" #include "parser/parse_node.h" #include "storage/buf.h" #include "utils/relcache.h" @@ -355,6 +356,16 @@ extern PGDLLIMPORT pg_atomic_uint32 *VacuumSharedCostBalance; extern PGDLLIMPORT pg_atomic_uint32 *VacuumActiveNWorkers; extern PGDLLIMPORT int VacuumCostBalanceLocal; +/* Snapshots of resource usage taken before a vacuum phase */ +typedef struct LVExtStatCounters +{ + WalUsage walusage; +} LVExtStatCounters; + +extern void extvac_stats_start(Relation rel, LVExtStatCounters *counters); +extern void extvac_stats_end(Relation rel, LVExtStatCounters *counters, + PgStat_CommonCounts *report); + extern PGDLLIMPORT bool VacuumFailsafeActive; extern PGDLLIMPORT double vacuum_cost_delay; extern PGDLLIMPORT int vacuum_cost_limit; diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 9ff55dfe94b..a984c1bc347 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -107,6 +107,9 @@ typedef enum ExtVacReportType typedef struct PgStat_CommonCounts { + int64 wal_records; + int64 wal_fpi; + uint64 wal_bytes; int64 tuples_deleted; } PgStat_CommonCounts; -- 2.39.5 (Apple Git-154)