From 9b66e77adb640500204dfdf0c1bffa6af7d9e3b7 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Mon, 20 Jul 2026 17:22:26 +0300 Subject: [PATCH v43 8/9] ext_vacuum_statistics: shared-buffer access counters and I/O timing. Add the buffer category: total shared-buffer traffic of a vacuum (total_blks_read, total_blks_hit, total_blks_dirtied, total_blks_written), the relation-level breakdown (rel_blks_read, rel_blks_hit) and the I/O times (blk_read_time, blk_write_time), for tables, indexes, and the per-database aggregate, sampled by the core machinery from pgBufferUsage and the relation's pgstat counts around the heap run and each index pass. The resource usage of index passes is subtracted from the heap report to avoid double-counting in aggregates. Authors: Alena Rybakina , Andrei Lepikhov , Andrei Zubkov --- contrib/ext_vacuum_statistics/README.md | 6 +- .../ext_vacuum_statistics--1.0.sql | 48 +++++- .../t/052_vacuum_extending_basic_test.pl | 23 ++- .../ext_vacuum_statistics/vacuum_statistics.c | 24 ++- doc/src/sgml/extvacuumstatistics.sgml | 143 +++++++++++++++++- src/backend/access/heap/vacuumlazy.c | 64 +++++++- src/include/commands/vacuum.h | 3 + src/include/pgstat.h | 8 + 8 files changed, 303 insertions(+), 16 deletions(-) diff --git a/contrib/ext_vacuum_statistics/README.md b/contrib/ext_vacuum_statistics/README.md index 8c8b42f014d..cff44301152 100644 --- a/contrib/ext_vacuum_statistics/README.md +++ b/contrib/ext_vacuum_statistics/README.md @@ -45,9 +45,9 @@ SELECT * FROM ext_vacuum_statistics.pg_stats_vacuum_database; Example output: ``` - relname | wal_records | tuples_deleted | pages_removed ------------+-------------+----------------+--------------- - mytable | 15 | 500 | 10 + relname | total_blks_read | total_blks_hit | wal_records | tuples_deleted | pages_removed +-----------+-----------------+----------------+-------------+----------------+--------------- + mytable | 120 | 340 | 15 | 500 | 10 ``` Reset statistics when needed: diff --git a/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql b/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql index d8d327741f8..51ff179ab9a 100644 --- a/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql +++ b/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql @@ -40,9 +40,17 @@ CREATE OR REPLACE FUNCTION ext_vacuum_statistics.pg_stats_get_vacuum_tables( IN dboid oid, IN reloid oid, OUT relid oid, + OUT total_blks_read bigint, + OUT total_blks_hit bigint, + OUT total_blks_dirtied bigint, + OUT total_blks_written bigint, OUT wal_records bigint, OUT wal_fpi bigint, OUT wal_bytes numeric, + OUT blk_read_time double precision, + OUT blk_write_time double precision, + OUT rel_blks_read bigint, + OUT rel_blks_hit bigint, OUT tuples_deleted bigint, OUT pages_scanned bigint, OUT pages_removed bigint, @@ -60,9 +68,17 @@ CREATE OR REPLACE FUNCTION ext_vacuum_statistics.pg_stats_get_vacuum_indexes( IN dboid oid, IN reloid oid, OUT relid oid, + OUT total_blks_read bigint, + OUT total_blks_hit bigint, + OUT total_blks_dirtied bigint, + OUT total_blks_written bigint, OUT wal_records bigint, OUT wal_fpi bigint, OUT wal_bytes numeric, + OUT blk_read_time double precision, + OUT blk_write_time double precision, + OUT rel_blks_read bigint, + OUT rel_blks_hit bigint, OUT tuples_deleted bigint, OUT pages_deleted bigint ) @@ -74,9 +90,15 @@ LANGUAGE C STRICT STABLE; CREATE OR REPLACE FUNCTION ext_vacuum_statistics.pg_stats_get_vacuum_database( IN dboid oid, OUT dbid oid, + OUT total_blks_read bigint, + OUT total_blks_hit bigint, + OUT total_blks_dirtied bigint, + OUT total_blks_written bigint, OUT wal_records bigint, OUT wal_fpi bigint, - OUT wal_bytes numeric + OUT wal_bytes numeric, + OUT blk_read_time double precision, + OUT blk_write_time double precision ) RETURNS SETOF record AS 'MODULE_PATHNAME', 'pg_stats_get_vacuum_database' @@ -89,9 +111,17 @@ SELECT ns.nspname AS schema, rel.relname AS relname, db.datname AS dbname, + stats.total_blks_read, + stats.total_blks_hit, + stats.total_blks_dirtied, + stats.total_blks_written, stats.wal_records, stats.wal_fpi, stats.wal_bytes, + stats.blk_read_time, + stats.blk_write_time, + stats.rel_blks_read, + stats.rel_blks_hit, stats.tuples_deleted, stats.pages_scanned, stats.pages_removed, @@ -118,9 +148,17 @@ SELECT ns.nspname AS schema, rel.relname AS indexrelname, db.datname AS dbname, + stats.total_blks_read, + stats.total_blks_hit, + stats.total_blks_dirtied, + stats.total_blks_written, stats.wal_records, stats.wal_fpi, stats.wal_bytes, + stats.blk_read_time, + stats.blk_write_time, + stats.rel_blks_read, + stats.rel_blks_hit, stats.tuples_deleted, stats.pages_deleted FROM pg_database db, @@ -140,9 +178,15 @@ CREATE VIEW ext_vacuum_statistics.pg_stats_vacuum_database AS SELECT db.oid AS dboid, db.datname AS dbname, + stats.total_blks_read AS db_blks_read, + stats.total_blks_hit AS db_blks_hit, + stats.total_blks_dirtied AS db_blks_dirtied, + stats.total_blks_written AS db_blks_written, stats.wal_records AS db_wal_records, stats.wal_fpi AS db_wal_fpi, - stats.wal_bytes AS db_wal_bytes + stats.wal_bytes AS db_wal_bytes, + stats.blk_read_time AS db_blk_read_time, + stats.blk_write_time AS db_blk_write_time FROM pg_database db LEFT JOIN LATERAL ext_vacuum_statistics.pg_stats_get_vacuum_database(db.oid) stats ON db.oid = stats.dbid; diff --git a/contrib/ext_vacuum_statistics/t/052_vacuum_extending_basic_test.pl b/contrib/ext_vacuum_statistics/t/052_vacuum_extending_basic_test.pl index 494c6cc8259..ba649bc25ee 100644 --- a/contrib/ext_vacuum_statistics/t/052_vacuum_extending_basic_test.pl +++ b/contrib/ext_vacuum_statistics/t/052_vacuum_extending_basic_test.pl @@ -263,16 +263,22 @@ sub fetch_error_base_db_vacuum_statistics { # fetch actual base database vacuum statistics my $base_statistics = $node->safe_psql( $database_name, - "SELECT db_wal_records, db_wal_fpi, db_wal_bytes + "SELECT db_blks_hit, db_blks_dirtied, + db_blks_written, db_wal_records, + db_wal_fpi, db_wal_bytes FROM ext_vacuum_statistics.pg_stats_vacuum_database, pg_database WHERE pg_database.datname = '$dbname' AND pg_database.oid = ext_vacuum_statistics.pg_stats_vacuum_database.dboid;" ); $base_statistics =~ s/\s*\|\s*/ /g; # transform " | " in space - my ($wal_records, $wal_fpi, $wal_bytes) = split /\s+/, $base_statistics; + my ($db_blks_hit, $total_blks_dirtied, $total_blks_written, + $wal_records, $wal_fpi, $wal_bytes) = split /\s+/, $base_statistics; diag( "BASE STATS MISMATCH FOR DATABASE $dbname:\n" . + " db_blks_hit = $db_blks_hit\n" . + " total_blks_dirtied = $total_blks_dirtied\n" . + " total_blks_written = $total_blks_written\n" . " wal_records = $wal_records\n" . " wal_fpi = $wal_fpi\n" . " wal_bytes = $wal_bytes\n" @@ -617,19 +623,28 @@ is($base_stats, 0, 'vacuum stats per all index objects from another database are #-------------------------------------------------------------------------------------- subtest 'Test 9: Check database-level vacuum statistics from the current and another database' => sub { +my $db_blk_hit = 0; +my $total_blks_dirtied = 0; +my $total_blks_written = 0; my $wal_records = 0; my $wal_fpi = 0; my $wal_bytes = 0; $base_stats = $node->safe_psql( $dbname, - "SELECT db_wal_records, db_wal_fpi, db_wal_bytes + "SELECT db_blks_hit, db_blks_dirtied, + db_blks_written, db_wal_records, + db_wal_fpi, db_wal_bytes FROM ext_vacuum_statistics.pg_stats_vacuum_database, pg_database WHERE pg_database.datname = '$dbname' AND pg_database.oid = ext_vacuum_statistics.pg_stats_vacuum_database.dboid;" ); $base_stats =~ s/\s*\|\s*/ /g; # transform " | " into space - ($wal_records, $wal_fpi, $wal_bytes) = split /\s+/, $base_stats; + ($db_blk_hit, $total_blks_dirtied, $total_blks_written, $wal_records, $wal_fpi, $wal_bytes) + = split /\s+/, $base_stats; +ok($db_blk_hit > 0, 'db_blks_hit is more than 0'); +ok($total_blks_dirtied > 0, 'total_blks_dirtied is more than 0'); +ok($total_blks_written > 0, 'total_blks_written is more than 0'); ok($wal_records > 0, 'wal_records is more than 0'); ok($wal_fpi > 0, 'wal_fpi is more than 0'); ok($wal_bytes > 0, 'wal_bytes is more than 0'); diff --git a/contrib/ext_vacuum_statistics/vacuum_statistics.c b/contrib/ext_vacuum_statistics/vacuum_statistics.c index f8f991e6460..5ef871c0a33 100644 --- a/contrib/ext_vacuum_statistics/vacuum_statistics.c +++ b/contrib/ext_vacuum_statistics/vacuum_statistics.c @@ -80,6 +80,14 @@ static const PgStat_KindInfo extvac_db_kind_info = { static inline void pgstat_accumulate_common(PgStat_CommonCounts * dst, const PgStat_CommonCounts * src) { + dst->total_blks_read += src->total_blks_read; + dst->total_blks_hit += src->total_blks_hit; + dst->total_blks_dirtied += src->total_blks_dirtied; + dst->total_blks_written += src->total_blks_written; + dst->blks_fetched += src->blks_fetched; + dst->blks_hit += src->blks_hit; + dst->blk_read_time += src->blk_read_time; + dst->blk_write_time += src->blk_write_time; dst->wal_records += src->wal_records; dst->wal_fpi += src->wal_fpi; dst->wal_bytes += src->wal_bytes; @@ -320,7 +328,7 @@ extvac_reset_db_entry(PG_FUNCTION_ARGS) /* * Output vacuum statistics (tables, indexes, or per-database aggregates). */ -#define EXTVAC_COMMON_STAT_COLS 3 +#define EXTVAC_COMMON_STAT_COLS 9 static void tuplestore_put_common(PgStat_CommonCounts * vacuum_ext, @@ -329,6 +337,10 @@ tuplestore_put_common(PgStat_CommonCounts * vacuum_ext, char buf[256]; const int base PG_USED_FOR_ASSERTS_ONLY = *i; + values[(*i)++] = Int64GetDatum(vacuum_ext->total_blks_read); + values[(*i)++] = Int64GetDatum(vacuum_ext->total_blks_hit); + values[(*i)++] = Int64GetDatum(vacuum_ext->total_blks_dirtied); + values[(*i)++] = Int64GetDatum(vacuum_ext->total_blks_written); values[(*i)++] = Int64GetDatum(vacuum_ext->wal_records); values[(*i)++] = Int64GetDatum(vacuum_ext->wal_fpi); snprintf(buf, sizeof buf, UINT64_FORMAT, vacuum_ext->wal_bytes); @@ -336,11 +348,13 @@ tuplestore_put_common(PgStat_CommonCounts * vacuum_ext, CStringGetDatum(buf), ObjectIdGetDatum(0), Int32GetDatum(-1)); + values[(*i)++] = Float8GetDatum(vacuum_ext->blk_read_time); + values[(*i)++] = Float8GetDatum(vacuum_ext->blk_write_time); Assert((*i - base) == EXTVAC_COMMON_STAT_COLS); } -#define EXTVAC_HEAP_STAT_COLS 11 -#define EXTVAC_IDX_STAT_COLS 6 +#define EXTVAC_HEAP_STAT_COLS 19 +#define EXTVAC_IDX_STAT_COLS 14 #define EXTVAC_MAX_STAT_COLS Max(EXTVAC_HEAP_STAT_COLS, EXTVAC_IDX_STAT_COLS) static void @@ -355,6 +369,8 @@ tuplestore_put_for_relation(Oid relid, Tuplestorestate *tupstore, values[i++] = ObjectIdGetDatum(relid); tuplestore_put_common(&vacuum_ext->common, values, nulls, &i); + values[i++] = Int64GetDatum(vacuum_ext->common.blks_fetched - vacuum_ext->common.blks_hit); + values[i++] = Int64GetDatum(vacuum_ext->common.blks_hit); if (vacuum_ext->type == PGSTAT_EXTVAC_TABLE) { @@ -431,7 +447,7 @@ pg_stats_vacuum(FunctionCallInfo fcinfo, int type) { if (OidIsValid(dbid)) { -#define EXTVAC_DB_STAT_COLS 4 +#define EXTVAC_DB_STAT_COLS 10 Datum values[EXTVAC_DB_STAT_COLS]; bool nulls[EXTVAC_DB_STAT_COLS]; int i = 0; diff --git a/doc/src/sgml/extvacuumstatistics.sgml b/doc/src/sgml/extvacuumstatistics.sgml index 23471350c5a..1832842e4f5 100644 --- a/doc/src/sgml/extvacuumstatistics.sgml +++ b/doc/src/sgml/extvacuumstatistics.sgml @@ -110,6 +110,38 @@ Name of the database containing this table + + + total_blks_read int8 + + + Number of database blocks read by vacuum operations performed on this table + + + + + total_blks_hit int8 + + + Number of times database blocks were found in the buffer cache by vacuum operations + + + + + total_blks_dirtied int8 + + + Number of database blocks dirtied by vacuum operations + + + + + total_blks_written int8 + + + Number of database blocks written by vacuum operations + + wal_records int8 @@ -134,6 +166,38 @@ Total amount of WAL bytes generated by vacuum operations + + + blk_read_time float8 + + + Time spent reading blocks by vacuum operations, in milliseconds + + + + + blk_write_time float8 + + + Time spent writing blocks by vacuum operations, in milliseconds + + + + + rel_blks_read int8 + + + Number of blocks vacuum operations read from this table + + + + + rel_blks_hit int8 + + + Number of times blocks of this table were found in the buffer cache by vacuum + + tuples_deleted int8 @@ -255,6 +319,42 @@ Name of the database containing this index + + + total_blks_read int8 + + + Number of database blocks read by vacuum operations while processing + this index + + + + + total_blks_hit int8 + + + Number of times database blocks were found in the buffer cache by + vacuum operations while processing this index + + + + + total_blks_dirtied int8 + + + Number of database blocks dirtied by vacuum operations while + processing this index + + + + + total_blks_written int8 + + + Number of database blocks written by vacuum operations while + processing this index + + wal_records int8 @@ -282,6 +382,41 @@ processing this index + + + blk_read_time float8 + + + Time spent reading blocks by vacuum operations while processing this + index, in milliseconds + + + + + blk_write_time float8 + + + Time spent writing blocks by vacuum operations while processing this + index, in milliseconds + + + + + rel_blks_read int8 + + + Number of blocks of this index read by vacuum operations + + + + + rel_blks_hit int8 + + + Number of blocks of this index found in the buffer cache by vacuum + operations + + tuples_deleted int8 @@ -316,8 +451,12 @@ contains one row for each database in the cluster, showing aggregate vacuum statistics for that database. Columns include dboid, dbname, - and WAL stats (db_wal_records, - db_wal_fpi, db_wal_bytes). + db_blks_read, db_blks_hit, + db_blks_dirtied, db_blks_written, + WAL stats (db_wal_records, + db_wal_fpi, db_wal_bytes), + and I/O timing (db_blk_read_time, + db_blk_write_time). diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 3626f92b652..6b6d9a6a94f 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -503,6 +503,30 @@ static void restore_vacuum_error_info(LVRelState *vacrel, /* Extended vacuum statistics functions */ +/* + * extvac_pending_blocks - Read the pending block counters of a relation. + * + * Tables and indexes keep those counters in different members of their + * pending stats entry, so pick the one matching the entry's stats kind. + */ +static void +extvac_pending_blocks(Relation rel, PgStat_Counter *fetched, + PgStat_Counter *hit) +{ + PgStat_RelationStatus *pgstat_info = rel->pgstat_info; + + if (pgstat_info->kind == PGSTAT_KIND_INDEX) + { + *fetched = pgstat_info->idx.blocks_fetched; + *hit = pgstat_info->idx.blocks_hit; + } + else + { + *fetched = pgstat_info->tab.counts.blocks_fetched; + *hit = pgstat_info->tab.counts.blocks_hit; + } +} + /* * extvac_stats_start - Snapshot resource usage before a vacuum phase. */ @@ -511,6 +535,11 @@ extvac_stats_start(Relation rel, LVExtStatCounters * counters) { memset(counters, 0, sizeof(LVExtStatCounters)); counters->walusage = pgWalUsage; + counters->bufusage = pgBufferUsage; + + if (rel->pgstat_info && pgstat_track_counts) + extvac_pending_blocks(rel, &counters->blocks_fetched, + &counters->blocks_hit); } /* @@ -521,13 +550,34 @@ extvac_stats_end(Relation rel, LVExtStatCounters * counters, PgStat_CommonCounts * report) { WalUsage walusage; + BufferUsage bufusage; memset(&walusage, 0, sizeof(WalUsage)); WalUsageAccumDiff(&walusage, &pgWalUsage, &counters->walusage); - + memset(&bufusage, 0, sizeof(BufferUsage)); + BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &counters->bufusage); + + report->total_blks_read = bufusage.local_blks_read + bufusage.shared_blks_read; + report->total_blks_hit = bufusage.local_blks_hit + bufusage.shared_blks_hit; + report->total_blks_dirtied = bufusage.local_blks_dirtied + bufusage.shared_blks_dirtied; + report->total_blks_written = bufusage.shared_blks_written; + report->blk_read_time = INSTR_TIME_GET_MILLISEC(bufusage.local_blk_read_time) + + INSTR_TIME_GET_MILLISEC(bufusage.shared_blk_read_time); + report->blk_write_time = INSTR_TIME_GET_MILLISEC(bufusage.local_blk_write_time) + + INSTR_TIME_GET_MILLISEC(bufusage.shared_blk_write_time); report->wal_records = walusage.wal_records; report->wal_fpi = walusage.wal_fpi; report->wal_bytes = walusage.wal_bytes; + + if (rel->pgstat_info && pgstat_track_counts) + { + PgStat_Counter blocks_fetched; + PgStat_Counter blocks_hit; + + extvac_pending_blocks(rel, &blocks_fetched, &blocks_hit); + report->blks_fetched = blocks_fetched - counters->blocks_fetched; + report->blks_hit = blocks_hit - counters->blocks_hit; + } } /* @@ -552,6 +602,12 @@ accumulate_heap_vacuum_statistics(LVRelState *vacrel, PgStat_VacuumRelationCount * them twice otherwise. Parallel workers report their own usage with * their index passes, so it never enters the leader's counters. */ + extVacStats->common.total_blks_read -= vacrel->extVacReportIdx.common.total_blks_read; + extVacStats->common.total_blks_hit -= vacrel->extVacReportIdx.common.total_blks_hit; + extVacStats->common.total_blks_dirtied -= vacrel->extVacReportIdx.common.total_blks_dirtied; + extVacStats->common.total_blks_written -= vacrel->extVacReportIdx.common.total_blks_written; + extVacStats->common.blk_read_time -= vacrel->extVacReportIdx.common.blk_read_time; + extVacStats->common.blk_write_time -= vacrel->extVacReportIdx.common.blk_write_time; extVacStats->common.wal_records -= vacrel->extVacReportIdx.common.wal_records; extVacStats->common.wal_fpi -= vacrel->extVacReportIdx.common.wal_fpi; extVacStats->common.wal_bytes -= vacrel->extVacReportIdx.common.wal_bytes; @@ -565,6 +621,12 @@ static void accumulate_idxs_vacuum_statistics(LVRelState *vacrel, PgStat_VacuumRelationCounts * extVacIdxStats) { + vacrel->extVacReportIdx.common.total_blks_read += extVacIdxStats->common.total_blks_read; + vacrel->extVacReportIdx.common.total_blks_hit += extVacIdxStats->common.total_blks_hit; + vacrel->extVacReportIdx.common.total_blks_dirtied += extVacIdxStats->common.total_blks_dirtied; + vacrel->extVacReportIdx.common.total_blks_written += extVacIdxStats->common.total_blks_written; + vacrel->extVacReportIdx.common.blk_read_time += extVacIdxStats->common.blk_read_time; + vacrel->extVacReportIdx.common.blk_write_time += extVacIdxStats->common.blk_write_time; vacrel->extVacReportIdx.common.wal_records += extVacIdxStats->common.wal_records; vacrel->extVacReportIdx.common.wal_fpi += extVacIdxStats->common.wal_fpi; vacrel->extVacReportIdx.common.wal_bytes += extVacIdxStats->common.wal_bytes; diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 758b5ebad38..4c3c04d5afd 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -368,6 +368,9 @@ extern PGDLLIMPORT int VacuumCostBalanceLocal; typedef struct LVExtStatCounters { WalUsage walusage; + BufferUsage bufusage; + PgStat_Counter blocks_fetched; + PgStat_Counter blocks_hit; } LVExtStatCounters; extern void extvac_stats_start(Relation rel, LVExtStatCounters *counters); diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 3cb24981192..c03a5caa492 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -107,6 +107,14 @@ typedef enum ExtVacReportType typedef struct PgStat_CommonCounts { + int64 total_blks_read; + int64 total_blks_hit; + int64 total_blks_dirtied; + int64 total_blks_written; + int64 blks_fetched; + int64 blks_hit; + double blk_read_time; + double blk_write_time; int64 wal_records; int64 wal_fpi; uint64 wal_bytes; -- 2.50.1 (Apple Git-155)