From 098e381ca88e2600cb8e8528e3f54f588d4c43a8 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Thu, 4 Sep 2025 18:16:52 +0300 Subject: [PATCH] Core patch for vacuum statistics. --- src/backend/access/heap/vacuumlazy.c | 309 ++++++++++++++++++- src/backend/access/heap/visibilitymap.c | 10 + src/backend/commands/vacuum.c | 4 + src/backend/commands/vacuumparallel.c | 12 + src/backend/utils/activity/pgstat_relation.c | 36 ++- src/backend/utils/adt/pgstatfuncs.c | 6 + src/backend/utils/error/elog.c | 13 + src/include/catalog/pg_proc.dat | 8 + src/include/commands/vacuum.h | 26 ++ src/include/pgstat.h | 119 ++++++- src/include/utils/elog.h | 1 + src/test/regress/expected/rules.out | 12 +- 12 files changed, 545 insertions(+), 11 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 932701d8420..330f9f90a6b 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -289,6 +289,8 @@ typedef struct LVRelState /* Error reporting state */ char *dbname; char *relnamespace; + Oid reloid; + Oid indoid; char *relname; char *indname; /* Current index name */ BlockNumber blkno; /* used only for heap operations */ @@ -407,6 +409,10 @@ typedef struct LVRelState * been permanently disabled. */ BlockNumber eager_scan_remaining_fails; + + int32 wraparound_failsafe_count; /* number of emergency vacuums to prevent anti-wraparound shutdown */ + + PgStat_VacuumRelationCounts extVacReportIdx; } LVRelState; @@ -474,6 +480,209 @@ static void update_vacuum_error_info(LVRelState *vacrel, static void restore_vacuum_error_info(LVRelState *vacrel, const LVSavedErrInfo *saved_vacrel); +/* ---------- + * extvac_stats_start() - + * + * Save cut-off values of extended vacuum counters before start of a relation + * processing. + * ---------- + */ +static void +extvac_stats_start(Relation rel, LVExtStatCounters *counters) +{ + TimestampTz starttime; + + memset(counters, 0, sizeof(LVExtStatCounters)); + + starttime = GetCurrentTimestamp(); + + counters->starttime = starttime; + counters->walusage = pgWalUsage; + counters->bufusage = pgBufferUsage; + counters->VacuumDelayTime = VacuumDelayTime; + counters->blocks_fetched = 0; + counters->blocks_hit = 0; + + if (!rel->pgstat_info || !pgstat_track_counts) + /* + * if something goes wrong or user doesn't want to track a database + * activity - just suppress it. + */ + return; + + counters->blocks_fetched = rel->pgstat_info->counts.blocks_fetched; + counters->blocks_hit = rel->pgstat_info->counts.blocks_hit; +} + +/* ---------- + * extvac_stats_end() - + * + * Called to finish an extended vacuum statistic gathering and form a report. + * ---------- + */ +static void +extvac_stats_end(Relation rel, LVExtStatCounters *counters, + PgStat_VacuumRelationCounts *report) +{ + WalUsage walusage; + BufferUsage bufusage; + TimestampTz endtime; + long secs; + int usecs; + + /* Calculate diffs of global stat parameters on WAL and buffer usage. */ + memset(&walusage, 0, sizeof(WalUsage)); + WalUsageAccumDiff(&walusage, &pgWalUsage, &counters->walusage); + + memset(&bufusage, 0, sizeof(BufferUsage)); + BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &counters->bufusage); + + endtime = GetCurrentTimestamp(); + TimestampDifference(counters->starttime, endtime, &secs, &usecs); + + /* + * Fill additional statistics on a vacuum processing operation. + */ + report->common.total_blks_read += bufusage.local_blks_read + bufusage.shared_blks_read; + report->common.total_blks_hit += bufusage.local_blks_hit + bufusage.shared_blks_hit; + report->common.total_blks_dirtied += bufusage.local_blks_dirtied + bufusage.shared_blks_dirtied; + report->common.total_blks_written += bufusage.shared_blks_written; + + report->common.wal_records += walusage.wal_records; + report->common.wal_fpi += walusage.wal_fpi; + report->common.wal_bytes += walusage.wal_bytes; + + report->common.blk_read_time += INSTR_TIME_GET_MILLISEC(bufusage.local_blk_read_time); + report->common.blk_read_time += INSTR_TIME_GET_MILLISEC(bufusage.shared_blk_read_time); + report->common.blk_write_time += INSTR_TIME_GET_MILLISEC(bufusage.local_blk_write_time); + report->common.blk_write_time += INSTR_TIME_GET_MILLISEC(bufusage.shared_blk_write_time); + report->common.delay_time += VacuumDelayTime - counters->VacuumDelayTime; + + report->common.total_time += secs * 1000. + usecs / 1000.; + + if (!rel->pgstat_info || !pgstat_track_counts) + /* + * if something goes wrong or an user doesn't want to track a database + * activity - just suppress it. + */ + return; + + report->common.blks_fetched += + rel->pgstat_info->counts.blocks_fetched - counters->blocks_fetched; + report->common.blks_hit += + rel->pgstat_info->counts.blocks_hit - counters->blocks_hit; +} + +void +extvac_stats_start_idx(Relation rel, IndexBulkDeleteResult *stats, + LVExtStatCountersIdx *counters) +{ + /* Set initial values for common heap and index statistics*/ + extvac_stats_start(rel, &counters->common); + counters->pages_deleted = counters->tuples_removed = 0; + + if (stats != NULL) + { + /* + * XXX: Why do we need this code here? If it is needed, I feel lack of + * comments, describing the reason. + */ + counters->tuples_removed = stats->tuples_removed; + counters->pages_deleted = stats->pages_deleted; + } +} + +void +extvac_stats_end_idx(Relation rel, IndexBulkDeleteResult *stats, + LVExtStatCountersIdx *counters, PgStat_VacuumRelationCounts *report) +{ + memset(report, 0, sizeof(PgStat_VacuumRelationCounts)); + + extvac_stats_end(rel, &counters->common, report); + report->type = PGSTAT_EXTVAC_INDEX; + + if (stats != NULL) + { + /* + * if something goes wrong or an user doesn't want to track a database + * activity - just suppress it. + */ + + /* Fill index-specific extended stats fields */ + report->index.tuples_deleted = + stats->tuples_removed - counters->tuples_removed; + report->index.pages_deleted = + stats->pages_deleted - counters->pages_deleted; + } +} + +/* Accumulate vacuum statistics for heap. + * + * Because of complexity of vacuum processing: it switch procesing between + * the heap relation to index relations and visa versa, we need to store + * gathered statistics information for heap relations several times before + * the vacuum starts processing the indexes again. + * + * It is necessary to gather correct statistics information for heap and indexes + * otherwice the index statistics information would be added to his parent heap + * statistics information and it would be difficult to analyze it later. + * + * We can't subtract union vacuum statistics information for index from the heap relations + * because of total and delay time time statistics collecting during parallel vacuum + * procudure. +*/ +static void +accumulate_heap_vacuum_statistics(LVRelState *vacrel, PgStat_VacuumRelationCounts *extVacStats) +{ + /* Fill heap-specific extended stats fields */ + extVacStats->type = PGSTAT_EXTVAC_TABLE; + extVacStats->table.pages_scanned = vacrel->scanned_pages; + extVacStats->table.pages_removed = vacrel->removed_pages; + extVacStats->table.vm_new_frozen_pages = vacrel->vm_new_frozen_pages; + extVacStats->table.vm_new_visible_pages = vacrel->vm_new_visible_pages; + extVacStats->table.vm_new_visible_frozen_pages = vacrel->vm_new_visible_frozen_pages; + extVacStats->table.tuples_deleted = vacrel->tuples_deleted; + extVacStats->table.tuples_frozen = vacrel->tuples_frozen; + extVacStats->table.recently_dead_tuples = vacrel->recently_dead_tuples; + extVacStats->table.recently_dead_tuples = vacrel->recently_dead_tuples; + extVacStats->table.missed_dead_tuples = vacrel->missed_dead_tuples; + extVacStats->table.missed_dead_pages = vacrel->missed_dead_pages; + extVacStats->table.index_vacuum_count = vacrel->num_index_scans; + extVacStats->common.wraparound_failsafe_count = vacrel->wraparound_failsafe_count; + + extVacStats->common.blk_read_time -= vacrel->extVacReportIdx.common.blk_read_time; + extVacStats->common.blk_write_time -= vacrel->extVacReportIdx.common.blk_write_time; + extVacStats->common.total_blks_dirtied -= vacrel->extVacReportIdx.common.total_blks_dirtied; + extVacStats->common.total_blks_hit -= vacrel->extVacReportIdx.common.total_blks_hit; + extVacStats->common.total_blks_read -= vacrel->extVacReportIdx.common.total_blks_read; + extVacStats->common.total_blks_written -= vacrel->extVacReportIdx.common.total_blks_written; + extVacStats->common.wal_bytes -= vacrel->extVacReportIdx.common.wal_bytes; + extVacStats->common.wal_fpi -= vacrel->extVacReportIdx.common.wal_fpi; + extVacStats->common.wal_records -= vacrel->extVacReportIdx.common.wal_records; + + extVacStats->common.total_time -= vacrel->extVacReportIdx.common.total_time; + extVacStats->common.delay_time -= vacrel->extVacReportIdx.common.delay_time; + +} + +static void +accumulate_idxs_vacuum_statistics(LVRelState *vacrel, PgStat_VacuumRelationCounts *extVacIdxStats) +{ + + /* Fill heap-specific extended stats fields */ + vacrel->extVacReportIdx.common.blk_read_time += extVacIdxStats->common.blk_read_time; + vacrel->extVacReportIdx.common.blk_write_time += extVacIdxStats->common.blk_write_time; + vacrel->extVacReportIdx.common.total_blks_dirtied += extVacIdxStats->common.total_blks_dirtied; + vacrel->extVacReportIdx.common.total_blks_hit += extVacIdxStats->common.total_blks_hit; + vacrel->extVacReportIdx.common.total_blks_read += extVacIdxStats->common.total_blks_read; + vacrel->extVacReportIdx.common.total_blks_written += extVacIdxStats->common.total_blks_written; + vacrel->extVacReportIdx.common.wal_bytes += extVacIdxStats->common.wal_bytes; + vacrel->extVacReportIdx.common.wal_fpi += extVacIdxStats->common.wal_fpi; + vacrel->extVacReportIdx.common.wal_records += extVacIdxStats->common.wal_records; + vacrel->extVacReportIdx.common.delay_time += extVacIdxStats->common.delay_time; + + vacrel->extVacReportIdx.common.total_time += extVacIdxStats->common.total_time; +} /* @@ -632,6 +841,13 @@ heap_vacuum_rel(Relation rel, const VacuumParams params, BufferUsage startbufferusage = pgBufferUsage; ErrorContextCallback errcallback; char **indnames = NULL; + LVExtStatCounters extVacCounters; + PgStat_VacuumRelationCounts ExtVacReport; + PgStat_VacuumRelationCounts allzero; + + /* Initialize vacuum statistics */ + memset(&allzero, 0, sizeof(PgStat_VacuumRelationCounts)); + ExtVacReport = allzero; verbose = (params.options & VACOPT_VERBOSE) != 0; instrument = (verbose || (AmAutoVacuumWorkerProcess() && @@ -652,6 +868,7 @@ heap_vacuum_rel(Relation rel, const VacuumParams params, pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); + extvac_stats_start(rel, &extVacCounters); /* * Setup error traceback support for ereport() first. The idea is to set * up an error context callback to display additional information on any @@ -668,6 +885,7 @@ heap_vacuum_rel(Relation rel, const VacuumParams params, vacrel->dbname = get_database_name(MyDatabaseId); vacrel->relnamespace = get_namespace_name(RelationGetNamespace(rel)); vacrel->relname = pstrdup(RelationGetRelationName(rel)); + vacrel->reloid = RelationGetRelid(rel); vacrel->indname = NULL; vacrel->phase = VACUUM_ERRCB_PHASE_UNKNOWN; vacrel->verbose = verbose; @@ -676,6 +894,8 @@ heap_vacuum_rel(Relation rel, const VacuumParams params, errcallback.previous = error_context_stack; error_context_stack = &errcallback; + memset(&vacrel->extVacReportIdx, 0, sizeof(PgStat_VacuumRelationCounts)); + /* Set up high level stuff about rel and its indexes */ vacrel->rel = rel; vac_open_indexes(vacrel->rel, RowExclusiveLock, &vacrel->nindexes, @@ -776,6 +996,7 @@ heap_vacuum_rel(Relation rel, const VacuumParams params, vacrel->aggressive = vacuum_get_cutoffs(rel, params, &vacrel->cutoffs); vacrel->rel_pages = orig_rel_pages = RelationGetNumberOfBlocks(rel); vacrel->vistest = GlobalVisTestFor(rel); + vacrel->wraparound_failsafe_count = 0; /* Initialize state used to track oldest extant XID/MXID */ vacrel->NewRelfrozenXid = vacrel->cutoffs.OldestXmin; @@ -924,6 +1145,9 @@ heap_vacuum_rel(Relation rel, const VacuumParams params, vacrel->NewRelfrozenXid, vacrel->NewRelminMxid, &frozenxid_updated, &minmulti_updated, false); + /* Make generic extended vacuum stats report */ + extvac_stats_end(rel, &extVacCounters, &ExtVacReport); + /* * Report results to the cumulative stats system, too. * @@ -934,12 +1158,20 @@ heap_vacuum_rel(Relation rel, const VacuumParams params, * soon in cases where the failsafe prevented significant amounts of heap * vacuuming. */ + /* Make generic extended vacuum stats report and + * fill heap-specific extended stats fields. + */ + extvac_stats_end(vacrel->rel, &extVacCounters, &ExtVacReport); + accumulate_heap_vacuum_statistics(vacrel, &ExtVacReport); + pgstat_report_vacuum(RelationGetRelid(rel), - rel->rd_rel->relisshared, - Max(vacrel->new_live_tuples, 0), - vacrel->recently_dead_tuples + - vacrel->missed_dead_tuples, - starttime); + rel->rd_rel->relisshared, + Max(vacrel->new_live_tuples, 0), + vacrel->recently_dead_tuples + + vacrel->missed_dead_tuples, + starttime, + &ExtVacReport); + pgstat_progress_end_command(); if (instrument) @@ -2631,10 +2863,20 @@ lazy_vacuum_all_indexes(LVRelState *vacrel) } else { + LVExtStatCounters counters; + PgStat_VacuumRelationCounts PgStat_VacuumRelationCounts; + + memset(&PgStat_VacuumRelationCounts, 0, sizeof(PgStat_VacuumRelationCounts)); + + extvac_stats_start(vacrel->rel, &counters); + /* Outsource everything to parallel variant */ parallel_vacuum_bulkdel_all_indexes(vacrel->pvs, old_live_tuples, vacrel->num_index_scans); + extvac_stats_end(vacrel->rel, &counters, &PgStat_VacuumRelationCounts); + accumulate_idxs_vacuum_statistics(vacrel, &PgStat_VacuumRelationCounts); + /* * Do a postcheck to consider applying wraparound failsafe now. Note * that parallel VACUUM only gets the precheck and this postcheck. @@ -2961,6 +3203,7 @@ lazy_check_wraparound_failsafe(LVRelState *vacrel) int64 progress_val[2] = {0, 0}; VacuumFailsafeActive = true; + vacrel->wraparound_failsafe_count ++; /* * Abandon use of a buffer access strategy to allow use of all of @@ -3043,10 +3286,20 @@ lazy_cleanup_all_indexes(LVRelState *vacrel) } else { + LVExtStatCounters counters; + PgStat_VacuumRelationCounts PgStat_VacuumRelationCounts; + + memset(&PgStat_VacuumRelationCounts, 0, sizeof(PgStat_VacuumRelationCounts)); + + extvac_stats_start(vacrel->rel, &counters); + /* Outsource everything to parallel variant */ parallel_vacuum_cleanup_all_indexes(vacrel->pvs, reltuples, vacrel->num_index_scans, estimated_count); + + extvac_stats_end(vacrel->rel, &counters, &PgStat_VacuumRelationCounts); + accumulate_idxs_vacuum_statistics(vacrel, &PgStat_VacuumRelationCounts); } /* Reset the progress counters */ @@ -3072,6 +3325,11 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, { IndexVacuumInfo ivinfo; LVSavedErrInfo saved_err_info; + LVExtStatCountersIdx extVacCounters; + PgStat_VacuumRelationCounts PgStat_VacuumRelationCounts; + + /* Set initial statistics values to gather vacuum statistics for the index */ + extvac_stats_start_idx(indrel, istat, &extVacCounters); ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; @@ -3090,6 +3348,7 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, */ Assert(vacrel->indname == NULL); vacrel->indname = pstrdup(RelationGetRelationName(indrel)); + vacrel->indoid = RelationGetRelid(indrel); update_vacuum_error_info(vacrel, &saved_err_info, VACUUM_ERRCB_PHASE_VACUUM_INDEX, InvalidBlockNumber, InvalidOffsetNumber); @@ -3098,6 +3357,16 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, istat = vac_bulkdel_one_index(&ivinfo, istat, vacrel->dead_items, vacrel->dead_items_info); + /* Make extended vacuum stats report for index */ + extvac_stats_end_idx(indrel, istat, &extVacCounters, &PgStat_VacuumRelationCounts); + + if (!ParallelVacuumIsActive(vacrel)) + accumulate_idxs_vacuum_statistics(vacrel, &PgStat_VacuumRelationCounts); + + pgstat_report_vacuum(RelationGetRelid(indrel), + indrel->rd_rel->relisshared, + 0, 0, 0, &PgStat_VacuumRelationCounts); + /* Revert to the previous phase information for error traceback */ restore_vacuum_error_info(vacrel, &saved_err_info); pfree(vacrel->indname); @@ -3122,6 +3391,11 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, { IndexVacuumInfo ivinfo; LVSavedErrInfo saved_err_info; + LVExtStatCountersIdx extVacCounters; + PgStat_VacuumRelationCounts PgStat_VacuumRelationCounts; + + /* Set initial statistics values to gather vacuum statistics for the index */ + extvac_stats_start_idx(indrel, istat, &extVacCounters); ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; @@ -3141,12 +3415,22 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, */ Assert(vacrel->indname == NULL); vacrel->indname = pstrdup(RelationGetRelationName(indrel)); + vacrel->indoid = RelationGetRelid(indrel); update_vacuum_error_info(vacrel, &saved_err_info, VACUUM_ERRCB_PHASE_INDEX_CLEANUP, InvalidBlockNumber, InvalidOffsetNumber); istat = vac_cleanup_one_index(&ivinfo, istat); + /* Make extended vacuum stats report for index */ + extvac_stats_end_idx(indrel, istat, &extVacCounters, &PgStat_VacuumRelationCounts); + if (!ParallelVacuumIsActive(vacrel)) + accumulate_idxs_vacuum_statistics(vacrel, &PgStat_VacuumRelationCounts); + + pgstat_report_vacuum(RelationGetRelid(indrel), + indrel->rd_rel->relisshared, + 0, 0, 0, &PgStat_VacuumRelationCounts); + /* Revert to the previous phase information for error traceback */ restore_vacuum_error_info(vacrel, &saved_err_info); pfree(vacrel->indname); @@ -3759,6 +4043,9 @@ vacuum_error_callback(void *arg) switch (errinfo->phase) { case VACUUM_ERRCB_PHASE_SCAN_HEAP: + if(geterrelevel() == ERROR) + pgstat_report_vacuum_error(errinfo->reloid, errinfo->rel->rd_rel->relisshared, PGSTAT_EXTVAC_TABLE); + if (BlockNumberIsValid(errinfo->blkno)) { if (OffsetNumberIsValid(errinfo->offnum)) @@ -3774,6 +4061,9 @@ vacuum_error_callback(void *arg) break; case VACUUM_ERRCB_PHASE_VACUUM_HEAP: + if(geterrelevel() == ERROR) + pgstat_report_vacuum_error(errinfo->reloid, errinfo->rel->rd_rel->relisshared, PGSTAT_EXTVAC_TABLE); + if (BlockNumberIsValid(errinfo->blkno)) { if (OffsetNumberIsValid(errinfo->offnum)) @@ -3789,16 +4079,25 @@ vacuum_error_callback(void *arg) break; case VACUUM_ERRCB_PHASE_VACUUM_INDEX: + if(geterrelevel() == ERROR) + pgstat_report_vacuum_error(errinfo->reloid, errinfo->rel->rd_rel->relisshared, PGSTAT_EXTVAC_INDEX); + errcontext("while vacuuming index \"%s\" of relation \"%s.%s\"", errinfo->indname, errinfo->relnamespace, errinfo->relname); break; case VACUUM_ERRCB_PHASE_INDEX_CLEANUP: + if(geterrelevel() == ERROR) + pgstat_report_vacuum_error(errinfo->reloid, errinfo->rel->rd_rel->relisshared, PGSTAT_EXTVAC_INDEX); + errcontext("while cleaning up index \"%s\" of relation \"%s.%s\"", errinfo->indname, errinfo->relnamespace, errinfo->relname); break; case VACUUM_ERRCB_PHASE_TRUNCATE: + if(geterrelevel() == ERROR) + pgstat_report_vacuum_error(errinfo->reloid, errinfo->rel->rd_rel->relisshared, PGSTAT_EXTVAC_TABLE); + if (BlockNumberIsValid(errinfo->blkno)) errcontext("while truncating relation \"%s.%s\" to %u blocks", errinfo->relnamespace, errinfo->relname, errinfo->blkno); diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c index 953ad4a4843..a21e77cd551 100644 --- a/src/backend/access/heap/visibilitymap.c +++ b/src/backend/access/heap/visibilitymap.c @@ -91,6 +91,7 @@ #include "access/xloginsert.h" #include "access/xlogutils.h" #include "miscadmin.h" +#include "pgstat.h" #include "port/pg_bitutils.h" #include "storage/bufmgr.h" #include "storage/smgr.h" @@ -160,6 +161,15 @@ visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags if (map[mapByte] & mask) { + /* + * As part of vacuum stats, track how often all-visible or all-frozen + * bits are cleared. + */ + if (map[mapByte] >> mapOffset & flags & VISIBILITYMAP_ALL_VISIBLE) + pgstat_count_vm_rev_all_visible(rel); + if (map[mapByte] >> mapOffset & flags & VISIBILITYMAP_ALL_FROZEN) + pgstat_count_vm_rev_all_frozen(rel); + map[mapByte] &= ~mask; MarkBufferDirty(vmbuf); diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index 733ef40ae7c..d8776ff1901 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -116,6 +116,9 @@ pg_atomic_uint32 *VacuumSharedCostBalance = NULL; pg_atomic_uint32 *VacuumActiveNWorkers = NULL; int VacuumCostBalanceLocal = 0; +/* Cumulative storage to report total vacuum delay time. */ +double VacuumDelayTime = 0; /* msec. */ + /* non-export function prototypes */ static List *expand_vacuum_rel(VacuumRelation *vrel, MemoryContext vac_context, int options); @@ -2533,6 +2536,7 @@ vacuum_delay_point(bool is_analyze) exit(1); VacuumCostBalance = 0; + VacuumDelayTime += msec; /* * Balance and update limit values for autovacuum workers. We must do diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c index 0feea1d30ec..b5461ec661b 100644 --- a/src/backend/commands/vacuumparallel.c +++ b/src/backend/commands/vacuumparallel.c @@ -868,6 +868,8 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, IndexBulkDeleteResult *istat = NULL; IndexBulkDeleteResult *istat_res; IndexVacuumInfo ivinfo; + LVExtStatCountersIdx extVacCounters; + PgStat_VacuumRelationCounts extVacReport; /* * Update the pointer to the corresponding bulk-deletion result if someone @@ -876,6 +878,9 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, if (indstats->istat_updated) istat = &(indstats->istat); + /* Set initial statistics values to gather vacuum statistics for the index */ + extvac_stats_start_idx(indrel, &(indstats->istat), &extVacCounters); + ivinfo.index = indrel; ivinfo.heaprel = pvs->heaprel; ivinfo.analyze_only = false; @@ -904,6 +909,12 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, RelationGetRelationName(indrel)); } + /* Make extended vacuum stats report for index */ + extvac_stats_end_idx(indrel, istat_res, &extVacCounters, &extVacReport); + pgstat_report_vacuum(RelationGetRelid(indrel), + indrel->rd_rel->relisshared, + 0, 0, 0, &extVacReport); + /* * Copy the index bulk-deletion result returned from ambulkdelete and * amvacuumcleanup to the DSM segment if it's the first cycle because they @@ -1054,6 +1065,7 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) /* Set cost-based vacuum delay */ VacuumUpdateCosts(); VacuumCostBalance = 0; + VacuumDelayTime = 0; VacuumCostBalanceLocal = 0; VacuumSharedCostBalance = &(shared->cost_balance); VacuumActiveNWorkers = &(shared->active_nworkers); diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index 69df741cbf6..33a4009f746 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -203,13 +203,39 @@ pgstat_drop_relation(Relation rel) } } +/* --------- + * pgstat_report_vacuum_error() - + * + * Tell the collector about an (auto)vacuum interruption. + * --------- + */ +void +pgstat_report_vacuum_error(Oid tableoid, bool shared, ExtVacReportType m_type) +{ + PgStat_VacuumRelationCounts params; + + if (!pgstat_track_counts) + return; + + if (set_report_vacuum_hook) + { + memset(¶ms, 0, sizeof(PgStat_VacuumRelationCounts)); + + params.common.interrupts_count++; + + (*set_report_vacuum_hook) (tableoid, shared, ¶ms); + } +} + +set_report_vacuum_hook_type set_report_vacuum_hook = NULL; + /* * Report that the table was just vacuumed and flush IO statistics. */ void pgstat_report_vacuum(Oid tableoid, bool shared, PgStat_Counter livetuples, PgStat_Counter deadtuples, - TimestampTz starttime) + TimestampTz starttime, PgStat_VacuumRelationCounts *params) { PgStat_EntryRef *entry_ref; PgStatShared_Relation *shtabentry; @@ -235,6 +261,11 @@ pgstat_report_vacuum(Oid tableoid, bool shared, tabentry->live_tuples = livetuples; tabentry->dead_tuples = deadtuples; + if (set_report_vacuum_hook) + { + (*set_report_vacuum_hook) (tableoid, shared, params); + } + /* * It is quite possible that a non-aggressive VACUUM ended up skipping * various pages, however, we'll zero the insert counter here regardless. @@ -881,6 +912,9 @@ pgstat_relation_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) tabentry->blocks_fetched += lstats->counts.blocks_fetched; tabentry->blocks_hit += lstats->counts.blocks_hit; + tabentry->rev_all_frozen_pages += lstats->counts.rev_all_frozen_pages; + tabentry->rev_all_visible_pages += lstats->counts.rev_all_visible_pages; + /* Clamp live_tuples in case of negative delta_live_tuples */ tabentry->live_tuples = Max(tabentry->live_tuples, 0); /* Likewise for dead_tuples */ diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index c756c2bebaa..9482bf80721 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -106,6 +106,12 @@ PG_STAT_GET_RELENTRY_INT64(tuples_updated) /* pg_stat_get_vacuum_count */ PG_STAT_GET_RELENTRY_INT64(vacuum_count) +/* pg_stat_get_rev_frozen_pages */ +PG_STAT_GET_RELENTRY_INT64(rev_all_frozen_pages) + +/* pg_stat_get_rev_all_visible_pages */ +PG_STAT_GET_RELENTRY_INT64(rev_all_visible_pages) + #define PG_STAT_GET_RELENTRY_FLOAT8(stat) \ Datum \ CppConcat(pg_stat_get_,stat)(PG_FUNCTION_ARGS) \ diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index b7b9692f8c8..f0ecf86e514 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -1627,6 +1627,19 @@ getinternalerrposition(void) return edata->internalpos; } +/* + * Return elevel of errors + */ +int +geterrelevel(void) +{ + ErrorData *edata = &errordata[errordata_stack_depth]; + + /* we don't bother incrementing recursion_depth */ + CHECK_STACK_DEPTH(); + + return edata->elevel; +} /* * Functions to allow construction of error message strings separately from diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 118d6da1ace..e0c7cf29b3a 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -12576,4 +12576,12 @@ proargnames => '{pid,io_id,io_generation,state,operation,off,length,target,handle_data_len,raw_result,result,target_desc,f_sync,f_localmem,f_buffered}', prosrc => 'pg_get_aios' }, + { oid => '8002', descr => 'statistics: number of times the all-visible pages in the visibility map was removed for pages of table', + proname => 'pg_stat_get_rev_all_visible_pages', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', + prosrc => 'pg_stat_get_rev_all_visible_pages' }, + { oid => '8003', descr => 'statistics: number of times the all-frozen pages in the visibility map was removed for pages of table', + proname => 'pg_stat_get_rev_all_frozen_pages', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', + prosrc => 'pg_stat_get_rev_all_frozen_pages' }, ] diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 14eeccbd718..bc9df1433c2 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -25,6 +25,7 @@ #include "storage/buf.h" #include "storage/lock.h" #include "utils/relcache.h" +#include "pgstat.h" /* * Flags for amparallelvacuumoptions to control the participation of bulkdelete @@ -295,6 +296,26 @@ typedef struct VacDeadItemsInfo int64 num_items; /* current # of entries */ } VacDeadItemsInfo; +/* + * Counters and usage data for extended stats tracking. + */ +typedef struct LVExtStatCounters +{ + TimestampTz starttime; + WalUsage walusage; + BufferUsage bufusage; + double VacuumDelayTime; + PgStat_Counter blocks_fetched; + PgStat_Counter blocks_hit; +} LVExtStatCounters; + +typedef struct LVExtStatCountersIdx +{ + LVExtStatCounters common; + int64 pages_deleted; + int64 tuples_removed; +} LVExtStatCountersIdx; + /* GUC parameters */ extern PGDLLIMPORT int default_statistics_target; /* PGDLLIMPORT for PostGIS */ extern PGDLLIMPORT int vacuum_freeze_min_age; @@ -327,6 +348,7 @@ extern PGDLLIMPORT double vacuum_max_eager_freeze_failure_rate; extern PGDLLIMPORT pg_atomic_uint32 *VacuumSharedCostBalance; extern PGDLLIMPORT pg_atomic_uint32 *VacuumActiveNWorkers; extern PGDLLIMPORT int VacuumCostBalanceLocal; +extern PGDLLIMPORT double VacuumDelayTime; extern PGDLLIMPORT bool VacuumFailsafeActive; extern PGDLLIMPORT double vacuum_cost_delay; @@ -407,4 +429,8 @@ extern double anl_random_fract(void); extern double anl_init_selection_state(int n); extern double anl_get_next_S(double t, int n, double *stateptr); +extern void extvac_stats_start_idx(Relation rel, IndexBulkDeleteResult *stats, + LVExtStatCountersIdx *counters); +extern void extvac_stats_end_idx(Relation rel, IndexBulkDeleteResult *stats, + LVExtStatCountersIdx *counters, PgStat_VacuumRelationCounts *report); #endif /* VACUUM_H */ diff --git a/src/include/pgstat.h b/src/include/pgstat.h index f402b17295c..ed6b3dc1d6f 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -100,6 +100,98 @@ typedef struct PgStat_FunctionCallUsage instr_time start; } PgStat_FunctionCallUsage; + +/* Type of ExtVacReport */ +typedef enum ExtVacReportType +{ + PGSTAT_EXTVAC_INVALID = 0, + PGSTAT_EXTVAC_TABLE = 1, + PGSTAT_EXTVAC_INDEX = 2, + PGSTAT_EXTVAC_DB = 3, +} ExtVacReportType; + +typedef struct PgStat_CommonCounts +{ + /* blocks */ + int64 total_blks_read; + int64 total_blks_hit; + int64 total_blks_dirtied; + int64 total_blks_written; + + /* heap blocks */ + int64 blks_fetched; + int64 blks_hit; + + /* WAL */ + int64 wal_records; + int64 wal_fpi; + uint64 wal_bytes; + + /* Time */ + double blk_read_time; + double blk_write_time; + double delay_time; + double total_time; + + /* failsafe */ + int32 wraparound_failsafe_count; + int32 interrupts_count; +} PgStat_CommonCounts; + +/* ---------- + * + * PgStat_VacuumRelationCounts + * + * Additional statistics of vacuum processing over a relation. + * pages_removed is the amount by which the physically shrank, + * if any (ie the change in its total size on disk) + * pages_deleted refer to free space within the index file + * ---------- + */ +typedef struct PgStat_VacuumRelationCounts +{ + PgStat_CommonCounts common; + + ExtVacReportType type; /* heap, index, etc. */ + + /* ---------- + * + * There are separate metrics of statistic for tables and indexes, + * which collect during vacuum. + * The union operator allows to combine these statistics + * so that each metric is assigned to a specific class of collected statistics. + * Such a combined structure was called per_type_stats. + * The name of the structure itself is not used anywhere, + * it exists only for understanding the code. + * ---------- + */ + union + { + struct + { + int64 tuples_frozen; /* tuples frozen up by vacuum */ + int64 recently_dead_tuples; /* deleted tuples that are still visible to some transaction */ + int64 missed_dead_tuples; /* tuples not pruned by vacuum due to failure to get a cleanup lock */ + int64 pages_scanned; /* heap pages examined (not skipped by VM) */ + int64 pages_removed; /* heap pages removed by vacuum "truncation" */ + int64 pages_frozen; /* pages marked in VM as frozen */ + int64 pages_all_visible; /* pages marked in VM as all-visible */ + int64 vm_new_frozen_pages; /* pages marked in VM as frozen */ + int64 vm_new_visible_pages; /* pages marked in VM as all-visible */ + int64 vm_new_visible_frozen_pages; /* pages marked in VM as all-visible and frozen */ + int64 missed_dead_pages; /* pages with missed dead tuples */ + int64 index_vacuum_count; /* number of index vacuumings */ + int64 tuples_deleted; + } table; + struct + { + int64 tuples_deleted; + int64 pages_deleted; /* number of pages deleted by vacuum */ + } index; + } /* per_type_stats */; +} PgStat_VacuumRelationCounts; + + /* ---------- * PgStat_BackendSubEntry Non-flushed subscription stats. * ---------- @@ -153,6 +245,9 @@ typedef struct PgStat_TableCounts PgStat_Counter blocks_fetched; PgStat_Counter blocks_hit; + + PgStat_Counter rev_all_visible_pages; + PgStat_Counter rev_all_frozen_pages; } PgStat_TableCounts; /* ---------- @@ -211,7 +306,7 @@ typedef struct PgStat_TableXactStatus * ------------------------------------------------------------ */ -#define PGSTAT_FILE_FORMAT_ID 0x01A5BCB7 +#define PGSTAT_FILE_FORMAT_ID 0x01A5BCB8 typedef struct PgStat_ArchiverStats { @@ -453,6 +548,9 @@ typedef struct PgStat_StatTabEntry PgStat_Counter total_autovacuum_time; PgStat_Counter total_analyze_time; PgStat_Counter total_autoanalyze_time; + + PgStat_Counter rev_all_visible_pages; + PgStat_Counter rev_all_frozen_pages; } PgStat_StatTabEntry; /* ------ @@ -660,10 +758,11 @@ extern void pgstat_unlink_relation(Relation rel); extern void pgstat_report_vacuum(Oid tableoid, bool shared, PgStat_Counter livetuples, PgStat_Counter deadtuples, - TimestampTz starttime); + TimestampTz starttime, PgStat_VacuumRelationCounts *params); extern void pgstat_report_analyze(Relation rel, PgStat_Counter livetuples, PgStat_Counter deadtuples, bool resetcounter, TimestampTz starttime); +extern void pgstat_report_vacuum_error(Oid tableoid, bool shared, ExtVacReportType m_type); /* * If stats are enabled, but pending data hasn't been prepared yet, call @@ -711,6 +810,17 @@ extern void pgstat_report_analyze(Relation rel, if (pgstat_should_count_relation(rel)) \ (rel)->pgstat_info->counts.blocks_hit++; \ } while (0) +/* accumulate unfrozen all-visible and all-frozen pages */ +#define pgstat_count_vm_rev_all_visible(rel) \ + do { \ + if (pgstat_should_count_relation(rel)) \ + (rel)->pgstat_info->counts.rev_all_visible_pages++; \ + } while (0) +#define pgstat_count_vm_rev_all_frozen(rel) \ + do { \ + if (pgstat_should_count_relation(rel)) \ + (rel)->pgstat_info->counts.rev_all_frozen_pages++; \ + } while (0) extern void pgstat_count_heap_insert(Relation rel, PgStat_Counter n); extern void pgstat_count_heap_update(Relation rel, bool hot, bool newpage); @@ -838,4 +948,9 @@ extern PGDLLIMPORT PgStat_Counter pgStatTransactionIdleTime; /* updated by the traffic cop and in errfinish() */ extern PGDLLIMPORT SessionEndType pgStatSessionEndCause; +/* Hook for plugins to get control in set_rel_pathlist() */ +typedef void (*set_report_vacuum_hook_type) (Oid tableoid, bool shared, PgStat_VacuumRelationCounts *params); +extern PGDLLIMPORT set_report_vacuum_hook_type set_report_vacuum_hook; + + #endif /* PGSTAT_H */ diff --git a/src/include/utils/elog.h b/src/include/utils/elog.h index 675f4f5f469..356dadd6b0a 100644 --- a/src/include/utils/elog.h +++ b/src/include/utils/elog.h @@ -230,6 +230,7 @@ extern int geterrcode(void); extern int geterrposition(void); extern int getinternalerrposition(void); +extern int geterrelevel(void); /*---------- * Old-style error reporting API: to be used in this way: diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 35e8aad7701..4731ca2121e 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1833,7 +1833,9 @@ pg_stat_all_tables| SELECT c.oid AS relid, pg_stat_get_total_vacuum_time(c.oid) AS total_vacuum_time, pg_stat_get_total_autovacuum_time(c.oid) AS total_autovacuum_time, pg_stat_get_total_analyze_time(c.oid) AS total_analyze_time, - pg_stat_get_total_autoanalyze_time(c.oid) AS total_autoanalyze_time + pg_stat_get_total_autoanalyze_time(c.oid) AS total_autoanalyze_time, + pg_stat_get_rev_all_frozen_pages(c.oid) AS rev_all_frozen_pages, + pg_stat_get_rev_all_visible_pages(c.oid) AS rev_all_visible_pages FROM ((pg_class c LEFT JOIN pg_index i ON ((c.oid = i.indrelid))) LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))) @@ -2232,7 +2234,9 @@ pg_stat_sys_tables| SELECT relid, total_vacuum_time, total_autovacuum_time, total_analyze_time, - total_autoanalyze_time + total_autoanalyze_time, + rev_all_frozen_pages, + rev_all_visible_pages FROM pg_stat_all_tables WHERE ((schemaname = ANY (ARRAY['pg_catalog'::name, 'information_schema'::name])) OR (schemaname ~ '^pg_toast'::text)); pg_stat_user_functions| SELECT p.oid AS funcid, @@ -2284,7 +2288,9 @@ pg_stat_user_tables| SELECT relid, total_vacuum_time, total_autovacuum_time, total_analyze_time, - total_autoanalyze_time + total_autoanalyze_time, + rev_all_frozen_pages, + rev_all_visible_pages FROM pg_stat_all_tables WHERE ((schemaname <> ALL (ARRAY['pg_catalog'::name, 'information_schema'::name])) AND (schemaname !~ '^pg_toast'::text)); pg_stat_wal| SELECT wal_records, -- 2.34.1