diff --git a/src/backend/access/brin/brin.c b/src/backend/access/brin/brin.c index f521bb9635..97b2f8bc13 100644 --- a/src/backend/access/brin/brin.c +++ b/src/backend/access/brin/brin.c @@ -39,6 +39,8 @@ #include "utils/index_selfuncs.h" #include "utils/memutils.h" #include "utils/rel.h" +#include "commands/progress.h" +#include "pgstat.h" /* @@ -77,7 +79,7 @@ static void brinsummarize(Relation index, Relation heapRel, BlockNumber pageRang static void form_and_insert_tuple(BrinBuildState *state); static void union_tuples(BrinDesc *bdesc, BrinMemTuple *a, BrinTuple *b); -static void brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy); +static void brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy, bool report_progress); static bool add_values_to_range(Relation idxRel, BrinDesc *bdesc, BrinMemTuple *dtup, Datum *values, bool *nulls); static bool check_null_keys(BrinValues *bval, ScanKey *nullkeys, int nnullkeys); @@ -953,7 +955,7 @@ brinvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats) heapRel = table_open(IndexGetRelation(RelationGetRelid(info->index), false), AccessShareLock); - brin_vacuum_scan(info->index, info->strategy); + brin_vacuum_scan(info->index, info->strategy, info->report_progress); brinsummarize(info->index, heapRel, BRIN_ALL_BLOCKRANGES, false, &stats->num_index_tuples, &stats->num_index_tuples); @@ -1635,16 +1637,24 @@ union_tuples(BrinDesc *bdesc, BrinMemTuple *a, BrinTuple *b) * and such. */ static void -brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy) +brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy, bool report_progress) { BlockNumber nblocks; BlockNumber blkno; + const int initprog_index[] = { + PROGRESS_SCAN_BLOCKS_DONE, + PROGRESS_SCAN_BLOCKS_TOTAL + }; + int64 initprog_val[2]; /* * Scan the index in physical order, and clean up any possible mess in * each page. */ nblocks = RelationGetNumberOfBlocks(idxrel); + if (report_progress) + pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL, + nblocks); for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -1656,9 +1666,20 @@ brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy) brin_page_cleanup(idxrel, buf); + if (report_progress) + pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE, + blkno + 1); + ReleaseBuffer(buf); } + if (report_progress) + { + initprog_val[0] = 0; + initprog_val[1] = 0; + pgstat_progress_update_multi_param(2, initprog_index, initprog_val); + } + /* * Update all upper pages in the index's FSM, as well. This ensures not * only that we propagate leaf-page FSM updates made by brin_page_cleanup, diff --git a/src/backend/access/gin/ginvacuum.c b/src/backend/access/gin/ginvacuum.c index a276eb020b..714586040a 100644 --- a/src/backend/access/gin/ginvacuum.c +++ b/src/backend/access/gin/ginvacuum.c @@ -24,6 +24,8 @@ #include "storage/lmgr.h" #include "storage/predicate.h" #include "utils/memutils.h" +#include "commands/progress.h" +#include "pgstat.h" struct GinVacuumState { @@ -571,6 +573,14 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, Buffer buffer; BlockNumber rootOfPostingTree[BLCKSZ / (sizeof(IndexTupleData) + sizeof(ItemId))]; uint32 nRoot; + BlockNumber num_pages; + bool needLock; + int blocks_scanned = 0; + const int initprog_index[] = { + PROGRESS_SCAN_BLOCKS_DONE, + PROGRESS_SCAN_BLOCKS_TOTAL + }; + int64 initprog_val[2]; gvs.tmpCxt = AllocSetContextCreate(CurrentMemoryContext, "Gin vacuum temporary context", @@ -635,6 +645,19 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, RBM_NORMAL, info->strategy); } + needLock = !RELATION_IS_LOCAL(index); + + /* Get the current relation length */ + if (needLock) + LockRelationForExtension(index, ExclusiveLock); + num_pages = RelationGetNumberOfBlocks(index); + if (needLock) + UnlockRelationForExtension(index, ExclusiveLock); + + if (info->report_progress) + pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL, + num_pages); + /* right now we found leftmost page in entry's BTree */ for (;;) @@ -676,9 +699,20 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno, RBM_NORMAL, info->strategy); + blocks_scanned++; + if (info->report_progress) + pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE, + blocks_scanned + 1); LockBuffer(buffer, GIN_EXCLUSIVE); } + if (info->report_progress) + { + initprog_val[0] = 0; + initprog_val[1] = 0; + pgstat_progress_update_multi_param(2, initprog_index, initprog_val); + } + MemoryContextDelete(gvs.tmpCxt); return gvs.result; @@ -694,6 +728,12 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats) BlockNumber totFreePages; GinState ginstate; GinStatsData idxStat; + int blocks_scanned = 0; + const int initprog_index[] = { + PROGRESS_SCAN_BLOCKS_DONE, + PROGRESS_SCAN_BLOCKS_TOTAL + }; + int64 initprog_val[2]; /* * In an autovacuum analyze, we want to clean up pending insertions. @@ -744,6 +784,9 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats) totFreePages = 0; + if (info->report_progress) + pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL, + npages); for (blkno = GIN_ROOT_BLKNO; blkno < npages; blkno++) { Buffer buffer; @@ -774,9 +817,21 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats) idxStat.nEntries += PageGetMaxOffsetNumber(page); } + blocks_scanned++; + if (info->report_progress) + pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE, + blocks_scanned + 1); + UnlockReleaseBuffer(buffer); } + if (info->report_progress) + { + initprog_val[0] = 0; + initprog_val[1] = 0; + pgstat_progress_update_multi_param(2, initprog_index, initprog_val); + } + /* Update the metapage with accurate page and entry counts */ idxStat.nTotalPages = npages; ginUpdateStats(info->index, &idxStat, false); diff --git a/src/backend/access/gist/gistvacuum.c b/src/backend/access/gist/gistvacuum.c index 0663193531..e7d13c9eb6 100644 --- a/src/backend/access/gist/gistvacuum.c +++ b/src/backend/access/gist/gistvacuum.c @@ -23,6 +23,8 @@ #include "storage/indexfsm.h" #include "storage/lmgr.h" #include "utils/memutils.h" +#include "commands/progress.h" +#include "pgstat.h" /* Working state needed by gistbulkdelete */ typedef struct @@ -131,6 +133,11 @@ gistvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, bool needLock; BlockNumber blkno; MemoryContext oldctx; + const int initprog_index[] = { + PROGRESS_SCAN_BLOCKS_DONE, + PROGRESS_SCAN_BLOCKS_TOTAL + }; + int64 initprog_val[2]; /* * Reset fields that track information about the entire index now. This @@ -215,9 +222,26 @@ gistvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, /* Quit if we've scanned the whole relation */ if (blkno >= num_pages) break; + + if (info->report_progress) + pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL, + num_pages); + /* Iterate over pages, then loop back to recheck length */ for (; blkno < num_pages; blkno++) + { gistvacuumpage(&vstate, blkno, blkno); + if (info->report_progress) + pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE, + blkno + 1); + } + } + + if (info->report_progress) + { + initprog_val[0] = 0; + initprog_val[1] = 0; + pgstat_progress_update_multi_param(2, initprog_index, initprog_val); } /* diff --git a/src/backend/access/hash/hash.c b/src/backend/access/hash/hash.c index 81c7da7ec6..2b4eed6aae 100644 --- a/src/backend/access/hash/hash.c +++ b/src/backend/access/hash/hash.c @@ -31,6 +31,7 @@ #include "utils/builtins.h" #include "utils/index_selfuncs.h" #include "utils/rel.h" +#include "storage/lmgr.h" /* Working state for hashbuild and its callback */ typedef struct @@ -469,9 +470,21 @@ hashbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, Buffer metabuf = InvalidBuffer; HashMetaPage metap; HashMetaPage cachedmetap; + int blocks_scanned; + int bucket_blocks_scanned; + BlockNumber num_pages; + bool needLock; + const int initprog_index[] = { + PROGRESS_SCAN_BLOCKS_DONE, + PROGRESS_SCAN_BLOCKS_TOTAL + }; + int64 initprog_val[2]; + tuples_removed = 0; num_index_tuples = 0; + blocks_scanned = 0; + bucket_blocks_scanned = 0; /* * We need a copy of the metapage so that we can use its hashm_spares[] @@ -489,6 +502,19 @@ hashbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, cur_bucket = 0; cur_maxbucket = orig_maxbucket; + needLock = !RELATION_IS_LOCAL(rel); + + /* Get the current relation length */ + if (needLock) + LockRelationForExtension(rel, ExclusiveLock); + num_pages = RelationGetNumberOfBlocks(rel); + if (needLock) + UnlockRelationForExtension(rel, ExclusiveLock); + + if (info->report_progress) + pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL, + num_pages); + loop_top: while (cur_bucket <= cur_maxbucket) { @@ -504,6 +530,7 @@ loop_top: bucket_blkno = BUCKET_TO_BLKNO(cachedmetap, cur_bucket); blkno = bucket_blkno; + blocks_scanned++; /* * We need to acquire a cleanup lock on the primary bucket page to out @@ -550,10 +577,14 @@ loop_top: cachedmetap->hashm_highmask, cachedmetap->hashm_lowmask, &tuples_removed, &num_index_tuples, split_cleanup, - callback, callback_state); + callback, callback_state, info->report_progress, + &bucket_blocks_scanned); _hash_dropbuf(rel, bucket_buf); + pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE, + blocks_scanned + bucket_blocks_scanned); + /* Advance to next bucket */ cur_bucket++; } @@ -633,6 +664,13 @@ loop_top: stats->tuples_removed += tuples_removed; /* hashvacuumcleanup will fill in num_pages */ + if (info->report_progress) + { + initprog_val[0] = 0; + initprog_val[1] = 0; + pgstat_progress_update_multi_param(2, initprog_index, initprog_val); + } + return stats; } @@ -686,7 +724,8 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf, uint32 maxbucket, uint32 highmask, uint32 lowmask, double *tuples_removed, double *num_index_tuples, bool split_cleanup, - IndexBulkDeleteCallback callback, void *callback_state) + IndexBulkDeleteCallback callback, void *callback_state, + bool report_progress, int *bucket_blocks_scanned) { BlockNumber blkno; Buffer buf; @@ -718,6 +757,8 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf, page = BufferGetPage(buf); opaque = (HashPageOpaque) PageGetSpecialPointer(page); + bucket_blocks_scanned++; + /* Scan each tuple in page */ maxoffno = PageGetMaxOffsetNumber(page); for (offno = FirstOffsetNumber; @@ -916,4 +957,5 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf, bstrategy); else LockBuffer(bucket_buf, BUFFER_LOCK_UNLOCK); + } diff --git a/src/backend/access/hash/hashpage.c b/src/backend/access/hash/hashpage.c index 159646c7c3..538d153df7 100644 --- a/src/backend/access/hash/hashpage.c +++ b/src/backend/access/hash/hashpage.c @@ -758,7 +758,7 @@ restart_expand: hashbucketcleanup(rel, old_bucket, buf_oblkno, start_oblkno, NULL, maxbucket, highmask, lowmask, NULL, NULL, true, - NULL, NULL); + NULL, NULL, NULL, NULL); _hash_dropbuf(rel, buf_oblkno); @@ -1326,7 +1326,7 @@ _hash_splitbucket(Relation rel, hashbucketcleanup(rel, obucket, bucket_obuf, BufferGetBlockNumber(bucket_obuf), NULL, maxbucket, highmask, lowmask, NULL, NULL, true, - NULL, NULL); + NULL, NULL, NULL, NULL); } else { diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index db6becfed5..ca26d5344e 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -517,6 +517,9 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); + pgstat_progress_update_param(PROGRESS_VACUUM_LEADER_PID, + MyProcPid); + vacuum_set_xid_limits(rel, params->freeze_min_age, params->freeze_table_age, @@ -3021,12 +3024,17 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, IndexVacuumInfo ivinfo; PGRUsage ru0; LVSavedErrInfo saved_err_info; + const int initprog_index[] = { + PROGRESS_VACUUM_PHASE, + PROGRESS_VACUUM_CURRENT_INDRELID + }; + int64 initprog_val[2]; pg_rusage_init(&ru0); ivinfo.index = indrel; ivinfo.analyze_only = false; - ivinfo.report_progress = false; + ivinfo.report_progress = true; ivinfo.estimated_count = true; ivinfo.message_level = elevel; ivinfo.num_heap_tuples = reltuples; @@ -3044,10 +3052,19 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, VACUUM_ERRCB_PHASE_VACUUM_INDEX, InvalidBlockNumber, InvalidOffsetNumber); + /* Report that we're vacuuming the index, advertising the indrelid */ + initprog_val[0] = PROGRESS_VACUUM_PHASE_VACUUM_INDEX; + initprog_val[1] = RelationGetRelid(indrel); + pgstat_progress_update_multi_param(2, initprog_index, initprog_val); + /* Do bulk deletion */ istat = index_bulk_delete(&ivinfo, istat, lazy_tid_reaped, (void *) vacrel->dead_items); + /* Report that we're done vacuuming the index */ + pgstat_progress_update_param(PROGRESS_VACUUM_CURRENT_INDRELID, + 0); + ereport(elevel, (errmsg("scanned index \"%s\" to remove %d row versions", vacrel->indname, vacrel->dead_items->num_items), @@ -3078,12 +3095,17 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, IndexVacuumInfo ivinfo; PGRUsage ru0; LVSavedErrInfo saved_err_info; + const int initprog_index[] = { + PROGRESS_VACUUM_PHASE, + PROGRESS_VACUUM_CURRENT_INDRELID + }; + int64 initprog_val[2]; pg_rusage_init(&ru0); ivinfo.index = indrel; ivinfo.analyze_only = false; - ivinfo.report_progress = false; + ivinfo.report_progress = true; ivinfo.estimated_count = estimated_count; ivinfo.message_level = elevel; @@ -3102,8 +3124,18 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, VACUUM_ERRCB_PHASE_INDEX_CLEANUP, InvalidBlockNumber, InvalidOffsetNumber); + /* Report that we're cleaning the index, advertising the indrelid */ + initprog_val[0] = PROGRESS_VACUUM_PHASE_INDEX_CLEANUP; + initprog_val[1] = RelationGetRelid(indrel); + pgstat_progress_update_multi_param(2, initprog_index, initprog_val); + istat = index_vacuum_cleanup(&ivinfo, istat); + /* Report that we're done cleaning the index */ + initprog_val[0] = 0; + initprog_val[1] = 0; + pgstat_progress_update_multi_param(2, initprog_index, initprog_val); + if (istat) { ereport(elevel, @@ -4145,6 +4177,7 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) char *sharedquery; LVRelState vacrel; ErrorContextCallback errcallback; + PGPROC *leader = MyProc->lockGroupLeader; /* * A parallel vacuum worker must have only PROC_IN_VACUUM flag since we @@ -4170,6 +4203,14 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) */ rel = table_open(lvshared->relid, ShareUpdateExclusiveLock); + /* + *Track progress of current index being vacuumed + */ + pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, + RelationGetRelid(rel)); + + pgstat_progress_update_param(PROGRESS_VACUUM_LEADER_PID, leader->pid); + /* * Open all indexes. indrels are sorted in order by OID, which should be * matched to the leader's one. @@ -4241,6 +4282,7 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) vac_close_indexes(nindexes, indrels, RowExclusiveLock); table_close(rel, ShareUpdateExclusiveLock); FreeAccessStrategy(vacrel.bstrategy); + pgstat_progress_end_command(); pfree(vacrel.indstats); } diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c index dfce06dc49..15c11fcab8 100644 --- a/src/backend/access/nbtree/nbtree.c +++ b/src/backend/access/nbtree/nbtree.c @@ -909,6 +909,11 @@ btvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, BlockNumber num_pages; BlockNumber scanblkno; bool needLock; + const int initprog_index[] = { + PROGRESS_SCAN_BLOCKS_DONE, + PROGRESS_SCAN_BLOCKS_TOTAL + }; + int64 initprog_val[2]; /* * Reset fields that track information about the entire index now. This @@ -997,10 +1002,17 @@ btvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, btvacuumpage(&vstate, scanblkno); if (info->report_progress) pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE, - scanblkno); + scanblkno + 1); } } + if (info->report_progress) + { + initprog_val[0] = 0; + initprog_val[1] = 0; + pgstat_progress_update_multi_param(2, initprog_index, initprog_val); + } + /* Set statistics num_pages field to final size of index */ stats->num_pages = num_pages; diff --git a/src/backend/access/spgist/spgvacuum.c b/src/backend/access/spgist/spgvacuum.c index 76fb0374c4..c71345fceb 100644 --- a/src/backend/access/spgist/spgvacuum.c +++ b/src/backend/access/spgist/spgvacuum.c @@ -27,6 +27,8 @@ #include "storage/indexfsm.h" #include "storage/lmgr.h" #include "utils/snapmgr.h" +#include "commands/progress.h" +#include "pgstat.h" /* Entry in pending-list of TIDs we need to revisit */ @@ -797,6 +799,12 @@ spgvacuumscan(spgBulkDeleteState *bds) bool needLock; BlockNumber num_pages, blkno; + int blocks_scanned = 0; + const int initprog_index[] = { + PROGRESS_SCAN_BLOCKS_DONE, + PROGRESS_SCAN_BLOCKS_TOTAL + }; + int64 initprog_val[2]; /* Finish setting up spgBulkDeleteState */ initSpGistState(&bds->spgstate, index); @@ -836,6 +844,11 @@ spgvacuumscan(spgBulkDeleteState *bds) /* Quit if we've scanned the whole relation */ if (blkno >= num_pages) break; + + if (bds->info->report_progress) + pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL, + num_pages); + /* Iterate over pages, then loop back to recheck length */ for (; blkno < num_pages; blkno++) { @@ -843,9 +856,21 @@ spgvacuumscan(spgBulkDeleteState *bds) /* empty the pending-list after each page */ if (bds->pendingList != NULL) spgprocesspending(bds); + + blocks_scanned++; + if (bds->info->report_progress) + pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE, + blocks_scanned + 1); } } + if (bds->info->report_progress) + { + initprog_val[0] = 0; + initprog_val[1] = 0; + pgstat_progress_update_multi_param(2, initprog_index, initprog_val); + } + /* Propagate local lastUsedPages cache to metablock */ SpGistUpdateMetaPage(index); diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 61b515cdb8..9e0dc39314 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1128,6 +1128,17 @@ CREATE VIEW pg_stat_progress_vacuum AS FROM pg_stat_get_progress_info('VACUUM') AS S LEFT JOIN pg_database D ON S.datid = D.oid; +CREATE VIEW pg_stat_progress_vacuum_worker AS + SELECT + S.pid, + S.param9 leader_pid, + S.param8 AS indrelid, + S.param16 index_blks_total, + S.param17 AS index_blks_scanned + FROM pg_stat_get_progress_info('VACUUM') AS S + LEFT JOIN pg_database D ON S.datid = D.oid + WHERE S.param8 > 0; + CREATE VIEW pg_stat_progress_cluster AS SELECT S.pid AS pid, diff --git a/src/include/access/hash.h b/src/include/access/hash.h index 1cce865be2..c00bb76e3e 100644 --- a/src/include/access/hash.h +++ b/src/include/access/hash.h @@ -478,6 +478,7 @@ extern void hashbucketcleanup(Relation rel, Bucket cur_bucket, uint32 maxbucket, uint32 highmask, uint32 lowmask, double *tuples_removed, double *num_index_tuples, bool split_cleanup, - IndexBulkDeleteCallback callback, void *callback_state); + IndexBulkDeleteCallback callback, void *callback_state, + bool report_progress, int *bucket_blocks_scanned); #endif /* HASH_H */ diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h index d7bf16368b..4387a7c1f1 100644 --- a/src/include/commands/progress.h +++ b/src/include/commands/progress.h @@ -25,6 +25,8 @@ #define PROGRESS_VACUUM_NUM_INDEX_VACUUMS 4 #define PROGRESS_VACUUM_MAX_DEAD_TUPLES 5 #define PROGRESS_VACUUM_NUM_DEAD_TUPLES 6 +#define PROGRESS_VACUUM_CURRENT_INDRELID 7 +#define PROGRESS_VACUUM_LEADER_PID 8 /* Phases of vacuum (as advertised via PROGRESS_VACUUM_PHASE) */ #define PROGRESS_VACUUM_PHASE_SCAN_HEAP 1