From 35aac7704eabeaefd5ab76bb18c0e68d29388be1 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Sun, 25 Aug 2024 17:09:21 +0300 Subject: [PATCH 2/4] Machinery for grabbing an extended vacuum statistics on heap and index relations. Remember, statistic on heap and index relations a bit different (see ExtVacReport to find out more information). The concept of the ExtVacReport structure has been complicated to store statistic information for two kinds of relations: for heap and index relations. ExtVacReportType variable helps to determine what the kind is considering now. --- src/backend/access/heap/vacuumlazy.c | 99 +++++++++-- src/backend/catalog/system_views.sql | 41 +++++ src/backend/utils/activity/pgstat.c | 45 +++-- src/backend/utils/activity/pgstat_relation.c | 3 +- src/backend/utils/adt/pgstatfuncs.c | 99 ++++++----- src/include/catalog/pg_proc.dat | 9 + src/include/pgstat.h | 53 ++++-- .../vacuum-extending-in-repetable-read.out | 7 +- .../vacuum-extending-in-repetable-read.spec | 2 +- src/test/regress/expected/opr_sanity.out | 7 +- src/test/regress/expected/rules.out | 26 +++ .../expected/vacuum_index_statistics.out | 158 ++++++++++++++++++ .../expected/vacuum_tables_statistics.out | 3 +- src/test/regress/parallel_schedule | 1 + .../regress/sql/vacuum_index_statistics.sql | 128 ++++++++++++++ 15 files changed, 600 insertions(+), 81 deletions(-) create mode 100644 src/test/regress/expected/vacuum_index_statistics.out create mode 100644 src/test/regress/sql/vacuum_index_statistics.sql diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 3941ae26f2d..4e2ae78d255 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -168,6 +168,7 @@ typedef struct LVRelState char *dbname; char *relnamespace; Oid reloid; + Oid indoid; char *relname; char *indname; /* Current index name */ BlockNumber blkno; /* used only for heap operations */ @@ -246,6 +247,13 @@ typedef struct LVExtStatCounters PgStat_Counter blocks_hit; } LVExtStatCounters; +typedef struct LVExtStatCountersIdx +{ + LVExtStatCounters common; + int64 pages_deleted; + int64 tuples_removed; +} LVExtStatCountersIdx; + /* non-export function prototypes */ static void lazy_scan_heap(LVRelState *vacrel); static bool heap_vac_scan_next_block(LVRelState *vacrel, BlockNumber *blkno, @@ -408,6 +416,46 @@ extvac_stats_end(Relation rel, LVExtStatCounters *counters, rel->pgstat_info->counts.blocks_hit - counters->blocks_hit; } +static void +extvac_stats_start_idx(Relation rel, IndexBulkDeleteResult *stats, + LVExtStatCountersIdx *counters) +{ + extvac_stats_start(rel, &counters->common); + counters->pages_deleted = counters->tuples_removed = 0; + + if (stats != NULL) + { + /* + * XXX: Why do we need this code here? If it is needed, I feel lack of + * comments, describing the reason. + */ + counters->tuples_removed = stats->tuples_removed; + counters->pages_deleted = stats->pages_deleted; + } +} + +static void +extvac_stats_end_idx(Relation rel, IndexBulkDeleteResult *stats, + LVExtStatCountersIdx *counters, ExtVacReport *report) +{ + extvac_stats_end(rel, &counters->common, report); + report->type = PGSTAT_EXTVAC_INDEX; + + if (stats != NULL) + { + /* + * if something goes wrong or an user doesn't want to track a database + * activity - just suppress it. + */ + + /* Fill index-specific extended stats fields */ + report->index.tuples_deleted = + stats->tuples_removed - counters->tuples_removed; + report->index.pages_deleted = + stats->pages_deleted - counters->pages_deleted; + } +} + /* * heap_vacuum_rel() -- perform VACUUM for one heap relation * @@ -711,14 +759,15 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, extvac_stats_end(rel, &extVacCounters, &extVacReport); /* Fill heap-specific extended stats fields */ - extVacReport.pages_scanned = vacrel->scanned_pages; - extVacReport.pages_removed = vacrel->removed_pages; - extVacReport.pages_frozen = vacrel->set_frozen_pages; - extVacReport.pages_all_visible = vacrel->set_all_visible_pages; - extVacReport.tuples_deleted = vacrel->tuples_deleted; - extVacReport.tuples_frozen = vacrel->tuples_frozen; - extVacReport.dead_tuples = vacrel->recently_dead_tuples + vacrel->missed_dead_tuples; - extVacReport.index_vacuum_count = vacrel->num_index_scans; + extVacReport.type = PGSTAT_EXTVAC_HEAP; + extVacReport.heap.pages_scanned = vacrel->scanned_pages; + extVacReport.heap.pages_removed = vacrel->removed_pages; + extVacReport.heap.pages_frozen = vacrel->set_frozen_pages; + extVacReport.heap.pages_all_visible = vacrel->set_all_visible_pages; + extVacReport.heap.tuples_deleted = vacrel->tuples_deleted; + extVacReport.heap.tuples_frozen = vacrel->tuples_frozen; + extVacReport.heap.dead_tuples = vacrel->recently_dead_tuples + vacrel->missed_dead_tuples; + extVacReport.heap.index_vacuum_count = vacrel->num_index_scans; /* * Report results to the cumulative stats system, too. @@ -2583,6 +2632,10 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, { IndexVacuumInfo ivinfo; LVSavedErrInfo saved_err_info; + LVExtStatCountersIdx extVacCounters; + ExtVacReport extVacReport; + + extvac_stats_start_idx(indrel, istat, &extVacCounters); ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; @@ -2601,6 +2654,7 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, */ Assert(vacrel->indname == NULL); vacrel->indname = pstrdup(RelationGetRelationName(indrel)); + vacrel->indoid = RelationGetRelid(indrel); update_vacuum_error_info(vacrel, &saved_err_info, VACUUM_ERRCB_PHASE_VACUUM_INDEX, InvalidBlockNumber, InvalidOffsetNumber); @@ -2609,6 +2663,13 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, istat = vac_bulkdel_one_index(&ivinfo, istat, (void *) vacrel->dead_items, vacrel->dead_items_info); + /* Make extended vacuum stats report for index */ + extvac_stats_end_idx(indrel, istat, &extVacCounters, &extVacReport); + + pgstat_report_vacuum(RelationGetRelid(indrel), + indrel->rd_rel->relisshared, + 0, 0, &extVacReport); + /* Revert to the previous phase information for error traceback */ restore_vacuum_error_info(vacrel, &saved_err_info); pfree(vacrel->indname); @@ -2633,6 +2694,10 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, { IndexVacuumInfo ivinfo; LVSavedErrInfo saved_err_info; + LVExtStatCountersIdx extVacCounters; + ExtVacReport extVacReport; + + extvac_stats_start_idx(indrel, istat, &extVacCounters); ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; @@ -2652,12 +2717,20 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, */ Assert(vacrel->indname == NULL); vacrel->indname = pstrdup(RelationGetRelationName(indrel)); + vacrel->indoid = RelationGetRelid(indrel); update_vacuum_error_info(vacrel, &saved_err_info, VACUUM_ERRCB_PHASE_INDEX_CLEANUP, InvalidBlockNumber, InvalidOffsetNumber); istat = vac_cleanup_one_index(&ivinfo, istat); + /* Make extended vacuum stats report for index */ + extvac_stats_end_idx(indrel, istat, &extVacCounters, &extVacReport); + + pgstat_report_vacuum(RelationGetRelid(indrel), + indrel->rd_rel->relisshared, + 0, 0, &extVacReport); + /* Revert to the previous phase information for error traceback */ restore_vacuum_error_info(vacrel, &saved_err_info); pfree(vacrel->indname); @@ -3274,7 +3347,7 @@ vacuum_error_callback(void *arg) { case VACUUM_ERRCB_PHASE_SCAN_HEAP: if(geterrelevel() >= ERROR) - pgstat_report_vacuum_error(errinfo->reloid); + pgstat_report_vacuum_error(errinfo->reloid, PGSTAT_EXTVAC_HEAP); if (BlockNumberIsValid(errinfo->blkno)) { if (OffsetNumberIsValid(errinfo->offnum)) @@ -3291,7 +3364,7 @@ vacuum_error_callback(void *arg) case VACUUM_ERRCB_PHASE_VACUUM_HEAP: if(geterrelevel() >= ERROR) - pgstat_report_vacuum_error(errinfo->reloid); + pgstat_report_vacuum_error(errinfo->reloid, PGSTAT_EXTVAC_HEAP); if (BlockNumberIsValid(errinfo->blkno)) { if (OffsetNumberIsValid(errinfo->offnum)) @@ -3307,16 +3380,22 @@ vacuum_error_callback(void *arg) break; case VACUUM_ERRCB_PHASE_VACUUM_INDEX: + if(geterrelevel() >= ERROR) + pgstat_report_vacuum_error(errinfo->indoid, PGSTAT_EXTVAC_INDEX); errcontext("while vacuuming index \"%s\" of relation \"%s.%s\"", errinfo->indname, errinfo->relnamespace, errinfo->relname); break; case VACUUM_ERRCB_PHASE_INDEX_CLEANUP: + if(geterrelevel() >= ERROR) + pgstat_report_vacuum_error(errinfo->indoid, PGSTAT_EXTVAC_INDEX); errcontext("while cleaning up index \"%s\" of relation \"%s.%s\"", errinfo->indname, errinfo->relnamespace, errinfo->relname); break; case VACUUM_ERRCB_PHASE_TRUNCATE: + if(geterrelevel() >= ERROR) + pgstat_report_vacuum_error(errinfo->reloid, PGSTAT_EXTVAC_HEAP); if (BlockNumberIsValid(errinfo->blkno)) errcontext("while truncating relation \"%s.%s\" to %u blocks", errinfo->relnamespace, errinfo->relname, errinfo->blkno); diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index e84d6881403..ee759cdc5c3 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1425,3 +1425,44 @@ WHERE db.datname = current_database() AND rel.oid = stats.relid AND ns.oid = rel.relnamespace; + +CREATE VIEW pg_stat_vacuum_indexes AS +SELECT + rel.oid as relid, + ns.nspname AS "schema", + rel.relname AS relname, + + stats.total_blks_read, + stats.total_blks_hit, + stats.total_blks_dirtied, + stats.total_blks_written, + + stats.rel_blks_read, + stats.rel_blks_hit, + + stats.pages_deleted, + stats.tuples_deleted, + + stats.wal_records, + stats.wal_fpi, + stats.wal_bytes, + + stats.blk_read_time, + stats.blk_write_time, + + stats.delay_time, + stats.system_time, + stats.user_time, + stats.total_time, + + stats.interrupts +FROM + pg_database db, + pg_class rel, + pg_namespace ns, + pg_stat_vacuum_indexes(rel.oid) stats +WHERE + db.datname = current_database() AND + rel.oid = stats.relid AND + ns.oid = rel.relnamespace; + diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 4e8d8f8dc77..a6f971b5a68 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -851,17 +851,33 @@ pgstat_accumulate_extvac_stats(ExtVacReport *dst, ExtVacReport *src, if (!accumulate_reltype_specific_info) return; - dst->blks_fetched += src->blks_fetched; - dst->blks_hit += src->blks_hit; - - dst->pages_scanned += src->pages_scanned; - dst->pages_removed += src->pages_removed; - dst->pages_frozen += src->pages_frozen; - dst->pages_all_visible += src->pages_all_visible; - dst->tuples_deleted += src->tuples_deleted; - dst->tuples_frozen += src->tuples_frozen; - dst->dead_tuples += src->dead_tuples; - dst->index_vacuum_count += src->index_vacuum_count; + if (dst->type == PGSTAT_EXTVAC_INVALID) + dst->type = src->type; + + Assert(src->type == PGSTAT_EXTVAC_INVALID || src->type == dst->type); + + if (dst->type == src->type) + { + dst->blks_fetched += src->blks_fetched; + dst->blks_hit += src->blks_hit; + + if (dst->type == PGSTAT_EXTVAC_HEAP) + { + dst->heap.pages_scanned += src->heap.pages_scanned; + dst->heap.pages_removed += src->heap.pages_removed; + dst->heap.pages_frozen += src->heap.pages_frozen; + dst->heap.pages_all_visible += src->heap.pages_all_visible; + dst->heap.tuples_deleted += src->heap.tuples_deleted; + dst->heap.tuples_frozen += src->heap.tuples_frozen; + dst->heap.dead_tuples += src->heap.dead_tuples; + dst->heap.index_vacuum_count += src->heap.index_vacuum_count; + } + else if (dst->type == PGSTAT_EXTVAC_INDEX) + { + dst->index.pages_deleted += src->index.pages_deleted; + dst->index.tuples_deleted += src->index.tuples_deleted; + } + } } /* ------------------------------------------------------------ @@ -1108,7 +1124,8 @@ pgstat_update_snapshot(PgStat_Kind kind) PG_TRY(); { pgstat_fetch_consistency = PGSTAT_FETCH_CONSISTENCY_SNAPSHOT; - pgstat_build_snapshot(PGSTAT_KIND_RELATION); + if (kind == PGSTAT_KIND_RELATION) + pgstat_build_snapshot(PGSTAT_KIND_RELATION); } PG_FINALLY(); { @@ -1163,6 +1180,10 @@ pgstat_build_snapshot(PgStat_Kind statKind) if (p->dropped) continue; + if (statKind != PGSTAT_KIND_INVALID && statKind != p->key.kind) + /* Load stat of specific type, if defined */ + continue; + Assert(pg_atomic_read_u32(&p->refcount) > 0); stats_data = dsa_get_address(pgStatLocal.dsa, p->body); diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index d40d43cdb4a..5b06b04faad 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -211,7 +211,7 @@ pgstat_drop_relation(Relation rel) * --------- */ void -pgstat_report_vacuum_error(Oid tableoid) +pgstat_report_vacuum_error(Oid tableoid, ExtVacReportType m_type) { PgStat_EntryRef *entry_ref; PgStatShared_Relation *shtabentry; @@ -228,6 +228,7 @@ pgstat_report_vacuum_error(Oid tableoid) tabentry = &shtabentry->stats; tabentry->vacuum_ext.interrupts++; + tabentry->vacuum_ext.type = m_type; pgstat_unlock_entry(entry_ref); } diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 1df271286e6..915c3e59bfa 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2070,17 +2070,19 @@ pg_stat_have_stats(PG_FUNCTION_ARGS) } #define EXTVACHEAPSTAT_COLUMNS 27 +#define EXTVACIDXSTAT_COLUMNS 19 +#define EXTVACSTAT_COLUMNS Max(EXTVACHEAPSTAT_COLUMNS, EXTVACIDXSTAT_COLUMNS) static void tuplestore_put_for_relation(Oid relid, ReturnSetInfo *rsinfo, PgStat_StatTabEntry *tabentry) { - Datum values[EXTVACHEAPSTAT_COLUMNS]; - bool nulls[EXTVACHEAPSTAT_COLUMNS]; + Datum values[EXTVACSTAT_COLUMNS]; + bool nulls[EXTVACSTAT_COLUMNS]; char buf[256]; int i = 0; - memset(nulls, 0, EXTVACHEAPSTAT_COLUMNS * sizeof(bool)); + memset(nulls, 0, EXTVACSTAT_COLUMNS * sizeof(bool)); values[i++] = ObjectIdGetDatum(relid); @@ -2093,16 +2095,25 @@ tuplestore_put_for_relation(Oid relid, ReturnSetInfo *rsinfo, tabentry->vacuum_ext.blks_hit); values[i++] = Int64GetDatum(tabentry->vacuum_ext.blks_hit); - values[i++] = Int64GetDatum(tabentry->vacuum_ext.pages_scanned); - values[i++] = Int64GetDatum(tabentry->vacuum_ext.pages_removed); - values[i++] = Int64GetDatum(tabentry->vacuum_ext.pages_frozen); - values[i++] = Int64GetDatum(tabentry->vacuum_ext.pages_all_visible); - values[i++] = Int64GetDatum(tabentry->vacuum_ext.tuples_deleted); - values[i++] = Int64GetDatum(tabentry->vacuum_ext.tuples_frozen); - values[i++] = Int64GetDatum(tabentry->vacuum_ext.dead_tuples); - values[i++] = Int64GetDatum(tabentry->vacuum_ext.index_vacuum_count); - values[i++] = Int64GetDatum(tabentry->rev_all_frozen_pages); - values[i++] = Int64GetDatum(tabentry->rev_all_visible_pages); + if (tabentry->vacuum_ext.type == PGSTAT_EXTVAC_HEAP) + { + values[i++] = Int64GetDatum(tabentry->vacuum_ext.heap.pages_scanned); + values[i++] = Int64GetDatum(tabentry->vacuum_ext.heap.pages_removed); + values[i++] = Int64GetDatum(tabentry->vacuum_ext.heap.pages_frozen); + values[i++] = Int64GetDatum(tabentry->vacuum_ext.heap.pages_all_visible); + values[i++] = Int64GetDatum(tabentry->vacuum_ext.heap.tuples_deleted); + values[i++] = Int64GetDatum(tabentry->vacuum_ext.heap.tuples_frozen); + values[i++] = Int64GetDatum(tabentry->vacuum_ext.heap.dead_tuples); + values[i++] = Int64GetDatum(tabentry->vacuum_ext.heap.index_vacuum_count); + values[i++] = Int64GetDatum(tabentry->rev_all_frozen_pages); + values[i++] = Int64GetDatum(tabentry->rev_all_visible_pages); + + } + else if (tabentry->vacuum_ext.type == PGSTAT_EXTVAC_INDEX) + { + values[i++] = Int64GetDatum(tabentry->vacuum_ext.index.pages_deleted); + values[i++] = Int64GetDatum(tabentry->vacuum_ext.index.tuples_deleted); + } values[i++] = Int64GetDatum(tabentry->vacuum_ext.wal_records); values[i++] = Int64GetDatum(tabentry->vacuum_ext.wal_fpi); @@ -2130,10 +2141,9 @@ tuplestore_put_for_relation(Oid relid, ReturnSetInfo *rsinfo, * Get the vacuum statistics for the heap tables or indexes. */ static void -pg_stats_vacuum(FunctionCallInfo fcinfo, int ncolumns) +pg_stats_vacuum(FunctionCallInfo fcinfo, ExtVacReportType type, int ncolumns) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - Oid relid = PG_GETARG_OID(0); PgStat_StatTabEntry *tabentry; InitMaterializedSRF(fcinfo, 0); @@ -2146,35 +2156,37 @@ pg_stats_vacuum(FunctionCallInfo fcinfo, int ncolumns) Assert(rsinfo->setDesc->natts == ncolumns); Assert(rsinfo->setResult != NULL); - /* Load table statistics for specified database. */ - if (OidIsValid(relid)) + if (type == PGSTAT_EXTVAC_INDEX || type == PGSTAT_EXTVAC_HEAP) { - tabentry = pgstat_fetch_stat_tabentry(relid); - if (tabentry == NULL) - /* Table don't exists or isn't an heap relation. */ - return; + Oid relid = PG_GETARG_OID(0); - tuplestore_put_for_relation(relid, rsinfo, tabentry); - } - else - { - SnapshotIterator hashiter; - PgStat_SnapshotEntry *entry; - - /* Iterate the snapshot */ - InitSnapshotIterator(pgStatLocal.snapshot.stats, &hashiter); + /* Load table statistics for specified relation. */ + if (OidIsValid(relid)) + { + tabentry = pgstat_fetch_stat_tabentry(relid); + if (tabentry == NULL || tabentry->vacuum_ext.type != type) + /* Table don't exists or isn't an heap relation. */ + return; - while ((entry = ScanStatSnapshot(pgStatLocal.snapshot.stats, &hashiter)) != NULL) + tuplestore_put_for_relation(relid, rsinfo, tabentry); + } + else { - Oid reloid; + SnapshotIterator hashiter; + PgStat_SnapshotEntry *entry; + + /* Iterate the snapshot */ + InitSnapshotIterator(pgStatLocal.snapshot.stats, &hashiter); - CHECK_FOR_INTERRUPTS(); + while ((entry = ScanStatSnapshot(pgStatLocal.snapshot.stats, &hashiter)) != NULL) + { + CHECK_FOR_INTERRUPTS(); - tabentry = (PgStat_StatTabEntry *) entry->data; - reloid = entry->key.objoid; + tabentry = (PgStat_StatTabEntry *) entry->data; - if (tabentry != NULL) - tuplestore_put_for_relation(reloid, rsinfo, tabentry); + if (tabentry != NULL && tabentry->vacuum_ext.type == type) + tuplestore_put_for_relation(relid, rsinfo, tabentry); + } } } } @@ -2185,7 +2197,18 @@ pg_stats_vacuum(FunctionCallInfo fcinfo, int ncolumns) Datum pg_stat_vacuum_tables(PG_FUNCTION_ARGS) { - pg_stats_vacuum(fcinfo, EXTVACHEAPSTAT_COLUMNS); + pg_stats_vacuum(fcinfo, PGSTAT_EXTVAC_HEAP, EXTVACHEAPSTAT_COLUMNS); PG_RETURN_VOID(); } + +/* + * Get the vacuum statistics for the indexes. + */ +Datum +pg_stat_vacuum_indexes(PG_FUNCTION_ARGS) +{ + pg_stats_vacuum(fcinfo, PGSTAT_EXTVAC_INDEX, EXTVACIDXSTAT_COLUMNS); + + PG_RETURN_VOID(); + } \ No newline at end of file diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 2023270f923..604b4f44930 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -12263,4 +12263,13 @@ proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', proargnames => '{reloid,relid,total_blks_read,total_blks_hit,total_blks_dirtied,total_blks_written,rel_blks_read,rel_blks_hit,pages_scanned,pages_removed,pages_frozen,pages_all_visible,tuples_deleted,tuples_frozen,dead_tuples,index_vacuum_count,rev_all_frozen_pages,rev_all_visible_pages,wal_records,wal_fpi,wal_bytes,blk_read_time,blk_write_time,delay_time,system_time,user_time,total_time,interrupts}', prosrc => 'pg_stat_vacuum_tables' }, +{ oid => '8002', + descr => 'pg_stat_vacuum_indexes return stats values', + proname => 'pg_stat_vacuum_indexes', provolatile => 's', prorettype => 'record',proisstrict => 'f', + proretset => 't', + proargtypes => 'oid', + proallargtypes => '{oid,oid,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,numeric,float8,float8,float8,float8,float8,float8,int4}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{reloid,relid,total_blks_read,total_blks_hit,total_blks_dirtied,total_blks_written,rel_blks_read,rel_blks_hit,pages_deleted,tuples_deleted,wal_records,wal_fpi,wal_bytes,blk_read_time,blk_write_time,delay_time,system_time,user_time,total_time,interrupts}', + prosrc => 'pg_stat_vacuum_indexes' } ] diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 4492a0572c6..762b53b88ed 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -167,11 +167,20 @@ typedef struct PgStat_BackendSubEntry PgStat_Counter sync_error_count; } PgStat_BackendSubEntry; + +/* Type of ExtVacReport */ +typedef enum ExtVacReportType +{ + PGSTAT_EXTVAC_INVALID = 0, + PGSTAT_EXTVAC_HEAP = 1, + PGSTAT_EXTVAC_INDEX = 2 +} ExtVacReportType; + /* ---------- * * ExtVacReport * - * Additional statistics of vacuum processing over a heap relation. + * Additional statistics of vacuum processing over a relation. * pages_removed is the amount by which the physically shrank, * if any (ie the change in its total size on disk) * pages_deleted refer to free space within the index file @@ -203,14 +212,38 @@ typedef struct ExtVacReport /* Interruptions on any errors. */ int32 interrupts; - int64 pages_scanned; /* number of pages we examined */ - int64 pages_removed; /* number of pages removed by vacuum */ - int64 pages_frozen; /* number of pages marked in VM as frozen */ - int64 pages_all_visible; /* number of pages marked in VM as all-visible */ - int64 tuples_deleted; /* tuples deleted by vacuum */ - int64 tuples_frozen; /* tuples frozen up by vacuum */ - int64 dead_tuples; /* number of deleted tuples which vacuum cannot clean up by vacuum operation */ - int64 index_vacuum_count; /* number of index vacuumings */ + ExtVacReportType type; /* heap, index, etc. */ + + /* ---------- + * + * There are separate metrics of statistic for tables and indexes, + * which collect during vacuum. + * The union operator allows to combine these statistics + * so that each metric is assigned to a specific class of collected statistics. + * Such a combined structure was called per_type_stats. + * The name of the structure itself is not used anywhere, + * it exists only for understanding the code. + * ---------- + */ + union + { + struct + { + int64 pages_scanned; /* number of pages we examined */ + int64 pages_removed; /* number of pages removed by vacuum */ + int64 pages_frozen; /* number of pages marked in VM as frozen */ + int64 pages_all_visible; /* number of pages marked in VM as all-visible */ + int64 tuples_deleted; /* tuples deleted by vacuum */ + int64 tuples_frozen; /* tuples frozen up by vacuum */ + int64 dead_tuples; /* number of deleted tuples which vacuum cannot clean up by vacuum operation */ + int64 index_vacuum_count; /* number of index vacuumings */ + } heap; + struct + { + int64 pages_deleted; /* number of pages deleted by vacuum */ + int64 tuples_deleted; /* tuples deleted by vacuum */ + } index; + } /* per_type_stats */; } ExtVacReport; /* ---------- @@ -689,7 +722,7 @@ extern void pgstat_report_vacuum(Oid tableoid, bool shared, extern void pgstat_report_analyze(Relation rel, PgStat_Counter livetuples, PgStat_Counter deadtuples, bool resetcounter); -extern void pgstat_report_vacuum_error(Oid tableoid); +extern void pgstat_report_vacuum_error(Oid tableoid, ExtVacReportType m_type); /* * If stats are enabled, but pending data hasn't been prepared yet, call diff --git a/src/test/isolation/expected/vacuum-extending-in-repetable-read.out b/src/test/isolation/expected/vacuum-extending-in-repetable-read.out index 7cdb79c0ec4..93fe15c01f9 100644 --- a/src/test/isolation/expected/vacuum-extending-in-repetable-read.out +++ b/src/test/isolation/expected/vacuum-extending-in-repetable-read.out @@ -9,10 +9,9 @@ step s2_print_vacuum_stats_table: FROM pg_stat_vacuum_tables vt, pg_class c WHERE vt.relname = 'test_vacuum_stat_isolation' AND vt.relid = c.oid; -relname |tuples_deleted|dead_tuples|tuples_frozen ---------------------------+--------------+-----------+------------- -test_vacuum_stat_isolation| 0| 0| 0 -(1 row) +relname|tuples_deleted|dead_tuples|tuples_frozen +-------+--------------+-----------+------------- +(0 rows) step s1_begin_repeatable_read: BEGIN transaction ISOLATION LEVEL REPEATABLE READ; diff --git a/src/test/isolation/specs/vacuum-extending-in-repetable-read.spec b/src/test/isolation/specs/vacuum-extending-in-repetable-read.spec index 7d31ddbece9..bca3e8516b2 100644 --- a/src/test/isolation/specs/vacuum-extending-in-repetable-read.spec +++ b/src/test/isolation/specs/vacuum-extending-in-repetable-read.spec @@ -48,4 +48,4 @@ permutation s1_commit s2_checkpoint s2_vacuum - s2_print_vacuum_stats_table + s2_print_vacuum_stats_table \ No newline at end of file diff --git a/src/test/regress/expected/opr_sanity.out b/src/test/regress/expected/opr_sanity.out index 9ae743eae0c..5d72b970b03 100644 --- a/src/test/regress/expected/opr_sanity.out +++ b/src/test/regress/expected/opr_sanity.out @@ -32,10 +32,11 @@ WHERE p1.prolang = 0 OR p1.prorettype = 0 OR prokind NOT IN ('f', 'a', 'w', 'p') OR provolatile NOT IN ('i', 's', 'v') OR proparallel NOT IN ('s', 'r', 'u'); - oid | proname -------+----------------------- + oid | proname +------+------------------------ 8001 | pg_stat_vacuum_tables -(1 row) + 8002 | pg_stat_vacuum_indexes +(2 rows) -- prosrc should never be null; it can be empty only if prosqlbody isn't null SELECT p1.oid, p1.proname diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index cc0b5bde0a1..265eb597d69 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2229,6 +2229,32 @@ pg_stat_user_tables| SELECT relid, autoanalyze_count FROM pg_stat_all_tables WHERE ((schemaname <> ALL (ARRAY['pg_catalog'::name, 'information_schema'::name])) AND (schemaname !~ '^pg_toast'::text)); +pg_stat_vacuum_indexes| SELECT rel.oid AS relid, + ns.nspname AS schema, + rel.relname, + stats.total_blks_read, + stats.total_blks_hit, + stats.total_blks_dirtied, + stats.total_blks_written, + stats.rel_blks_read, + stats.rel_blks_hit, + stats.pages_deleted, + stats.tuples_deleted, + stats.wal_records, + stats.wal_fpi, + stats.wal_bytes, + stats.blk_read_time, + stats.blk_write_time, + stats.delay_time, + stats.system_time, + stats.user_time, + stats.total_time, + stats.interrupts + FROM pg_database db, + pg_class rel, + pg_namespace ns, + LATERAL pg_stat_vacuum_indexes(db.oid, rel.oid) stats(relid, total_blks_read, total_blks_hit, total_blks_dirtied, total_blks_written, rel_blks_read, rel_blks_hit, pages_deleted, tuples_deleted, wal_records, wal_fpi, wal_bytes, blk_read_time, blk_write_time, delay_time, system_time, user_time, total_time, interrupts) + WHERE ((db.datname = current_database()) AND (rel.oid = stats.relid) AND (ns.oid = rel.relnamespace)); pg_stat_vacuum_tables| SELECT rel.oid AS relid, ns.nspname AS schema, rel.relname, diff --git a/src/test/regress/expected/vacuum_index_statistics.out b/src/test/regress/expected/vacuum_index_statistics.out new file mode 100644 index 00000000000..a0da8d25f1a --- /dev/null +++ b/src/test/regress/expected/vacuum_index_statistics.out @@ -0,0 +1,158 @@ +-- +-- Test cumulative vacuum stats system +-- +-- Check the wall statistics collected during vacuum operation: +-- number of frozen and visible pages set by vacuum; +-- number of frozen and visible pages removed by backend. +-- Statistic wal_fpi is not displayed in this test because its behavior is unstable. +-- +-- conditio sine qua non +SHOW track_counts; -- must be on + track_counts +-------------- + on +(1 row) + +-- not enabled by default, but we want to test it... +SET track_functions TO 'all'; +-- ensure pending stats are flushed +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +\set sample_size 10000 +SET vacuum_freeze_min_age = 0; +SET vacuum_freeze_table_age = 0; +--SET stats_fetch_consistency = snapshot; +CREATE TABLE vestat (x int primary key) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; +SELECT oid AS ioid from pg_class where relname = 'vestat_pkey' \gset +DELETE FROM vestat WHERE x % 2 = 0; +-- Before the first vacuum execution extended stats view is empty. +SELECT vt.relname,relpages,pages_deleted,tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; + relname | relpages | pages_deleted | tuples_deleted +---------+----------+---------------+---------------- +(0 rows) + +SELECT relpages AS irp +FROM pg_class c +WHERE relname = 'vestat_pkey' \gset +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP ON) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; +-- The table and index extended vacuum statistics should show us that +-- vacuum frozed pages and clean up pages, but pages_removed stayed the same +-- because of not full table have cleaned up +SELECT vt.relname,relpages-:irp = 0 AS relpages,pages_deleted = 0 AS pages_deleted,tuples_deleted > 0 AS tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; + relname | relpages | pages_deleted | tuples_deleted +-------------+----------+---------------+---------------- + vestat_pkey | t | t | t +(1 row) + +SELECT vt.relname,relpages AS irp,pages_deleted AS ipd,tuples_deleted AS itd +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid \gset +-- Store WAL advances into variables +SELECT wal_records AS iwr,wal_bytes AS iwb,wal_fpi AS ifpi FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset +-- Look into WAL records deltas. +SELECT wal_records > 0 AS diWR, wal_bytes > 0 AS diWB +FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey'; + diwr | diwb +------+------ + t | t +(1 row) + +DELETE FROM vestat;; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP ON) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; +-- pages_removed must be increased +SELECT vt.relname,relpages-:irp = 0 AS relpages,pages_deleted-:ipd > 0 AS pages_deleted,tuples_deleted-:itd > 0 AS tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; + relname | relpages | pages_deleted | tuples_deleted +-------------+----------+---------------+---------------- + vestat_pkey | t | t | t +(1 row) + +SELECT vt.relname,relpages AS irp,pages_deleted AS ipd,tuples_deleted AS itd +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid \gset +-- Store WAL advances into variables +SELECT wal_records-:iwr AS diwr, wal_bytes-:iwb AS diwb, wal_fpi-:ifpi AS difpi +FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset +-- WAL advance should be detected. +SELECT :diwr > 0 AS diWR, :diwb > 0 AS diWB; + diwr | diwb +------+------ + t | t +(1 row) + +-- Store WAL advances into variables +SELECT wal_records AS iwr,wal_bytes AS iwb,wal_fpi AS ifpi FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +DELETE FROM vestat WHERE x % 2 = 0; +-- VACUUM FULL doesn't report to stat collector. So, no any advancements of statistics +-- are detected here. +VACUUM FULL vestat; +-- It is necessary to check the wal statistics +CHECKPOINT; +-- Store WAL advances into variables +SELECT wal_records-:iwr AS diwr2, wal_bytes-:iwb AS diwb2, wal_fpi-:ifpi AS difpi2 +FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset +-- WAL and other statistics advance should not be detected. +SELECT :diwr2=0 AS diWR, :difpi2=0 AS iFPI, :diwb2=0 AS diWB; + diwr | ifpi | diwb +------+------+------ + t | t | t +(1 row) + +SELECT vt.relname,relpages-:irp < 0 AS relpages,pages_deleted-:ipd = 0 AS pages_deleted,tuples_deleted-:itd = 0 AS tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; + relname | relpages | pages_deleted | tuples_deleted +-------------+----------+---------------+---------------- + vestat_pkey | t | t | t +(1 row) + +SELECT vt.relname,relpages AS irp,pages_deleted AS ipd,tuples_deleted AS itd +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid \gset +-- Store WAL advances into variables +SELECT wal_records AS iwr,wal_bytes AS iwb,wal_fpi AS ifpi FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset +DELETE FROM vestat; +TRUNCATE vestat; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP ON) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; +-- Store WAL advances into variables after removing all tuples from the table +SELECT wal_records-:iwr AS diwr3, wal_bytes-:iwb AS diwb3, wal_fpi-:ifpi AS difpi3 +FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset +--There are nothing changed +SELECT :diwr3=0 AS diWR, :difpi3=0 AS iFPI, :diwb3=0 AS diWB; + diwr | ifpi | diwb +------+------+------ + t | t | t +(1 row) + +-- +-- Now, the table and index is compressed into zero number of pages. Check it +-- in vacuum extended statistics. +-- The pages_frozen, pages_scanned values shouldn't be changed +-- +SELECT vt.relname,relpages-:irp = 0 AS relpages,pages_deleted-:ipd = 0 AS pages_deleted,tuples_deleted-:itd = 0 AS tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; + relname | relpages | pages_deleted | tuples_deleted +-------------+----------+---------------+---------------- + vestat_pkey | f | t | t +(1 row) + +DROP TABLE vestat; diff --git a/src/test/regress/expected/vacuum_tables_statistics.out b/src/test/regress/expected/vacuum_tables_statistics.out index 1a7d04b0590..b85a5cab9af 100644 --- a/src/test/regress/expected/vacuum_tables_statistics.out +++ b/src/test/regress/expected/vacuum_tables_statistics.out @@ -37,8 +37,7 @@ FROM pg_stat_vacuum_tables vt, pg_class c WHERE vt.relname = 'vestat' AND vt.relid = c.oid; relname | pages_frozen | tuples_deleted | relpages | pages_scanned | pages_removed ---------+--------------+----------------+----------+---------------+--------------- - vestat | 0 | 0 | 455 | 0 | 0 -(1 row) +(0 rows) SELECT relpages AS rp FROM pg_class c diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index f8a4bcccc9d..b9408a43f71 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -140,4 +140,5 @@ test: tablespace # ---------- # Check vacuum statistics # ---------- +test: vacuum_index_statistics test: vacuum_tables_statistics \ No newline at end of file diff --git a/src/test/regress/sql/vacuum_index_statistics.sql b/src/test/regress/sql/vacuum_index_statistics.sql new file mode 100644 index 00000000000..9113fd26e6f --- /dev/null +++ b/src/test/regress/sql/vacuum_index_statistics.sql @@ -0,0 +1,128 @@ +-- +-- Test cumulative vacuum stats system +-- +-- Check the wall statistics collected during vacuum operation: +-- number of frozen and visible pages set by vacuum; +-- number of frozen and visible pages removed by backend. +-- Statistic wal_fpi is not displayed in this test because its behavior is unstable. +-- +-- conditio sine qua non +SHOW track_counts; -- must be on +-- not enabled by default, but we want to test it... +SET track_functions TO 'all'; + + +-- ensure pending stats are flushed +SELECT pg_stat_force_next_flush(); + +\set sample_size 10000 +SET vacuum_freeze_min_age = 0; +SET vacuum_freeze_table_age = 0; +--SET stats_fetch_consistency = snapshot; +CREATE TABLE vestat (x int primary key) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; + +SELECT oid AS ioid from pg_class where relname = 'vestat_pkey' \gset + +DELETE FROM vestat WHERE x % 2 = 0; +-- Before the first vacuum execution extended stats view is empty. +SELECT vt.relname,relpages,pages_deleted,tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; +SELECT relpages AS irp +FROM pg_class c +WHERE relname = 'vestat_pkey' \gset + +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP ON) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; + +-- The table and index extended vacuum statistics should show us that +-- vacuum frozed pages and clean up pages, but pages_removed stayed the same +-- because of not full table have cleaned up +SELECT vt.relname,relpages-:irp = 0 AS relpages,pages_deleted = 0 AS pages_deleted,tuples_deleted > 0 AS tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; +SELECT vt.relname,relpages AS irp,pages_deleted AS ipd,tuples_deleted AS itd +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid \gset + +-- Store WAL advances into variables +SELECT wal_records AS iwr,wal_bytes AS iwb,wal_fpi AS ifpi FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset + +-- Look into WAL records deltas. +SELECT wal_records > 0 AS diWR, wal_bytes > 0 AS diWB +FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey'; + +DELETE FROM vestat;; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP ON) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; + +-- pages_removed must be increased +SELECT vt.relname,relpages-:irp = 0 AS relpages,pages_deleted-:ipd > 0 AS pages_deleted,tuples_deleted-:itd > 0 AS tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; +SELECT vt.relname,relpages AS irp,pages_deleted AS ipd,tuples_deleted AS itd +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid \gset + +-- Store WAL advances into variables +SELECT wal_records-:iwr AS diwr, wal_bytes-:iwb AS diwb, wal_fpi-:ifpi AS difpi +FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset + +-- WAL advance should be detected. +SELECT :diwr > 0 AS diWR, :diwb > 0 AS diWB; + +-- Store WAL advances into variables +SELECT wal_records AS iwr,wal_bytes AS iwb,wal_fpi AS ifpi FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset + +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +DELETE FROM vestat WHERE x % 2 = 0; +-- VACUUM FULL doesn't report to stat collector. So, no any advancements of statistics +-- are detected here. +VACUUM FULL vestat; +-- It is necessary to check the wal statistics +CHECKPOINT; + +-- Store WAL advances into variables +SELECT wal_records-:iwr AS diwr2, wal_bytes-:iwb AS diwb2, wal_fpi-:ifpi AS difpi2 +FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset + +-- WAL and other statistics advance should not be detected. +SELECT :diwr2=0 AS diWR, :difpi2=0 AS iFPI, :diwb2=0 AS diWB; + +SELECT vt.relname,relpages-:irp < 0 AS relpages,pages_deleted-:ipd = 0 AS pages_deleted,tuples_deleted-:itd = 0 AS tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; +SELECT vt.relname,relpages AS irp,pages_deleted AS ipd,tuples_deleted AS itd +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid \gset + +-- Store WAL advances into variables +SELECT wal_records AS iwr,wal_bytes AS iwb,wal_fpi AS ifpi FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset + +DELETE FROM vestat; +TRUNCATE vestat; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP ON) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; + +-- Store WAL advances into variables after removing all tuples from the table +SELECT wal_records-:iwr AS diwr3, wal_bytes-:iwb AS diwb3, wal_fpi-:ifpi AS difpi3 +FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset + +--There are nothing changed +SELECT :diwr3=0 AS diWR, :difpi3=0 AS iFPI, :diwb3=0 AS diWB; + +-- +-- Now, the table and index is compressed into zero number of pages. Check it +-- in vacuum extended statistics. +-- The pages_frozen, pages_scanned values shouldn't be changed +-- +SELECT vt.relname,relpages-:irp = 0 AS relpages,pages_deleted-:ipd = 0 AS pages_deleted,tuples_deleted-:itd = 0 AS tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; + +DROP TABLE vestat; -- 2.34.1