From bdf712ead5b8cc88ac32f92e1801f3d5a04e9aa9 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Fri, 10 Jan 2025 09:57:13 +0300 Subject: [PATCH 1/4] Machinery for grabbing an extended vacuum statistics on heap relations. Value of total_blks_hit, total_blks_read, total_blks_dirtied are number of hitted, missed and dirtied pages in shared buffers during a vacuum operation respectively. total_blks_dirtied means 'dirtied only by this action'. So, if this page was dirty before the vacuum operation, it doesn't count this page as 'dirtied'. The tuples_deleted parameter is the number of tuples cleaned up by the vacuum operation. The delay_time value means total vacuum sleep time in vacuum delay point. The pages_removed value is the number of pages by which the physical data storage of the relation was reduced. The value of pages_deleted parameter is the number of freed pages in the table (file size may not have changed). Tracking of IO during an (auto)vacuum operation. Introduced variables blk_read_time and blk_write_time tracks only access to buffer pages and flushing them to disk. Reading operation is trivial, but writing measurement technique is not obvious. So, during a vacuum writing time can be zero incremented because no any flushing operations were performed. System time and user time are parameters that describes how much time a vacuum operation has spent in executing of code in user space and kernel space accordingly. Also, accumulate total time of a vacuum that is a diff between timestamps in start and finish points in the vacuum code. Remember about idle time, when vacuum waited for IO and locks, so total time isn't equal a sum of user and system time, but no less. pages_frozen is a number of pages that are marked as frozen in vm during vacuum. This parameter is incremented if page is marked as all-frozen. pages_all_visible is a number of pages that are marked as all-visible in vm during vacuum. wraparound_failsafe_count is a number of times when the vacuum starts urgent cleanup to prevent wraparound problem which is critical for the database. Authors: Alena Rybakina , Andrei Lepikhov , Andrei Zubkov Reviewed-by: Dilip Kumar , Masahiko Sawada , Ilia Evdokimov , jian he , Kirill Reshke , Alexander Korotkov , Jim Nasby , Sami Imseih --- src/backend/access/heap/vacuumlazy.c | 150 +++++++++++- src/backend/access/heap/visibilitymap.c | 10 + src/backend/catalog/system_views.sql | 52 +++- src/backend/commands/vacuum.c | 4 + src/backend/commands/vacuumparallel.c | 1 + src/backend/utils/activity/pgstat.c | 12 +- src/backend/utils/activity/pgstat_relation.c | 47 +++- src/backend/utils/adt/pgstatfuncs.c | 147 ++++++++++++ src/backend/utils/error/elog.c | 13 + src/backend/utils/misc/guc_tables.c | 9 + src/backend/utils/misc/postgresql.conf.sample | 1 + src/include/catalog/pg_proc.dat | 18 ++ src/include/commands/vacuum.h | 1 + src/include/pgstat.h | 82 ++++++- src/include/utils/elog.h | 1 + src/include/utils/pgstat_internal.h | 1 - .../vacuum-extending-in-repetable-read.out | 53 +++++ src/test/isolation/isolation_schedule | 1 + .../vacuum-extending-in-repetable-read.spec | 53 +++++ src/test/regress/expected/rules.out | 44 +++- .../expected/vacuum_tables_statistics.out | 225 ++++++++++++++++++ src/test/regress/parallel_schedule | 5 + .../regress/sql/vacuum_tables_statistics.sql | 180 ++++++++++++++ 23 files changed, 1092 insertions(+), 18 deletions(-) create mode 100644 src/test/isolation/expected/vacuum-extending-in-repetable-read.out create mode 100644 src/test/isolation/specs/vacuum-extending-in-repetable-read.spec create mode 100644 src/test/regress/expected/vacuum_tables_statistics.out create mode 100644 src/test/regress/sql/vacuum_tables_statistics.sql diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 09fab08b8e1..03179f8ecaa 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -165,6 +165,7 @@ typedef struct LVRelState /* Error reporting state */ char *dbname; char *relnamespace; + Oid reloid; char *relname; char *indname; /* Current index name */ BlockNumber blkno; /* used only for heap operations */ @@ -229,6 +230,8 @@ typedef struct LVRelState BlockNumber next_unskippable_block; /* next unskippable block */ bool next_unskippable_allvis; /* its visibility status */ Buffer next_unskippable_vmbuffer; /* buffer containing its VM bit */ + + int32 wraparound_failsafe_count; /* the number of times to prevent workaround problem */ } LVRelState; /* Struct for saving and restoring vacuum error information. */ @@ -239,6 +242,18 @@ typedef struct LVSavedErrInfo VacErrPhase phase; } LVSavedErrInfo; +/* + * Counters and usage data for extended stats tracking. + */ +typedef struct LVExtStatCounters +{ + TimestampTz starttime; + WalUsage walusage; + BufferUsage bufusage; + double VacuumDelayTime; + PgStat_Counter blocks_fetched; + PgStat_Counter blocks_hit; +} LVExtStatCounters; /* non-export function prototypes */ static void lazy_scan_heap(LVRelState *vacrel); @@ -292,6 +307,106 @@ static void update_vacuum_error_info(LVRelState *vacrel, static void restore_vacuum_error_info(LVRelState *vacrel, const LVSavedErrInfo *saved_vacrel); +/* ---------- + * extvac_stats_start() - + * + * Save cut-off values of extended vacuum counters before start of a relation + * processing. + * ---------- + */ +static void +extvac_stats_start(Relation rel, LVExtStatCounters *counters) +{ + TimestampTz starttime; + + if(!pgstat_track_vacuum_statistics) + return; + + memset(counters, 0, sizeof(LVExtStatCounters)); + + starttime = GetCurrentTimestamp(); + + counters->starttime = starttime; + counters->walusage = pgWalUsage; + counters->bufusage = pgBufferUsage; + counters->VacuumDelayTime = VacuumDelayTime; + counters->blocks_fetched = 0; + counters->blocks_hit = 0; + + if (!rel->pgstat_info || !pgstat_track_counts) + /* + * if something goes wrong or user doesn't want to track a database + * activity - just suppress it. + */ + return; + + counters->blocks_fetched = rel->pgstat_info->counts.blocks_fetched; + counters->blocks_hit = rel->pgstat_info->counts.blocks_hit; +} + +/* ---------- + * extvac_stats_end() - + * + * Called to finish an extended vacuum statistic gathering and form a report. + * ---------- + */ +static void +extvac_stats_end(Relation rel, LVExtStatCounters *counters, + ExtVacReport *report) +{ + WalUsage walusage; + BufferUsage bufusage; + TimestampTz endtime; + long secs; + int usecs; + + if(!pgstat_track_vacuum_statistics) + return; + + /* Calculate diffs of global stat parameters on WAL and buffer usage. */ + memset(&walusage, 0, sizeof(WalUsage)); + WalUsageAccumDiff(&walusage, &pgWalUsage, &counters->walusage); + + memset(&bufusage, 0, sizeof(BufferUsage)); + BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &counters->bufusage); + + endtime = GetCurrentTimestamp(); + TimestampDifference(counters->starttime, endtime, &secs, &usecs); + + memset(report, 0, sizeof(ExtVacReport)); + + /* + * Fill additional statistics on a vacuum processing operation. + */ + report->total_blks_read = bufusage.local_blks_read + bufusage.shared_blks_read; + report->total_blks_hit = bufusage.local_blks_hit + bufusage.shared_blks_hit; + report->total_blks_dirtied = bufusage.local_blks_dirtied + bufusage.shared_blks_dirtied; + report->total_blks_written = bufusage.shared_blks_written; + + report->wal_records = walusage.wal_records; + report->wal_fpi = walusage.wal_fpi; + report->wal_bytes = walusage.wal_bytes; + + report->blk_read_time = INSTR_TIME_GET_MILLISEC(bufusage.local_blk_read_time); + report->blk_read_time += INSTR_TIME_GET_MILLISEC(bufusage.shared_blk_read_time); + report->blk_write_time = INSTR_TIME_GET_MILLISEC(bufusage.local_blk_write_time); + report->blk_write_time = INSTR_TIME_GET_MILLISEC(bufusage.shared_blk_write_time); + report->delay_time = VacuumDelayTime - counters->VacuumDelayTime; + + report->total_time = secs * 1000. + usecs / 1000.; + + if (!rel->pgstat_info || !pgstat_track_counts) + /* + * if something goes wrong or an user doesn't want to track a database + * activity - just suppress it. + */ + return; + + report->blks_fetched = + rel->pgstat_info->counts.blocks_fetched - counters->blocks_fetched; + report->blks_hit = + rel->pgstat_info->counts.blocks_hit - counters->blocks_hit; +} /* * heap_vacuum_rel() -- perform VACUUM for one heap relation @@ -324,7 +439,14 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, WalUsage startwalusage = pgWalUsage; BufferUsage startbufferusage = pgBufferUsage; ErrorContextCallback errcallback; + LVExtStatCounters extVacCounters; + ExtVacReport extVacReport; char **indnames = NULL; + ExtVacReport allzero; + + /* Initialize vacuum statistics */ + memset(&allzero, 0, sizeof(ExtVacReport)); + extVacReport = allzero; verbose = (params->options & VACOPT_VERBOSE) != 0; instrument = (verbose || (AmAutoVacuumWorkerProcess() && @@ -342,7 +464,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); - + extvac_stats_start(rel, &extVacCounters); /* * Setup error traceback support for ereport() first. The idea is to set * up an error context callback to display additional information on any @@ -359,6 +481,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, vacrel->dbname = get_database_name(MyDatabaseId); vacrel->relnamespace = get_namespace_name(RelationGetNamespace(rel)); vacrel->relname = pstrdup(RelationGetRelationName(rel)); + vacrel->reloid = RelationGetRelid(rel); vacrel->indname = NULL; vacrel->phase = VACUUM_ERRCB_PHASE_UNKNOWN; vacrel->verbose = verbose; @@ -446,6 +569,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, vacrel->vm_new_visible_pages = 0; vacrel->vm_new_visible_frozen_pages = 0; vacrel->vm_new_frozen_pages = 0; + vacrel->wraparound_failsafe_count = 0; /* * Get cutoffs that determine which deleted tuples are considered DEAD, @@ -591,6 +715,26 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, vacrel->NewRelfrozenXid, vacrel->NewRelminMxid, &frozenxid_updated, &minmulti_updated, false); + /* Make generic extended vacuum stats report */ + extvac_stats_end(rel, &extVacCounters, &extVacReport); + + if(pgstat_track_vacuum_statistics) + { + /* Fill heap-specific extended stats fields */ + extVacReport.pages_scanned = vacrel->scanned_pages; + extVacReport.pages_removed = vacrel->removed_pages; + extVacReport.vm_new_frozen_pages = vacrel->vm_new_frozen_pages; + extVacReport.vm_new_visible_pages = vacrel->vm_new_visible_pages; + extVacReport.vm_new_visible_frozen_pages = vacrel->vm_new_visible_frozen_pages; + extVacReport.tuples_deleted = vacrel->tuples_deleted; + extVacReport.tuples_frozen = vacrel->tuples_frozen; + extVacReport.recently_dead_tuples = vacrel->recently_dead_tuples; + extVacReport.missed_dead_tuples = vacrel->missed_dead_tuples; + extVacReport.missed_dead_pages = vacrel->missed_dead_pages; + extVacReport.index_vacuum_count = vacrel->num_index_scans; + extVacReport.wraparound_failsafe_count = vacrel->wraparound_failsafe_count; + } + /* * Report results to the cumulative stats system, too. * @@ -605,7 +749,8 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, rel->rd_rel->relisshared, Max(vacrel->new_live_tuples, 0), vacrel->recently_dead_tuples + - vacrel->missed_dead_tuples); + vacrel->missed_dead_tuples, + &extVacReport); pgstat_progress_end_command(); if (instrument) @@ -2418,6 +2563,7 @@ lazy_check_wraparound_failsafe(LVRelState *vacrel) int64 progress_val[2] = {0, 0}; VacuumFailsafeActive = true; + vacrel->wraparound_failsafe_count ++; /* * Abandon use of a buffer access strategy to allow use of all of diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c index 745a04ef26e..07623a045fa 100644 --- a/src/backend/access/heap/visibilitymap.c +++ b/src/backend/access/heap/visibilitymap.c @@ -91,6 +91,7 @@ #include "access/xloginsert.h" #include "access/xlogutils.h" #include "miscadmin.h" +#include "pgstat.h" #include "port/pg_bitutils.h" #include "storage/bufmgr.h" #include "storage/smgr.h" @@ -160,6 +161,15 @@ visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags if (map[mapByte] & mask) { + /* + * As part of vacuum stats, track how often all-visible or all-frozen + * bits are cleared. + */ + if (map[mapByte] >> mapOffset & flags & VISIBILITYMAP_ALL_VISIBLE) + pgstat_count_vm_rev_all_visible(rel); + if (map[mapByte] >> mapOffset & flags & VISIBILITYMAP_ALL_FROZEN) + pgstat_count_vm_rev_all_frozen(rel); + map[mapByte] &= ~mask; MarkBufferDirty(vmbuf); diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 7a595c84db9..43ac27ed5b4 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -691,7 +691,9 @@ CREATE VIEW pg_stat_all_tables AS pg_stat_get_vacuum_count(C.oid) AS vacuum_count, pg_stat_get_autovacuum_count(C.oid) AS autovacuum_count, pg_stat_get_analyze_count(C.oid) AS analyze_count, - pg_stat_get_autoanalyze_count(C.oid) AS autoanalyze_count + pg_stat_get_autoanalyze_count(C.oid) AS autoanalyze_count, + pg_stat_get_rev_all_frozen_pages(C.oid) AS rev_all_frozen_pages, + pg_stat_get_rev_all_visible_pages(C.oid) AS rev_all_visible_pages FROM pg_class C LEFT JOIN pg_index I ON C.oid = I.indrelid LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace) @@ -1381,3 +1383,51 @@ CREATE VIEW pg_stat_subscription_stats AS CREATE VIEW pg_wait_events AS SELECT * FROM pg_get_wait_events(); +-- +-- Show extended cumulative statistics on a vacuum operation over all tables and +-- databases of the instance. +-- Use Invalid Oid "0" as an input relation id to get stat on each table in a +-- database. +-- + +CREATE VIEW pg_stat_vacuum_tables AS +SELECT + ns.nspname AS schemaname, + rel.relname AS relname, + stats.relid as relid, + + stats.total_blks_read AS total_blks_read, + stats.total_blks_hit AS total_blks_hit, + stats.total_blks_dirtied AS total_blks_dirtied, + stats.total_blks_written AS total_blks_written, + + stats.rel_blks_read AS rel_blks_read, + stats.rel_blks_hit AS rel_blks_hit, + + stats.pages_scanned AS pages_scanned, + stats.pages_removed AS pages_removed, + stats.vm_new_frozen_pages AS vm_new_frozen_pages, + stats.vm_new_visible_pages AS vm_new_visible_pages, + stats.vm_new_visible_frozen_pages AS vm_new_visible_frozen_pages, + stats.missed_dead_pages AS missed_dead_pages, + stats.tuples_deleted AS tuples_deleted, + stats.tuples_frozen AS tuples_frozen, + stats.recently_dead_tuples AS recently_dead_tuples, + stats.missed_dead_tuples AS missed_dead_tuples, + + stats.wraparound_failsafe AS wraparound_failsafe, + stats.index_vacuum_count AS index_vacuum_count, + stats.wal_records AS wal_records, + stats.wal_fpi AS wal_fpi, + stats.wal_bytes AS wal_bytes, + + stats.blk_read_time AS blk_read_time, + stats.blk_write_time AS blk_write_time, + + stats.delay_time AS delay_time, + stats.total_time AS total_time + +FROM pg_class rel + JOIN pg_namespace ns ON ns.oid = rel.relnamespace, + LATERAL pg_stat_get_vacuum_tables(rel.oid) stats +WHERE rel.relkind = 'r'; diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index e6745e6145c..1ea277b2c16 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -102,6 +102,9 @@ pg_atomic_uint32 *VacuumSharedCostBalance = NULL; pg_atomic_uint32 *VacuumActiveNWorkers = NULL; int VacuumCostBalanceLocal = 0; +/* Cumulative storage to report total vacuum delay time. */ +double VacuumDelayTime = 0; /* msec. */ + /* non-export function prototypes */ static List *expand_vacuum_rel(VacuumRelation *vrel, MemoryContext vac_context, int options); @@ -2418,6 +2421,7 @@ vacuum_delay_point(void) exit(1); VacuumCostBalance = 0; + VacuumDelayTime += msec; /* * Balance and update limit values for autovacuum workers. We must do diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c index 0d92e694d6a..672f8f4bfe8 100644 --- a/src/backend/commands/vacuumparallel.c +++ b/src/backend/commands/vacuumparallel.c @@ -1047,6 +1047,7 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) /* Set cost-based vacuum delay */ VacuumUpdateCosts(); VacuumCostBalance = 0; + VacuumDelayTime = 0; VacuumCostBalanceLocal = 0; VacuumSharedCostBalance = &(shared->cost_balance); VacuumActiveNWorkers = &(shared->active_nworkers); diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 34520535d54..d21b9302c29 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -191,7 +191,7 @@ static void pgstat_reset_after_failure(void); static bool pgstat_flush_pending_entries(bool nowait); static void pgstat_prep_snapshot(void); -static void pgstat_build_snapshot(void); +static void pgstat_build_snapshot(PgStat_Kind statKind); static void pgstat_build_snapshot_fixed(PgStat_Kind kind); static inline bool pgstat_is_kind_valid(PgStat_Kind kind); @@ -204,7 +204,7 @@ static inline bool pgstat_is_kind_valid(PgStat_Kind kind); bool pgstat_track_counts = false; int pgstat_fetch_consistency = PGSTAT_FETCH_CONSISTENCY_CACHE; - +bool pgstat_track_vacuum_statistics = true; /* ---------- * state shared with pgstat_*.c @@ -261,7 +261,6 @@ static bool pgstat_is_initialized = false; static bool pgstat_is_shutdown = false; #endif - /* * The different kinds of built-in statistics. * @@ -911,7 +910,6 @@ pgstat_reset_of_kind(PgStat_Kind kind) pgstat_reset_entries_of_kind(kind, ts); } - /* ------------------------------------------------------------ * Fetching of stats * ------------------------------------------------------------ @@ -980,7 +978,7 @@ pgstat_fetch_entry(PgStat_Kind kind, Oid dboid, uint64 objid) /* if we need to build a full snapshot, do so */ if (pgstat_fetch_consistency == PGSTAT_FETCH_CONSISTENCY_SNAPSHOT) - pgstat_build_snapshot(); + pgstat_build_snapshot(PGSTAT_KIND_INVALID); /* if caching is desired, look up in cache */ if (pgstat_fetch_consistency > PGSTAT_FETCH_CONSISTENCY_NONE) @@ -1096,7 +1094,7 @@ pgstat_snapshot_fixed(PgStat_Kind kind) pgstat_clear_snapshot(); if (pgstat_fetch_consistency == PGSTAT_FETCH_CONSISTENCY_SNAPSHOT) - pgstat_build_snapshot(); + pgstat_build_snapshot(PGSTAT_KIND_INVALID); else pgstat_build_snapshot_fixed(kind); @@ -1147,7 +1145,7 @@ pgstat_prep_snapshot(void) } static void -pgstat_build_snapshot(void) +pgstat_build_snapshot(PgStat_Kind statKind) { dshash_seq_status hstat; PgStatShared_HashEntry *p; diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index 09247ba0971..458bd4ece49 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -47,6 +47,8 @@ static void add_tabstat_xact_level(PgStat_TableStatus *pgstat_info, int nest_lev static void ensure_tabstat_xact_level(PgStat_TableStatus *pgstat_info); static void save_truncdrop_counters(PgStat_TableXactStatus *trans, bool is_drop); static void restore_truncdrop_counters(PgStat_TableXactStatus *trans); +static void pgstat_accumulate_extvac_stats(ExtVacReport *dst, ExtVacReport *src, + bool accumulate_reltype_specific_info); /* @@ -208,7 +210,8 @@ pgstat_drop_relation(Relation rel) */ void pgstat_report_vacuum(Oid tableoid, bool shared, - PgStat_Counter livetuples, PgStat_Counter deadtuples) + PgStat_Counter livetuples, PgStat_Counter deadtuples, + ExtVacReport *params) { PgStat_EntryRef *entry_ref; PgStatShared_Relation *shtabentry; @@ -232,6 +235,8 @@ pgstat_report_vacuum(Oid tableoid, bool shared, tabentry->live_tuples = livetuples; tabentry->dead_tuples = deadtuples; + pgstat_accumulate_extvac_stats(&tabentry->vacuum_ext, params, true); + /* * It is quite possible that a non-aggressive VACUUM ended up skipping * various pages, however, we'll zero the insert counter here regardless. @@ -859,6 +864,9 @@ pgstat_relation_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) tabentry->blocks_fetched += lstats->counts.blocks_fetched; tabentry->blocks_hit += lstats->counts.blocks_hit; + tabentry->rev_all_frozen_pages += lstats->counts.rev_all_frozen_pages; + tabentry->rev_all_visible_pages += lstats->counts.rev_all_visible_pages; + /* Clamp live_tuples in case of negative delta_live_tuples */ tabentry->live_tuples = Max(tabentry->live_tuples, 0); /* Likewise for dead_tuples */ @@ -982,3 +990,40 @@ restore_truncdrop_counters(PgStat_TableXactStatus *trans) trans->tuples_deleted = trans->deleted_pre_truncdrop; } } + +static void +pgstat_accumulate_extvac_stats(ExtVacReport *dst, ExtVacReport *src, + bool accumulate_reltype_specific_info) +{ + dst->total_blks_read += src->total_blks_read; + dst->total_blks_hit += src->total_blks_hit; + dst->total_blks_dirtied += src->total_blks_dirtied; + dst->total_blks_written += src->total_blks_written; + dst->wal_bytes += src->wal_bytes; + dst->wal_fpi += src->wal_fpi; + dst->wal_records += src->wal_records; + dst->blk_read_time += src->blk_read_time; + dst->blk_write_time += src->blk_write_time; + dst->delay_time += src->delay_time; + dst->total_time += src->total_time; + + if (!accumulate_reltype_specific_info) + return; + + dst->blks_fetched += src->blks_fetched; + dst->blks_hit += src->blks_hit; + + dst->pages_scanned += src->pages_scanned; + dst->pages_removed += src->pages_removed; + dst->vm_new_frozen_pages += src->vm_new_frozen_pages; + dst->vm_new_visible_pages += src->vm_new_visible_pages; + dst->vm_new_visible_frozen_pages += src->vm_new_visible_frozen_pages; + dst->tuples_deleted += src->tuples_deleted; + dst->tuples_frozen += src->tuples_frozen; + dst->recently_dead_tuples += src->recently_dead_tuples; + dst->index_vacuum_count += src->index_vacuum_count; + dst->wraparound_failsafe_count += src->wraparound_failsafe_count; + dst->missed_dead_pages += src->missed_dead_pages; + dst->missed_dead_tuples += src->missed_dead_tuples; + +} \ No newline at end of file diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 5f8d20a406d..da31d9d3e29 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -106,6 +106,12 @@ PG_STAT_GET_RELENTRY_INT64(tuples_updated) /* pg_stat_get_vacuum_count */ PG_STAT_GET_RELENTRY_INT64(vacuum_count) +/* pg_stat_get_rev_frozen_pages */ +PG_STAT_GET_RELENTRY_INT64(rev_all_frozen_pages) + +/* pg_stat_get_rev_all_visible_pages */ +PG_STAT_GET_RELENTRY_INT64(rev_all_visible_pages) + #define PG_STAT_GET_RELENTRY_TIMESTAMPTZ(stat) \ Datum \ CppConcat(pg_stat_get_,stat)(PG_FUNCTION_ARGS) \ @@ -2172,3 +2178,144 @@ pg_stat_have_stats(PG_FUNCTION_ARGS) PG_RETURN_BOOL(pgstat_have_entry(kind, dboid, objid)); } + + +/* + * Get the vacuum statistics for the heap tables. + */ +Datum +pg_stat_get_vacuum_tables(PG_FUNCTION_ARGS) +{ + #define PG_STAT_GET_VACUUM_TABLES_STATS_COLS 26 + + Oid relid = PG_GETARG_OID(0); + PgStat_StatTabEntry *tabentry; + ExtVacReport *extvacuum; + TupleDesc tupdesc; + Datum values[PG_STAT_GET_VACUUM_TABLES_STATS_COLS] = {0}; + bool nulls[PG_STAT_GET_VACUUM_TABLES_STATS_COLS] = {0}; + char buf[256]; + int i = 0; + ExtVacReport allzero; + + /* Initialise attributes information in the tuple descriptor */ + tupdesc = CreateTemplateTupleDesc(PG_STAT_GET_VACUUM_TABLES_STATS_COLS); + + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "relid", + INT4OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "total_blks_read", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "total_blks_hit", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "total_blks_dirtied", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "total_blks_written", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "rel_blks_read", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "rel_blks_hit", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "pages_scanned", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "pages_removed", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "vm_new_frozen_pages", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "vm_new_visible_pages", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "vm_new_visible_frozen_pages", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "missed_dead_pages", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "tuples_deleted", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "tuples_frozen", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "recently_dead_tuples", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "missed_dead_tuples", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "wraparound_failsafe_count", + INT4OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "index_vacuum_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "wal_records", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "wal_fpi", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "wal_bytes", + NUMERICOID, -1, 0); + + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "blk_read_time", + FLOAT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "blk_write_time", + FLOAT8OID, -1, 0); + + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "delay_time", + FLOAT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "total_time", + FLOAT8OID, -1, 0); + + Assert(i == PG_STAT_GET_VACUUM_TABLES_STATS_COLS); + + BlessTupleDesc(tupdesc); + + tabentry = pgstat_fetch_stat_tabentry(relid); + + if (tabentry == NULL) + { + /* If the subscription is not found, initialise its stats */ + memset(&allzero, 0, sizeof(ExtVacReport)); + extvacuum = &allzero; + } + else + { + extvacuum = &(tabentry->vacuum_ext); + } + + i = 0; + + values[i++] = ObjectIdGetDatum(relid); + + values[i++] = Int64GetDatum(extvacuum->total_blks_read); + values[i++] = Int64GetDatum(extvacuum->total_blks_hit); + values[i++] = Int64GetDatum(extvacuum->total_blks_dirtied); + values[i++] = Int64GetDatum(extvacuum->total_blks_written); + + values[i++] = Int64GetDatum(extvacuum->blks_fetched - + extvacuum->blks_hit); + values[i++] = Int64GetDatum(extvacuum->blks_hit); + + values[i++] = Int64GetDatum(extvacuum->pages_scanned); + values[i++] = Int64GetDatum(extvacuum->pages_removed); + values[i++] = Int64GetDatum(extvacuum->vm_new_frozen_pages); + values[i++] = Int64GetDatum(extvacuum->vm_new_visible_pages); + values[i++] = Int64GetDatum(extvacuum->vm_new_visible_frozen_pages); + values[i++] = Int64GetDatum(extvacuum->missed_dead_pages); + values[i++] = Int64GetDatum(extvacuum->tuples_deleted); + values[i++] = Int64GetDatum(extvacuum->tuples_frozen); + values[i++] = Int64GetDatum(extvacuum->recently_dead_tuples); + values[i++] = Int64GetDatum(extvacuum->missed_dead_tuples); + values[i++] = Int32GetDatum(extvacuum->wraparound_failsafe_count); + values[i++] = Int64GetDatum(extvacuum->index_vacuum_count); + + values[i++] = Int64GetDatum(extvacuum->wal_records); + values[i++] = Int64GetDatum(extvacuum->wal_fpi); + + /* Convert to numeric, like pg_stat_statements */ + snprintf(buf, sizeof buf, UINT64_FORMAT, extvacuum->wal_bytes); + values[i++] = DirectFunctionCall3(numeric_in, + CStringGetDatum(buf), + ObjectIdGetDatum(0), + Int32GetDatum(-1)); + + values[i++] = Float8GetDatum(extvacuum->blk_read_time); + values[i++] = Float8GetDatum(extvacuum->blk_write_time); + values[i++] = Float8GetDatum(extvacuum->delay_time); + values[i++] = Float8GetDatum(extvacuum->total_time); + + Assert(i == PG_STAT_GET_VACUUM_TABLES_STATS_COLS); + + /* Returns the record as Datum */ + PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls))); +} \ No newline at end of file diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index 860bbd40d42..4da8d3f87fd 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -1619,6 +1619,19 @@ getinternalerrposition(void) return edata->internalpos; } +/* + * Return elevel of errors + */ +int +geterrelevel(void) +{ + ErrorData *edata = &errordata[errordata_stack_depth]; + + /* we don't bother incrementing recursion_depth */ + CHECK_STACK_DEPTH(); + + return edata->elevel; +} /* * Functions to allow construction of error message strings separately from diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index c9d8cd796a8..f2d31e174b4 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -1478,6 +1478,15 @@ struct config_bool ConfigureNamesBool[] = false, NULL, NULL, NULL }, + { + {"track_vacuum_statistics", PGC_SUSET, STATS_CUMULATIVE, + gettext_noop("Collects vacuum statistics for table relations."), + NULL + }, + &pgstat_track_vacuum_statistics, + true, + NULL, NULL, NULL + }, { {"track_wal_io_timing", PGC_SUSET, STATS_CUMULATIVE, gettext_noop("Collects timing statistics for WAL I/O activity."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b2bc43383db..522e56f1a83 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -644,6 +644,7 @@ #track_wal_io_timing = off #track_functions = none # none, pl, all #stats_fetch_consistency = cache # cache, none, snapshot +#track_vacuum_statistics = off # - Monitoring - diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index b37e8a6f882..0d9584a9fec 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -12429,4 +12429,22 @@ proargtypes => 'int2', prosrc => 'gist_stratnum_identity' }, +{ oid => '8001', + descr => 'pg_stat_get_vacuum_tables returns vacuum stats values for table', + proname => 'pg_stat_get_vacuum_tables', prorows => 1000, provolatile => 's', prorettype => 'record',proisstrict => 'f', + proretset => 't', + proargtypes => 'oid', + proallargtypes => '{oid,oid,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int4,int8,int8,int8,numeric,float8,float8,float8,float8}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{reloid,relid,total_blks_read,total_blks_hit,total_blks_dirtied,total_blks_written,rel_blks_read,rel_blks_hit,pages_scanned,pages_removed,vm_new_frozen_pages,vm_new_visible_pages,vm_new_visible_frozen_pages,missed_dead_pages,tuples_deleted,tuples_frozen,recently_dead_tuples,missed_dead_tuples,wraparound_failsafe,index_vacuum_count,wal_records,wal_fpi,wal_bytes,blk_read_time,blk_write_time,delay_time,total_time}', + prosrc => 'pg_stat_get_vacuum_tables' }, + + { oid => '8002', descr => 'statistics: number of times the all-visible pages in the visibility map was removed for pages of table', + proname => 'pg_stat_get_rev_all_visible_pages', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', + prosrc => 'pg_stat_get_rev_all_visible_pages' }, + { oid => '8003', descr => 'statistics: number of times the all-frozen pages in the visibility map was removed for pages of table', + proname => 'pg_stat_get_rev_all_frozen_pages', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', + prosrc => 'pg_stat_get_rev_all_frozen_pages' }, ] diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 12d0b61950d..94d599767df 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -308,6 +308,7 @@ extern PGDLLIMPORT int vacuum_multixact_failsafe_age; extern PGDLLIMPORT pg_atomic_uint32 *VacuumSharedCostBalance; extern PGDLLIMPORT pg_atomic_uint32 *VacuumActiveNWorkers; extern PGDLLIMPORT int VacuumCostBalanceLocal; +extern PGDLLIMPORT double VacuumDelayTime; extern PGDLLIMPORT bool VacuumFailsafeActive; extern PGDLLIMPORT double vacuum_cost_delay; diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 6475889c58c..0266993dbf2 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -167,6 +167,53 @@ typedef struct PgStat_BackendSubEntry PgStat_Counter conflict_count[CONFLICT_NUM_TYPES]; } PgStat_BackendSubEntry; +/* ---------- + * + * ExtVacReport + * + * Additional statistics of vacuum processing over a heap relation. + * pages_removed is the amount by which the physically shrank, + * if any (ie the change in its total size on disk) + * pages_deleted refer to free space within the index file + * ---------- + */ +typedef struct ExtVacReport +{ + /* number of blocks missed, hit, dirtied and written during a vacuum of specific relation */ + int64 total_blks_read; + int64 total_blks_hit; + int64 total_blks_dirtied; + int64 total_blks_written; + + /* blocks missed and hit for just the heap during a vacuum of specific relation */ + int64 blks_fetched; + int64 blks_hit; + + /* Vacuum WAL usage stats */ + int64 wal_records; /* wal usage: number of WAL records */ + int64 wal_fpi; /* wal usage: number of WAL full page images produced */ + uint64 wal_bytes; /* wal usage: size of WAL records produced */ + + /* Time stats. */ + double blk_read_time; /* time spent reading pages, in msec */ + double blk_write_time; /* time spent writing pages, in msec */ + double delay_time; /* how long vacuum slept in vacuum delay point, in msec */ + double total_time; /* total time of a vacuum operation, in msec */ + + int64 pages_scanned; /* heap pages examined (not skipped by VM) */ + int64 pages_removed; /* heap pages removed by vacuum "truncation" */ + int64 vm_new_frozen_pages; /* pages marked in VM as frozen */ + int64 vm_new_visible_pages; /* pages marked in VM as all-visible */ + int64 vm_new_visible_frozen_pages; /* pages marked in VM as all-visible and frozen */ + int64 missed_dead_tuples; /* tuples not pruned by vacuum due to failure to get a cleanup lock */ + int64 missed_dead_pages; /* pages with missed dead tuples */ + int64 tuples_deleted; /* tuples deleted by vacuum */ + int64 tuples_frozen; /* tuples frozen up by vacuum */ + int64 recently_dead_tuples; /* deleted tuples that are still visible to some transaction */ + int64 index_vacuum_count; /* the number of index vacuumings */ + int32 wraparound_failsafe_count; /* the number of times to prevent workaround problem */ +} ExtVacReport; + /* ---------- * PgStat_TableCounts The actual per-table counts kept by a backend * @@ -209,6 +256,16 @@ typedef struct PgStat_TableCounts PgStat_Counter blocks_fetched; PgStat_Counter blocks_hit; + + PgStat_Counter rev_all_visible_pages; + PgStat_Counter rev_all_frozen_pages; + + /* + * Additional cumulative stat on vacuum operations. + * Use an expensive structure as an abstraction for different types of + * relations. + */ + ExtVacReport vacuum_ext; } PgStat_TableCounts; /* ---------- @@ -267,7 +324,7 @@ typedef struct PgStat_TableXactStatus * ------------------------------------------------------------ */ -#define PGSTAT_FILE_FORMAT_ID 0x01A5BCB0 +#define PGSTAT_FILE_FORMAT_ID 0x01A5BCB1 typedef struct PgStat_ArchiverStats { @@ -429,6 +486,8 @@ typedef struct PgStat_StatDBEntry PgStat_Counter parallel_workers_launched; TimestampTz stat_reset_timestamp; + + ExtVacReport vacuum_ext; /* extended vacuum statistics */ } PgStat_StatDBEntry; typedef struct PgStat_StatFuncEntry @@ -502,6 +561,11 @@ typedef struct PgStat_StatTabEntry PgStat_Counter analyze_count; TimestampTz last_autoanalyze_time; /* autovacuum initiated */ PgStat_Counter autoanalyze_count; + + PgStat_Counter rev_all_visible_pages; + PgStat_Counter rev_all_frozen_pages; + + ExtVacReport vacuum_ext; } PgStat_StatTabEntry; typedef struct PgStat_WalStats @@ -676,7 +740,8 @@ extern void pgstat_assoc_relation(Relation rel); extern void pgstat_unlink_relation(Relation rel); extern void pgstat_report_vacuum(Oid tableoid, bool shared, - PgStat_Counter livetuples, PgStat_Counter deadtuples); + PgStat_Counter livetuples, PgStat_Counter deadtuples, + ExtVacReport *params); extern void pgstat_report_analyze(Relation rel, PgStat_Counter livetuples, PgStat_Counter deadtuples, bool resetcounter); @@ -727,6 +792,17 @@ extern void pgstat_report_analyze(Relation rel, if (pgstat_should_count_relation(rel)) \ (rel)->pgstat_info->counts.blocks_hit++; \ } while (0) +/* accumulate unfrozen all-visible and all-frozen pages */ +#define pgstat_count_vm_rev_all_visible(rel) \ + do { \ + if (pgstat_should_count_relation(rel)) \ + (rel)->pgstat_info->counts.rev_all_visible_pages++; \ + } while (0) +#define pgstat_count_vm_rev_all_frozen(rel) \ + do { \ + if (pgstat_should_count_relation(rel)) \ + (rel)->pgstat_info->counts.rev_all_frozen_pages++; \ + } while (0) extern void pgstat_count_heap_insert(Relation rel, PgStat_Counter n); extern void pgstat_count_heap_update(Relation rel, bool hot, bool newpage); @@ -744,7 +820,6 @@ extern PgStat_StatTabEntry *pgstat_fetch_stat_tabentry_ext(bool shared, Oid reloid); extern PgStat_TableStatus *find_tabstat_entry(Oid rel_id); - /* * Functions in pgstat_replslot.c */ @@ -815,6 +890,7 @@ extern PgStat_WalStats *pgstat_fetch_stat_wal(void); extern PGDLLIMPORT bool pgstat_track_counts; extern PGDLLIMPORT int pgstat_track_functions; extern PGDLLIMPORT int pgstat_fetch_consistency; +extern PGDLLIMPORT bool pgstat_track_vacuum_statistics; /* diff --git a/src/include/utils/elog.h b/src/include/utils/elog.h index 7161f5c6ad6..9e080747a92 100644 --- a/src/include/utils/elog.h +++ b/src/include/utils/elog.h @@ -230,6 +230,7 @@ extern int geterrlevel(void); extern int geterrposition(void); extern int getinternalerrposition(void); +extern int geterrelevel(void); /*---------- * Old-style error reporting API: to be used in this way: diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 4bb8e5c53ab..4f836e7fca0 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -600,7 +600,6 @@ extern PgStat_EntryRef *pgstat_fetch_pending_entry(PgStat_Kind kind, extern void *pgstat_fetch_entry(PgStat_Kind kind, Oid dboid, uint64 objid); extern void pgstat_snapshot_fixed(PgStat_Kind kind); - /* * Functions in pgstat_archiver.c */ diff --git a/src/test/isolation/expected/vacuum-extending-in-repetable-read.out b/src/test/isolation/expected/vacuum-extending-in-repetable-read.out new file mode 100644 index 00000000000..87f7e40b4a6 --- /dev/null +++ b/src/test/isolation/expected/vacuum-extending-in-repetable-read.out @@ -0,0 +1,53 @@ +unused step name: s2_delete +Parsed test spec with 2 sessions + +starting permutation: s2_insert s2_print_vacuum_stats_table s1_begin_repeatable_read s2_update s2_insert_interrupt s2_vacuum s2_print_vacuum_stats_table s1_commit s2_checkpoint s2_vacuum s2_print_vacuum_stats_table +step s2_insert: INSERT INTO test_vacuum_stat_isolation(id, ival) SELECT ival, ival%10 FROM generate_series(1,1000) As ival; +step s2_print_vacuum_stats_table: + SELECT + vt.relname, vt.tuples_deleted, vt.recently_dead_tuples, vt.missed_dead_tuples, vt.missed_dead_pages, vt.tuples_frozen + FROM pg_stat_vacuum_tables vt, pg_class c + WHERE vt.relname = 'test_vacuum_stat_isolation' AND vt.relid = c.oid; + +relname |tuples_deleted|recently_dead_tuples|missed_dead_tuples|missed_dead_pages|tuples_frozen +--------------------------+--------------+--------------------+------------------+-----------------+------------- +test_vacuum_stat_isolation| 0| 0| 0| 0| 0 +(1 row) + +step s1_begin_repeatable_read: + BEGIN transaction ISOLATION LEVEL REPEATABLE READ; + select count(ival) from test_vacuum_stat_isolation where id>900; + +count +----- + 100 +(1 row) + +step s2_update: UPDATE test_vacuum_stat_isolation SET ival = ival + 2 where id > 900; +step s2_insert_interrupt: INSERT INTO test_vacuum_stat_isolation values (1,1); +step s2_vacuum: VACUUM test_vacuum_stat_isolation; +step s2_print_vacuum_stats_table: + SELECT + vt.relname, vt.tuples_deleted, vt.recently_dead_tuples, vt.missed_dead_tuples, vt.missed_dead_pages, vt.tuples_frozen + FROM pg_stat_vacuum_tables vt, pg_class c + WHERE vt.relname = 'test_vacuum_stat_isolation' AND vt.relid = c.oid; + +relname |tuples_deleted|recently_dead_tuples|missed_dead_tuples|missed_dead_pages|tuples_frozen +--------------------------+--------------+--------------------+------------------+-----------------+------------- +test_vacuum_stat_isolation| 0| 100| 0| 0| 0 +(1 row) + +step s1_commit: COMMIT; +step s2_checkpoint: CHECKPOINT; +step s2_vacuum: VACUUM test_vacuum_stat_isolation; +step s2_print_vacuum_stats_table: + SELECT + vt.relname, vt.tuples_deleted, vt.recently_dead_tuples, vt.missed_dead_tuples, vt.missed_dead_pages, vt.tuples_frozen + FROM pg_stat_vacuum_tables vt, pg_class c + WHERE vt.relname = 'test_vacuum_stat_isolation' AND vt.relid = c.oid; + +relname |tuples_deleted|recently_dead_tuples|missed_dead_tuples|missed_dead_pages|tuples_frozen +--------------------------+--------------+--------------------+------------------+-----------------+------------- +test_vacuum_stat_isolation| 100| 100| 0| 0| 101 +(1 row) + diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule index 143109aa4da..e93dd4f626c 100644 --- a/src/test/isolation/isolation_schedule +++ b/src/test/isolation/isolation_schedule @@ -95,6 +95,7 @@ test: timeouts test: vacuum-concurrent-drop test: vacuum-conflict test: vacuum-skip-locked +test: vacuum-extending-in-repetable-read test: stats test: horizons test: predicate-hash diff --git a/src/test/isolation/specs/vacuum-extending-in-repetable-read.spec b/src/test/isolation/specs/vacuum-extending-in-repetable-read.spec new file mode 100644 index 00000000000..5893d89573d --- /dev/null +++ b/src/test/isolation/specs/vacuum-extending-in-repetable-read.spec @@ -0,0 +1,53 @@ +# Test for checking recently_dead_tuples, tuples_deleted and frozen tuples in pg_stat_vacuum_tables. +# recently_dead_tuples values are counted when vacuum hasn't cleared tuples because they were deleted recently. +# recently_dead_tuples aren't increased after releasing lock compared with tuples_deleted, which increased +# by the value of the cleared tuples that the vacuum managed to clear. + +setup +{ + CREATE TABLE test_vacuum_stat_isolation(id int, ival int) WITH (autovacuum_enabled = off); + SET track_io_timing = on; + SET track_vacuum_statistics TO 'on'; +} + +teardown +{ + DROP TABLE test_vacuum_stat_isolation CASCADE; + RESET track_io_timing; + RESET track_vacuum_statistics; +} + +session s1 +step s1_begin_repeatable_read { + BEGIN transaction ISOLATION LEVEL REPEATABLE READ; + select count(ival) from test_vacuum_stat_isolation where id>900; + } +step s1_commit { COMMIT; } + +session s2 +step s2_insert { INSERT INTO test_vacuum_stat_isolation(id, ival) SELECT ival, ival%10 FROM generate_series(1,1000) As ival; } +step s2_update { UPDATE test_vacuum_stat_isolation SET ival = ival + 2 where id > 900; } +step s2_delete { DELETE FROM test_vacuum_stat_isolation where id > 900; } +step s2_insert_interrupt { INSERT INTO test_vacuum_stat_isolation values (1,1); } +step s2_vacuum { VACUUM test_vacuum_stat_isolation; } +step s2_checkpoint { CHECKPOINT; } +step s2_print_vacuum_stats_table +{ + SELECT + vt.relname, vt.tuples_deleted, vt.recently_dead_tuples, vt.missed_dead_tuples, vt.missed_dead_pages, vt.tuples_frozen + FROM pg_stat_vacuum_tables vt, pg_class c + WHERE vt.relname = 'test_vacuum_stat_isolation' AND vt.relid = c.oid; +} + +permutation + s2_insert + s2_print_vacuum_stats_table + s1_begin_repeatable_read + s2_update + s2_insert_interrupt + s2_vacuum + s2_print_vacuum_stats_table + s1_commit + s2_checkpoint + s2_vacuum + s2_print_vacuum_stats_table \ No newline at end of file diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 3014d047fef..b49f7936f93 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1804,7 +1804,9 @@ pg_stat_all_tables| SELECT c.oid AS relid, pg_stat_get_vacuum_count(c.oid) AS vacuum_count, pg_stat_get_autovacuum_count(c.oid) AS autovacuum_count, pg_stat_get_analyze_count(c.oid) AS analyze_count, - pg_stat_get_autoanalyze_count(c.oid) AS autoanalyze_count + pg_stat_get_autoanalyze_count(c.oid) AS autoanalyze_count, + pg_stat_get_rev_all_frozen_pages(c.oid) AS rev_all_frozen_pages, + pg_stat_get_rev_all_visible_pages(c.oid) AS rev_all_visible_pages FROM ((pg_class c LEFT JOIN pg_index i ON ((c.oid = i.indrelid))) LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))) @@ -2188,7 +2190,9 @@ pg_stat_sys_tables| SELECT relid, vacuum_count, autovacuum_count, analyze_count, - autoanalyze_count + autoanalyze_count, + rev_all_frozen_pages, + rev_all_visible_pages FROM pg_stat_all_tables WHERE ((schemaname = ANY (ARRAY['pg_catalog'::name, 'information_schema'::name])) OR (schemaname ~ '^pg_toast'::text)); pg_stat_user_functions| SELECT p.oid AS funcid, @@ -2236,9 +2240,43 @@ pg_stat_user_tables| SELECT relid, vacuum_count, autovacuum_count, analyze_count, - autoanalyze_count + autoanalyze_count, + rev_all_frozen_pages, + rev_all_visible_pages FROM pg_stat_all_tables WHERE ((schemaname <> ALL (ARRAY['pg_catalog'::name, 'information_schema'::name])) AND (schemaname !~ '^pg_toast'::text)); +pg_stat_vacuum_tables| SELECT ns.nspname AS schemaname, + rel.relname, + stats.relid, + stats.total_blks_read, + stats.total_blks_hit, + stats.total_blks_dirtied, + stats.total_blks_written, + stats.rel_blks_read, + stats.rel_blks_hit, + stats.pages_scanned, + stats.pages_removed, + stats.vm_new_frozen_pages, + stats.vm_new_visible_pages, + stats.vm_new_visible_frozen_pages, + stats.missed_dead_pages, + stats.tuples_deleted, + stats.tuples_frozen, + stats.recently_dead_tuples, + stats.missed_dead_tuples, + stats.wraparound_failsafe, + stats.index_vacuum_count, + stats.wal_records, + stats.wal_fpi, + stats.wal_bytes, + stats.blk_read_time, + stats.blk_write_time, + stats.delay_time, + stats.total_time + FROM (pg_class rel + JOIN pg_namespace ns ON ((ns.oid = rel.relnamespace))), + LATERAL pg_stat_get_vacuum_tables(rel.oid) stats(relid, total_blks_read, total_blks_hit, total_blks_dirtied, total_blks_written, rel_blks_read, rel_blks_hit, pages_scanned, pages_removed, vm_new_frozen_pages, vm_new_visible_pages, vm_new_visible_frozen_pages, missed_dead_pages, tuples_deleted, tuples_frozen, recently_dead_tuples, missed_dead_tuples, wraparound_failsafe, index_vacuum_count, wal_records, wal_fpi, wal_bytes, blk_read_time, blk_write_time, delay_time, total_time) + WHERE (rel.relkind = 'r'::"char"); pg_stat_wal| SELECT wal_records, wal_fpi, wal_bytes, diff --git a/src/test/regress/expected/vacuum_tables_statistics.out b/src/test/regress/expected/vacuum_tables_statistics.out new file mode 100644 index 00000000000..0c05a812dd1 --- /dev/null +++ b/src/test/regress/expected/vacuum_tables_statistics.out @@ -0,0 +1,225 @@ +-- +-- Test cumulative vacuum stats system +-- +-- Check the wall statistics collected during vacuum operation: +-- number of frozen and visible pages set by vacuum; +-- number of frozen and visible pages removed by backend. +-- Statistic wal_fpi is not displayed in this test because its behavior is unstable. +-- +-- conditio sine qua non +SHOW track_counts; -- must be on + track_counts +-------------- + on +(1 row) + +\set sample_size 10000 +-- not enabled by default, but we want to test it... +SET track_functions TO 'all'; +-- Test that vacuum statistics will be empty when parameter is off. +SET track_vacuum_statistics TO 'off'; +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; +DELETE FROM vestat WHERE x % 2 = 0; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; +-- Must be empty. +SELECT relname,total_blks_read, total_blks_hit, total_blks_dirtied, total_blks_written,rel_blks_read, rel_blks_hit, +pages_scanned, pages_removed, vm_new_frozen_pages, vm_new_visible_pages, vm_new_visible_frozen_pages, missed_dead_pages, +tuples_deleted, tuples_frozen, recently_dead_tuples, missed_dead_tuples, index_vacuum_count, +wal_records, wal_fpi, wal_bytes, blk_read_time, blk_write_time,delay_time, total_time +FROM pg_stat_vacuum_tables vt +WHERE vt.relname = 'vestat'; + relname | total_blks_read | total_blks_hit | total_blks_dirtied | total_blks_written | rel_blks_read | rel_blks_hit | pages_scanned | pages_removed | vm_new_frozen_pages | vm_new_visible_pages | vm_new_visible_frozen_pages | missed_dead_pages | tuples_deleted | tuples_frozen | recently_dead_tuples | missed_dead_tuples | index_vacuum_count | wal_records | wal_fpi | wal_bytes | blk_read_time | blk_write_time | delay_time | total_time +---------+-----------------+----------------+--------------------+--------------------+---------------+--------------+---------------+---------------+---------------------+----------------------+-----------------------------+-------------------+----------------+---------------+----------------------+--------------------+--------------------+-------------+---------+-----------+---------------+----------------+------------+------------ + vestat | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 +(1 row) + +RESET track_vacuum_statistics; +DROP TABLE vestat CASCADE; +SHOW track_vacuum_statistics; -- must be on + track_vacuum_statistics +------------------------- + on +(1 row) + +-- ensure pending stats are flushed +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +--SET stats_fetch_consistency = snapshot; +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; +SELECT oid AS roid from pg_class where relname = 'vestat' \gset +DELETE FROM vestat WHERE x % 2 = 0; +-- Before the first vacuum execution extended stats view is empty. +SELECT vt.relname,vm_new_frozen_pages,tuples_deleted,relpages,pages_scanned,pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; + relname | vm_new_frozen_pages | tuples_deleted | relpages | pages_scanned | pages_removed +---------+---------------------+----------------+----------+---------------+--------------- + vestat | 0 | 0 | 455 | 0 | 0 +(1 row) + +SELECT relpages AS rp +FROM pg_class c +WHERE relname = 'vestat' \gset +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP OFF) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; +-- The table and index extended vacuum statistics should show us that +-- vacuum frozed pages and clean up pages, but pages_removed stayed the same +-- because of not full table have cleaned up +SELECT vt.relname,vm_new_frozen_pages > 0 AS vm_new_frozen_pages,tuples_deleted > 0 AS tuples_deleted,relpages-:rp = 0 AS relpages,pages_scanned > 0 AS pages_scanned,pages_removed = 0 AS pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; + relname | vm_new_frozen_pages | tuples_deleted | relpages | pages_scanned | pages_removed +---------+---------------------+----------------+----------+---------------+--------------- + vestat | f | t | t | t | t +(1 row) + +SELECT vm_new_frozen_pages AS fp,tuples_deleted AS td,relpages AS rp, pages_scanned AS ps, pages_removed AS pr +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid \gset +-- Store WAL advances into variables +SELECT wal_records AS hwr,wal_bytes AS hwb,wal_fpi AS hfpi FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset +-- Look into WAL records deltas. +SELECT wal_records > 0 AS dWR, wal_bytes > 0 AS dWB +FROM pg_stat_vacuum_tables WHERE relname = 'vestat'; + dwr | dwb +-----+----- + t | t +(1 row) + +DELETE FROM vestat;; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP OFF) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; +-- pages_removed must be increased +SELECT vt.relname,vm_new_frozen_pages-:fp > 0 AS vm_new_frozen_pages,tuples_deleted-:td > 0 AS tuples_deleted,relpages -:rp = 0 AS relpages,pages_scanned-:ps > 0 AS pages_scanned,pages_removed-:pr > 0 AS pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; + relname | vm_new_frozen_pages | tuples_deleted | relpages | pages_scanned | pages_removed +---------+---------------------+----------------+----------+---------------+--------------- + vestat | f | t | f | t | t +(1 row) + +SELECT vm_new_frozen_pages AS fp,tuples_deleted AS td,relpages AS rp, pages_scanned AS ps, pages_removed AS pr +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid \gset +-- Store WAL advances into variables +SELECT wal_records-:hwr AS dwr, wal_bytes-:hwb AS dwb, wal_fpi-:hfpi AS dfpi +FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset +-- WAL advance should be detected. +SELECT :dwr > 0 AS dWR, :dwb > 0 AS dWB; + dwr | dwb +-----+----- + t | t +(1 row) + +-- Store WAL advances into variables +SELECT wal_records AS hwr,wal_bytes AS hwb,wal_fpi AS hfpi FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +DELETE FROM vestat WHERE x % 2 = 0; +-- VACUUM FULL doesn't report to stat collector. So, no any advancements of statistics +-- are detected here. +VACUUM FULL vestat; +-- It is necessary to check the wal statistics +CHECKPOINT; +-- Store WAL advances into variables +SELECT wal_records-:hwr AS dwr2, wal_bytes-:hwb AS dwb2, wal_fpi-:hfpi AS dfpi2 +FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset +-- WAL and other statistics advance should not be detected. +SELECT :dwr2=0 AS dWR, :dfpi2=0 AS dFPI, :dwb2=0 AS dWB; + dwr | dfpi | dwb +-----+------+----- + t | t | t +(1 row) + +SELECT vt.relname,vm_new_frozen_pages-:fp = 0 AS vm_new_frozen_pages,tuples_deleted-:td = 0 AS tuples_deleted,relpages -:rp < 0 AS relpages,pages_scanned-:ps = 0 AS pages_scanned,pages_removed-:pr = 0 AS pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; + relname | vm_new_frozen_pages | tuples_deleted | relpages | pages_scanned | pages_removed +---------+---------------------+----------------+----------+---------------+--------------- + vestat | t | t | f | t | t +(1 row) + +SELECT vm_new_frozen_pages AS fp,tuples_deleted AS td,relpages AS rp, pages_scanned AS ps,pages_removed AS pr +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid \gset +-- Store WAL advances into variables +SELECT wal_records AS hwr,wal_bytes AS hwb,wal_fpi AS hfpi FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset +DELETE FROM vestat; +TRUNCATE vestat; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP OFF) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; +-- Store WAL advances into variables after removing all tuples from the table +SELECT wal_records-:hwr AS dwr3, wal_bytes-:hwb AS dwb3, wal_fpi-:hfpi AS dfpi3 +FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset +--There are nothing changed +SELECT :dwr3>0 AS dWR, :dfpi3=0 AS dFPI, :dwb3>0 AS dWB; + dwr | dfpi | dwb +-----+------+----- + t | t | t +(1 row) + +-- +-- Now, the table and index is compressed into zero number of pages. Check it +-- in vacuum extended statistics. +-- The vm_new_frozen_pages, pages_scanned values shouldn't be changed +-- +SELECT vt.relname,vm_new_frozen_pages-:fp = 0 AS vm_new_frozen_pages,tuples_deleted-:td = 0 AS tuples_deleted,relpages -:rp = 0 AS relpages,pages_scanned-:ps = 0 AS pages_scanned,pages_removed-:pr = 0 AS pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; + relname | vm_new_frozen_pages | tuples_deleted | relpages | pages_scanned | pages_removed +---------+---------------------+----------------+----------+---------------+--------------- + vestat | t | t | f | t | t +(1 row) + +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; +-- must be empty +SELECT vm_new_frozen_pages, vm_new_visible_pages, rev_all_frozen_pages,rev_all_visible_pages,vm_new_visible_frozen_pages +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; + vm_new_frozen_pages | vm_new_visible_pages | rev_all_frozen_pages | rev_all_visible_pages | vm_new_visible_frozen_pages +---------------------+----------------------+----------------------+-----------------------+----------------------------- + 0 | 910 | 0 | 0 | 455 +(1 row) + +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; +-- backend defreezed pages +SELECT vm_new_frozen_pages > 0 AS vm_new_frozen_pages,vm_new_visible_pages > 0 AS vm_new_visible_pages,vm_new_visible_frozen_pages > 0 AS vm_new_visible_frozen_pages,rev_all_frozen_pages = 0 AS rev_all_frozen_pages,rev_all_visible_pages = 0 AS rev_all_visible_pages +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; + vm_new_frozen_pages | vm_new_visible_pages | vm_new_visible_frozen_pages | rev_all_frozen_pages | rev_all_visible_pages +---------------------+----------------------+-----------------------------+----------------------+----------------------- + f | t | t | t | t +(1 row) + +SELECT vm_new_frozen_pages AS pf, vm_new_visible_pages AS pv,vm_new_visible_frozen_pages AS pvf, rev_all_frozen_pages AS hafp,rev_all_visible_pages AS havp +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid \gset +UPDATE vestat SET x = x + 1001; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; +SELECT vm_new_frozen_pages > :pf AS vm_new_frozen_pages,vm_new_visible_pages > :pv AS vm_new_visible_pages,vm_new_visible_frozen_pages > :pvf AS vm_new_visible_frozen_pages,rev_all_frozen_pages > :hafp AS rev_all_frozen_pages,rev_all_visible_pages > :havp AS rev_all_visible_pages +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; + vm_new_frozen_pages | vm_new_visible_pages | vm_new_visible_frozen_pages | rev_all_frozen_pages | rev_all_visible_pages +---------------------+----------------------+-----------------------------+----------------------+----------------------- + f | t | f | f | f +(1 row) + +SELECT vm_new_frozen_pages AS pf, vm_new_visible_pages AS pv, vm_new_visible_frozen_pages AS pvf, rev_all_frozen_pages AS hafp,rev_all_visible_pages AS havp +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid \gset +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; +-- vacuum freezed pages +SELECT vm_new_frozen_pages = :pf AS vm_new_frozen_pages,vm_new_visible_pages = :pv AS vm_new_visible_pages,vm_new_visible_frozen_pages = :pvf AS vm_new_visible_frozen_pages, rev_all_frozen_pages = :hafp AS rev_all_frozen_pages,rev_all_visible_pages = :havp AS rev_all_visible_pages +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; + vm_new_frozen_pages | vm_new_visible_pages | vm_new_visible_frozen_pages | rev_all_frozen_pages | rev_all_visible_pages +---------------------+----------------------+-----------------------------+----------------------+----------------------- + t | t | t | t | t +(1 row) + +DROP TABLE vestat CASCADE; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 1edd9e45ebb..d11f6b7ef4b 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -136,3 +136,8 @@ test: fast_default # run tablespace test at the end because it drops the tablespace created during # setup that other tests may use. test: tablespace + +# ---------- +# Check vacuum statistics +# ---------- +test: vacuum_tables_statistics \ No newline at end of file diff --git a/src/test/regress/sql/vacuum_tables_statistics.sql b/src/test/regress/sql/vacuum_tables_statistics.sql new file mode 100644 index 00000000000..8ad69108ca1 --- /dev/null +++ b/src/test/regress/sql/vacuum_tables_statistics.sql @@ -0,0 +1,180 @@ +-- +-- Test cumulative vacuum stats system +-- +-- Check the wall statistics collected during vacuum operation: +-- number of frozen and visible pages set by vacuum; +-- number of frozen and visible pages removed by backend. +-- Statistic wal_fpi is not displayed in this test because its behavior is unstable. +-- + +-- conditio sine qua non +SHOW track_counts; -- must be on +\set sample_size 10000 + +-- not enabled by default, but we want to test it... +SET track_functions TO 'all'; + +-- Test that vacuum statistics will be empty when parameter is off. +SET track_vacuum_statistics TO 'off'; + +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; + +DELETE FROM vestat WHERE x % 2 = 0; + +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; + +-- Must be empty. +SELECT relname,total_blks_read, total_blks_hit, total_blks_dirtied, total_blks_written,rel_blks_read, rel_blks_hit, +pages_scanned, pages_removed, vm_new_frozen_pages, vm_new_visible_pages, vm_new_visible_frozen_pages, missed_dead_pages, +tuples_deleted, tuples_frozen, recently_dead_tuples, missed_dead_tuples, index_vacuum_count, +wal_records, wal_fpi, wal_bytes, blk_read_time, blk_write_time,delay_time, total_time +FROM pg_stat_vacuum_tables vt +WHERE vt.relname = 'vestat'; + +RESET track_vacuum_statistics; +DROP TABLE vestat CASCADE; + +SHOW track_vacuum_statistics; -- must be on + +-- ensure pending stats are flushed +SELECT pg_stat_force_next_flush(); + +--SET stats_fetch_consistency = snapshot; +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; + +SELECT oid AS roid from pg_class where relname = 'vestat' \gset + +DELETE FROM vestat WHERE x % 2 = 0; +-- Before the first vacuum execution extended stats view is empty. +SELECT vt.relname,vm_new_frozen_pages,tuples_deleted,relpages,pages_scanned,pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; +SELECT relpages AS rp +FROM pg_class c +WHERE relname = 'vestat' \gset + +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP OFF) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; + +-- The table and index extended vacuum statistics should show us that +-- vacuum frozed pages and clean up pages, but pages_removed stayed the same +-- because of not full table have cleaned up +SELECT vt.relname,vm_new_frozen_pages > 0 AS vm_new_frozen_pages,tuples_deleted > 0 AS tuples_deleted,relpages-:rp = 0 AS relpages,pages_scanned > 0 AS pages_scanned,pages_removed = 0 AS pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; +SELECT vm_new_frozen_pages AS fp,tuples_deleted AS td,relpages AS rp, pages_scanned AS ps, pages_removed AS pr +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid \gset + +-- Store WAL advances into variables +SELECT wal_records AS hwr,wal_bytes AS hwb,wal_fpi AS hfpi FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset + +-- Look into WAL records deltas. +SELECT wal_records > 0 AS dWR, wal_bytes > 0 AS dWB +FROM pg_stat_vacuum_tables WHERE relname = 'vestat'; + +DELETE FROM vestat;; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP OFF) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; + +-- pages_removed must be increased +SELECT vt.relname,vm_new_frozen_pages-:fp > 0 AS vm_new_frozen_pages,tuples_deleted-:td > 0 AS tuples_deleted,relpages -:rp = 0 AS relpages,pages_scanned-:ps > 0 AS pages_scanned,pages_removed-:pr > 0 AS pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; +SELECT vm_new_frozen_pages AS fp,tuples_deleted AS td,relpages AS rp, pages_scanned AS ps, pages_removed AS pr +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid \gset + +-- Store WAL advances into variables +SELECT wal_records-:hwr AS dwr, wal_bytes-:hwb AS dwb, wal_fpi-:hfpi AS dfpi +FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset + +-- WAL advance should be detected. +SELECT :dwr > 0 AS dWR, :dwb > 0 AS dWB; + +-- Store WAL advances into variables +SELECT wal_records AS hwr,wal_bytes AS hwb,wal_fpi AS hfpi FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset + +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +DELETE FROM vestat WHERE x % 2 = 0; +-- VACUUM FULL doesn't report to stat collector. So, no any advancements of statistics +-- are detected here. +VACUUM FULL vestat; +-- It is necessary to check the wal statistics +CHECKPOINT; + +-- Store WAL advances into variables +SELECT wal_records-:hwr AS dwr2, wal_bytes-:hwb AS dwb2, wal_fpi-:hfpi AS dfpi2 +FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset + +-- WAL and other statistics advance should not be detected. +SELECT :dwr2=0 AS dWR, :dfpi2=0 AS dFPI, :dwb2=0 AS dWB; + +SELECT vt.relname,vm_new_frozen_pages-:fp = 0 AS vm_new_frozen_pages,tuples_deleted-:td = 0 AS tuples_deleted,relpages -:rp < 0 AS relpages,pages_scanned-:ps = 0 AS pages_scanned,pages_removed-:pr = 0 AS pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; +SELECT vm_new_frozen_pages AS fp,tuples_deleted AS td,relpages AS rp, pages_scanned AS ps,pages_removed AS pr +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid \gset + +-- Store WAL advances into variables +SELECT wal_records AS hwr,wal_bytes AS hwb,wal_fpi AS hfpi FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset + +DELETE FROM vestat; +TRUNCATE vestat; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP OFF) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; + +-- Store WAL advances into variables after removing all tuples from the table +SELECT wal_records-:hwr AS dwr3, wal_bytes-:hwb AS dwb3, wal_fpi-:hfpi AS dfpi3 +FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset + +--There are nothing changed +SELECT :dwr3>0 AS dWR, :dfpi3=0 AS dFPI, :dwb3>0 AS dWB; + +-- +-- Now, the table and index is compressed into zero number of pages. Check it +-- in vacuum extended statistics. +-- The vm_new_frozen_pages, pages_scanned values shouldn't be changed +-- +SELECT vt.relname,vm_new_frozen_pages-:fp = 0 AS vm_new_frozen_pages,tuples_deleted-:td = 0 AS tuples_deleted,relpages -:rp = 0 AS relpages,pages_scanned-:ps = 0 AS pages_scanned,pages_removed-:pr = 0 AS pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; + +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; + +-- must be empty +SELECT vm_new_frozen_pages, vm_new_visible_pages, rev_all_frozen_pages,rev_all_visible_pages,vm_new_visible_frozen_pages +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; + +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; + +-- backend defreezed pages +SELECT vm_new_frozen_pages > 0 AS vm_new_frozen_pages,vm_new_visible_pages > 0 AS vm_new_visible_pages,vm_new_visible_frozen_pages > 0 AS vm_new_visible_frozen_pages,rev_all_frozen_pages = 0 AS rev_all_frozen_pages,rev_all_visible_pages = 0 AS rev_all_visible_pages +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; +SELECT vm_new_frozen_pages AS pf, vm_new_visible_pages AS pv,vm_new_visible_frozen_pages AS pvf, rev_all_frozen_pages AS hafp,rev_all_visible_pages AS havp +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid \gset + +UPDATE vestat SET x = x + 1001; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; + +SELECT vm_new_frozen_pages > :pf AS vm_new_frozen_pages,vm_new_visible_pages > :pv AS vm_new_visible_pages,vm_new_visible_frozen_pages > :pvf AS vm_new_visible_frozen_pages,rev_all_frozen_pages > :hafp AS rev_all_frozen_pages,rev_all_visible_pages > :havp AS rev_all_visible_pages +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; +SELECT vm_new_frozen_pages AS pf, vm_new_visible_pages AS pv, vm_new_visible_frozen_pages AS pvf, rev_all_frozen_pages AS hafp,rev_all_visible_pages AS havp +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid \gset + +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; + +-- vacuum freezed pages +SELECT vm_new_frozen_pages = :pf AS vm_new_frozen_pages,vm_new_visible_pages = :pv AS vm_new_visible_pages,vm_new_visible_frozen_pages = :pvf AS vm_new_visible_frozen_pages, rev_all_frozen_pages = :hafp AS rev_all_frozen_pages,rev_all_visible_pages = :havp AS rev_all_visible_pages +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; + +DROP TABLE vestat CASCADE; -- 2.34.1