From 5fe3cb2abf25ec64eb43ecf6b70885183a89fe2b Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Tue, 11 Jun 2024 13:55:39 +0300 Subject: [PATCH 4/4] Machinery for grabbing an extended vacuum statistics on databases. It transmits vacuum statistical information about each table and accumulates it for the database which the table belonged. --- src/backend/catalog/system_views.sql | 27 +++++++ src/backend/utils/activity/pgstat.c | 2 + src/backend/utils/activity/pgstat_database.c | 1 + src/backend/utils/activity/pgstat_relation.c | 16 ++++ src/backend/utils/adt/pgstatfuncs.c | 79 +++++++++++++++++++ src/include/catalog/pg_proc.dat | 9 +++ src/include/pgstat.h | 3 +- src/test/regress/expected/opr_sanity.out | 7 +- src/test/regress/expected/rules.out | 17 ++++ ...ut => vacuum_tables_and_db_statistics.out} | 78 ++++++++++++++++++ src/test/regress/parallel_schedule | 2 +- ...ql => vacuum_tables_and_db_statistics.sql} | 66 +++++++++++++++- 12 files changed, 301 insertions(+), 6 deletions(-) rename src/test/regress/expected/{vacuum_tables_statistics.out => vacuum_tables_and_db_statistics.out} (76%) rename src/test/regress/sql/{vacuum_tables_statistics.sql => vacuum_tables_and_db_statistics.sql} (77%) diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 05f8fc07108..1b60f0dd81a 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1466,3 +1466,30 @@ WHERE rel.oid = stats.relid AND ns.oid = rel.relnamespace; +CREATE VIEW pg_stat_vacuum_database AS +SELECT + db.oid as dboid, + + stats.db_blks_read, + stats.db_blks_hit, + stats.total_blks_dirtied, + stats.total_blks_written, + + stats.wal_records, + stats.wal_fpi, + stats.wal_bytes, + + stats.blk_read_time, + stats.blk_write_time, + + stats.delay_time, + stats.system_time, + stats.user_time, + stats.total_time, + + stats.interrupts +FROM + pg_database db LEFT JOIN pg_stat_vacuum_database(db.oid) stats +ON + db.oid = stats.dboid; + diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 75890b18988..3c50bea379c 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -1099,6 +1099,8 @@ pgstat_update_snapshot(PgStat_Kind kind) pgstat_fetch_consistency = PGSTAT_FETCH_CONSISTENCY_SNAPSHOT; if (kind == PGSTAT_KIND_RELATION) pgstat_build_snapshot(PGSTAT_KIND_RELATION); + else if (kind == PGSTAT_KIND_DATABASE) + pgstat_build_snapshot(PGSTAT_KIND_DATABASE); } PG_FINALLY(); { diff --git a/src/backend/utils/activity/pgstat_database.c b/src/backend/utils/activity/pgstat_database.c index 29bc0909748..a060d1a4042 100644 --- a/src/backend/utils/activity/pgstat_database.c +++ b/src/backend/utils/activity/pgstat_database.c @@ -430,6 +430,7 @@ pgstat_database_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) pgstat_unlock_entry(entry_ref); memset(pendingent, 0, sizeof(*pendingent)); + memset(&(pendingent)->vacuum_ext, 0, sizeof(ExtVacReport)); return true; } diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index 5b06b04faad..cc09aba571f 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -217,6 +217,7 @@ pgstat_report_vacuum_error(Oid tableoid, ExtVacReportType m_type) PgStatShared_Relation *shtabentry; PgStat_StatTabEntry *tabentry; Oid dboid = MyDatabaseId; + PgStat_StatDBEntry *dbentry; /* pending database entry */ if (!pgstat_track_counts) return; @@ -230,6 +231,10 @@ pgstat_report_vacuum_error(Oid tableoid, ExtVacReportType m_type) tabentry->vacuum_ext.interrupts++; tabentry->vacuum_ext.type = m_type; pgstat_unlock_entry(entry_ref); + + dbentry = pgstat_prep_database_pending(dboid); + dbentry->vacuum_ext.interrupts++; + dbentry->vacuum_ext.type = m_type; } /* @@ -243,6 +248,7 @@ pgstat_report_vacuum(Oid tableoid, bool shared, PgStat_EntryRef *entry_ref; PgStatShared_Relation *shtabentry; PgStat_StatTabEntry *tabentry; + PgStatShared_Database *dbentry; Oid dboid = (shared ? InvalidOid : MyDatabaseId); TimestampTz ts; @@ -296,6 +302,16 @@ pgstat_report_vacuum(Oid tableoid, bool shared, * VACUUM command has processed all tables and committed. */ pgstat_flush_io(false); + if (dboid != InvalidOid) + { + entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_DATABASE, + dboid, InvalidOid, false); + dbentry = (PgStatShared_Database *) entry_ref->shared_stats; + + pgstat_accumulate_extvac_stats(&dbentry->stats.vacuum_ext, params, false); + pgstat_unlock_entry(entry_ref); + } + } /* diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index b868b917ceb..0c490ba5f1a 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2036,6 +2036,7 @@ pg_stat_have_stats(PG_FUNCTION_ARGS) #define EXTVACHEAPSTAT_COLUMNS 27 #define EXTVACIDXSTAT_COLUMNS 19 +#define EXTVACDBSTAT_COLUMNS 15 #define EXTVACSTAT_COLUMNS Max(EXTVACHEAPSTAT_COLUMNS, EXTVACIDXSTAT_COLUMNS) static Oid CurrentDatabaseId = InvalidOid; @@ -2076,6 +2077,48 @@ fetch_dbstat_tabentry(Oid dbid, Oid relid) return tabentry; } +static void +tuplestore_put_for_database(Oid dbid, Tuplestorestate *tupstore, + TupleDesc tupdesc, PgStatShared_Database *dbentry, int ncolumns) +{ + Datum values[EXTVACDBSTAT_COLUMNS]; + bool nulls[EXTVACDBSTAT_COLUMNS]; + char buf[256]; + int i = 0; + + memset(nulls, 0, EXTVACDBSTAT_COLUMNS * sizeof(bool)); + + values[i++] = ObjectIdGetDatum(dbid); + + values[i++] = Int64GetDatum(dbentry->stats.vacuum_ext.total_blks_read); + values[i++] = Int64GetDatum(dbentry->stats.vacuum_ext.total_blks_hit); + values[i++] = Int64GetDatum(dbentry->stats.vacuum_ext.total_blks_dirtied); + values[i++] = Int64GetDatum(dbentry->stats.vacuum_ext.total_blks_written); + + values[i++] = Int64GetDatum(dbentry->stats.vacuum_ext.wal_records); + values[i++] = Int64GetDatum(dbentry->stats.vacuum_ext.wal_fpi); + + /* Convert to numeric, like pg_stat_statements */ + snprintf(buf, sizeof buf, UINT64_FORMAT, dbentry->stats.vacuum_ext.wal_bytes); + values[i++] = DirectFunctionCall3(numeric_in, + CStringGetDatum(buf), + ObjectIdGetDatum(0), + Int32GetDatum(-1)); + + values[i++] = Float8GetDatum(dbentry->stats.vacuum_ext.blk_read_time); + values[i++] = Float8GetDatum(dbentry->stats.vacuum_ext.blk_write_time); + values[i++] = Float8GetDatum(dbentry->stats.vacuum_ext.delay_time); + values[i++] = Float8GetDatum(dbentry->stats.vacuum_ext.system_time); + values[i++] = Float8GetDatum(dbentry->stats.vacuum_ext.user_time); + values[i++] = Float8GetDatum(dbentry->stats.vacuum_ext.total_time); + values[i++] = Int32GetDatum(dbentry->stats.vacuum_ext.interrupts); + + + Assert(i == ncolumns); + + tuplestore_putvalues(tupstore, tupdesc, values, nulls); +} + static void tuplestore_put_for_relation(Oid relid, Tuplestorestate *tupstore, TupleDesc tupdesc, PgStat_StatTabEntry *tabentry, int ncolumns) @@ -2220,6 +2263,31 @@ pg_stats_vacuum(FunctionCallInfo fcinfo, ExtVacReportType type, int ncolumns) } } } + else if (type == PGSTAT_EXTVAC_DB) + { + PgStatShared_Database *dbentry; + PgStat_EntryRef *entry_ref; + Oid storedMyDatabaseId = MyDatabaseId; + + pgstat_update_snapshot(PGSTAT_KIND_DATABASE); + MyDatabaseId = storedMyDatabaseId; + + if (OidIsValid(dbid)) + { + entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_DATABASE, + dbid, InvalidOid, false); + dbentry = (PgStatShared_Database *) entry_ref->shared_stats; + + if (dbentry == NULL) + /* Table doesn't exist or isn't a heap relation */ + PG_RETURN_NULL(); + + tuplestore_put_for_database(dbid, tupstore, tupdesc, dbentry, ncolumns); + pgstat_unlock_entry(entry_ref); + } + else + PG_RETURN_NULL(); + } PG_RETURN_NULL(); } @@ -2244,3 +2312,14 @@ pg_stat_vacuum_indexes(PG_FUNCTION_ARGS) PG_RETURN_NULL(); } + +/* + * Get the vacuum statistics for the database. + */ +Datum +pg_stat_vacuum_database(PG_FUNCTION_ARGS) +{ + return pg_stats_vacuum(fcinfo, PGSTAT_EXTVAC_DB, EXTVACDBSTAT_COLUMNS); + + PG_RETURN_NULL(); +} diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 6613d2775f9..8a8f21840c0 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -12264,4 +12264,13 @@ proargmodes => '{i,i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', proargnames => '{dboid,reloid,relid,total_blks_read,total_blks_hit,total_blks_dirtied,total_blks_written,rel_blks_read,rel_blks_hit,pages_deleted,tuples_deleted,wal_records,wal_fpi,wal_bytes,blk_read_time,blk_write_time,delay_time,system_time,user_time,total_time,interrupts}', prosrc => 'pg_stat_vacuum_indexes' } +{ oid => '8003', + descr => 'pg_stat_vacuum_database return stats values', + proname => 'pg_stat_vacuum_database', provolatile => 's', prorettype => 'record',proisstrict => 'f', + proretset => 't', + proargtypes => 'oid', + proallargtypes => '{oid,oid,int8,int8,int8,int8,int8,int8,numeric,float8,float8,float8,float8,float8,float8,int4}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{dbid,dboid,db_blks_read,db_blks_hit,total_blks_dirtied,total_blks_written,wal_records,wal_fpi,wal_bytes,blk_read_time,blk_write_time,delay_time,system_time,user_time,total_time,interrupts}', + prosrc => 'pg_stat_vacuum_database' }, ] diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 762b53b88ed..110e9472f3c 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -173,7 +173,8 @@ typedef enum ExtVacReportType { PGSTAT_EXTVAC_INVALID = 0, PGSTAT_EXTVAC_HEAP = 1, - PGSTAT_EXTVAC_INDEX = 2 + PGSTAT_EXTVAC_INDEX = 2, + PGSTAT_EXTVAC_DB = 3, } ExtVacReportType; /* ---------- diff --git a/src/test/regress/expected/opr_sanity.out b/src/test/regress/expected/opr_sanity.out index 1127c99177b..696602d974c 100644 --- a/src/test/regress/expected/opr_sanity.out +++ b/src/test/regress/expected/opr_sanity.out @@ -32,11 +32,12 @@ WHERE p1.prolang = 0 OR p1.prorettype = 0 OR prokind NOT IN ('f', 'a', 'w', 'p') OR provolatile NOT IN ('i', 's', 'v') OR proparallel NOT IN ('s', 'r', 'u'); - oid | proname -------+------------------------ + oid | proname +------+------------------------- 8001 | pg_stat_vacuum_tables 8002 | pg_stat_vacuum_indexes -(2 rows) + 8003 | pg_stat_vacuum_database +(3 rows) -- prosrc should never be null; it can be empty only if prosqlbody isn't null SELECT p1.oid, p1.proname diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index bc56f352d14..51763846607 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2230,6 +2230,23 @@ pg_stat_user_tables| SELECT relid, autoanalyze_count FROM pg_stat_all_tables WHERE ((schemaname <> ALL (ARRAY['pg_catalog'::name, 'information_schema'::name])) AND (schemaname !~ '^pg_toast'::text)); +pg_stat_vacuum_database| SELECT db.oid AS dboid, + stats.db_blks_read, + stats.db_blks_hit, + stats.total_blks_dirtied, + stats.total_blks_written, + stats.wal_records, + stats.wal_fpi, + stats.wal_bytes, + stats.blk_read_time, + stats.blk_write_time, + stats.delay_time, + stats.system_time, + stats.user_time, + stats.total_time, + stats.interrupts + FROM (pg_database db + LEFT JOIN LATERAL pg_stat_vacuum_database(db.oid) stats(dboid, db_blks_read, db_blks_hit, total_blks_dirtied, total_blks_written, wal_records, wal_fpi, wal_bytes, blk_read_time, blk_write_time, delay_time, system_time, user_time, total_time, interrupts) ON ((db.oid = stats.dboid))); pg_stat_vacuum_indexes| SELECT rel.oid AS relid, ns.nspname AS schema, rel.relname, diff --git a/src/test/regress/expected/vacuum_tables_statistics.out b/src/test/regress/expected/vacuum_tables_and_db_statistics.out similarity index 76% rename from src/test/regress/expected/vacuum_tables_statistics.out rename to src/test/regress/expected/vacuum_tables_and_db_statistics.out index b85a5cab9af..6146a27bdb5 100644 --- a/src/test/regress/expected/vacuum_tables_statistics.out +++ b/src/test/regress/expected/vacuum_tables_and_db_statistics.out @@ -6,6 +6,9 @@ -- number of frozen and visible pages removed by backend. -- Statistic wal_fpi is not displayed in this test because its behavior is unstable. -- +CREATE DATABASE statistic_vacuum_database; +CREATE DATABASE statistic_vacuum_database1; +\c statistic_vacuum_database; -- conditio sine qua non SHOW track_counts; -- must be on track_counts @@ -196,4 +199,79 @@ FROM pg_stat_vacuum_tables WHERE relname = 'vestat'; t | t | t | t (1 row) +-- Now check vacuum statistics for current database +SELECT pg_database.datname, + db_blks_hit > 0 AS db_blks_hit, + total_blks_dirtied > 0 AS total_blks_dirtied, + total_blks_written > 0 AS total_blks_written, + wal_records > 0 AS wal_records, + wal_fpi > 0 AS wal_fpi, + wal_bytes > 0 AS wal_bytes, + user_time > 0 AS user_time, + total_time > 0 AS total_time +FROM +pg_stat_vacuum_database, pg_database +WHERE pg_database.datname = current_database() and pg_database.oid = pg_stat_vacuum_database.dboid; + datname | db_blks_hit | total_blks_dirtied | total_blks_written | wal_records | wal_fpi | wal_bytes | user_time | total_time +---------------------------+-------------+--------------------+--------------------+-------------+---------+-----------+-----------+------------ + statistic_vacuum_database | t | t | t | t | t | t | t | t +(1 row) + +DROP TABLE vestat CASCADE; +-- ensure pending stats are flushed +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; +UPDATE vestat SET x = 10001; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; +\c statistic_vacuum_database1; +-- Now check vacuum statistics for postgres database from another database +SELECT pg_database.datname, + db_blks_hit > 0 AS db_blks_hit, + total_blks_dirtied > 0 AS total_blks_dirtied, + total_blks_written > 0 AS total_blks_written, + wal_records > 0 AS wal_records, + wal_fpi > 0 AS wal_fpi, + wal_bytes > 0 AS wal_bytes, + user_time > 0 AS user_time, + total_time > 0 AS total_time +FROM +pg_stat_vacuum_database, pg_database +WHERE pg_database.datname = 'statistic_vacuum_database' and pg_database.oid = pg_stat_vacuum_database.dboid; + datname | db_blks_hit | total_blks_dirtied | total_blks_written | wal_records | wal_fpi | wal_bytes | user_time | total_time +---------------------------+-------------+--------------------+--------------------+-------------+---------+-----------+-----------+------------ + statistic_vacuum_database | t | t | t | t | t | t | t | t +(1 row) + +\c statistic_vacuum_database +RESET vacuum_freeze_min_age; +RESET vacuum_freeze_table_age; DROP TABLE vestat CASCADE; +\c statistic_vacuum_database1; +SELECT count(*) +FROM pg_database d +CROSS JOIN pg_stat_vacuum_tables(d.oid, 0) +WHERE oid = 0; -- must be 0 + count +------- + 0 +(1 row) + +SELECT count(*) +FROM pg_database d +CROSS JOIN pg_stat_vacuum_database(0) +WHERE oid = 0; -- must be 0 + count +------- + 0 +(1 row) + +\c postgres +DROP DATABASE statistic_vacuum_database1; +DROP DATABASE statistic_vacuum_database; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index b9408a43f71..129b1102028 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -141,4 +141,4 @@ test: tablespace # Check vacuum statistics # ---------- test: vacuum_index_statistics -test: vacuum_tables_statistics \ No newline at end of file +test: vacuum_tables_and_db_statistics \ No newline at end of file diff --git a/src/test/regress/sql/vacuum_tables_statistics.sql b/src/test/regress/sql/vacuum_tables_and_db_statistics.sql similarity index 77% rename from src/test/regress/sql/vacuum_tables_statistics.sql rename to src/test/regress/sql/vacuum_tables_and_db_statistics.sql index 41e387dd304..da59acf604f 100644 --- a/src/test/regress/sql/vacuum_tables_statistics.sql +++ b/src/test/regress/sql/vacuum_tables_and_db_statistics.sql @@ -7,6 +7,10 @@ -- Statistic wal_fpi is not displayed in this test because its behavior is unstable. -- +CREATE DATABASE statistic_vacuum_database; +CREATE DATABASE statistic_vacuum_database1; +\c statistic_vacuum_database; + -- conditio sine qua non SHOW track_counts; -- must be on -- not enabled by default, but we want to test it... @@ -155,4 +159,64 @@ VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; SELECT pages_frozen = :pf AS pages_frozen,pages_all_visible = :pv AS pages_all_visible,rev_all_frozen_pages = :hafp AS rev_all_frozen_pages,rev_all_visible_pages = :havp AS rev_all_visible_pages FROM pg_stat_vacuum_tables WHERE relname = 'vestat'; -DROP TABLE vestat CASCADE; \ No newline at end of file +-- Now check vacuum statistics for current database +SELECT pg_database.datname, + db_blks_hit > 0 AS db_blks_hit, + total_blks_dirtied > 0 AS total_blks_dirtied, + total_blks_written > 0 AS total_blks_written, + wal_records > 0 AS wal_records, + wal_fpi > 0 AS wal_fpi, + wal_bytes > 0 AS wal_bytes, + user_time > 0 AS user_time, + total_time > 0 AS total_time +FROM +pg_stat_vacuum_database, pg_database +WHERE pg_database.datname = current_database() and pg_database.oid = pg_stat_vacuum_database.dboid; + +DROP TABLE vestat CASCADE; + +-- ensure pending stats are flushed +SELECT pg_stat_force_next_flush(); + +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; +UPDATE vestat SET x = 10001; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; + +\c statistic_vacuum_database1; + +-- Now check vacuum statistics for postgres database from another database +SELECT pg_database.datname, + db_blks_hit > 0 AS db_blks_hit, + total_blks_dirtied > 0 AS total_blks_dirtied, + total_blks_written > 0 AS total_blks_written, + wal_records > 0 AS wal_records, + wal_fpi > 0 AS wal_fpi, + wal_bytes > 0 AS wal_bytes, + user_time > 0 AS user_time, + total_time > 0 AS total_time +FROM +pg_stat_vacuum_database, pg_database +WHERE pg_database.datname = 'statistic_vacuum_database' and pg_database.oid = pg_stat_vacuum_database.dboid; + +\c statistic_vacuum_database + +RESET vacuum_freeze_min_age; +RESET vacuum_freeze_table_age; +DROP TABLE vestat CASCADE; + +\c statistic_vacuum_database1; +SELECT count(*) +FROM pg_database d +CROSS JOIN pg_stat_vacuum_tables(d.oid, 0) +WHERE oid = 0; -- must be 0 + +SELECT count(*) +FROM pg_database d +CROSS JOIN pg_stat_vacuum_database(0) +WHERE oid = 0; -- must be 0 + +\c postgres +DROP DATABASE statistic_vacuum_database1; +DROP DATABASE statistic_vacuum_database; -- 2.34.1