From 662a0d70cf927f8525152ad7d6356a8113bada8d Mon Sep 17 00:00:00 2001 From: Melanie Plageman Date: Thu, 12 Nov 2026 14:28:08 -0500 Subject: [PATCH v3] Exclude new pages from VACUUM reltuples estimates VACUUM always scans PageIsNew blocks because they have no visibility map bits. They could already distort estimates when other pages were skipped by repeated VACUUMs. On-access VM setting (b46e1e54d) makes this more likely by allowing populated pages to be skipped by every vacuum. Exclude PageIsNew blocks from the sampled pages and sampleable population that is used to extrapolate reltuples for the unscanned pages. Reported-by: Vaibhav Jain Co-authored-by: Vaibhav Jain Co-authored-by: Melanie Plageman Discussion: https://postgr.es/m/flat/CABa-Az2YXuX%2B_JH8VL89i7oA%2BDC74GvfkSsM9%2B92fGCeWYVCqg%40mail.gmail.com Backpatch-through: 19 --- contrib/pgstattuple/pgstatapprox.c | 16 ++++- src/backend/access/heap/vacuumlazy.c | 16 +++-- src/backend/commands/vacuum.c | 26 ++++++-- src/include/commands/vacuum.h | 1 + src/test/regress/expected/vacuum.out | 93 ++++++++++++++++++++++++++++ src/test/regress/sql/vacuum.sql | 43 +++++++++++++ 6 files changed, 183 insertions(+), 12 deletions(-) diff --git a/contrib/pgstattuple/pgstatapprox.c b/contrib/pgstattuple/pgstatapprox.c index ade1b3bcebd..bed904206d7 100644 --- a/contrib/pgstattuple/pgstatapprox.c +++ b/contrib/pgstattuple/pgstatapprox.c @@ -57,6 +57,7 @@ typedef struct StatApproxReadStreamPrivate BlockNumber current_blocknum; BlockNumber nblocks; BlockNumber scanned; /* count of pages actually read */ + BlockNumber scanned_new; /* count of scanned PageIsNew pages */ Buffer vmbuffer; /* for VM lookups */ } StatApproxReadStreamPrivate; @@ -133,6 +134,7 @@ statapprox_heap(Relation rel, output_type *stat) p.current_blocknum = 0; p.nblocks = nblocks; p.scanned = 0; + p.scanned_new = 0; p.vmbuffer = InvalidBuffer; /* @@ -168,7 +170,14 @@ statapprox_heap(Relation rel, output_type *stat) stat->free_space += PageGetExactFreeSpace(page); - if (PageIsNew(page) || PageIsEmpty(page)) + if (PageIsNew(page)) + { + p.scanned_new++; + UnlockReleaseBuffer(buf); + continue; + } + + if (PageIsEmpty(page)) { UnlockReleaseBuffer(buf); continue; @@ -246,6 +255,7 @@ statapprox_heap(Relation rel, output_type *stat) * we already accounted for the space in those pages, too. */ stat->tuple_count = vac_estimate_reltuples(rel, nblocks, p.scanned, + p.scanned_new, stat->tuple_count); /* It's not clear if we could get -1 here, but be safe. */ @@ -256,6 +266,10 @@ statapprox_heap(Relation rel, output_type *stat) */ if (nblocks != 0) { + /* + * Includes PageIsNew blocks that were read but excluded from the + * live-tuple estimate. + */ stat->scanned_percent = 100.0 * p.scanned / nblocks; stat->tuple_percent = 100.0 * stat->tuple_len / stat->table_len; stat->dead_tuple_percent = 100.0 * stat->dead_tuple_len / stat->table_len; diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 30a3e1b7063..1393c57ad8f 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -305,6 +305,7 @@ typedef struct LVRelState BlockNumber rel_pages; /* total number of pages */ BlockNumber scanned_pages; /* # pages examined (not skipped via VM) */ + BlockNumber scanned_new_pages; /* # scanned pages that were PageIsNew */ /* * Count of all-visible blocks eagerly scanned (for logging only). This @@ -740,6 +741,7 @@ heap_vacuum_rel(Relation rel, const VacuumParams *params, /* Initialize page counters explicitly (be tidy) */ vacrel->scanned_pages = 0; + vacrel->scanned_new_pages = 0; vacrel->eager_scanned_pages = 0; vacrel->removed_pages = 0; vacrel->new_frozen_tuple_pages = 0; @@ -955,8 +957,8 @@ heap_vacuum_rel(Relation rel, const VacuumParams *params, /* * Now actually update rel's pg_class entry. * - * new_live_tuples can be -1 when VACUUM skipped nearly all pages and had - * no prior tuple-density estimate. + * new_live_tuples can be -1 when VACUUM skipped nearly all initialized + * pages and had no prior tuple-density estimate. */ vac_update_relstats(rel, new_rel_pages, vacrel->new_live_tuples, new_rel_allvisible, new_rel_allfrozen, @@ -1594,9 +1596,11 @@ lazy_scan_heap(LVRelState *vacrel) rel_pages); /* now we can compute the new value for pg_class.reltuples */ - vacrel->new_live_tuples = vac_estimate_reltuples(vacrel->rel, rel_pages, - vacrel->scanned_pages, - vacrel->live_tuples); + vacrel->new_live_tuples = + vac_estimate_reltuples(vacrel->rel, rel_pages, + vacrel->scanned_pages, + vacrel->scanned_new_pages, + vacrel->live_tuples); /* * Also compute the total number of surviving heap entries. In the @@ -1893,6 +1897,8 @@ lazy_scan_new_or_empty(LVRelState *vacrel, Buffer buf, BlockNumber blkno, if (PageIsNew(page)) { + vacrel->scanned_new_pages++; + /* * All-zeroes pages can be left over if either a backend extends the * relation by a single page, but crashes before the newly initialized diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index ef9e091d9fe..9472fe52a84 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -1352,6 +1352,11 @@ vacuum_xid_failsafe_check(const struct VacuumCutoffs *cutoffs) * the old value of pg_class.reltuples/pg_class.relpages as a measurement * of the tuple density in the unscanned pages. * + * scanned_new_pages is the number of scanned PageIsNew blocks. Such + * blocks contain no tuples and are not a representative sample, so + * exclude them from the sampled pages and sampleable population used to + * extrapolate reltuples. + * * Note: scanned_tuples should count only *live* tuples, since * pg_class.reltuples is defined that way. */ @@ -1359,18 +1364,27 @@ double vac_estimate_reltuples(Relation relation, BlockNumber total_pages, BlockNumber scanned_pages, + BlockNumber scanned_new_pages, double scanned_tuples) { BlockNumber old_rel_pages = relation->rd_rel->relpages; + BlockNumber sampled_pages; + BlockNumber sampleable_pages; double old_rel_tuples = relation->rd_rel->reltuples; double old_density; double unscanned_pages; double total_tuples; + Assert(scanned_new_pages <= scanned_pages); + Assert(scanned_new_pages <= total_pages); + /* If we did scan the whole table, just use the count as-is */ if (scanned_pages >= total_pages) return scanned_tuples; + sampled_pages = scanned_pages - scanned_new_pages; + sampleable_pages = total_pages - scanned_new_pages; + /* * With no prior density, a tiny non-random sample is not sufficient to * estimate the relation's tuple count. This can happen when on-access @@ -1378,11 +1392,11 @@ vac_estimate_reltuples(Relation relation, */ if (old_rel_tuples < 0 || old_rel_pages == 0) { - if (scanned_pages <= 1 || - scanned_pages < (double) total_pages * 0.02) + if (sampled_pages <= 1 || + sampled_pages < (double) sampleable_pages * 0.02) return -1; - return floor((scanned_tuples / scanned_pages) * total_pages + 0.5); + return floor((scanned_tuples / sampled_pages) * sampleable_pages + 0.5); } /* @@ -1402,8 +1416,8 @@ vac_estimate_reltuples(Relation relation, * (Note: we might be returning -1 here.) */ if (old_rel_pages == total_pages && - (scanned_pages <= 1 || - scanned_pages < (double) total_pages * 0.02)) + (sampled_pages <= 1 || + sampled_pages < (double) sampleable_pages * 0.02)) return old_rel_tuples; /* @@ -1413,7 +1427,7 @@ vac_estimate_reltuples(Relation relation, * and finally add on the number of tuples in the scanned pages. */ old_density = old_rel_tuples / old_rel_pages; - unscanned_pages = (double) total_pages - (double) scanned_pages; + unscanned_pages = (double) sampleable_pages - (double) sampled_pages; total_tuples = old_density * unscanned_pages + scanned_tuples; return floor(total_tuples + 0.5); } diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index d8ac5fd1fb2..404307b6f6c 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -379,6 +379,7 @@ extern void vac_close_indexes(int nindexes, Relation *Irel, LOCKMODE lockmode); extern double vac_estimate_reltuples(Relation relation, BlockNumber total_pages, BlockNumber scanned_pages, + BlockNumber scanned_new_pages, double scanned_tuples); extern void vac_update_relstats(Relation relation, BlockNumber num_pages, diff --git a/src/test/regress/expected/vacuum.out b/src/test/regress/expected/vacuum.out index d4696bc3325..906b958520d 100644 --- a/src/test/regress/expected/vacuum.out +++ b/src/test/regress/expected/vacuum.out @@ -730,3 +730,96 @@ SELECT pg_column_toast_chunk_id(f1) = :'id_2_chunk' AS same_chunk (1 row) DROP TABLE vac_rewrite_toast; +-- Test estimates when on-access VM setting lets VACUUM skip most pages. +CREATE TEMP TABLE vac_reltuples_new AS + SELECT g AS a, repeat('x', 100) AS b FROM generate_series(1, 4000) g; +SELECT count(*) FROM vac_reltuples_new; + count +------- + 4000 +(1 row) + +VACUUM vac_reltuples_new; +SELECT reltuples = -1 AS reltuples_unknown + FROM pg_class WHERE oid = 'vac_reltuples_new'::regclass; + reltuples_unknown +------------------- + t +(1 row) + +ANALYZE vac_reltuples_new; +SELECT reltuples = 4000 AS reltuples_updated + FROM pg_class WHERE oid = 'vac_reltuples_new'::regclass; + reltuples_updated +------------------- + t +(1 row) + +-- Likewise for a relation that was empty when it was last vacuumed. +TRUNCATE vac_reltuples_new; +VACUUM vac_reltuples_new; +INSERT INTO vac_reltuples_new + SELECT g, repeat('x', 100) FROM generate_series(1, 4000) g; +SELECT count(*) FROM vac_reltuples_new; + count +------- + 4000 +(1 row) + +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +VACUUM vac_reltuples_new; +SELECT reltuples = -1 AS reltuples_unknown + FROM pg_class WHERE oid = 'vac_reltuples_new'::regclass; + reltuples_unknown +------------------- + t +(1 row) + +SELECT n_live_tup = 4000 AS n_live_tup_preserved + FROM pg_stat_all_tables WHERE relid = 'vac_reltuples_new'::regclass; + n_live_tup_preserved +---------------------- + t +(1 row) + +-- Preserve a valid estimate when the only scanned tail pages were new. +CREATE TEMP TABLE vac_reltuples_known AS + SELECT g AS a, repeat('x', 100) AS b FROM generate_series(1, 4000) g; +ANALYZE vac_reltuples_known; +SELECT count(*) FROM vac_reltuples_known; + count +------- + 4000 +(1 row) + +VACUUM vac_reltuples_known; +SELECT reltuples = 4000 AS reltuples_preserved + FROM pg_class WHERE oid = 'vac_reltuples_known'::regclass; + reltuples_preserved +--------------------- + t +(1 row) + +-- Scale a valid prior density when the relation has grown. +INSERT INTO vac_reltuples_known + SELECT g, repeat('x', 100) FROM generate_series(4001, 8000) g; +SELECT count(*) FROM vac_reltuples_known; + count +------- + 8000 +(1 row) + +VACUUM vac_reltuples_known; +SELECT reltuples BETWEEN 7600 AND 8400 AS reltuples_ok + FROM pg_class WHERE oid = 'vac_reltuples_known'::regclass; + reltuples_ok +-------------- + t +(1 row) + +DROP TABLE vac_reltuples_new, vac_reltuples_known; diff --git a/src/test/regress/sql/vacuum.sql b/src/test/regress/sql/vacuum.sql index 247b8e23b23..2a401d497ea 100644 --- a/src/test/regress/sql/vacuum.sql +++ b/src/test/regress/sql/vacuum.sql @@ -525,3 +525,46 @@ SELECT id, pg_column_toast_chunk_id(f1) IS NULL AS f1_chunk_null, SELECT pg_column_toast_chunk_id(f1) = :'id_2_chunk' AS same_chunk FROM vac_rewrite_toast WHERE id = 2; DROP TABLE vac_rewrite_toast; + +-- Test estimates when on-access VM setting lets VACUUM skip most pages. +CREATE TEMP TABLE vac_reltuples_new AS + SELECT g AS a, repeat('x', 100) AS b FROM generate_series(1, 4000) g; +SELECT count(*) FROM vac_reltuples_new; +VACUUM vac_reltuples_new; +SELECT reltuples = -1 AS reltuples_unknown + FROM pg_class WHERE oid = 'vac_reltuples_new'::regclass; +ANALYZE vac_reltuples_new; +SELECT reltuples = 4000 AS reltuples_updated + FROM pg_class WHERE oid = 'vac_reltuples_new'::regclass; + +-- Likewise for a relation that was empty when it was last vacuumed. +TRUNCATE vac_reltuples_new; +VACUUM vac_reltuples_new; +INSERT INTO vac_reltuples_new + SELECT g, repeat('x', 100) FROM generate_series(1, 4000) g; +SELECT count(*) FROM vac_reltuples_new; +SELECT pg_stat_force_next_flush(); +VACUUM vac_reltuples_new; +SELECT reltuples = -1 AS reltuples_unknown + FROM pg_class WHERE oid = 'vac_reltuples_new'::regclass; +SELECT n_live_tup = 4000 AS n_live_tup_preserved + FROM pg_stat_all_tables WHERE relid = 'vac_reltuples_new'::regclass; + +-- Preserve a valid estimate when the only scanned tail pages were new. +CREATE TEMP TABLE vac_reltuples_known AS + SELECT g AS a, repeat('x', 100) AS b FROM generate_series(1, 4000) g; +ANALYZE vac_reltuples_known; +SELECT count(*) FROM vac_reltuples_known; +VACUUM vac_reltuples_known; +SELECT reltuples = 4000 AS reltuples_preserved + FROM pg_class WHERE oid = 'vac_reltuples_known'::regclass; + +-- Scale a valid prior density when the relation has grown. +INSERT INTO vac_reltuples_known + SELECT g, repeat('x', 100) FROM generate_series(4001, 8000) g; +SELECT count(*) FROM vac_reltuples_known; +VACUUM vac_reltuples_known; +SELECT reltuples BETWEEN 7600 AND 8400 AS reltuples_ok + FROM pg_class WHERE oid = 'vac_reltuples_known'::regclass; + +DROP TABLE vac_reltuples_new, vac_reltuples_known; -- 2.43.0