From f6d6f61e992820a16708b50d8ecba91567f55e0c Mon Sep 17 00:00:00 2001 From: Jakub Wartak Date: Wed, 8 Apr 2026 14:01:59 +0200 Subject: [PATCH v4b] amcheck: attempt to detect lost heap segments based on index tuples bt_index_check() family of functions reads the whole btree index and builds bloom filter along the way. Later it reads the heap table and for every tuple there it probes using bloom filter if the index has entry or not. However in conditions where some segments of the main fork have been removed as discussed in [0] (e.g. by rogue action, filesystem corruption/fsck, missing full backup restore) this logic - after startup - cannot reliably detect that relation is 'short' as the RelationGetNumberOfBlocks() will stop counting on the last available heap segment, therefore not full heap segment will be probed against btree bloom filter. This patch teaches amcheck to verify during btree scan if the currently processed index tuple does not point to heap block above current relation size. [0] - https://www.postgresql.org/message-id/flat/013D63E2-5D75-492E-85FF-1D5CC0148C82%40gmail.com Author: Jakub Wartak Discussion: https://www.postgresql.org/message-id/flat/432626F9-65DF-4F0D-B345-26CFC3E2CFAC@yandex-team.ru --- contrib/amcheck/meson.build | 1 + .../t/008_verify_nbtree_lost_segments.pl | 115 ++++++++++++++++++ contrib/amcheck/verify_nbtree.c | 51 ++++++++ 3 files changed, 167 insertions(+) create mode 100644 contrib/amcheck/t/008_verify_nbtree_lost_segments.pl diff --git a/contrib/amcheck/meson.build b/contrib/amcheck/meson.build index 220b1ce1d59..28e3e17b447 100644 --- a/contrib/amcheck/meson.build +++ b/contrib/amcheck/meson.build @@ -52,6 +52,7 @@ tests += { 't/005_pitr.pl', 't/006_verify_gin.pl', 't/007_verify_nbtree_indexallkeysmatch.pl', + 't/008_verify_nbtree_lost_segments.pl', ], }, } diff --git a/contrib/amcheck/t/008_verify_nbtree_lost_segments.pl b/contrib/amcheck/t/008_verify_nbtree_lost_segments.pl new file mode 100644 index 00000000000..7a02f106693 --- /dev/null +++ b/contrib/amcheck/t/008_verify_nbtree_lost_segments.pl @@ -0,0 +1,115 @@ + +# Copyright (c) 2023-2026, PostgreSQL Global Development Group + +# This regression test checks the behavior of the btree validation in the +# presence of missing relation segments. +# +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('test'); +$node->init; +$node->append_conf('postgresql.conf', 'autovacuum = off'); +$node->start; + +# Create two tables, one with unique index and another to test +# posting list (btree duplicates). +$node->safe_psql( + 'postgres', q( + CREATE EXTENSION amcheck; + CREATE TABLE missingsegs_test1 AS + SELECT * FROM generate_series(1, 3000) id; + CREATE TABLE missingsegs_test2 AS + SELECT 10 AS id FROM generate_series(1, 3000); + CREATE UNIQUE INDEX bttest_unique_idx1 ON missingsegs_test1 (id); + CREATE INDEX bttest_idx2 ON missingsegs_test2 (id); +)); + +my ($result, $stdout, $stderr); + +# We have not yet broken the index, so we should get no corruption +$result = $node->safe_psql( + 'postgres', q( + SELECT bt_index_check('bttest_unique_idx1', true, true, true); +)); +is($result, '', 'run amcheck on non-broken bttest_unique_idx1'); + +$result = $node->safe_psql( + 'postgres', q( + SELECT bt_index_check('bttest_idx2', true, true, true); +)); +is($result, '', 'run amcheck on non-broken bttest_idx2'); + +# Break the relations, simulating rogue action or just fsck moving files +# into the /lost+found. +my $relpath1 = relation_filepath('missingsegs_test1'); +my $relpath2 = relation_filepath('missingsegs_test2'); +$node->stop; +corrupt_segment($relpath1.".1"); +corrupt_segment($relpath2.".1"); +$node->start; + +$result = $node->safe_psql( + 'postgres', q( + SET enable_indexscan TO off; + SET enable_indexonlyscan TO off; + SELECT count(id) FROM missingsegs_test1; +)); +cmp_ok( + '3000', '>', $result, + "ensure there is missing data on missingsegs_test1"); + +$result = $node->safe_psql( + 'postgres', q( + SET enable_indexscan TO off; + SET enable_indexonlyscan TO off; + SELECT count(id) FROM missingsegs_test2; +)); +cmp_ok( + '3000', '>', $result, + "ensure there is missing data on missingsegs_test2"); + +($result, $stdout, $stderr) = $node->psql( + 'postgres', q(SELECT bt_index_check('bttest_unique_idx1', true, true, true);) +); +like( + $stderr, + qr/index line pointer in index "bttest_unique_idx1" points to missing page in table "missingsegs_test1"/, + 'detected corrupted segments for missingsegs_test1'); + +($result, $stdout, $stderr) = $node->psql( + 'postgres', q(SELECT bt_index_check('bttest_idx2', true, true, true);) +); +like( + $stderr, + qr/index line pointer in index "bttest_idx2" points to missing page in table "missingsegs_test2"/, + 'detected corrupted segments for missingsegs_test2'); + +$node->stop; +done_testing(); + +# Returns the filesystem path for the named relation. +sub relation_filepath +{ + my ($relname) = @_; + + my $pgdata = $node->data_dir; + my $rel = $node->safe_psql('postgres', + qq(SELECT pg_relation_filepath('$relname'))); + die "path not found for relation $relname" unless defined $rel; + return "$pgdata/$rel"; +} + +# Rename segment so that it is in accessible +sub corrupt_segment +{ + my ($relpath) = @_; + my $destrelpath = $relpath . ".BAK"; + + rename($relpath, $destrelpath) + or BAIL_OUT("rename failed: $!"); +} + diff --git a/contrib/amcheck/verify_nbtree.c b/contrib/amcheck/verify_nbtree.c index 7243b83977d..41176e98c59 100644 --- a/contrib/amcheck/verify_nbtree.c +++ b/contrib/amcheck/verify_nbtree.c @@ -149,6 +149,8 @@ typedef struct BtreeCheckState bloom_filter *heapfilter; /* Debug counter for index tuples verified */ int64 indextuplesverified; + /* Short heap segments verification */ + BlockNumber heapnblocks; } BtreeCheckState; /* @@ -441,6 +443,7 @@ bt_check_every_level(Relation rel, Relation heaprel, bool heapkeyspace, state->rel = rel; state->heaprel = heaprel; state->heapkeyspace = heapkeyspace; + state->heapnblocks = RelationGetNumberOfBlocks(state->heaprel); state->readonly = readonly; state->heapallindexed = heapallindexed; state->indexallkeysmatch = indexallkeysmatch; @@ -1608,6 +1611,54 @@ bt_target_page_check(BtreeCheckState *state) } } + /* Check if leaf page tuples point to valid heap tuples */ + if(P_ISLEAF(topaque) && !ItemIdIsDead(itemid)) { + int nposting = 1; + + if(BTreeTupleIsPosting(itup)) + nposting = BTreeTupleGetNPosting(itup); + + for (int i = 0; i < nposting; i++) + { + ItemPointer htid; + BlockNumber heapblk; + OffsetNumber heapoff; + + if (nposting > 1) + htid = BTreeTupleGetPostingN(itup, i); + else + htid = BTreeTupleGetPointsToTID(itup); + + heapblk = ItemPointerGetBlockNumber(htid); + heapoff = ItemPointerGetOffsetNumber(htid); + + /* + * Does heapblk goes beyond RelationGetNumberOfBlocks() - potentially + * indicating missing relation segment? + */ + if (heapblk >= state->heapnblocks) { + /* We may need to recheck our cached value as we operate with ASL */ + state->heapnblocks = RelationGetNumberOfBlocks(state->heaprel); + if(heapblk >= state->heapnblocks) { + char *postingoff = ""; + if(nposting > 1) + postingoff = psprintf("posting list offset=%d", i); + + ereport(ERROR, + (errcode(ERRCODE_INDEX_CORRUPTED), + errmsg("index line pointer in index \"%s\" points to missing page in table \"%s\"", + RelationGetRelationName(state->rel), + RelationGetRelationName(state->heaprel)), + errdetail_internal("Index tid=(%u,%u) %s points to heap tid=(%u,%u) but heap has only %u blocks.", + state->targetblock, offset, postingoff, + heapblk, heapoff, + state->heapnblocks)), + errhint("this can be caused by lost relation segment (missing or removed file).")); + } + } + } + } + /* Verify each index tuple points to heap tuple with same key */ if (state->indexallkeysmatch && P_ISLEAF(topaque) && !ItemIdIsDead(itemid)) { -- 2.43.0