BUG #19776: pg_combinebackup ignores the truncation_block_length of an intermediate incremental backup

The following bug has been logged on the website:

Bug reference: 19776
Logged by: Ke
Email address: kehan5800(at)gmail(dot)com
PostgreSQL version: 18.6
Operating system: Ubuntu 22.04 (Linux 5.15, x86_64)
Description:

This happens with a chain of a full backup F and two incremental backups, I1
and I2. VACUUM truncates a table between F and I1, and COPY makes it grow
again
between I1 and I2. "pg_combinebackup F I1 I2" then fills blocks that did not
exist when I1 was taken with their contents from F, so rows deleted before
I1
reappear in the restored table. Sequential scans return those rows, the
primary
key index does not contain them, and bt_index_check() fails. pg_verifybackup
reports all three input backups as valid, and "pg_combinebackup F I1" alone
restores correctly.

Reproduced on master (45fc51fcbf) and 18.6, built from source with
./configure --without-icu (contrib amcheck installed).

---- repro.sh: PGBIN=/usr/local/pgsql/bin sh repro.sh /tmp/chain ----
#!/bin/sh
set -e
W=$1; P=5460
rm -rf "$W"; mkdir -p "$W"
"$PGBIN/initdb" -D "$W/src" -A trust --no-locale >/dev/null
cat >> "$W/src/postgresql.conf" <<EOC
port = $P
unix_socket_directories = '$W'
listen_addresses = ''
summarize_wal = on
autovacuum = off
EOC
"$PGBIN/pg_ctl" -D "$W/src" -l "$W/src.log" -w start >/dev/null
q() { "$PGBIN/psql" -X -h "$W" -p "$1" -d postgres -Atq -c "$2"; }
SEQ="SET enable_indexscan = off; SET enable_indexonlyscan = off; SET
enable_bitmapscan = off;"
q $P "CREATE TABLE t (id int PRIMARY KEY, pad text)"
q $P "INSERT INTO t SELECT g, repeat('h', 20) FROM generate_series(1, 10000)
g"
q $P "INSERT INTO t SELECT 1000000 + g, repeat('x', 20) FROM
generate_series(1, 20000) g"
q $P "VACUUM (FREEZE) t"
"$PGBIN/pg_basebackup" -h "$W" -p $P -D "$W/F" -c fast
q $P "DELETE FROM t WHERE id > 1000000"
q $P "VACUUM t" # truncates the heap
"$PGBIN/pg_basebackup" -h "$W" -p $P -D "$W/I1" -c fast --incremental
"$W/F/backup_manifest"
seq 2000001 2005000 | awk '{print $1 "\tnew"}' > "$W/copy.txt"
q $P "COPY t FROM '$W/copy.txt'" # regrows the heap
"$PGBIN/pg_basebackup" -h "$W" -p $P -D "$W/I2" -c fast --incremental
"$W/I1/backup_manifest"
echo "source: $(q $P "$SEQ SELECT count(*), count(*) FILTER (WHERE id
BETWEEN 1000001 AND 1999999), pg_relation_size('t') / 8192 FROM t")"
REL=$(q $P "SELECT pg_relation_filepath('t')")
"$PGBIN/pg_ctl" -D "$W/src" -w stop >/dev/null
for b in I1 I2; do
set -- $(od -A n -t u4 -N 12 "$W/$b/$(dirname $REL)/INCREMENTAL.$(basename
$REL)")
echo "$b/$(dirname $REL)/INCREMENTAL.$(basename $REL): $2 blocks,
truncation_block_length $3"
done
"$PGBIN/pg_combinebackup" -o "$W/R" "$W/F" "$W/I1" "$W/I2"
echo "port = $((P + 1))" >> "$W/R/postgresql.conf"
"$PGBIN/pg_ctl" -D "$W/R" -l "$W/R.log" -w start >/dev/null
P=$((P + 1))
echo "restored: $(q $P "$SEQ SELECT count(*), count(*) FILTER (WHERE id
BETWEEN 1000001 AND 1999999), pg_relation_size('t') / 8192 FROM t")"
echo "restored, blocks holding deleted rows: $(q $P "$SEQ SELECT
min((ctid::text::point)[0]) || '..' || max((ctid::text::point)[0]) FROM t
WHERE id BETWEEN 1000001 AND 1999999")"
echo "restored, index scan: $(q $P "SET enable_seqscan = off; SET
enable_bitmapscan = off; SELECT count(*) FROM t WHERE id BETWEEN 1000001 AND
1999999")"
q $P "CREATE EXTENSION amcheck"
q $P "SELECT bt_index_check('t_pkey', true)" 2>&1 || true
"$PGBIN/pg_ctl" -D "$W/R" -w stop >/dev/null
---- end ----

Output on master (identical on 18.6):

source: 15000|0|110
I1/base/5/INCREMENTAL.16384: 1 blocks, truncation_block_length 74
I2/base/5/INCREMENTAL.16384: 23 blocks, truncation_block_length 110
restored: 16904|1904|110
restored, blocks holding deleted rows: 96..109
restored, index scan: 0
ERROR: heap tuple (96,1) from table "t" lacks matching index tuple within
index "t_pkey"
HINT: Retrying verification using the function bt_index_parent_check()
might provide a more specific error.

Expected: "restored: 15000|0|110", and bt_index_check() succeeds.

In the source cluster, blocks 96..109 of t are all-zero pages that COPY's
bulk
extension added and no WAL record touched, so neither I1 nor I2 contains
them.
The two INCREMENTAL files are consistent with that. I1 records that the file
had 74 blocks at that point, and I2 has truncation_block_length 110 and
holds
the WAL-logged blocks.

reconstruct_from_incremental_file() walks the chain from newest to oldest.
For
every older backup, it uses the newest file's truncation_block_length (110)
as
the bound for the blocks it may take from that backup:

https://github.com/postgres/postgres/blob/45fc51fcbf4534ea2f6f85e36f8155dbd5ba9851/src/bin/pg_combinebackup/reconstruct.c#L227
https://github.com/postgres/postgres/blob/45fc51fcbf4534ea2f6f85e36f8155dbd5ba9851/src/bin/pg_combinebackup/reconstruct.c#L267

It never lowers that bound to an intermediate file's
truncation_block_length.
So blocks 96..109 come from F, where they hold the rows deleted before I1.
They
should be zero-filled, because I1 says that the file ended at block 74.

Using the running minimum of the truncation lengths fixes it. With this
patch, pg_combinebackup run on the same (unchanged) three backups writes
blocks 96..109 as zeroes, and the script shows "restored: 15000|0|110" with
no
bt_index_check() error:

--- a/src/bin/pg_combinebackup/reconstruct.c
+++ b/src/bin/pg_combinebackup/reconstruct.c
@@ -105,6 +105,7 @@ reconstruct_from_incremental_file(char *input_filename,
rfile **sourcemap;
off_t *offsetmap;
unsigned block_length;
+ unsigned limit; /* older backups may supply
blocks below this */
unsigned sidx = n_prior_backups;
bool full_copy_possible = true;
int copy_source_index = -1;
@@ -128,6 +129,7 @@ reconstruct_from_incremental_file(char *input_filename,
latest_source = make_incremental_rfile(input_filename);
source[n_prior_backups] = latest_source;
block_length = find_reconstructed_block_length(latest_source);
+ limit = latest_source->truncation_block_length;

/*
* For each block in the output file, we need to know from which
file we
@@ -224,7 +226,7 @@ reconstruct_from_incremental_file(char *input_filename,
* later backup is one that we can't validate.
*/
blocklength = sb.st_size / BLCKSZ;
- for (b = 0; b <
latest_source->truncation_block_length; ++b)
+ for (b = 0; b < limit; ++b)
{
if (sourcemap[b] == NULL && b < blocklength)
{
@@ -238,7 +240,7 @@ reconstruct_from_incremental_file(char *input_filename,
* should be exactly as long as the source file is.
If so, a full
* copy is acceptable, otherwise not.
*/
- if (full_copy_possible)
+ if (full_copy_possible && limit ==
latest_source->truncation_block_length)
{
uint64 expected_length;

@@ -264,7 +266,7 @@ reconstruct_from_incremental_file(char *input_filename,
{
BlockNumber b = s->relative_block_numbers[i];

- if (b < latest_source->truncation_block_length &&
+ if (b < limit &&
sourcemap[b] == NULL)
{
sourcemap[b] = s;
@@ -278,6 +280,9 @@ reconstruct_from_incremental_file(char *input_filename,
full_copy_possible = false;
}
}
+ /* blocks at or above this file's truncation length did not
exist then */
+ if (s->truncation_block_length < limit)
+ limit = s->truncation_block_length;
}

/*

Related: "Incremental backup inconsistency after vacuum truncation + COPY
regrowth" (pgsql-hackers, 2026-07-29, no replies so far,
https://www.postgresql.org/message-id/CAPnDWbagY0Y91qZmJkuJX4+byVJ-dJysnx4QmW23wgR_AvECVg@mail.gmail.com)
covers the truncation and the COPY regrowth within a single incremental
backup.
In that case the INCREMENTAL file itself cannot tell the zero pages apart,
and
the change proposed there is on the server side. In the chain case shown
here,
the information is in I1 and pg_combinebackup does not use it. A
pg_combinebackup fix would also make chains that already exist restore
correctly.