From dc15f09d732c0f662ef2c387ff90881ca4f10d84 Mon Sep 17 00:00:00 2001
From: Amit Kapila <akapila@postgresql.org>
Date: Tue, 1 Sep 2026 16:16:43 +0530
Subject: [PATCH v2] Avoid oversized WAL records when flattening REPLICA
 IDENTITY FULL tuples

REPLICA IDENTITY FULL unconditionally inlines every out-of-line column
value into the old tuple via toast_flatten_tuple() before WAL-logging
it.  When the combined size is large enough, this can fail outright
with "invalid memory alloc request size", or, in a narrower window,
succeed but produce a WAL record exceeding XLogRecordMaxSize.  That
failure is only discovered inside XLogInsert(), after heap_update()/
heap_delete() have already entered the critical section, turning what
should be an ordinary ERROR into a PANIC.

Add BuildFullIdentityTuple(), which cheaply sums the out-of-line
columns' va_rawsize (no I/O) before flattening.  Below a safety
margin under XLogRecordMaxSize, behavior is unchanged.  Above it, each
out-of-line value's bytes are instead persisted durably in WAL via the
new toast_save_wal_only_datum(), keeping only a small toast pointer in
the tuple.  That function streams the value's existing chunks into
freshly inserted chunk rows under a new toast id, then deletes those
rows again within the same transaction, so nothing is left live for
vacuum to deal with, while the resulting small insert/delete WAL
records stay well clear of any record-size limit regardless of the
value's size.

ReorderBufferToastReplace() previously only resolved such placeholder
pointers in change->newtuple.  Split its per-tuple logic into
ReorderBufferToastReplaceTuple() and call it for oldtuple too, so the
new placeholder pointers are resolved there as well, including for
plain DELETEs, which have only an oldtuple.
---
 src/backend/access/common/toast_internals.c   | 132 +++++++++++++++
 src/backend/access/heap/heapam.c              |  84 +++++++++-
 .../replication/logical/reorderbuffer.c       | 153 +++++++++++-------
 src/include/access/toast_internals.h          |   2 +
 4 files changed, 309 insertions(+), 62 deletions(-)

diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c
index 77d42e7ed65..3fb3232f3d0 100644
--- a/src/backend/access/common/toast_internals.c
+++ b/src/backend/access/common/toast_internals.c
@@ -438,6 +438,138 @@ toast_delete_datum(Relation rel, Datum value, bool is_speculative)
 	table_close(toastrel, NoLock);
 }
 
+/* ----------
+ * toast_save_wal_only_datum -
+ *
+ *	Persist an already out-of-line datum's bytes durably in WAL, via the
+ *	ordinary toast chunking path, without leaving a live row behind in the
+ *	toast relation, and without ever materializing the whole (potentially
+ *	huge) value in memory at once.
+ *
+ *	Each of the value's existing chunks is read and immediately
+ *	re-inserted, verbatim, under a freshly allocated toast value id; the
+ *	new chunks are then deleted again within the same transaction.  The
+ *	delete goes through heap_delete(), whose own visibility check
+ *	(HeapTupleSatisfiesUpdate()) treats a tuple inserted by the current
+ *	command as not yet visible to that same command, so a
+ *	CommandCounterIncrement() between the inserts and the delete is
+ *	required for the delete to find them; without it, heap_delete() raises
+ *	"attempted to delete invisible tuple".  Physically, the net effect on
+ *	the toast relation is the same as if nothing had happened once the
+ *	current transaction commits and the dead chunk rows are eventually
+ *	vacuumed; the only lasting effect is the sequence of small INSERT (and
+ *	DELETE) WAL records the chunking produces, each safely below any WAL
+ *	record size limit regardless of how large the datum is.
+ *
+ *	Reusing the value's original toast pointer id is not an option: doing so
+ *	would collide with the still-live chunks under the toast relation's
+ *	(valueid, chunkseq) unique index.
+ *
+ *	This exists for callers that need a datum's value to survive in WAL for
+ *	logical decoding's sake (see ReorderBufferToastReplace()), without
+ *	wanting to either flatten arbitrarily large data inline into one tuple
+ *	or leave a toast row referenced by nothing once the current transaction
+ *	ends.
+ *
+ * rel: the main relation we're working with (not the toast rel!)
+ * value: an on-disk external toast pointer
+ * ----------
+ */
+Datum
+toast_save_wal_only_datum(Relation rel, Datum value, uint32 options)
+{
+	varlena    *attr = (varlena *) DatumGetPointer(value);
+	varatt_external toast_pointer;
+	varatt_external new_toast_pointer;
+	Relation	toastrel;
+	Relation   *toastidxs;
+	TupleDesc	toasttupDesc;
+	ScanKeyData toastkey;
+	SysScanDesc toastscan;
+	HeapTuple	ttup;
+	CommandId	mycid = GetCurrentCommandId(true);
+	int			num_indexes;
+	int			validIndex;
+	varlena    *result;
+
+	Assert(VARATT_IS_EXTERNAL_ONDISK(attr));
+
+	/* Must copy to access aligned fields */
+	VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+
+	toastrel = table_open(toast_pointer.va_toastrelid, RowExclusiveLock);
+	toasttupDesc = toastrel->rd_att;
+
+	validIndex = toast_open_indexes(toastrel, RowExclusiveLock,
+									&toastidxs, &num_indexes);
+
+	/* the new pointer describes the same bytes under a fresh value id */
+	new_toast_pointer = toast_pointer;
+	new_toast_pointer.va_valueid =
+		GetNewOidWithIndex(toastrel,
+						   RelationGetRelid(toastidxs[validIndex]),
+						   (AttrNumber) 1);
+
+	ScanKeyInit(&toastkey,
+				(AttrNumber) 1,
+				BTEqualStrategyNumber, F_OIDEQ,
+				ObjectIdGetDatum(toast_pointer.va_valueid));
+
+	toastscan = systable_beginscan_ordered(toastrel, toastidxs[validIndex],
+										   get_toast_snapshot(), 1, &toastkey);
+
+	while ((ttup = systable_getnext_ordered(toastscan, ForwardScanDirection)) != NULL)
+	{
+		Datum		t_values[3];
+		bool		t_isnull[3] = {0};
+		HeapTuple	newchunktup;
+		bool		isnull;
+
+		/* copy the existing chunk's sequence number and bytes verbatim */
+		t_values[0] = ObjectIdGetDatum(new_toast_pointer.va_valueid);
+		t_values[1] = fastgetattr(ttup, 2, toasttupDesc, &isnull);
+		Assert(!isnull);
+		t_values[2] = fastgetattr(ttup, 3, toasttupDesc, &isnull);
+		Assert(!isnull);
+
+		newchunktup = heap_form_tuple(toasttupDesc, t_values, t_isnull);
+
+		heap_insert(toastrel, newchunktup, mycid, options, NULL);
+
+		for (int i = 0; i < num_indexes; i++)
+		{
+			if (toastidxs[i]->rd_index->indisready)
+				index_insert(toastidxs[i], t_values, t_isnull,
+							&(newchunktup->t_self),
+							toastrel,
+							toastidxs[i]->rd_index->indisunique ?
+							UNIQUE_CHECK_YES : UNIQUE_CHECK_NO,
+							false, NULL);
+		}
+
+		heap_freetuple(newchunktup);
+	}
+
+	systable_endscan_ordered(toastscan);
+	toast_close_indexes(toastidxs, num_indexes, NoLock);
+	table_close(toastrel, NoLock);
+
+	result = (varlena *) palloc(TOAST_POINTER_SIZE);
+	SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK);
+	memcpy(VARDATA_EXTERNAL(result), &new_toast_pointer, sizeof(new_toast_pointer));
+
+	/*
+	 * Make the just-inserted chunks visible to the delete below; see the
+	 * file header comment for why this is required.
+	 */
+	CommandCounterIncrement();
+
+	/* remove the freshly-written chunks again; see comment above */
+	toast_delete_datum(rel, PointerGetDatum(result), false);
+
+	return PointerGetDatum(result);
+}
+
 /* ----------
  * toastrel_valueid_exists -
  *
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 72d6541734c..020a98ce1bd 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -31,15 +31,18 @@
  */
 #include "postgres.h"
 
+#include "access/detoast.h"
 #include "access/heapam.h"
 #include "access/heaptoast.h"
 #include "access/hio.h"
 #include "access/multixact.h"
 #include "access/subtrans.h"
 #include "access/syncscan.h"
+#include "access/toast_internals.h"
 #include "access/valid.h"
 #include "access/visibilitymap.h"
 #include "access/xloginsert.h"
+#include "access/xlogrecord.h"
 #include "catalog/pg_database.h"
 #include "catalog/pg_database_d.h"
 #include "commands/vacuum.h"
@@ -112,6 +115,8 @@ static int	bottomup_sort_and_shrink(TM_IndexDeleteOp *delstate);
 static XLogRecPtr log_heap_new_cid(Relation relation, HeapTuple tup);
 static HeapTuple ExtractReplicaIdentity(Relation relation, HeapTuple tp, bool key_required,
 										bool *copy);
+static HeapTuple BuildFullIdentityTuple(Relation relation, HeapTuple tp,
+										TupleDesc desc);
 
 
 /*
@@ -9321,6 +9326,80 @@ log_heap_new_cid(Relation relation, HeapTuple tup)
 	return recptr;
 }
 
+/*
+ * Above this combined size of out-of-line column values, BuildFullIdentityTuple
+ * persists them via WAL-only toast chunks rather than inlining them, to stay
+ * safely clear of XLogRecordMaxSize.  The slack below XLogRecordMaxSize
+ * covers the old tuple's other columns, its header, and the WAL record's
+ * own framing.
+ */
+#define FULL_IDENTITY_INLINE_LIMIT \
+	((Size) XLogRecordMaxSize - (64 * 1024 * 1024))
+
+/*
+ * Build the flattened REPLICA IDENTITY FULL old tuple for heap_update() or
+ * heap_delete() to WAL-log, for a tuple that HeapTupleHasExternal() found to
+ * have out-of-line column values.
+ *
+ * Ordinarily every out-of-line value is inlined via toast_flatten_tuple(),
+ * as before this function existed.  But heap_update()/heap_delete() build
+ * this tuple, and hence insert the record containing it, inside a critical
+ * section; if inlining produced a tuple large enough to push the resulting
+ * WAL record past XLogRecordMaxSize, the resulting ERROR would be promoted
+ * to a PANIC.  Once the out-of-line values' combined size approaches that
+ * limit, this instead keeps a small toast pointer for each such value,
+ * persisting the actual bytes via toast_save_wal_only_datum() so they
+ * remain available to logical decoding without ever inlining them into one
+ * tuple.  ReorderBufferToastReplace() resolves such pointers from the
+ * transaction's buffered toast chunks when reconstructing the old tuple.
+ */
+static HeapTuple
+BuildFullIdentityTuple(Relation relation, HeapTuple tp, TupleDesc desc)
+{
+	Datum		values[MaxHeapAttributeNumber];
+	bool		nulls[MaxHeapAttributeNumber];
+	Size		total_external_size = 0;
+
+	heap_deform_tuple(tp, desc, values, nulls);
+
+	for (int i = 0; i < desc->natts; i++)
+	{
+		CompactAttribute *att;
+		varatt_external toast_pointer;
+
+		if (nulls[i])
+			continue;
+
+		att = TupleDescCompactAttr(desc, i);
+		if (att->attlen != -1 ||
+			!VARATT_IS_EXTERNAL_ONDISK(DatumGetPointer(values[i])))
+			continue;
+
+		VARATT_EXTERNAL_GET_POINTER(toast_pointer, DatumGetPointer(values[i]));
+		total_external_size += toast_pointer.va_rawsize;
+	}
+
+	if (total_external_size <= FULL_IDENTITY_INLINE_LIMIT)
+		return toast_flatten_tuple(tp, desc);
+
+	for (int i = 0; i < desc->natts; i++)
+	{
+		CompactAttribute *att;
+
+		if (nulls[i])
+			continue;
+
+		att = TupleDescCompactAttr(desc, i);
+		if (att->attlen != -1 ||
+			!VARATT_IS_EXTERNAL_ONDISK(DatumGetPointer(values[i])))
+			continue;
+
+		values[i] = toast_save_wal_only_datum(relation, values[i], 0);
+	}
+
+	return heap_form_tuple(desc, values, nulls);
+}
+
 /*
  * Build a heap tuple representing the configured REPLICA IDENTITY to represent
  * the old tuple in an UPDATE or DELETE.
@@ -9357,12 +9436,13 @@ ExtractReplicaIdentity(Relation relation, HeapTuple tp, bool key_required,
 	{
 		/*
 		 * When logging the entire old tuple, it very well could contain
-		 * toasted columns. If so, force them to be inlined.
+		 * toasted columns. If so, force them to be inlined, subject to a
+		 * safety check; see BuildFullIdentityTuple.
 		 */
 		if (HeapTupleHasExternal(tp))
 		{
 			*copy = true;
-			tp = toast_flatten_tuple(tp, desc);
+			tp = BuildFullIdentityTuple(relation, tp, desc);
 		}
 		return tp;
 	}
diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c
index 900864afc6d..e2e6a00be18 100644
--- a/src/backend/replication/logical/reorderbuffer.c
+++ b/src/backend/replication/logical/reorderbuffer.c
@@ -303,6 +303,9 @@ static void ReorderBufferToastInitHash(ReorderBuffer *rb, ReorderBufferTXN *txn)
 static void ReorderBufferToastReset(ReorderBuffer *rb, ReorderBufferTXN *txn);
 static void ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
 									  Relation relation, ReorderBufferChange *change);
+static void ReorderBufferToastReplaceTuple(ReorderBuffer *rb, Relation relation,
+											Relation toast_rel, HeapTuple tup,
+											ReorderBufferTXN *txn);
 static void ReorderBufferToastAppendChunk(ReorderBuffer *rb, ReorderBufferTXN *txn,
 										  Relation relation, ReorderBufferChange *change);
 
@@ -5087,79 +5090,35 @@ ReorderBufferToastAppendChunk(ReorderBuffer *rb, ReorderBufferTXN *txn,
 }
 
 /*
- * Rejigger change->newtuple to point to in-memory toast tuples instead of
- * on-disk toast tuples that may no longer exist (think DROP TABLE or VACUUM).
+ * Rejigger one of a change's tuples (newtuple or oldtuple) to point to
+ * in-memory toast tuples instead of on-disk toast tuples that may no longer
+ * exist (think DROP TABLE or VACUUM).
  *
  * We cannot replace unchanged toast tuples though, so those will still point
  * to on-disk toast data.
  *
- * While updating the existing change with detoasted tuple data, we need to
- * update the memory accounting info, because the change size will differ.
- * Otherwise the accounting may get out of sync, triggering serialization
- * at unexpected times.
- *
- * We simply subtract size of the change before rejiggering the tuple, and
- * then add the new size. This makes it look like the change was removed
- * and then added back, except it only tweaks the accounting info.
- *
- * In particular it can't trigger serialization, which would be pointless
- * anyway as it happens during commit processing right before handing
- * the change to the output plugin.
+ * relation is the table tup belongs to; toast_rel is its already-open toast
+ * relation.
  */
 static void
-ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
-						  Relation relation, ReorderBufferChange *change)
+ReorderBufferToastReplaceTuple(ReorderBuffer *rb, Relation relation,
+								Relation toast_rel, HeapTuple tup,
+								ReorderBufferTXN *txn)
 {
-	TupleDesc	desc;
+	TupleDesc	desc = RelationGetDescr(relation);
+	TupleDesc	toast_desc = RelationGetDescr(toast_rel);
 	int			natt;
 	Datum	   *attrs;
 	bool	   *isnull;
 	bool	   *free;
 	HeapTuple	tmphtup;
-	Relation	toast_rel;
-	TupleDesc	toast_desc;
-	MemoryContext oldcontext;
-	HeapTuple	newtup;
-	Size		old_size;
-
-	/* no toast tuples changed */
-	if (txn->toast_hash == NULL)
-		return;
-
-	/*
-	 * We're going to modify the size of the change. So, to make sure the
-	 * accounting is correct we record the current change size and then after
-	 * re-computing the change we'll subtract the recorded size and then
-	 * re-add the new change size at the end. We don't immediately subtract
-	 * the old size because if there is any error before we add the new size,
-	 * we will release the changes and that will update the accounting info
-	 * (subtracting the size from the counters). And we don't want to
-	 * underflow there.
-	 */
-	old_size = ReorderBufferChangeSize(change);
-
-	oldcontext = MemoryContextSwitchTo(rb->context);
-
-	/* we should only have toast tuples in an INSERT or UPDATE */
-	Assert(change->data.tp.newtuple);
-
-	desc = RelationGetDescr(relation);
-
-	toast_rel = RelationIdGetRelation(relation->rd_rel->reltoastrelid);
-	if (!RelationIsValid(toast_rel))
-		elog(ERROR, "could not open toast relation with OID %u (base relation \"%s\")",
-			 relation->rd_rel->reltoastrelid, RelationGetRelationName(relation));
-
-	toast_desc = RelationGetDescr(toast_rel);
 
 	/* should we allocate from stack instead? */
 	attrs = palloc0_array(Datum, desc->natts);
 	isnull = palloc0_array(bool, desc->natts);
 	free = palloc0_array(bool, desc->natts);
 
-	newtup = change->data.tp.newtuple;
-
-	heap_deform_tuple(newtup, desc, attrs, isnull);
+	heap_deform_tuple(tup, desc, attrs, isnull);
 
 	for (natt = 0; natt < desc->natts; natt++)
 	{
@@ -5258,19 +5217,24 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
 	 * Build tuple in separate memory & copy tuple back into the tuplebuf
 	 * passed to the output plugin. We can't directly heap_fill_tuple() into
 	 * the tuplebuf because attrs[] will point back into the current content.
+	 *
+	 * Note that tup can legitimately be larger than MaxHeapTupleSize here:
+	 * for a replica identity old tuple, ExtractReplicaIdentity() may have
+	 * inlined out-of-line column values (always for REPLICA IDENTITY FULL,
+	 * or when the identity key itself is toasted), and
+	 * ReorderBufferAllocTupleBuf() already sized tup's buffer to match the
+	 * WAL record's actual length, not to a page-sized bound.
 	 */
 	tmphtup = heap_form_tuple(desc, attrs, isnull);
-	Assert(newtup->t_len <= MaxHeapTupleSize);
-	Assert(newtup->t_data == (HeapTupleHeader) ((char *) newtup + HEAPTUPLESIZE));
+	Assert(tup->t_data == (HeapTupleHeader) ((char *) tup + HEAPTUPLESIZE));
 
-	memcpy(newtup->t_data, tmphtup->t_data, tmphtup->t_len);
-	newtup->t_len = tmphtup->t_len;
+	memcpy(tup->t_data, tmphtup->t_data, tmphtup->t_len);
+	tup->t_len = tmphtup->t_len;
 
 	/*
 	 * free resources we won't further need, more persistent stuff will be
 	 * free'd in ReorderBufferToastReset().
 	 */
-	RelationClose(toast_rel);
 	pfree(tmphtup);
 	for (natt = 0; natt < desc->natts; natt++)
 	{
@@ -5280,6 +5244,75 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
 	pfree(attrs);
 	pfree(free);
 	pfree(isnull);
+}
+
+/*
+ * Rejigger change->newtuple and/or change->oldtuple to point to in-memory
+ * toast tuples instead of on-disk toast tuples that may no longer exist
+ * (think DROP TABLE or VACUUM); see ReorderBufferToastReplaceTuple().
+ *
+ * While updating the existing change with detoasted tuple data, we need to
+ * update the memory accounting info, because the change size will differ.
+ * Otherwise the accounting may get out of sync, triggering serialization
+ * at unexpected times.
+ *
+ * We simply subtract size of the change before rejiggering the tuple, and
+ * then add the new size. This makes it look like the change was removed
+ * and then added back, except it only tweaks the accounting info.
+ *
+ * In particular it can't trigger serialization, which would be pointless
+ * anyway as it happens during commit processing right before handing
+ * the change to the output plugin.
+ */
+static void
+ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
+						  Relation relation, ReorderBufferChange *change)
+{
+	Relation	toast_rel;
+	MemoryContext oldcontext;
+	Size		old_size;
+
+	/* no toast tuples changed */
+	if (txn->toast_hash == NULL)
+		return;
+
+	/*
+	 * We're going to modify the size of the change. So, to make sure the
+	 * accounting is correct we record the current change size and then after
+	 * re-computing the change we'll subtract the recorded size and then
+	 * re-add the new change size at the end. We don't immediately subtract
+	 * the old size because if there is any error before we add the new size,
+	 * we will release the changes and that will update the accounting info
+	 * (subtracting the size from the counters). And we don't want to
+	 * underflow there.
+	 */
+	old_size = ReorderBufferChangeSize(change);
+
+	oldcontext = MemoryContextSwitchTo(rb->context);
+
+	/*
+	 * We should have a tuple to work with for an INSERT, UPDATE or DELETE.
+	 * A DELETE only has an oldtuple, and can reach here because
+	 * REPLICA IDENTITY FULL's flattening step now sometimes leaves an
+	 * out-of-line placeholder pointer in the old tuple; see
+	 * BuildFullIdentityTuple() in heapam.c.
+	 */
+	Assert(change->data.tp.newtuple || change->data.tp.oldtuple);
+
+	toast_rel = RelationIdGetRelation(relation->rd_rel->reltoastrelid);
+	if (!RelationIsValid(toast_rel))
+		elog(ERROR, "could not open toast relation with OID %u (base relation \"%s\")",
+			 relation->rd_rel->reltoastrelid, RelationGetRelationName(relation));
+
+	if (change->data.tp.newtuple)
+		ReorderBufferToastReplaceTuple(rb, relation, toast_rel,
+									   change->data.tp.newtuple, txn);
+
+	if (change->data.tp.oldtuple)
+		ReorderBufferToastReplaceTuple(rb, relation, toast_rel,
+									   change->data.tp.oldtuple, txn);
+
+	RelationClose(toast_rel);
 
 	MemoryContextSwitchTo(oldcontext);
 
diff --git a/src/include/access/toast_internals.h b/src/include/access/toast_internals.h
index bf45889a642..697c6efb2d5 100644
--- a/src/include/access/toast_internals.h
+++ b/src/include/access/toast_internals.h
@@ -51,6 +51,8 @@ extern Oid	toast_get_valid_index(Oid toastoid, LOCKMODE lock);
 extern void toast_delete_datum(Relation rel, Datum value, bool is_speculative);
 extern Datum toast_save_datum(Relation rel, Datum value,
 							  varlena *oldexternal, uint32 options);
+extern Datum toast_save_wal_only_datum(Relation rel, Datum value,
+									   uint32 options);
 
 extern int	toast_open_indexes(Relation toastrel,
 							   LOCKMODE lock,
-- 
2.54.0

