From 1491d9971625eb950bbac3da6688bdb505652060 Mon Sep 17 00:00:00 2001 From: Nikhil Sontakke Date: Wed, 12 Aug 2026 17:33:34 +0530 Subject: [PATCH 1/2] Refuse a logical replication INSERT that is missing column data A row filter can transform an UPDATE into an INSERT when the new row moves into the filter, but an INSERT has no local row to inherit values from and so must carry every published column. pgoutput_row_filter() copies unchanged out-of-line values over from the old tuple, which only works for the replica identity columns: unless the replica identity is FULL, any other column is still an external on-disk pointer, and logicalrep_write_tuple() sends it as LOGICALREP_COLUMN_UNCHANGED. The publisher cannot do better. An unchanged out-of-line value is not written to WAL and is not reachable through the historic snapshot, so the output plugin never sees it. slot_store_data() has no case for an unchanged column and stores a NULL, which loses the value silently where the column is nullable, and fails with a confusing constraint violation where it is not. Refuse the change instead, naming the relation and the column, so that the failure is visible and can be stepped over with ALTER SUBSCRIPTION ... SKIP. The check goes in apply_handle_insert() rather than slot_store_data(), because the latter is also used for old tuples and to materialise an UPDATE's new tuple for conflict reporting, where an unchanged column is legitimate. It is the only place an INSERT's tuple is stored: a partitioned target routes the already-stored slot through apply_handle_tuple_routing(). Also document the restriction, and that REPLICA IDENTITY FULL avoids it. Add a test to 028_row_filter.pl. The existing tab_rowfilter_toast case there covers an out-of-line column that is part of the replica identity, which works; the new one covers the adjacent case where it is not, and checks the REPLICA IDENTITY FULL workaround alongside it. Author: Nikhil Sontakke Reported-by: Shinya Kato Backpatch-through: 15 --- doc/src/sgml/logical-replication.sgml | 21 +++++ src/backend/replication/logical/worker.c | 26 ++++++ src/test/subscription/t/028_row_filter.pl | 96 +++++++++++++++++++++++ 3 files changed, 143 insertions(+) diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index 3a61e2d6889..8830a94cf7a 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -957,6 +957,27 @@ HINT: To initiate replication, you must manually create the replication slot, e + + + An UPDATE that is transformed into an + INSERT must send a value for every published column, + because the subscriber has no existing row to take values from. A column + cannot be sent if all of the following hold: it was not modified by the + UPDATE, it is stored out-of-line (see + ), and it is not part of the + replica identity. + The publisher has only a pointer to such a value; it is not written to + WAL and cannot be retrieved during decoding. Applying + the change is refused in that case, and replication stops with an error + naming the table and column. + + + Setting REPLICA IDENTITY FULL on the published table + avoids this, because the whole old row is then logged with out-of-line + values inlined, which lets the publisher supply the missing value. + + + diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 92ea1d0df24..68473469fcf 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -2694,6 +2694,32 @@ apply_handle_insert(StringInfo s) /* Set relation for error callback */ apply_error_callback_arg.rel = rel; + /* + * An INSERT has no local row to inherit from, so every column must have + * arrived with a value. A column marked unchanged reaches us when a row + * filter turns an UPDATE into an INSERT and the column is stored + * out-of-line, unchanged, and outside the replica identity: the publisher + * has only a TOAST pointer for it, and the value is neither in WAL nor + * reachable through the historic snapshot. Storing it would silently + * substitute a NULL, so refuse the change instead. + * + * All of the publisher's columns are known to exist here, since + * logicalrep_rel_open() rejects a target relation missing any of them. + */ + for (int i = 0; i < newtup.ncols; i++) + { + if (newtup.colstatus[i] != LOGICALREP_COLUMN_UNCHANGED) + continue; + + ereport(ERROR, + errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("incomplete tuple received for logical replication target relation \"%s.%s\"", + rel->remoterel.nspname, rel->remoterel.relname), + errdetail("Column \"%s\" was sent as unchanged, which an INSERT cannot express.", + rel->remoterel.attnames[i]), + errhint("This can happen when a row filter transforms an UPDATE into an INSERT and the column is stored out-of-line and is not part of the replica identity. Setting REPLICA IDENTITY FULL on the publisher avoids it.")); + } + /* Initialize the executor state. */ edata = create_edata_for_relation(rel); estate = edata->estate; diff --git a/src/test/subscription/t/028_row_filter.pl b/src/test/subscription/t/028_row_filter.pl index c666ae00483..ae468154a7d 100644 --- a/src/test/subscription/t/028_row_filter.pl +++ b/src/test/subscription/t/028_row_filter.pl @@ -799,6 +799,102 @@ is($result, qq(), 'check replicated rows to tab_rowfilter_viaroot_part_1'); # Testcase end: FOR TABLE with row filter publications # ====================================================== +# ====================================================== +# Testcase start: out-of-line column outside the replica identity +# +# An UPDATE that moves a row into the row filter is transformed into an +# INSERT, and an INSERT must carry a value for every published column. A +# column that was not modified by the UPDATE, is stored out-of-line, and is +# not part of the replica identity cannot be supplied: the new tuple holds +# only a pointer to it, the value is not written to WAL, and decoding cannot +# fetch it afterwards. +# +# REPLICA IDENTITY FULL is not affected, because the whole old row is then +# logged with out-of-line values inlined, which leaves the transformation a +# usable value. Both cases are checked here, the working one first, since the +# other one stops the apply worker. +# +# tab_rowfilter_toast above covers the case where the out-of-line column is +# part of the replica identity, which works for the same reason. + +# STORAGE EXTERNAL disables compression for the column, so the value is +# certain to be stored out-of-line rather than compressed into the tuple. +$node_publisher->safe_psql('postgres', + "CREATE TABLE tab_rowfilter_toast_full (a int PRIMARY KEY, b text)"); +$node_publisher->safe_psql('postgres', + "ALTER TABLE tab_rowfilter_toast_full ALTER COLUMN b SET STORAGE EXTERNAL" +); +$node_publisher->safe_psql('postgres', + "ALTER TABLE tab_rowfilter_toast_full REPLICA IDENTITY FULL"); +$node_publisher->safe_psql('postgres', + "CREATE TABLE tab_rowfilter_toast_pk (a int PRIMARY KEY, b text)"); +$node_publisher->safe_psql('postgres', + "ALTER TABLE tab_rowfilter_toast_pk ALTER COLUMN b SET STORAGE EXTERNAL"); + +# The rows start outside the filter, so nothing is copied initially. +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_rowfilter_toast_full VALUES (1, repeat('1234567890', 500))" +); +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_rowfilter_toast_pk VALUES (1, repeat('1234567890', 500))" +); + +# Guard against the value silently staying inline, which would leave the rest +# of this testcase proving nothing. +$result = $node_publisher->safe_psql('postgres', + "SELECT pg_relation_size(reltoastrelid) > 0 FROM pg_class WHERE relname = 'tab_rowfilter_toast_pk'" +); +is($result, qq(t), + 'column b of tab_rowfilter_toast_pk is stored out-of-line'); + +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION tap_pub_toast_ri FOR TABLE tab_rowfilter_toast_full WHERE (a > 10), tab_rowfilter_toast_pk WHERE (a > 10)" +); + +$node_subscriber->safe_psql('postgres', + "CREATE TABLE tab_rowfilter_toast_full (a int PRIMARY KEY, b text)"); +$node_subscriber->safe_psql('postgres', + "CREATE TABLE tab_rowfilter_toast_pk (a int PRIMARY KEY, b text)"); +$node_subscriber->safe_psql('postgres', + "CREATE SUBSCRIPTION tap_sub_toast_ri CONNECTION '$publisher_connstr application_name=tap_sub_toast_ri' PUBLICATION tap_pub_toast_ri" +); +$node_subscriber->wait_for_subscription_sync($node_publisher, + 'tap_sub_toast_ri'); + +# The row moves into the filter, so this UPDATE arrives as an INSERT. Column +# b is unchanged and out-of-line, but REPLICA IDENTITY FULL makes it available. +$node_publisher->safe_psql('postgres', + "UPDATE tab_rowfilter_toast_full SET a = 11 WHERE a = 1"); +$node_publisher->wait_for_catchup('tap_sub_toast_ri'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT a, length(b) FROM tab_rowfilter_toast_full"); +is($result, qq(11|5000), + 'unchanged out-of-line column replicated with REPLICA IDENTITY FULL'); + +# The same UPDATE on a table whose replica identity is the primary key cannot +# carry column b at all. Refuse the change rather than storing a NULL. +my $log_offset = -s $node_subscriber->logfile; + +$node_publisher->safe_psql('postgres', + "UPDATE tab_rowfilter_toast_pk SET a = 11 WHERE a = 1"); + +$node_subscriber->wait_for_log( + qr/incomplete tuple received for logical replication target relation "public\.tab_rowfilter_toast_pk"/, + $log_offset); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*) FROM tab_rowfilter_toast_pk"); +is($result, qq(0), + 'no row applied when an unchanged out-of-line column cannot be sent'); + +# The apply worker cannot get past this transaction, so stop it retrying. +$node_subscriber->safe_psql('postgres', + "ALTER SUBSCRIPTION tap_sub_toast_ri DISABLE"); + +# Testcase end: out-of-line column outside the replica identity +# ====================================================== + $node_subscriber->stop('fast'); $node_publisher->stop('fast'); -- 2.50.1 (Apple Git-155)