From d81ed92c169a86753f426a1049f68198823c34c9 Mon Sep 17 00:00:00 2001 From: Vignesh C Date: Wed, 28 Jan 2026 10:58:56 +0530 Subject: [PATCH v38] Restrict EXCEPT TABLE to root partitioned tables Only root partitioned tables can be specified in the EXCEPT TABLE clause. Specifying a root partitioned table excludes all partitions belonging to that partition hierarchy from publication, irrespective of the option publish_via_partition_root. This is based on approach 3 discussed at: https://www.postgresql.org/message-id/CAJpy0uD81HRrMYr7S-6AV4W2PtbGKM-nf2D89zsoMHJ9jZssUg@mail.gmail.com This patch is a topup patch on top of 0001 patch. --- doc/src/sgml/ref/create_publication.sgml | 15 +- src/backend/catalog/pg_publication.c | 26 +-- src/backend/commands/tablecmds.c | 9 + src/backend/replication/pgoutput/pgoutput.c | 39 ++-- src/backend/utils/cache/relcache.c | 13 +- .../t/037_rep_changes_except_table.pl | 177 ++++++++++-------- 6 files changed, 140 insertions(+), 139 deletions(-) diff --git a/doc/src/sgml/ref/create_publication.sgml b/doc/src/sgml/ref/create_publication.sgml index 1e091bb3c6d..61974f41fd9 100644 --- a/doc/src/sgml/ref/create_publication.sgml +++ b/doc/src/sgml/ref/create_publication.sgml @@ -205,16 +205,11 @@ CREATE PUBLICATION name tables are excluded. - For partitioned tables, when publish_via_partition_root - is set to true, specifying a root partitioned table in - EXCEPT TABLE excludes it and all its partitions from - replication. Specifying a leaf partition has no effect, as its changes are - still replicated via the root partitioned table. When - publish_via_partition_root is set to - false, specifying a root partitioned table has no - effect, as changes are replicated via the leaf partitions. Specifying a - leaf partition excludes only that partition from replication. The optional - * has no meaning for partitioned tables. + For partitioned tables, only the root partitioned table may be specified + in EXCEPT TABLE. Doing so excludes the root table and + all of its partitions from replication, regardless of the value of + publish_via_partition_root. The optional + * has no effect for partitioned tables. diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c index 09c69005122..0fdddd96704 100644 --- a/src/backend/catalog/pg_publication.c +++ b/src/backend/catalog/pg_publication.c @@ -469,24 +469,12 @@ publication_add_relation(Oid pubid, PublicationRelInfo *pri, } /* - * Handle the case where a partition is excluded by EXCEPT TABLE while - * publish_via_partition_root = true. + * Handle the case where a partition is excluded by EXCEPT TABLE */ - if (pub->alltables && pub->pubviaroot && pri->except && - targetrel->rd_rel->relispartition) - ereport(WARNING, - (errmsg("partition \"%s\" might be replicated as publish_via_partition_root is \"%s\"", - RelationGetRelationName(targetrel), "true"))); - - /* - * Handle the case where a partitioned table is excluded by EXCEPT TABLE - * while publish_via_partition_root = false. - */ - if (pub->alltables && !pub->pubviaroot && pri->except && - targetrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) - ereport(WARNING, - (errmsg("partitioned table \"%s\" might be replicated as publish_via_partition_root is \"%s\"", - RelationGetRelationName(targetrel), "false"))); + if (pub->alltables && pri->except && targetrel->rd_rel->relispartition) + ereport(ERROR, + (errmsg("partition \"%s\" cannot be excluded using EXCEPT TABLE", + RelationGetRelationName(targetrel)))); check_publication_add_relation(targetrel); @@ -960,8 +948,8 @@ GetAllPublicationRelations(Publication *pub, char relkind) if (relkind == RELKIND_RELATION) exceptlist = GetAllPublicationExcludedTables(pubid, pubviaroot ? - PUBLICATION_PART_ALL : - PUBLICATION_PART_ROOT); + PUBLICATION_PART_ROOT : + PUBLICATION_PART_LEAF); classRel = table_open(RelationRelationId, AccessShareLock); diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index a5351fc59c6..395b8c0c2a1 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -20325,6 +20325,7 @@ ATExecAttachPartition(List **wqueue, Relation rel, PartitionCmd *cmd, const char *trigger_name; Oid defaultPartOid; List *partBoundConstraint; + List *except_pubids = NIL; ParseState *pstate = make_parsestate(NULL); pstate->p_sourcetext = context->queryString; @@ -20449,6 +20450,14 @@ ATExecAttachPartition(List **wqueue, Relation rel, PartitionCmd *cmd, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot attach temporary relation of another session as partition"))); + /* Check if the partiton is part of EXCEPT list of any publication */ + GetRelationPublications(RelationGetRelid(attachrel), NULL, &except_pubids); + if (except_pubids != NIL) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot attach relation \"%s\" as partition because it is part of EXCEPT list in publication", + RelationGetRelationName(attachrel)))); + /* * Check if attachrel has any identity columns or any columns that aren't * in the parent. diff --git a/src/backend/replication/pgoutput/pgoutput.c b/src/backend/replication/pgoutput/pgoutput.c index 05802482c10..48dcf0fedc4 100644 --- a/src/backend/replication/pgoutput/pgoutput.c +++ b/src/backend/replication/pgoutput/pgoutput.c @@ -2206,16 +2206,11 @@ get_rel_sync_entry(PGOutputData *data, Relation relation) * root and set the ancestor level accordingly. * * If this is a FOR ALL TABLES publication and it has an EXCEPT - * TABLE list: - * - * 1. If pubviaroot is set and the relation is a partition, check - * whether the partition root is included in the EXCEPT TABLE - * list. If so, do not publish the change. - * - * 2. If pubviaroot is not set, check whether the relation itself - * is included in the EXCEPT TABLE list. If so, do not publish the - * change. - * + * TABLE list + * 1. For a normal table or a partitioned table, if it is part of + * the EXCEPT TABLE list, we don't publish it. + * 2. For a partition, if the topmost ancestor is part of + * the EXCEPT TABLE list, we don't publish it. * This is achieved by keeping the variable "publish" set to * false. And eventually, entry->pubactions will remain all false * for this publication. @@ -2223,27 +2218,23 @@ get_rel_sync_entry(PGOutputData *data, Relation relation) if (pub->alltables) { List *exceptpubids = NIL; + List *ancestors = get_partition_ancestors(relid); + Oid root_relid = relid; - if (pub->pubviaroot && am_partition) + if (am_partition) { - List *ancestors = get_partition_ancestors(relid); + root_relid = llast_oid(ancestors); + GetRelationPublications(root_relid, NULL, &exceptpubids); - pub_relid = llast_oid(ancestors); - ancestor_level = list_length(ancestors); + if (pub->pubviaroot) + { + pub_relid = root_relid; + ancestor_level = list_length(ancestors); + } } - GetRelationPublications(pub_relid, NULL, &exceptpubids); - if (!list_member_oid(exceptpubids, pub->oid)) publish = true; - else - { - /* Sanity check */ - Assert(entry->pubactions.pubinsert == false && - entry->pubactions.pubupdate == false && - entry->pubactions.pubdelete == false && - entry->pubactions.pubtruncate == false); - } list_free(exceptpubids); } diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index dc021dbb6cd..ec7f82fbcdc 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -5837,21 +5837,24 @@ RelationBuildPublicationDesc(Relation relation, PublicationDesc *pubdesc) /* Add publications that the ancestors are in too. */ ancestors = get_partition_ancestors(relid); + /* + * Only the topmost ancestor of a partitioned table can be specified + * in EXCEPT TABLES clause of a FOR ALL TABLES publication. So fetch + * the publications excluding the topmost ancestor only. + */ + GetRelationPublications(llast_oid(ancestors), NULL, &exceptpuboids); + foreach(lc, ancestors) { Oid ancestor = lfirst_oid(lc); List *ancestor_puboids = NIL; - List *ancestor_exceptpuboids = NIL; - GetRelationPublications(ancestor, &ancestor_puboids, - &ancestor_exceptpuboids); + GetRelationPublications(ancestor, &ancestor_puboids, NULL); puboids = list_concat_unique_oid(puboids, ancestor_puboids); schemaid = get_rel_namespace(ancestor); puboids = list_concat_unique_oid(puboids, GetSchemaPublications(schemaid)); - exceptpuboids = list_concat_unique_oid(exceptpuboids, - ancestor_exceptpuboids); } } diff --git a/src/test/subscription/t/037_rep_changes_except_table.pl b/src/test/subscription/t/037_rep_changes_except_table.pl index 95904ddd005..4c3c81462a5 100644 --- a/src/test/subscription/t/037_rep_changes_except_table.pl +++ b/src/test/subscription/t/037_rep_changes_except_table.pl @@ -88,8 +88,11 @@ $node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub_schema"); $node_publisher->safe_psql( 'postgres', qq( CREATE TABLE sch1.t1(a int) PARTITION BY RANGE(a); - CREATE TABLE sch1.part1 PARTITION OF sch1.t1 FOR VALUES FROM (0) TO (5); - CREATE TABLE sch1.part2 PARTITION OF sch1.t1 FOR VALUES FROM (6) TO (10); + CREATE TABLE sch1.part1 PARTITION OF sch1.t1 FOR VALUES FROM (0) TO (100); + CREATE TABLE sch1.part2(a int) PARTITION BY RANGE(a); + CREATE TABLE sch1.part2_1 PARTITION OF sch1.part2 FOR VALUES FROM (101) TO (150); + CREATE TABLE sch1.part2_2 PARTITION OF sch1.part2 FOR VALUES FROM (151) TO (200); + ALTER TABLE sch1.t1 ATTACH PARTITION sch1.part2 FOR VALUES FROM (101) TO (200); )); $node_subscriber->safe_psql( @@ -97,140 +100,152 @@ $node_subscriber->safe_psql( CREATE TABLE sch1.t1(a int); CREATE TABLE sch1.part1(a int); CREATE TABLE sch1.part2(a int); + CREATE TABLE sch1.part2_1(a int); + CREATE TABLE sch1.part2_2(a int); )); -# EXCEPT TABLE (sch1.part1) with publish_via_partition_root = false -# Excluding a partition while publish_via_partition_root = false prevents -# replication of rows inserted into the partitioned table for that particular -# partition. -$node_publisher->safe_psql( +# Partititions cannot be excluded using EXCEPT TABLE +my ($stdout, $stderr); +($result, $stdout, $stderr) = $node_publisher->psql( 'postgres', qq( - CREATE PUBLICATION tap_pub_part FOR ALL TABLES EXCEPT TABLE (sch1.part1) WITH (publish_via_partition_root = false); - INSERT INTO sch1.t1 VALUES (1), (6); + CREATE PUBLICATION tap_pub_part FOR ALL TABLES EXCEPT TABLE (sch1.part2) WITH (publish_via_partition_root = false); )); -$node_subscriber->safe_psql('postgres', - "CREATE SUBSCRIPTION tap_sub_part CONNECTION '$publisher_connstr' PUBLICATION tap_pub_part" -); -$node_subscriber->wait_for_subscription_sync($node_publisher, 'tap_sub_part'); -$node_publisher->safe_psql('postgres', - "INSERT INTO sch1.t1 VALUES (2), (7);"); -$node_publisher->wait_for_catchup('tap_sub_part'); - -$result = $node_subscriber->safe_psql('postgres', "SELECT * FROM sch1.t1"); -is($result, qq(), 'check rows on partitioned table'); +like( + $stderr, + qr/partition "part2" cannot be excluded using EXCEPT TABLE/, + 'partition "part2" cannot be excluded using EXCEPT TABLE'); -$result = $node_subscriber->safe_psql('postgres', "SELECT * FROM sch1.part1"); -is($result, qq(), 'check rows on excluded partition'); +($result, $stdout, $stderr) = $node_publisher->psql( + 'postgres', qq( + CREATE PUBLICATION tap_pub_part FOR ALL TABLES EXCEPT TABLE (sch1.part2) WITH (publish_via_partition_root = true); +)); +like( + $stderr, + qr/partition "part2" cannot be excluded using EXCEPT TABLE/, + 'partition "part2" cannot be excluded using EXCEPT TABLE'); -$result = $node_subscriber->safe_psql('postgres', "SELECT * FROM sch1.part2"); -is( $result, qq(6 -7), 'check rows on other partition'); +($result, $stdout, $stderr) = $node_publisher->psql( + 'postgres', qq( + CREATE PUBLICATION tap_pub_part FOR ALL TABLES EXCEPT TABLE (sch1.part2_1) WITH (publish_via_partition_root = false); +)); +like( + $stderr, + qr/partition "part2_1" cannot be excluded using EXCEPT TABLE/, + 'partition "part2_1" cannot be excluded using EXCEPT TABLE'); -$node_publisher->safe_psql('postgres', "TRUNCATE sch1.t1"); -$node_publisher->wait_for_catchup('tap_sub_part'); -$node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub_part"); -$node_publisher->safe_psql('postgres', "DROP PUBLICATION tap_pub_part;"); +($result, $stdout, $stderr) = $node_publisher->psql( + 'postgres', qq( + CREATE PUBLICATION tap_pub_part FOR ALL TABLES EXCEPT TABLE (sch1.part2_1) WITH (publish_via_partition_root = true); +)); +like( + $stderr, + qr/partition "part2_1" cannot be excluded using EXCEPT TABLE/, + 'partition "part2_1" cannot be excluded using EXCEPT TABLE'); -# EXCEPT TABLE (sch1.t1) with publish_via_partition_root = false -# Excluding the partitioned table still allows rows inserted into the -# partitioned table to be replicated via its partitions. +# Excluding the root partitioned table excludes all its partitions as well when +# publish_via_partition_root = false. $node_publisher->safe_psql( 'postgres', qq( CREATE PUBLICATION tap_pub_part FOR ALL TABLES EXCEPT TABLE (sch1.t1) WITH (publish_via_partition_root = false); - INSERT INTO sch1.t1 VALUES (1), (6); + INSERT INTO sch1.t1 VALUES (1), (101), (151); )); $node_subscriber->safe_psql('postgres', "CREATE SUBSCRIPTION tap_sub_part CONNECTION '$publisher_connstr' PUBLICATION tap_pub_part" ); $node_subscriber->wait_for_subscription_sync($node_publisher, 'tap_sub_part'); $node_publisher->safe_psql('postgres', - "INSERT INTO sch1.t1 VALUES (2), (7);"); + "SELECT slot_name FROM pg_replication_slot_advance('test_slot', pg_current_wal_lsn());" +); +$node_publisher->safe_psql('postgres', + "INSERT INTO sch1.t1 VALUES (2), (102), (152)"); + +# Verify that data inserted to the partitioned table is not published when it is +# excluded with publish_via_partition_root = true. +$result = $node_publisher->safe_psql('postgres', + "SELECT count(*) = 0 FROM pg_logical_slot_get_binary_changes('test_slot', NULL, NULL, 'proto_version', '1', 'publication_names', 'tap_pub_part')" +); $node_publisher->wait_for_catchup('tap_sub_part'); +# Check that no rows are replicated to subscriber $result = $node_subscriber->safe_psql('postgres', "SELECT * FROM sch1.t1"); -is($result, qq(), 'check rows on partitioned table'); +is($result, qq(), 'check rows on root table'); $result = $node_subscriber->safe_psql('postgres', "SELECT * FROM sch1.part1"); -is( $result, qq(1 -2), 'check rows on first partition'); +is($result, qq(), 'check rows on table sch1.part1'); $result = $node_subscriber->safe_psql('postgres', "SELECT * FROM sch1.part2"); -is( $result, qq(6 -7), 'check rows on second partition'); +is($result, qq(), 'check rows on table sch1.part2'); + +$result = + $node_subscriber->safe_psql('postgres', "SELECT * FROM sch1.part2_1"); +is($result, qq(), 'check rows on table sch1.part2_1'); + +$result = + $node_subscriber->safe_psql('postgres', "SELECT * FROM sch1.part2_2"); +is($result, qq(), 'check rows on table sch1.part2_2'); -$node_publisher->safe_psql('postgres', "TRUNCATE sch1.t1"); -$node_publisher->wait_for_catchup('tap_sub_part'); $node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub_part"); $node_publisher->safe_psql('postgres', "DROP PUBLICATION tap_pub_part;"); -$node_publisher->safe_psql('postgres', - "SELECT slot_name FROM pg_replication_slot_advance('test_slot', pg_current_wal_lsn());" -); -# EXCEPT TABLE (sch1.t1) with publish_via_partition_root = true -# When the partitioned table is excluded and publish_via_partition_root is true, -# no rows from the table or its partitions are replicated. +# Excluding the root partitioned table excludes all its partitions as well when +# publish_via_partition_root = true. $node_publisher->safe_psql( 'postgres', qq( - CREATE PUBLICATION tap_pub_part FOR ALL TABLES EXCEPT TABLE (sch1.t1) WITH (publish_via_partition_root); - INSERT INTO sch1.t1 VALUES (1), (6); + CREATE PUBLICATION tap_pub_part FOR ALL TABLES EXCEPT TABLE (sch1.t1) WITH (publish_via_partition_root = true); )); $node_subscriber->safe_psql('postgres', "CREATE SUBSCRIPTION tap_sub_part CONNECTION '$publisher_connstr' PUBLICATION tap_pub_part" ); $node_subscriber->wait_for_subscription_sync($node_publisher, 'tap_sub_part'); $node_publisher->safe_psql('postgres', - "INSERT INTO sch1.t1 VALUES (2), (7);"); -$node_publisher->wait_for_catchup('tap_sub_part'); + "SELECT slot_name FROM pg_replication_slot_advance('test_slot', pg_current_wal_lsn());" +); +$node_publisher->safe_psql('postgres', + "INSERT INTO sch1.t1 VALUES (3), (103), (153);"); # Verify that data inserted to the partitioned table is not published when it is # excluded with publish_via_partition_root = true. $result = $node_publisher->safe_psql('postgres', "SELECT count(*) = 0 FROM pg_logical_slot_get_binary_changes('test_slot', NULL, NULL, 'proto_version', '1', 'publication_names', 'tap_pub_part')" ); -is($result, qq(t), 'check no changes for excluded table in replication slot'); +$node_publisher->wait_for_catchup('tap_sub_part'); +# Check that no rows are replicated to subscriber $result = $node_subscriber->safe_psql('postgres', "SELECT * FROM sch1.t1"); -is($result, qq(), 'check rows on partitioned table'); +is($result, qq(), 'check rows on root table'); $result = $node_subscriber->safe_psql('postgres', "SELECT * FROM sch1.part1"); -is($result, qq(), 'check rows on first partition'); +is($result, qq(), 'check rows on table sch1.part1'); $result = $node_subscriber->safe_psql('postgres', "SELECT * FROM sch1.part2"); -is($result, qq(), 'check rows on second partition'); +is($result, qq(), 'check rows on table sch1.part2'); + +$result = + $node_subscriber->safe_psql('postgres', "SELECT * FROM sch1.part2_1"); +is($result, qq(), 'check rows on table sch1.part2_1'); + +$result = + $node_subscriber->safe_psql('postgres', "SELECT * FROM sch1.part2_2"); +is($result, qq(), 'check rows on table sch1.part2_2'); -$node_publisher->safe_psql('postgres', "TRUNCATE sch1.t1"); -$node_publisher->wait_for_catchup('tap_sub_part'); $node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub_part"); $node_publisher->safe_psql('postgres', "DROP PUBLICATION tap_pub_part;"); -# EXCEPT TABLE (sch1.part1) with publish_via_partition_root = true -# When a partition is excluded but publish_via_partition_root is true, -# rows published through the partitioned table can still be replicated. -$node_publisher->safe_psql( +# Cannot attach partition that is part of EXCEPT list in publication +$node_publisher->safe_psql('postgres', + "ALTER TABLE sch1.t1 DETACH PARTITION sch1.part2"); +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION tap_pub_part FOR ALL TABLES EXCEPT TABLE (sch1.part2) WITH (publish_via_partition_root = true)" +); +($result, $stdout, $stderr) = $node_publisher->psql( 'postgres', qq( - CREATE PUBLICATION tap_pub_part FOR ALL TABLES EXCEPT TABLE (sch1.part1) WITH (publish_via_partition_root); - INSERT INTO sch1.t1 VALUES (1), (6) + ALTER TABLE sch1.t1 ATTACH PARTITION sch1.part2 FOR VALUES FROM (101) TO (200); )); -$node_subscriber->safe_psql('postgres', - "CREATE SUBSCRIPTION tap_sub_part CONNECTION '$publisher_connstr' PUBLICATION tap_pub_part" +like( + $stderr, + qr/cannot attach relation "part2" as partition because it is part of EXCEPT list in publication/, + 'cannot attach relation "part2" as partition because it is part of EXCEPT list in publication' ); -$node_subscriber->wait_for_subscription_sync($node_publisher, 'tap_sub_part'); -$node_publisher->safe_psql('postgres', - "INSERT INTO sch1.t1 VALUES (2), (7);"); -$node_publisher->wait_for_catchup('tap_sub_part'); - -$result = - $node_subscriber->safe_psql('postgres', "SELECT * FROM sch1.t1 ORDER BY a"); -is( $result, qq(1 -2 -6 -7), 'check rows on partitioned table'); - -$result = $node_subscriber->safe_psql('postgres', "SELECT * FROM sch1.part1"); -is($result, qq(), 'check rows on excluded partition'); - -$result = $node_subscriber->safe_psql('postgres', "SELECT * FROM sch1.part2"); -is($result, qq(), 'check rows on other partition'); $node_subscriber->stop('fast'); $node_publisher->stop('fast'); -- 2.43.0