From 4248729b91c72d68c3deea6b540cefa04b2501c1 Mon Sep 17 00:00:00 2001 From: Nisha Moond Date: Mon, 4 May 2026 12:49:27 +0530 Subject: [PATCH v6 1/3] Support EXCEPT clause for schema-level publications Extend table exclusion support in publications to allow specific tables to be excluded from schema-level publications using an EXCEPT clause in CREATE PUBLICATION. Supported syntax: CREATE PUBLICATION FOR TABLES IN SCHEMA s EXCEPT (TABLE t1,...); --- doc/src/sgml/logical-replication.sgml | 3 +- doc/src/sgml/ref/create_publication.sgml | 22 +++- src/backend/catalog/pg_publication.c | 86 +++++++++++-- src/backend/commands/publicationcmds.c | 47 ++++++- src/backend/parser/gram.y | 52 +++++++- src/backend/replication/pgoutput/pgoutput.c | 21 +++- src/bin/psql/describe.c | 18 +++ src/bin/psql/tab-complete.in.c | 26 +++- src/include/catalog/pg_publication.h | 3 +- src/include/nodes/parsenodes.h | 2 + src/test/regress/expected/publication.out | 100 ++++++++++++++- src/test/regress/sql/publication.sql | 65 +++++++++- src/test/subscription/t/037_except.pl | 133 +++++++++++++++++++- 13 files changed, 545 insertions(+), 33 deletions(-) diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index 9e7868487de..1433d2660fe 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -117,7 +117,8 @@ or FOR ALL SEQUENCES. Unlike tables, sequences can be synchronized at any time. For more information, see . When a publication is - created with FOR ALL TABLES, a table or set of tables can + created with FOR ALL TABLES or + FOR TABLES IN SCHEMA, a table or set of tables can be explicitly excluded from publication using the EXCEPT clause. diff --git a/doc/src/sgml/ref/create_publication.sgml b/doc/src/sgml/ref/create_publication.sgml index f82d640e6ca..7fa0bd11f7b 100644 --- a/doc/src/sgml/ref/create_publication.sgml +++ b/doc/src/sgml/ref/create_publication.sgml @@ -28,7 +28,7 @@ CREATE PUBLICATION name where publication_object is one of: TABLE table_and_columns [, ... ] - TABLES IN SCHEMA { schema_name | CURRENT_SCHEMA } [, ... ] + TABLES IN SCHEMA tables_in_schema [, ... ] and publication_all_object is one of: @@ -39,6 +39,10 @@ CREATE PUBLICATION name table_object [ ( column_name [, ... ] ) ] [ WHERE ( expression ) ] +and tables_in_schema is: + + { schema_name | CURRENT_SCHEMA } [ EXCEPT ( except_table_object [, ... ] ) ] + and except_table_object is: TABLE table_object [, ... ] @@ -142,6 +146,8 @@ CREATE PUBLICATION name Marks the publication as one that replicates changes for all tables in the specified list of schemas, including tables created in the future. + Tables listed in the EXCEPT clause for a given schema + are excluded from the publication. @@ -173,7 +179,7 @@ CREATE PUBLICATION name Marks the publication as one that replicates changes for all tables in the database, including tables created in the future. Tables listed in - EXCEPT clause are excluded from the publication. + the EXCEPT clause are excluded from the publication. @@ -198,7 +204,8 @@ CREATE PUBLICATION name This clause specifies a list of tables to be excluded from the - publication. + publication. It can be used with FOR ALL TABLES or + FOR TABLES IN SCHEMA. For inherited tables, if ONLY is specified before the @@ -515,6 +522,15 @@ CREATE PUBLICATION production_publication FOR TABLE users, departments, TABLES I CREATE PUBLICATION sales_publication FOR TABLES IN SCHEMA marketing, sales; + + Create a publication that publishes all changes for all the tables present in + the schema sales, except + internal and drafts: + +CREATE PUBLICATION sales_filtered FOR TABLES IN SCHEMA sales EXCEPT (TABLE internal, drafts); + + + Create a publication that publishes all changes for table users, but replicates only columns user_id and diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c index 5c457d9aca8..6f945955901 100644 --- a/src/backend/catalog/pg_publication.c +++ b/src/backend/catalog/pg_publication.c @@ -446,7 +446,8 @@ GetPubPartitionOptionRelations(List *result, PublicationPartOpt pub_partopt, * ancestor is at the end of the list. */ Oid -GetTopMostAncestorInPublication(Oid puboid, List *ancestors, int *ancestor_level) +GetTopMostAncestorInPublication(Oid puboid, List *ancestors, + int *ancestor_level, List *except_pubids) { ListCell *lc; Oid topmost_relid = InvalidOid; @@ -473,7 +474,8 @@ GetTopMostAncestorInPublication(Oid puboid, List *ancestors, int *ancestor_level else { aschemaPubids = GetSchemaPublications(get_rel_namespace(ancestor)); - if (list_member_oid(aschemaPubids, puboid)) + if (list_member_oid(aschemaPubids, puboid) && + !list_member_oid(except_pubids, puboid)) { topmost_relid = ancestor; @@ -545,18 +547,30 @@ publication_add_relation(Oid pubid, PublicationRelInfo *pri, * duplicates, it's here just to provide nicer error message in common * case. The real protection is the unique key on the catalog. */ - if (SearchSysCacheExists2(PUBLICATIONRELMAP, ObjectIdGetDatum(relid), - ObjectIdGetDatum(pubid))) + tup = SearchSysCache2(PUBLICATIONRELMAP, ObjectIdGetDatum(relid), + ObjectIdGetDatum(pubid)); + + if (HeapTupleIsValid(tup)) { + bool is_except = ((Form_pg_publication_rel) GETSTRUCT(tup))->prexcept; + + ReleaseSysCache(tup); table_close(rel, RowExclusiveLock); if (if_not_exists) return InvalidObjectAddress; - ereport(ERROR, - (errcode(ERRCODE_DUPLICATE_OBJECT), - errmsg("relation \"%s\" is already member of publication \"%s\"", - RelationGetRelationName(targetrel), pub->name))); + if (is_except) + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("table \"%s\" cannot be added because it is excluded from publication \"%s\"", + RelationGetQualifiedRelationName(targetrel), + pub->name))); + else + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("relation \"%s\" is already member of publication \"%s\"", + RelationGetRelationName(targetrel), pub->name))); } check_publication_add_relation(pri); @@ -982,12 +996,13 @@ GetIncludedPublicationRelations(Oid pubid, PublicationPartOpt pub_partopt) * Gets list of table oids that were specified in the EXCEPT clause for a * publication. * - * This should only be used FOR ALL TABLES publications. + * This is used for FOR ALL TABLES and FOR TABLES IN SCHEMA publications, + * both of which support EXCEPT TABLE. */ List * GetExcludedPublicationTables(Oid pubid, PublicationPartOpt pub_partopt) { - Assert(GetPublication(pubid)->alltables); + Assert(GetPublication(pubid)->alltables || is_schema_publication(pubid)); return get_publication_relations(pubid, pub_partopt, true); } @@ -1232,22 +1247,67 @@ GetSchemaPublicationRelations(Oid schemaid, PublicationPartOpt pub_partopt) /* * Gets the list of all relations published by FOR TABLES IN SCHEMA - * publication. + * publication, excluding any tables listed in the EXCEPT clause. */ List * GetAllSchemaPublicationRelations(Oid pubid, PublicationPartOpt pub_partopt) { List *result = NIL; List *pubschemalist = GetPublicationSchemas(pubid); + List *exceptlist = NIL; ListCell *cell; + /* get the list of tables excluded via EXCEPT TABLE for this publication */ + if (pubschemalist != NIL) + exceptlist = GetExcludedPublicationTables(pubid, pub_partopt); + foreach(cell, pubschemalist) { Oid schemaid = lfirst_oid(cell); List *schemaRels = NIL; schemaRels = GetSchemaPublicationRelations(schemaid, pub_partopt); - result = list_concat(result, schemaRels); + + if (exceptlist != NIL) + { + /* filter out any tables that appear in the EXCEPT list */ + ListCell *rlc; + + foreach(rlc, schemaRels) + { + Oid relid = lfirst_oid(rlc); + bool excluded = list_member_oid(exceptlist, relid); + + /* + * Also exclude any relation whose partition ancestor is in + * the EXCEPT list. This matters when pub_partopt is + * PUBLICATION_PART_ROOT: the except list holds only the root + * OID, but the schema scan may also return individual + * partition relations that live in the same schema. + */ + if (!excluded && get_rel_relispartition(relid)) + { + List *ancestors = get_partition_ancestors(relid); + ListCell *alc; + + foreach(alc, ancestors) + { + if (list_member_oid(exceptlist, lfirst_oid(alc))) + { + excluded = true; + break; + } + } + list_free(ancestors); + } + + if (!excluded) + result = lappend_oid(result, relid); + } + list_free(schemaRels); + } + else + result = list_concat(result, schemaRels); } return result; @@ -1381,7 +1441,7 @@ is_table_publishable_in_publication(Oid relid, Publication *pub) * the publication, it should be included (return true). */ if (relispartition && - OidIsValid(GetTopMostAncestorInPublication(pub->oid, ancestors, NULL))) + OidIsValid(GetTopMostAncestorInPublication(pub->oid, ancestors, NULL, NIL))) return !pub->pubviaroot; /* diff --git a/src/backend/commands/publicationcmds.c b/src/backend/commands/publicationcmds.c index 440adb356ad..95186ca7377 100644 --- a/src/backend/commands/publicationcmds.c +++ b/src/backend/commands/publicationcmds.c @@ -305,7 +305,7 @@ pub_rf_contains_invalid_column(Oid pubid, Relation relation, List *ancestors, if (pubviaroot && relation->rd_rel->relispartition) { publish_as_relid - = GetTopMostAncestorInPublication(pubid, ancestors, NULL); + = GetTopMostAncestorInPublication(pubid, ancestors, NULL, NIL); if (!OidIsValid(publish_as_relid)) publish_as_relid = relid; @@ -389,7 +389,7 @@ pub_contains_invalid_column(Oid pubid, Relation relation, List *ancestors, */ if (pubviaroot && relation->rd_rel->relispartition) { - publish_as_relid = GetTopMostAncestorInPublication(pubid, ancestors, NULL); + publish_as_relid = GetTopMostAncestorInPublication(pubid, ancestors, NULL, NIL); if (!OidIsValid(publish_as_relid)) publish_as_relid = relid; @@ -959,6 +959,8 @@ CreatePublication(ParseState *pstate, CreatePublicationStmt *stmt) } else if (!stmt->for_all_sequences) { + List *explicitrelids = NIL; + /* FOR TABLES IN SCHEMA requires superuser */ if (schemaidlist != NIL && !superuser()) ereport(ERROR, @@ -977,6 +979,19 @@ CreatePublication(ParseState *pstate, CreatePublicationStmt *stmt) schemaidlist != NIL, publish_via_partition_root); + /* + * Collect explicit table OIDs now, before we close the relation + * list, so that except-table validation below can check for + * contradictions without relying on a catalog scan that might not + * yet see the just-inserted rows. + */ + if (exceptrelations != NIL) + { + foreach_ptr(PublicationRelInfo, pri, rels) + explicitrelids = lappend_oid(explicitrelids, + RelationGetRelid(pri->relation)); + } + PublicationAddTables(puboid, rels, true, NULL); CloseTableList(rels); } @@ -989,6 +1004,34 @@ CreatePublication(ParseState *pstate, CreatePublicationStmt *stmt) */ LockSchemaList(schemaidlist); PublicationAddSchemas(puboid, schemaidlist, true, NULL); + + if (exceptrelations != NIL) + { + List *exceptrels; + + exceptrels = OpenTableList(exceptrelations); + + /* + * Validate that each excluded table is not also in the + * explicit table list (which would be contradictory). Use the + * in-memory explicitrelids collected above rather than + * re-reading the catalog, which may not yet see the + * just-inserted rows. + */ + foreach_ptr(PublicationRelInfo, pri, exceptrels) + { + Oid except_relid = RelationGetRelid(pri->relation); + + if (list_member_oid(explicitrelids, except_relid)) + ereport(ERROR, + errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("table \"%s\" cannot appear in both the table list and the EXCEPT clause", + RelationGetQualifiedRelationName(pri->relation))); + } + + PublicationAddTables(puboid, exceptrels, true, NULL); + CloseTableList(exceptrels); + } } } diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index ff4e1388c55..4514ef7f9c2 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -58,6 +58,7 @@ #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "parser/parser.h" +#include "utils/builtins.h" #include "utils/datetime.h" #include "utils/xml.h" @@ -11272,7 +11273,7 @@ AlterOwnerStmt: ALTER AGGREGATE aggregate_with_argtypes OWNER TO RoleSpec * pub_obj is one of: * * TABLE table [, ...] - * TABLES IN SCHEMA schema [, ...] + * TABLES IN SCHEMA schema [EXCEPT (TABLE table [, ...] )] [, ...] * *****************************************************************************/ @@ -11332,23 +11333,26 @@ PublicationObjSpec: $$->pubtable->columns = $3; $$->pubtable->whereClause = $4; } - | TABLES IN_P SCHEMA ColId + | TABLES IN_P SCHEMA ColId opt_pub_except_clause { $$ = makeNode(PublicationObjSpec); $$->pubobjtype = PUBLICATIONOBJ_TABLES_IN_SCHEMA; $$->name = $4; + $$->except_tables = $5; $$->location = @4; } - | TABLES IN_P SCHEMA CURRENT_SCHEMA + | TABLES IN_P SCHEMA CURRENT_SCHEMA opt_pub_except_clause { $$ = makeNode(PublicationObjSpec); $$->pubobjtype = PUBLICATIONOBJ_TABLES_IN_CUR_SCHEMA; + $$->except_tables = $5; $$->location = @4; } - | ColId opt_column_list OptWhereClause + | ColId opt_column_list OptWhereClause opt_pub_except_clause { $$ = makeNode(PublicationObjSpec); $$->pubobjtype = PUBLICATIONOBJ_CONTINUATION; + $$->except_tables = $4; /* * If either a row filter or column list is specified, create * a PublicationTable object. @@ -11392,10 +11396,11 @@ PublicationObjSpec: $$->pubtable->columns = $2; $$->pubtable->whereClause = $3; } - | CURRENT_SCHEMA + | CURRENT_SCHEMA opt_pub_except_clause { $$ = makeNode(PublicationObjSpec); $$->pubobjtype = PUBLICATIONOBJ_CONTINUATION; + $$->except_tables = $2; $$->location = @1; } ; @@ -20784,6 +20789,8 @@ preprocess_pub_all_objtype_list(List *all_objects_list, List **pubobjects, /* * Process pubobjspec_list to check for errors in any of the objects and * convert PUBLICATIONOBJ_CONTINUATION into appropriate PublicationObjSpecType. + * Also flattens except_tables from TABLES IN SCHEMA nodes into the list so + * that ObjectsInPublicationToOids() sees them as top-level EXCEPT_TABLE entries. */ static void preprocess_pubobj_list(List *pubobjspec_list, core_yyscan_t yyscanner) @@ -20812,6 +20819,13 @@ preprocess_pubobj_list(List *pubobjspec_list, core_yyscan_t yyscanner) if (pubobj->pubobjtype == PUBLICATIONOBJ_TABLE) { + /* EXCEPT is not valid for table objects */ + if (pubobj->except_tables != NIL) + ereport(ERROR, + errcode(ERRCODE_SYNTAX_ERROR), + errmsg("EXCEPT is not allowed for TABLE publication objects"), + parser_errposition(pubobj->location)); + /* relation name or pubtable must be set for this type of object */ if (!pubobj->name && !pubobj->pubtable) ereport(ERROR, @@ -20860,6 +20874,34 @@ preprocess_pubobj_list(List *pubobjspec_list, core_yyscan_t yyscanner) errcode(ERRCODE_SYNTAX_ERROR), errmsg("invalid schema name"), parser_errposition(pubobj->location)); + + /* Flatten EXCEPT entries into the top-level list */ + foreach_ptr(PublicationObjSpec, eobj, pubobj->except_tables) + { + /* + * Unqualified names are implicitly qualified with the parent + * schema. Qualified names must match the parent schema — + * each EXCEPT clause is bound to exactly one schema, so + * cross-schema entries are rejected at parse time. + */ + if (pubobj->pubobjtype == PUBLICATIONOBJ_TABLES_IN_SCHEMA) + { + const char *eobj_schemaname = eobj->pubtable->relation->schemaname; + const char *eobj_relname = eobj->pubtable->relation->relname; + + if (eobj_schemaname == NULL) + eobj->pubtable->relation->schemaname = pubobj->name; + else if (strcmp(eobj_schemaname, pubobj->name) != 0) + ereport(ERROR, + errcode(ERRCODE_SYNTAX_ERROR), + errmsg("table \"%s\" in EXCEPT clause does not belong to schema \"%s\"", + quote_qualified_identifier(eobj_schemaname, eobj_relname), + pubobj->name), + parser_errposition(eobj->location)); + } + } + pubobjspec_list = list_concat(pubobjspec_list, pubobj->except_tables); + pubobj->except_tables = NIL; } prevobjtype = pubobj->pubobjtype; diff --git a/src/backend/replication/pgoutput/pgoutput.c b/src/backend/replication/pgoutput/pgoutput.c index 4ecfcbff7ab..b55d7ab7cd1 100644 --- a/src/backend/replication/pgoutput/pgoutput.c +++ b/src/backend/replication/pgoutput/pgoutput.c @@ -2097,6 +2097,7 @@ get_rel_sync_entry(PGOutputData *data, Relation relation) * are absorbed while decoding WAL. */ List *schemaPubids = GetSchemaPublications(schemaId); + List *schemaExceptPubids; ListCell *lc; Oid publish_as_relid = relid; int publish_ancestor_level = 0; @@ -2104,6 +2105,19 @@ get_rel_sync_entry(PGOutputData *data, Relation relation) char relkind = get_rel_relkind(relid); List *rel_publications = NIL; + /* + * For the schema EXCEPT check, we must look up the top-most ancestor + * rather than the relation itself. check_publication_add_relation() + * prevents individual partitions from appearing in the EXCEPT clause, + * so only a root (non-partition) table can have prexcept = true. + * Using the partition's own OID would always return NIL and miss the + * exclusion. + */ + Oid root_relid = am_partition ? + llast_oid(get_partition_ancestors(relid)) : relid; + + schemaExceptPubids = GetRelationExcludedPublications(root_relid); + /* Reload publications if needed before use. */ if (!publications_valid) { @@ -2267,7 +2281,8 @@ get_rel_sync_entry(PGOutputData *data, Relation relation) ancestor = GetTopMostAncestorInPublication(pub->oid, ancestors, - &level); + &level, + schemaExceptPubids); if (ancestor != InvalidOid) { @@ -2281,7 +2296,8 @@ get_rel_sync_entry(PGOutputData *data, Relation relation) } if (list_member_oid(pubids, pub->oid) || - list_member_oid(schemaPubids, pub->oid) || + (list_member_oid(schemaPubids, pub->oid) && + !list_member_oid(schemaExceptPubids, pub->oid)) || ancestor_published) publish = true; } @@ -2360,6 +2376,7 @@ get_rel_sync_entry(PGOutputData *data, Relation relation) list_free(pubids); list_free(schemaPubids); + list_free(schemaExceptPubids); list_free(rel_publications); entry->replicate_valid = true; diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index e1449654f96..e5b1a70e05e 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -7038,6 +7038,24 @@ describePublications(const char *pattern) if (!addFooterToPublicationDesc(&buf, _("Tables from schemas:"), true, &cont)) goto error_return; + + if (pset.sversion >= 190000) + { + /* + * Get tables in the EXCEPT clause for this schema + * publication. + */ + printfPQExpBuffer(&buf, + "SELECT concat(c.relnamespace::regnamespace, '.', c.relname)\n" + "FROM pg_catalog.pg_class c\n" + " JOIN pg_catalog.pg_publication_rel pr ON c.oid = pr.prrelid\n" + "WHERE pr.prpubid = '%s'\n" + " AND pr.prexcept\n" + "ORDER BY 1", pubid); + if (!addFooterToPublicationDesc(&buf, _("Except tables:"), + true, &cont)) + goto error_return; + } } } else diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c index 75132528f3a..2c652cf32a0 100644 --- a/src/bin/psql/tab-complete.in.c +++ b/src/bin/psql/tab-complete.in.c @@ -1065,6 +1065,15 @@ static const SchemaQuery Query_for_trigger_of_table = { "SELECT nspname FROM pg_catalog.pg_namespace "\ " WHERE nspname LIKE '%s'" +#define Query_for_list_of_tables_in_schema \ +"SELECT n.nspname || '.' || c.relname "\ +" FROM pg_catalog.pg_class c "\ +" JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace "\ +" WHERE c.relkind IN (" CppAsString2(RELKIND_RELATION) ", " \ + CppAsString2(RELKIND_PARTITIONED_TABLE) ") "\ +" AND (n.nspname || '.' || c.relname) LIKE '%s' "\ +" AND n.nspname = '%s'" + /* Use COMPLETE_WITH_QUERY_VERBATIM with these queries for GUC names: */ #define Query_for_list_of_alter_system_set_vars \ "SELECT pg_catalog.lower(name) FROM pg_catalog.pg_settings "\ @@ -3785,8 +3794,21 @@ match_previous_words(int pattern_id, COMPLETE_WITH_QUERY_PLUS(Query_for_list_of_schemas " AND nspname NOT LIKE E'pg\\\\_%%'", "CURRENT_SCHEMA"); - else if (Matches("CREATE", "PUBLICATION", MatchAny, "FOR", "TABLES", "IN", "SCHEMA", MatchAny) && (!ends_with(prev_wd, ','))) - COMPLETE_WITH("WITH ("); + else if (Matches("CREATE", "PUBLICATION", MatchAny, "FOR", "TABLES", "IN", "SCHEMA", MatchAny) && !ends_with(prev_wd, ',')) + COMPLETE_WITH("EXCEPT ( TABLE", "WITH ("); + else if (Matches("CREATE", "PUBLICATION", MatchAny, "FOR", "TABLES", "IN", "SCHEMA", MatchAny, "EXCEPT")) + COMPLETE_WITH("( TABLE"); + else if (Matches("CREATE", "PUBLICATION", MatchAny, "FOR", "TABLES", "IN", "SCHEMA", MatchAny, "EXCEPT", "(")) + COMPLETE_WITH("TABLE"); + else if (Matches("CREATE", "PUBLICATION", MatchAny, "FOR", "TABLES", "IN", "SCHEMA", MatchAny, "EXCEPT", "(", "TABLE")) + { + set_completion_reference(prev4_wd); + COMPLETE_WITH_QUERY_VERBATIM(Query_for_list_of_tables_in_schema); + } + else if (Matches("CREATE", "PUBLICATION", MatchAny, "FOR", "TABLES", "IN", "SCHEMA", MatchAny, "EXCEPT", "(", "TABLE", MatchAnyN) && ends_with(prev_wd, ',')) + COMPLETE_WITH_SCHEMA_QUERY(Query_for_list_of_tables); + else if (Matches("CREATE", "PUBLICATION", MatchAny, "FOR", "TABLES", "IN", "SCHEMA", MatchAny, "EXCEPT", "(", "TABLE", MatchAnyN) && !ends_with(prev_wd, ',')) + COMPLETE_WITH(")"); /* Complete "CREATE PUBLICATION [...] WITH" */ else if (Matches("CREATE", "PUBLICATION", MatchAnyN, "WITH", "(")) COMPLETE_WITH("publish", "publish_generated_columns", "publish_via_partition_root"); diff --git a/src/include/catalog/pg_publication.h b/src/include/catalog/pg_publication.h index 89b4bb14f62..53e3d7c6f3d 100644 --- a/src/include/catalog/pg_publication.h +++ b/src/include/catalog/pg_publication.h @@ -191,7 +191,8 @@ extern List *GetPubPartitionOptionRelations(List *result, PublicationPartOpt pub_partopt, Oid relid); extern Oid GetTopMostAncestorInPublication(Oid puboid, List *ancestors, - int *ancestor_level); + int *ancestor_level, + List *except_pubids); extern bool is_publishable_relation(Relation rel); extern bool is_schema_publication(Oid pubid); diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 91377a6cde3..98a03c0eeda 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -4493,6 +4493,8 @@ typedef struct PublicationObjSpec PublicationObjSpecType pubobjtype; /* type of this publication object */ char *name; PublicationTable *pubtable; + List *except_tables; /* tables specified in the EXCEPT clause (for + * TABLES IN SCHEMA) */ ParseLoc location; /* token location, or -1 if unknown */ } PublicationObjSpec; diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out index 29e54b214a0..77d77c89d80 100644 --- a/src/test/regress/expected/publication.out +++ b/src/test/regress/expected/publication.out @@ -270,6 +270,12 @@ CREATE PUBLICATION testpub_foralltables_excepttable2 FOR ALL TABLES EXCEPT (test ERROR: syntax error at or near "testpub_tbl1" LINE 1: ..._foralltables_excepttable2 FOR ALL TABLES EXCEPT (testpub_tb... ^ +-- fail - EXCEPT is not allowed for FOR TABLE publications +CREATE PUBLICATION testpub_except_err + FOR TABLE testpub_tbl1, testpub_tbl2 EXCEPT (TABLE testpub_tbl3); +ERROR: EXCEPT is not allowed for TABLE publication objects +LINE 2: FOR TABLE testpub_tbl1, testpub_tbl2 EXCEPT (TABLE testp... + ^ --------------------------------------------- -- SET ALL TABLES/SEQUENCES --------------------------------------------- @@ -470,7 +476,99 @@ HINT: Change the publication's EXCEPT clause using ALTER PUBLICATION ... SET AL RESET client_min_messages; DROP TABLE testpub_root, testpub_part1, tab_main; DROP PUBLICATION testpub8; ---- Tests for publications with SEQUENCES +--------------------------------------------- +-- EXCEPT tests for TABLES IN SCHEMA +--------------------------------------------- +SET client_min_messages = 'ERROR'; +-- Create tables in pub_test for these tests +CREATE TABLE pub_test.testpub_tbl_s1 (a int primary key, b text); +CREATE TABLE pub_test.testpub_tbl_s2 (x int primary key, y text); +-- Create same-named tables in public to verify unqualified EXCEPT entries +-- are qualified with the named schema, not public +CREATE TABLE testpub_nopk (foo int, bar int); +CREATE TABLE testpub_tbl_s1 (a int primary key, b text); +-- Basic: exclude one table from a schema publication +CREATE PUBLICATION testpub_schema_except1 + FOR TABLES IN SCHEMA pub_test EXCEPT (TABLE pub_test.testpub_tbl_s1); +\dRp+ testpub_schema_except1 + Publication testpub_schema_except1 + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root | Description +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+----------+------------- + regress_publication_user | f | f | t | t | t | t | none | f | +Tables from schemas: + "pub_test" +Except tables: + "pub_test.testpub_tbl_s1" + +-- Exclude multiple tables using unqualified names; same-named tables exist in +-- public to confirm unqualified names resolve to pub_test, not public +CREATE PUBLICATION testpub_schema_except2 + FOR TABLES IN SCHEMA pub_test EXCEPT (TABLE testpub_nopk, testpub_tbl_s1); +\dRp+ testpub_schema_except2 + Publication testpub_schema_except2 + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root | Description +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+----------+------------- + regress_publication_user | f | f | t | t | t | t | none | f | +Tables from schemas: + "pub_test" +Except tables: + "pub_test.testpub_nopk" + "pub_test.testpub_tbl_s1" + +-- fail: EXCEPT table belongs to a different schema +CREATE PUBLICATION testpub_except_wrongschema + FOR TABLES IN SCHEMA pub_test EXCEPT (TABLE public.testpub_tbl1); +ERROR: table "public.testpub_tbl1" in EXCEPT clause does not belong to schema "pub_test" +LINE 2: FOR TABLES IN SCHEMA pub_test EXCEPT (TABLE public.testp... + ^ +-- fail: cross-schema EXCEPT not allowed; each EXCEPT is bound to its immediate schema +CREATE PUBLICATION testpub_except_crossschema + FOR TABLES IN SCHEMA pub_test, public EXCEPT (TABLE pub_test.testpub_tbl_s1, public.testpub_tbl1); +ERROR: table "pub_test.testpub_tbl_s1" in EXCEPT clause does not belong to schema "public" +LINE 2: ...R TABLES IN SCHEMA pub_test, public EXCEPT (TABLE pub_test.t... + ^ +-- Multiple schemas each with their own EXCEPT clause +CREATE PUBLICATION testpub_schema_except_multi + FOR TABLES IN SCHEMA pub_test EXCEPT (TABLE pub_test.testpub_tbl_s1), + public EXCEPT (TABLE testpub_tbl1); +\dRp+ testpub_schema_except_multi + Publication testpub_schema_except_multi + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root | Description +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+----------+------------- + regress_publication_user | f | f | t | t | t | t | none | f | +Tables from schemas: + "pub_test" + "public" +Except tables: + "pub_test.testpub_tbl_s1" + "public.testpub_tbl1" + +-- fail: table appears in both the explicit table list and the EXCEPT clause +CREATE PUBLICATION testpub_except_conflict + FOR TABLE pub_test.testpub_tbl_s1, TABLES IN SCHEMA pub_test EXCEPT (TABLE pub_test.testpub_tbl_s1); +ERROR: table "pub_test.testpub_tbl_s1" cannot appear in both the table list and the EXCEPT clause +-- fail: nonexistent table in EXCEPT clause +CREATE PUBLICATION testpub_except_norel + FOR TABLES IN SCHEMA pub_test EXCEPT (TABLE pub_test.nonexistent_table); +ERROR: relation "pub_test.nonexistent_table" does not exist +-- fail: partition cannot appear in EXCEPT clause; only root tables are allowed +CREATE TABLE pub_test.testpub_parted_s (a int) PARTITION BY LIST (a); +CREATE TABLE pub_test.testpub_part_s PARTITION OF pub_test.testpub_parted_s FOR VALUES IN (1); +CREATE PUBLICATION testpub_except_partition + FOR TABLES IN SCHEMA pub_test EXCEPT (TABLE pub_test.testpub_part_s); +ERROR: cannot specify relation "pub_test.testpub_part_s" in the publication EXCEPT clause +DETAIL: This operation is not supported for individual partitions. +-- fail: TABLE keyword is required for the first entry in the EXCEPT clause +CREATE PUBLICATION testpub_except_nokw + FOR TABLES IN SCHEMA pub_test EXCEPT (testpub_nopk); +ERROR: syntax error at or near "testpub_nopk" +LINE 2: FOR TABLES IN SCHEMA pub_test EXCEPT (testpub_nopk); + ^ +RESET client_min_messages; +DROP TABLE pub_test.testpub_tbl_s1, pub_test.testpub_tbl_s2; +DROP TABLE pub_test.testpub_parted_s CASCADE; +DROP TABLE testpub_nopk, testpub_tbl_s1; +DROP PUBLICATION testpub_schema_except1, testpub_schema_except2, testpub_schema_except_multi; CREATE SEQUENCE regress_pub_seq0; CREATE SEQUENCE pub_test.regress_pub_seq1; -- FOR ALL SEQUENCES diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql index 041e14a4de6..5d8a4e2637e 100644 --- a/src/test/regress/sql/publication.sql +++ b/src/test/regress/sql/publication.sql @@ -123,6 +123,9 @@ CREATE PUBLICATION testpub_foralltables_excepttable1 FOR ALL TABLES EXCEPT (TABL \d testpub_tbl1 -- fail - first table in the EXCEPT list should use TABLE keyword CREATE PUBLICATION testpub_foralltables_excepttable2 FOR ALL TABLES EXCEPT (testpub_tbl1, testpub_tbl2); +-- fail - EXCEPT is not allowed for FOR TABLE publications +CREATE PUBLICATION testpub_except_err + FOR TABLE testpub_tbl1, testpub_tbl2 EXCEPT (TABLE testpub_tbl3); --------------------------------------------- -- SET ALL TABLES/SEQUENCES @@ -220,7 +223,67 @@ RESET client_min_messages; DROP TABLE testpub_root, testpub_part1, tab_main; DROP PUBLICATION testpub8; ---- Tests for publications with SEQUENCES +--------------------------------------------- +-- EXCEPT tests for TABLES IN SCHEMA +--------------------------------------------- +SET client_min_messages = 'ERROR'; +-- Create tables in pub_test for these tests +CREATE TABLE pub_test.testpub_tbl_s1 (a int primary key, b text); +CREATE TABLE pub_test.testpub_tbl_s2 (x int primary key, y text); +-- Create same-named tables in public to verify unqualified EXCEPT entries +-- are qualified with the named schema, not public +CREATE TABLE testpub_nopk (foo int, bar int); +CREATE TABLE testpub_tbl_s1 (a int primary key, b text); + +-- Basic: exclude one table from a schema publication +CREATE PUBLICATION testpub_schema_except1 + FOR TABLES IN SCHEMA pub_test EXCEPT (TABLE pub_test.testpub_tbl_s1); +\dRp+ testpub_schema_except1 + +-- Exclude multiple tables using unqualified names; same-named tables exist in +-- public to confirm unqualified names resolve to pub_test, not public +CREATE PUBLICATION testpub_schema_except2 + FOR TABLES IN SCHEMA pub_test EXCEPT (TABLE testpub_nopk, testpub_tbl_s1); +\dRp+ testpub_schema_except2 + +-- fail: EXCEPT table belongs to a different schema +CREATE PUBLICATION testpub_except_wrongschema + FOR TABLES IN SCHEMA pub_test EXCEPT (TABLE public.testpub_tbl1); + +-- fail: cross-schema EXCEPT not allowed; each EXCEPT is bound to its immediate schema +CREATE PUBLICATION testpub_except_crossschema + FOR TABLES IN SCHEMA pub_test, public EXCEPT (TABLE pub_test.testpub_tbl_s1, public.testpub_tbl1); + +-- Multiple schemas each with their own EXCEPT clause +CREATE PUBLICATION testpub_schema_except_multi + FOR TABLES IN SCHEMA pub_test EXCEPT (TABLE pub_test.testpub_tbl_s1), + public EXCEPT (TABLE testpub_tbl1); +\dRp+ testpub_schema_except_multi + +-- fail: table appears in both the explicit table list and the EXCEPT clause +CREATE PUBLICATION testpub_except_conflict + FOR TABLE pub_test.testpub_tbl_s1, TABLES IN SCHEMA pub_test EXCEPT (TABLE pub_test.testpub_tbl_s1); + +-- fail: nonexistent table in EXCEPT clause +CREATE PUBLICATION testpub_except_norel + FOR TABLES IN SCHEMA pub_test EXCEPT (TABLE pub_test.nonexistent_table); + +-- fail: partition cannot appear in EXCEPT clause; only root tables are allowed +CREATE TABLE pub_test.testpub_parted_s (a int) PARTITION BY LIST (a); +CREATE TABLE pub_test.testpub_part_s PARTITION OF pub_test.testpub_parted_s FOR VALUES IN (1); +CREATE PUBLICATION testpub_except_partition + FOR TABLES IN SCHEMA pub_test EXCEPT (TABLE pub_test.testpub_part_s); + +-- fail: TABLE keyword is required for the first entry in the EXCEPT clause +CREATE PUBLICATION testpub_except_nokw + FOR TABLES IN SCHEMA pub_test EXCEPT (testpub_nopk); + +RESET client_min_messages; +DROP TABLE pub_test.testpub_tbl_s1, pub_test.testpub_tbl_s2; +DROP TABLE pub_test.testpub_parted_s CASCADE; +DROP TABLE testpub_nopk, testpub_tbl_s1; +DROP PUBLICATION testpub_schema_except1, testpub_schema_except2, testpub_schema_except_multi; + CREATE SEQUENCE regress_pub_seq0; CREATE SEQUENCE pub_test.regress_pub_seq1; diff --git a/src/test/subscription/t/037_except.pl b/src/test/subscription/t/037_except.pl index 8c58d282eee..18c7b2c1fca 100644 --- a/src/test/subscription/t/037_except.pl +++ b/src/test/subscription/t/037_except.pl @@ -24,14 +24,17 @@ my $result; sub test_except_root_partition { - my ($pubviaroot) = @_; + my ($pubviaroot, $pubsql) = @_; + $pubsql //= + "CREATE PUBLICATION tap_pub_part FOR ALL TABLES EXCEPT (TABLE root1)"; + $pubsql .= " WITH (publish_via_partition_root = $pubviaroot)"; # If the root partitioned table is in the EXCEPT clause, all its # partitions are excluded from publication, regardless of the # publish_via_partition_root setting. $node_publisher->safe_psql( 'postgres', qq( - CREATE PUBLICATION tap_pub_part FOR ALL TABLES EXCEPT (TABLE root1) WITH (publish_via_partition_root = $pubviaroot); + $pubsql; INSERT INTO root1 VALUES (1), (101); )); $node_subscriber->safe_psql('postgres', @@ -223,6 +226,131 @@ $node_subscriber->safe_psql( test_except_root_partition('false'); test_except_root_partition('true'); +# Same validation using TABLES IN SCHEMA instead of FOR ALL TABLES. +my $schema_pub = + "CREATE PUBLICATION tap_pub_part FOR TABLES IN SCHEMA public EXCEPT (TABLE public.root1)"; +test_except_root_partition('false', $schema_pub); +test_except_root_partition('true', $schema_pub); + +# ============================================ +# EXCEPT test cases for TABLES IN SCHEMA +# ============================================ + +# Create a dedicated schema with two tables: one to be published and one to be +# excluded. Also create inherited tables to verify ONLY semantics. +$node_publisher->safe_psql( + 'postgres', qq( + CREATE SCHEMA sch1; + CREATE TABLE sch1.tab_published AS SELECT generate_series(1,5) AS a; + CREATE TABLE sch1.tab_excluded AS SELECT generate_series(1,5) AS a; + CREATE TABLE sch1.parent (a int); + CREATE TABLE sch1.child (b int) INHERITS (sch1.parent); +)); + +$node_subscriber->safe_psql( + 'postgres', qq( + CREATE SCHEMA sch1; + CREATE TABLE sch1.tab_published (a int); + CREATE TABLE sch1.tab_excluded (a int); + CREATE TABLE sch1.parent (a int); + CREATE TABLE sch1.child (b int) INHERITS (sch1.parent); +)); + +# Basic test: initial sync respects EXCEPT. +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION sch_pub FOR TABLES IN SCHEMA sch1 EXCEPT (TABLE sch1.tab_excluded)" +); +$node_subscriber->safe_psql('postgres', + "CREATE SUBSCRIPTION sch_sub CONNECTION '$publisher_connstr' PUBLICATION sch_pub" +); +$node_subscriber->wait_for_subscription_sync($node_publisher, 'sch_sub'); + +$result = + $node_subscriber->safe_psql('postgres', + "SELECT count(*) FROM sch1.tab_published"); +is($result, qq(5), + 'TABLES IN SCHEMA EXCEPT: initial sync copies included table'); +$result = + $node_subscriber->safe_psql('postgres', + "SELECT count(*) FROM sch1.tab_excluded"); +is($result, qq(0), + 'TABLES IN SCHEMA EXCEPT: initial sync skips excluded table'); + +# DML: only the included table should be replicated. +$node_publisher->safe_psql( + 'postgres', qq( + INSERT INTO sch1.tab_published VALUES (6); + INSERT INTO sch1.tab_excluded VALUES (6); +)); +$node_publisher->wait_for_catchup('sch_sub'); + +$result = + $node_subscriber->safe_psql('postgres', + "SELECT count(*) FROM sch1.tab_published"); +is($result, qq(6), + 'TABLES IN SCHEMA EXCEPT: DML on included table is replicated'); +$result = + $node_subscriber->safe_psql('postgres', + "SELECT count(*) FROM sch1.tab_excluded"); +is($result, qq(0), + 'TABLES IN SCHEMA EXCEPT: DML on excluded table is not replicated'); + +$node_subscriber->safe_psql('postgres', 'DROP SUBSCRIPTION sch_sub'); +$node_publisher->safe_psql('postgres', 'DROP PUBLICATION sch_pub'); + +# Inherited tables: excluding the parent (without ONLY) also excludes the child. +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION sch_pub FOR TABLES IN SCHEMA sch1 EXCEPT (TABLE sch1.parent)" +); +$node_subscriber->safe_psql('postgres', + "CREATE SUBSCRIPTION sch_sub CONNECTION '$publisher_connstr' PUBLICATION sch_pub" +); +$node_subscriber->wait_for_subscription_sync($node_publisher, 'sch_sub'); + +$node_publisher->safe_psql('postgres', + "INSERT INTO sch1.child VALUES (generate_series(1,5), generate_series(1,5))" +); +$node_publisher->wait_for_catchup('sch_sub'); + +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM sch1.child"); +is($result, qq(0), + 'TABLES IN SCHEMA EXCEPT: excluding parent (without ONLY) also excludes child' +); + +$node_subscriber->safe_psql('postgres', 'DROP SUBSCRIPTION sch_sub'); +$node_publisher->safe_psql('postgres', 'DROP PUBLICATION sch_pub'); + +# Test that EXCEPT (TABLE ONLY parent) excludes only the parent itself, not its +# child. Truncate child first so rows from the previous test are not copied by +# the initial table sync of the next subscription. +$node_publisher->safe_psql('postgres', 'TRUNCATE sch1.child'); +$node_subscriber->safe_psql('postgres', 'TRUNCATE sch1.child'); +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION sch_pub FOR TABLES IN SCHEMA sch1 EXCEPT (TABLE ONLY sch1.parent)" +); +$node_subscriber->safe_psql('postgres', + "CREATE SUBSCRIPTION sch_sub CONNECTION '$publisher_connstr' PUBLICATION sch_pub" +); +$node_subscriber->wait_for_subscription_sync($node_publisher, 'sch_sub'); + +$node_publisher->safe_psql('postgres', + "INSERT INTO sch1.child VALUES (generate_series(1,5), generate_series(1,5))" +); +$node_publisher->wait_for_catchup('sch_sub'); + +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM sch1.child"); +is($result, qq(5), + 'TABLES IN SCHEMA EXCEPT: ONLY parent in EXCEPT does not exclude child'); + +$node_subscriber->safe_psql('postgres', 'DROP SUBSCRIPTION sch_sub'); +$node_publisher->safe_psql('postgres', 'DROP PUBLICATION sch_pub'); + +# Cleanup schema tables before the multi-publication section. +$node_publisher->safe_psql('postgres', 'DROP SCHEMA sch1 CASCADE'); +$node_subscriber->safe_psql('postgres', 'DROP SCHEMA sch1 CASCADE'); + # ============================================ # Test when a subscription is subscribing to multiple publications # ============================================ @@ -254,6 +382,7 @@ $node_publisher->safe_psql( DROP PUBLICATION tap_pub2; TRUNCATE tab1; )); +$node_subscriber->safe_psql('postgres', 'DROP SUBSCRIPTION tap_sub'); $node_subscriber->safe_psql('postgres', qq(TRUNCATE tab1)); # OK when a table is excluded by pub1 EXCEPT clause, but it is included by pub2 -- 2.50.1 (Apple Git-155)