From 7a4ab83094fa1b243a23f74ffe9c6287b25bb425 Mon Sep 17 00:00:00 2001 From: "Chao Li (Evan)" Date: Fri, 14 Aug 2026 06:27:37 +0800 Subject: [PATCH v1] Support selective logical WAL for restricted replication slots The presence of a logical replication slot currently causes the extra WAL information required for logical decoding to be written for every eligible relation. This can generate substantial unnecessary WAL when only a small subset of tables is replicated. Add restricted logical replication slots, whose WAL requirements are derived from an immutable set of publications. Maintain a conservative mapping from restricted slots to their physical relations, including partitions and TOAST relations, and mark mapped relations in pg_class. A new restricted WAL mode writes logical tuple information only for marked relations when full logical WAL is not otherwise required. Keep existing slots unrestricted by default, preserving the current behavior. Restricted slots can be requested through CREATE SUBSCRIPTION, the replication protocol, or pg_create_logical_replication_slot(). Currently, restricted slots support only pgoutput and reject FOR ALL TABLES publications. Update mappings transactionally when publications expand, tables are created in published schemas, partitions are attached, or TOAST relations are created. Stale mappings are safe because they cause only unnecessary WAL and are removed lazily by autovacuum. Persist publication identities and restricted-slot initialization state, including crash-recovery markers. Also synchronize restricted-slot metadata for failover slots and validate the requested publication set when pgoutput starts. Author: Chao Li --- doc/src/sgml/catalogs.sgml | 69 + doc/src/sgml/config.sgml | 18 + doc/src/sgml/func/func-admin.sgml | 30 +- doc/src/sgml/protocol.sgml | 12 + doc/src/sgml/ref/alter_subscription.sgml | 7 + doc/src/sgml/ref/create_subscription.sgml | 20 + doc/src/sgml/system-views.sgml | 53 + src/backend/access/heap/heapam.c | 9 +- src/backend/access/transam/xlog.c | 10 +- src/backend/catalog/heap.c | 27 + src/backend/catalog/pg_subscription.c | 1 + src/backend/catalog/system_views.sql | 8 +- src/backend/catalog/toasting.c | 2 + src/backend/commands/publicationcmds.c | 10 + src/backend/commands/subscriptioncmds.c | 57 +- src/backend/commands/tablecmds.c | 23 + src/backend/postmaster/autovacuum.c | 3 + .../libpqwalreceiver/libpqwalreceiver.c | 28 + src/backend/replication/logical/Makefile | 1 + src/backend/replication/logical/logicalctl.c | 77 +- src/backend/replication/logical/meson.build | 1 + .../replication/logical/reorderbuffer.c | 5 +- src/backend/replication/logical/slotscope.c | 1423 +++++++++++++++++ src/backend/replication/logical/slotsync.c | 97 +- src/backend/replication/logical/tablesync.c | 1 + src/backend/replication/pgoutput/pgoutput.c | 37 + src/backend/replication/slot.c | 111 +- src/backend/replication/slotfuncs.c | 234 ++- src/backend/replication/walreceiver.c | 3 +- src/backend/replication/walsender.c | 68 +- src/backend/utils/init/postinit.c | 4 + src/backend/utils/misc/guc_parameters.dat | 9 + src/backend/utils/misc/guc_tables.c | 1 + src/bin/pg_dump/pg_dump.c | 14 + src/bin/pg_dump/pg_dump.h | 1 + src/bin/pg_dump/t/002_pg_dump.pl | 5 +- src/include/access/xlog.h | 7 +- src/include/catalog/Makefile | 1 + src/include/catalog/meson.build | 1 + src/include/catalog/pg_class.h | 3 + src/include/catalog/pg_proc.dat | 18 +- .../catalog/pg_restricted_slot_relation.h | 36 + src/include/catalog/pg_subscription.h | 5 + src/include/replication/logicalctl.h | 3 + src/include/replication/slot.h | 16 + src/include/replication/slotscope.h | 49 + src/include/replication/walreceiver.h | 18 +- src/include/utils/guc_hooks.h | 1 + src/include/utils/rel.h | 25 +- src/test/recovery/meson.build | 1 + .../t/040_standby_failover_slots_sync.pl | 147 ++ src/test/recovery/t/056_logical_slot_scope.pl | 647 ++++++++ src/test/regress/expected/oidjoins.out | 1 + src/test/regress/expected/rules.out | 9 +- src/tools/pgindent/typedefs.list | 4 + 55 files changed, 3357 insertions(+), 114 deletions(-) create mode 100644 src/backend/replication/logical/slotscope.c create mode 100644 src/include/catalog/pg_restricted_slot_relation.h create mode 100644 src/include/replication/slotscope.h create mode 100644 src/test/recovery/t/056_logical_slot_scope.pl diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 6066c4784f4..0a9d5f39aea 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -280,6 +280,11 @@ relation to publication mapping + + pg_restricted_slot_relation + restricted logical replication slot to relation mapping + + pg_range information about range types @@ -2131,6 +2136,17 @@ SCRAM-SHA-256$<iteration count>:&l + + + relhasrestrictedslots bool + + + True if this relation is (or recently was) required by a restricted + logical replication slot. This flag can remain true until autovacuum + removes obsolete slot mappings. + + + relisshared bool @@ -7136,6 +7152,48 @@ SCRAM-SHA-256$<iteration count>:&l + + + + + <structname>pg_restricted_slot_relation</structname> + + + pg_restricted_slot_relation + + + + The catalog pg_restricted_slot_relation records a + conservative mapping from restricted logical replication slots to + relations that must write logical tuple information. Rows belonging to + dropped or replaced slot incarnations can remain until autovacuum cleanup. + Rows made obsolete by publication contraction or partition detach are + retained conservatively and can remain indefinitely. + + + + <structname>pg_restricted_slot_relation</structname> Columns + + + + Column Type + Description + + + + rsrslotname name + Name of the restricted replication slot + + rsrrelid oid + (references pg_class.oid) + OID of a relation that must write logical tuple information, + or zero for a committed-initialization marker + + rsrincarnation int8 + Restricted slot incarnation associated with this mapping or + initialization marker + +
@@ -8690,6 +8748,17 @@ SCRAM-SHA-256$<iteration count>:&l + + + subunrestricted bool + + + If true, the associated replication slots on the upstream database are + unrestricted. If false, they are scoped to the subscription's + publications. + + + subretaindeadtuples bool diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 596fd45a3db..2aebbeb5697 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -12453,6 +12453,24 @@ dynamic_library_path = '/usr/local/lib/postgresql:$libdir' the value of effective_wal_level from the most upstream server in the replication chain. + + + + + restricted_wal_level (enum) + + restricted_wal_level configuration parameter + + + + + Reports whether restricted logical tuple logging is active. When this + value is logical while + is below + logical, logical tuple information is written only + for relations marked as required by a restricted logical replication + slot. This is a read-only parameter. + diff --git a/doc/src/sgml/func/func-admin.sgml b/doc/src/sgml/func/func-admin.sgml index 0eae1c1f616..90416e396a0 100644 --- a/doc/src/sgml/func/func-admin.sgml +++ b/doc/src/sgml/func/func-admin.sgml @@ -1077,7 +1077,7 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset pg_create_logical_replication_slot - pg_create_logical_replication_slot ( slot_name name, plugin name , temporary boolean, twophase boolean, failover boolean ) + pg_create_logical_replication_slot ( slot_name name, plugin name , temporary boolean, twophase boolean, failover boolean, publications text[] ) record ( slot_name name, lsn pg_lsn ) @@ -1098,9 +1098,32 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset failover, when set to true, specifies that this slot is enabled to be synced to the standbys so that logical replication can be resumed after - failover. A call to this function has the same effect as + failover. When the optional publications array + is supplied, it must contain at least one publication name and the + output plugin must be pgoutput. Such a slot is + restricted: logical tuple information is written only for relations + covered by one of the stored publications. Omitting the array creates + an unrestricted slot. A named publication that currently contains no + relations is allowed. The restriction + is retained when + is logical, even + though that setting causes logical tuple information to be written + for every eligible relation. A call to this function has the same effect as the replication protocol command CREATE_REPLICATION_SLOT ... LOGICAL. + Restricted slots cannot be created by this function inside an explicit + transaction block or subtransaction. + + + The slot stores publication identities, not a fixed relation list. + Publication expansion, partition attachment, and TOAST creation add + writer-side relation mappings synchronously. Mappings made obsolete + by dropping or recreating a slot are cleaned up later by autovacuum. + Mappings made obsolete by publication contraction or partition detach + are retained indefinitely; this is safe but can cause additional + logical tuple information to be written. At decoding startup, + pgoutput requires every requested publication to be + one of the publications stored by the slot. @@ -1151,7 +1174,8 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset is not copied and is set to false by default. This is to avoid the risk of being unable to continue logical replication after failover to standby where the slot is being synchronized. Copy of - an invalidated slot is not allowed. + an invalidated slot is not allowed. A restricted logical slot cannot be + copied inside an explicit transaction block or subtransaction. diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 49f81676712..45418274cb6 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -2446,6 +2446,18 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;" + + + PUBLICATION_NAMES publication_names + + + A nonempty comma-separated list of publication names. Supplying this + option creates a restricted logical slot and requires the + pgoutput output plugin. Omitting it creates an + unrestricted slot. + + + diff --git a/doc/src/sgml/ref/alter_subscription.sgml b/doc/src/sgml/ref/alter_subscription.sgml index 545264e8a0a..651bd107fef 100644 --- a/doc/src/sgml/ref/alter_subscription.sgml +++ b/doc/src/sgml/ref/alter_subscription.sgml @@ -143,6 +143,13 @@ ALTER SUBSCRIPTION name RENAME TO < REFRESH PUBLICATION. + + The publication list of a subscription created with + unrestricted_slot = false can only be reduced. + Publications removed from the subscription cannot be added again without + recreating the subscription. + + publication_option specifies additional options for this operation. The supported options are: diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index 25a81e2e62a..b707f88b171 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -164,6 +164,26 @@ CREATE SUBSCRIPTION subscription_name for examples. + + + + + unrestricted_slot (boolean) + + + Specifies whether the replication slot created on the publisher + writes logical tuple information for every eligible relation. The + default is true. When set to false, + the slot is scoped to the relations currently included in the + subscription's publications. + + + When create_slot is true, this + option also controls the main slot created on the publisher. When + create_slot is false, it does + not alter the separately created main slot, but still controls any + table synchronization slots created by this subscription. + diff --git a/doc/src/sgml/system-views.sgml b/doc/src/sgml/system-views.sgml index 5ea19d68622..8e5e4879b4f 100644 --- a/doc/src/sgml/system-views.sgml +++ b/doc/src/sgml/system-views.sgml @@ -3175,6 +3175,59 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx + + + + unrestricted bool + + + True if the logical slot writes logical tuple information for every + eligible relation, false if it is restricted by its stored + publications, or NULL for a physical slot. + + + + + + publication_oids oid[] + + + Publications stored by a restricted logical slot, or NULL + for an unrestricted or physical slot. + + + + + + restricted_scope_ready bool + + + True when a restricted logical slot's initial relation mappings have + committed and the slot can be used, false while initialization is in + progress, or NULL for an unrestricted or physical slot. + + + + + + restricted_scope_incarnation bigint + + + Internal identity of a restricted slot initialization, or + NULL for an unrestricted or physical slot. + + + + + + restricted_scope_ready_lsn pg_lsn + + + WAL location following the commit of a restricted slot's initial + relation mappings, or NULL while initialization is + incomplete or for an unrestricted or physical slot. + + diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index 72d6541734c..7ee61132223 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -2136,9 +2136,14 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid, /* * For logical decoding, we need the tuple even if we're doing a full * page write, so make sure it's included even if we take a full-page - * image. (XXX We could alternatively store a pointer into the FPW). + * image. A speculative confirmation has no logical-data flag of its + * own, so retain the speculative insertion whenever logical decoding + * is active, even if the relation is outside a restricted slot's + * scope. (XXX We could alternatively store a pointer into the FPW). */ - if (RelationIsLogicallyLogged(relation) && + if ((RelationIsLogicallyLogged(relation) || + ((options & HEAP_INSERT_SPECULATIVE) && + XLogLogicalInfoActive())) && !(options & HEAP_INSERT_NO_LOGICAL)) { xlrec.flags |= XLH_INSERT_CONTAINS_NEW_TUPLE; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index c3baca5193b..651f6c30314 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -5250,7 +5250,15 @@ show_effective_wal_level(void) if (RecoveryInProgress()) return IsXLogLogicalInfoEnabled() ? "logical" : "replica"; - return XLogLogicalInfoActive() ? "logical" : "replica"; + return XLogFullLogicalInfoActive() ? "logical" : "replica"; +} + +const char * +show_restricted_wal_level(void) +{ + if (wal_level == WAL_LEVEL_MINIMAL) + return "minimal"; + return IsXLogRestrictedInfoEnabled() ? "logical" : "replica"; } /* diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index b018f26545b..aeb95eb4b96 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -49,6 +49,7 @@ #include "catalog/pg_namespace.h" #include "catalog/pg_opclass.h" #include "catalog/pg_partitioned_table.h" +#include "catalog/pg_publication.h" #include "catalog/pg_statistic.h" #include "catalog/pg_subscription_rel.h" #include "catalog/pg_tablespace.h" @@ -67,6 +68,7 @@ #include "parser/parsetree.h" #include "partitioning/partdesc.h" #include "pgstat.h" +#include "replication/slotscope.h" #include "storage/lmgr.h" #include "storage/predicate.h" #include "utils/array.h" @@ -968,6 +970,8 @@ InsertPgClassTuple(Relation pg_class_desc, values[Anum_pg_class_relispopulated - 1] = BoolGetDatum(rd_rel->relispopulated); values[Anum_pg_class_relreplident - 1] = CharGetDatum(rd_rel->relreplident); values[Anum_pg_class_relispartition - 1] = BoolGetDatum(rd_rel->relispartition); + values[Anum_pg_class_relhasrestrictedslots - 1] = + BoolGetDatum(rd_rel->relhasrestrictedslots); values[Anum_pg_class_relrewrite - 1] = ObjectIdGetDatum(rd_rel->relrewrite); values[Anum_pg_class_relfrozenxid - 1] = TransactionIdGetDatum(rd_rel->relfrozenxid); values[Anum_pg_class_relminmxid - 1] = MultiXactIdGetDatum(rd_rel->relminmxid); @@ -1040,6 +1044,7 @@ AddNewRelationTuple(Relation pg_class_desc, /* relispartition is always set by updating this tuple later */ new_rel_reltup->relispartition = false; + new_rel_reltup->relhasrestrictedslots = false; /* fill rd_att's type ID with something sane even if reltype is zero */ new_rel_desc->rd_att->tdtypeid = new_type_oid ? new_type_oid : RECORDOID; @@ -1540,6 +1545,21 @@ heap_create_with_catalog(const char *relname, if (oncommit != ONCOMMIT_NOOP) register_on_commit_action(relid, oncommit); + /* A new table may be covered immediately by a schema publication. */ + if (!IsBootstrapProcessingMode() && + (relkind == RELKIND_RELATION || relkind == RELKIND_PARTITIONED_TABLE)) + { + List *publications; + List *relations = list_make1_oid(relid); + + CommandCounterIncrement(); + publications = GetSchemaPublications(relnamespace); + foreach_oid(pubid, publications) + LogicalSlotScopePublicationAddRelations(pubid, relations); + list_free(publications); + list_free(relations); + } + /* * ok, the relation has been cataloged, so close our relations and return * the OID of the newly created relation. @@ -1847,6 +1867,13 @@ heap_drop_with_catalog(Oid relid) */ rel = relation_open(relid, AccessExclusiveLock); + /* + * Remove any logical replication slot that is using this relation. This + * must be done before we remove the pg_class row, else the slot will be + * left pointing to a non-existent relation. + */ + LogicalSlotScopeRelationDrop(relid); + /* * There can no longer be anyone *else* touching the relation, but we * might still have open queries or cursors, or pending trigger events, in diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c index f1e8b624d8e..eed3582c806 100644 --- a/src/backend/catalog/pg_subscription.c +++ b/src/backend/catalog/pg_subscription.c @@ -129,6 +129,7 @@ GetSubscription(Oid subid, bool missing_ok) sub->passwordrequired = subform->subpasswordrequired; sub->runasowner = subform->subrunasowner; sub->failover = subform->subfailover; + sub->unrestricted = subform->subunrestricted; sub->retaindeadtuples = subform->subretaindeadtuples; sub->maxretention = subform->submaxretention; sub->retentionactive = subform->subretentionactive; diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 8612d99a890..a6dcfc2be04 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1123,7 +1123,12 @@ CREATE VIEW pg_replication_slots AS L.invalidation_reason, L.failover, L.synced, - L.slotsync_skip_reason + L.slotsync_skip_reason, + L.unrestricted, + L.publication_oids, + L.restricted_scope_ready, + L.restricted_scope_incarnation, + L.restricted_scope_ready_lsn FROM pg_get_replication_slots() AS L LEFT JOIN pg_database D ON (L.datoid = D.oid); @@ -1537,6 +1542,7 @@ REVOKE ALL ON pg_subscription FROM public; GRANT SELECT (oid, subdbid, subskiplsn, subname, subowner, subenabled, subbinary, substream, subtwophasestate, subdisableonerr, subpasswordrequired, subrunasowner, subfailover, + subunrestricted, subretaindeadtuples, submaxretention, subretentionactive, subserver, subconflictlogrelid, subconflictlogdest, subslotname, subsynccommit, subwalrcvtimeout, subpublications, suborigin) diff --git a/src/backend/catalog/toasting.c b/src/backend/catalog/toasting.c index 4aa52a4bd25..6822c1177ea 100644 --- a/src/backend/catalog/toasting.c +++ b/src/backend/catalog/toasting.c @@ -30,6 +30,7 @@ #include "catalog/toasting.h" #include "miscadmin.h" #include "nodes/makefuncs.h" +#include "replication/slotscope.h" #include "utils/fmgroids.h" #include "utils/rel.h" #include "utils/syscache.h" @@ -397,6 +398,7 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, * Make changes visible */ CommandCounterIncrement(); + LogicalSlotScopeNoteToastCreation(relOid, toast_relid); return true; } diff --git a/src/backend/commands/publicationcmds.c b/src/backend/commands/publicationcmds.c index 96838730fe1..23c95759bb0 100644 --- a/src/backend/commands/publicationcmds.c +++ b/src/backend/commands/publicationcmds.c @@ -38,6 +38,7 @@ #include "parser/parse_collate.h" #include "parser/parse_relation.h" #include "rewrite/rewriteHandler.h" +#include "replication/slotscope.h" #include "storage/lmgr.h" #include "utils/acl.h" #include "utils/builtins.h" @@ -2036,6 +2037,7 @@ PublicationAddTables(Oid pubid, List *rels, bool if_not_exists, AlterPublicationStmt *stmt) { ListCell *lc; + List *relids = NIL; foreach(lc, rels) { @@ -2049,6 +2051,7 @@ PublicationAddTables(Oid pubid, List *rels, bool if_not_exists, RelationGetRelationName(rel)); obj = publication_add_relation(pubid, pub_rel, if_not_exists, stmt); + relids = lappend_oid(relids, RelationGetRelid(rel)); if (stmt) { EventTriggerCollectSimpleCommand(obj, InvalidObjectAddress, @@ -2058,6 +2061,8 @@ PublicationAddTables(Oid pubid, List *rels, bool if_not_exists, obj.objectId, 0); } } + LogicalSlotScopePublicationAddRelations(pubid, relids); + list_free(relids); } /* @@ -2118,8 +2123,13 @@ PublicationAddSchemas(Oid pubid, List *schemas, bool if_not_exists, { Oid schemaid = lfirst_oid(lc); ObjectAddress obj; + List *relations; obj = publication_add_schema(pubid, schemaid, if_not_exists); + relations = GetSchemaPublicationRelations(schemaid, + PUBLICATION_PART_ALL); + LogicalSlotScopePublicationAddRelations(pubid, relations); + list_free(relations); if (stmt) { EventTriggerCollectSimpleCommand(obj, InvalidObjectAddress, diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index 6e805394808..dee887a3546 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -83,6 +83,7 @@ #define SUBOPT_LSN 0x00020000 #define SUBOPT_ORIGIN 0x00040000 #define SUBOPT_CONFLICT_LOG_DEST 0x00080000 +#define SUBOPT_UNRESTRICTED_SLOT 0x00100000 /* check if the 'val' has 'bits' set */ #define IsSet(val, bits) (((val) & (bits)) == (bits)) @@ -108,6 +109,7 @@ typedef struct SubOpts bool passwordrequired; bool runasowner; bool failover; + bool unrestricted; bool retaindeadtuples; int32 maxretention; char *origin; @@ -199,6 +201,8 @@ parse_subscription_options(ParseState *pstate, List *stmt_options, opts->runasowner = false; if (IsSet(supported_opts, SUBOPT_FAILOVER)) opts->failover = false; + if (IsSet(supported_opts, SUBOPT_UNRESTRICTED_SLOT)) + opts->unrestricted = true; if (IsSet(supported_opts, SUBOPT_RETAIN_DEAD_TUPLES)) opts->retaindeadtuples = false; if (IsSet(supported_opts, SUBOPT_MAX_RETENTION_DURATION)) @@ -350,6 +354,15 @@ parse_subscription_options(ParseState *pstate, List *stmt_options, opts->specified_opts |= SUBOPT_FAILOVER; opts->failover = defGetBoolean(defel); } + else if (IsSet(supported_opts, SUBOPT_UNRESTRICTED_SLOT) && + strcmp(defel->defname, "unrestricted_slot") == 0) + { + if (IsSet(opts->specified_opts, SUBOPT_UNRESTRICTED_SLOT)) + errorConflictingDefElem(defel, pstate); + + opts->specified_opts |= SUBOPT_UNRESTRICTED_SLOT; + opts->unrestricted = defGetBoolean(defel); + } else if (IsSet(supported_opts, SUBOPT_RETAIN_DEAD_TUPLES) && strcmp(defel->defname, "retain_dead_tuples") == 0) { @@ -539,6 +552,7 @@ parse_subscription_options(ParseState *pstate, List *stmt_options, "slot_name = NONE", "create_slot = false"))); } } + } /* @@ -691,6 +705,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, * Connection and publication should not be specified here. */ supported_opts = (SUBOPT_CONNECT | SUBOPT_ENABLED | SUBOPT_CREATE_SLOT | + SUBOPT_UNRESTRICTED_SLOT | SUBOPT_SLOT_NAME | SUBOPT_COPY_DATA | SUBOPT_SYNCHRONOUS_COMMIT | SUBOPT_BINARY | SUBOPT_STREAMING | SUBOPT_TWOPHASE_COMMIT | @@ -864,6 +879,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, values[Anum_pg_subscription_subpasswordrequired - 1] = BoolGetDatum(opts.passwordrequired); values[Anum_pg_subscription_subrunasowner - 1] = BoolGetDatum(opts.runasowner); values[Anum_pg_subscription_subfailover - 1] = BoolGetDatum(opts.failover); + values[Anum_pg_subscription_subunrestricted - 1] = BoolGetDatum(opts.unrestricted); values[Anum_pg_subscription_subretaindeadtuples - 1] = BoolGetDatum(opts.retaindeadtuples); values[Anum_pg_subscription_submaxretention - 1] = @@ -1065,7 +1081,8 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, twophase_enabled = true; walrcv_create_slot(wrconn, opts.slot_name, false, twophase_enabled, - opts.failover, CRS_NOEXPORT_SNAPSHOT, NULL); + opts.failover, opts.unrestricted, publications, + CRS_NOEXPORT_SNAPSHOT, NULL); if (twophase_enabled) UpdateTwoPhaseState(subid, LOGICALREP_TWOPHASE_STATE_ENABLED); @@ -1738,6 +1755,44 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, sub = GetSubscription(subid, false); + /* + * A restricted slot stores an immutable set of publication identities. + * The subscriber does not retain that original set, so only allow its + * current publication list to be reduced. This prevents the apply worker + * from failing asynchronously on an identity absent from the slot. + */ + if (!sub->unrestricted && + (stmt->kind == ALTER_SUBSCRIPTION_SET_PUBLICATION || + stmt->kind == ALTER_SUBSCRIPTION_ADD_PUBLICATION)) + { + if (stmt->kind == ALTER_SUBSCRIPTION_ADD_PUBLICATION) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot add publications to restricted subscription \"%s\"", + stmt->subname), + errhint("Drop and recreate the subscription to expand its publication list."))); + + foreach_node(String, publication, stmt->publication) + { + bool found = false; + + foreach_node(String, oldpublication, sub->publications) + { + if (strcmp(strVal(publication), strVal(oldpublication)) == 0) + { + found = true; + break; + } + } + if (!found) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot add publication \"%s\" to restricted subscription \"%s\"", + strVal(publication), stmt->subname), + errhint("Drop and recreate the subscription to expand its publication list."))); + } + } + /* * Determine in advance whether we need the original conninfo or not, so * that errors are generated consistently in cases where we do need it; diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 12beb37246f..9996014695e 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -49,6 +49,7 @@ #include "catalog/pg_opclass.h" #include "catalog/pg_policy.h" #include "catalog/pg_proc.h" +#include "catalog/pg_publication.h" #include "catalog/pg_publication_rel.h" #include "catalog/pg_rewrite.h" #include "catalog/pg_statistic_ext.h" @@ -92,6 +93,7 @@ #include "rewrite/rewriteDefine.h" #include "rewrite/rewriteHandler.h" #include "rewrite/rewriteManip.h" +#include "replication/slotscope.h" #include "storage/bufmgr.h" #include "storage/lmgr.h" #include "storage/lock.h" @@ -3658,6 +3660,11 @@ StoreCatalogInheritance1(Oid relationId, Oid parentOid, /* store the pg_inherits row */ StoreSingleInheritance(relationId, parentOid, seqNumber); + if (child_is_partition) + { + CommandCounterIncrement(); + CheckLogicalSlotScopeHierarchyChange(relationId, parentOid); + } /* * Store a dependency too @@ -19803,6 +19810,22 @@ AlterTableNamespaceInternal(Relation rel, Oid oldNspOid, Oid nspOid, false, objsMoved); table_close(classRel, RowExclusiveLock); + + /* Moving a table can make it a member of a schema publication. */ + if (!IsBootstrapProcessingMode() && oldNspOid != nspOid && + (rel->rd_rel->relkind == RELKIND_RELATION || + rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)) + { + List *publications; + List *relations = list_make1_oid(RelationGetRelid(rel)); + + CommandCounterIncrement(); + publications = GetSchemaPublications(nspOid); + foreach_oid(pubid, publications) + LogicalSlotScopePublicationAddRelations(pubid, relations); + list_free(publications); + list_free(relations); + } } /* diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index 874454891d3..5e297d4592d 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -89,6 +89,7 @@ #include "postmaster/autovacuum.h" #include "postmaster/interrupt.h" #include "postmaster/postmaster.h" +#include "replication/slotscope.h" #include "storage/aio_subsys.h" #include "storage/bufmgr.h" #include "storage/ipc.h" @@ -2667,6 +2668,8 @@ deleted: if (did_vacuum || !found_concurrent_worker) vac_update_datfrozenxid(); + LogicalSlotScopeCleanup(); + /* Finally close out the last transaction. */ CommitTransactionCommand(); } diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c index 029990d9fce..998ef9cba04 100644 --- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c +++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c @@ -42,6 +42,9 @@ PG_MODULE_MAGIC_EXT( .version = PG_VERSION ); +/* First server version supporting UNRESTRICTED and PUBLICATION_NAMES. */ +#define LOGICAL_SLOT_SCOPE_VERSION_NUM 200000 + struct WalReceiverConn { /* Current connection to the primary, if any */ @@ -85,6 +88,8 @@ static char *libpqrcv_create_slot(WalReceiverConn *conn, bool temporary, bool two_phase, bool failover, + bool unrestricted, + List *publications, CRSSnapshotAction snapshot_action, XLogRecPtr *lsn); static void libpqrcv_alter_slot(WalReceiverConn *conn, const char *slotname, @@ -924,6 +929,7 @@ libpqrcv_send(WalReceiverConn *conn, const char *buffer, int nbytes) static char * libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, bool temporary, bool two_phase, bool failover, + bool unrestricted, List *publications, CRSSnapshotAction snapshot_action, XLogRecPtr *lsn) { PGresult *res; @@ -932,6 +938,10 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, int use_new_options_syntax; use_new_options_syntax = (PQserverVersion(conn->streamConn) >= 150000); + if (!unrestricted && PQserverVersion(conn->streamConn) < LOGICAL_SLOT_SCOPE_VERSION_NUM) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("publisher does not support scoped logical replication slots"))); initStringInfo(&cmd); @@ -943,6 +953,9 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, if (conn->logical) { + StringInfoData pubnames; + ListCell *lc; + appendStringInfoString(&cmd, " LOGICAL pgoutput "); if (use_new_options_syntax) appendStringInfoChar(&cmd, '('); @@ -964,6 +977,21 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, appendStringInfoChar(&cmd, ' '); } + if (use_new_options_syntax && !unrestricted) + { + initStringInfo(&pubnames); + foreach(lc, publications) + { + if (lc != list_head(publications)) + appendStringInfoChar(&pubnames, ','); + appendStringInfoString(&pubnames, + quote_identifier(strVal(lfirst(lc)))); + } + appendStringInfo(&cmd, "PUBLICATION_NAMES %s, ", + quote_literal_cstr(pubnames.data)); + pfree(pubnames.data); + } + if (use_new_options_syntax) { switch (snapshot_action) diff --git a/src/backend/replication/logical/Makefile b/src/backend/replication/logical/Makefile index 455768a57f0..f064790b5a1 100644 --- a/src/backend/replication/logical/Makefile +++ b/src/backend/replication/logical/Makefile @@ -21,6 +21,7 @@ OBJS = \ launcher.o \ logical.o \ logicalctl.o \ + slotscope.o \ logicalfuncs.o \ message.o \ origin.o \ diff --git a/src/backend/replication/logical/logicalctl.c b/src/backend/replication/logical/logicalctl.c index e5340880fa7..96d0460961a 100644 --- a/src/backend/replication/logical/logicalctl.c +++ b/src/backend/replication/logical/logicalctl.c @@ -89,6 +89,7 @@ typedef struct LogicalDecodingCtlData * caches this value in XLogLogicalInfo for better performance. */ bool xlog_logical_info; + bool xlog_restricted_info; /* True if logical decoding is available in the system */ bool logical_decoding_enabled; @@ -113,6 +114,7 @@ const ShmemCallbacks LogicalDecodingCtlShmemCallbacks = { * transaction ends. See the comments for XLogLogicalInfoUpdatePending for details. */ bool XLogLogicalInfo = false; +bool XLogRestrictedInfo = false; /* * When receiving the PROCSIGNAL_BARRIER_UPDATE_XLOG_LOGICAL_INFO signal, if @@ -164,6 +166,7 @@ static inline void update_xlog_logical_info(void) { XLogLogicalInfo = IsXLogLogicalInfoEnabled(); + XLogRestrictedInfo = IsXLogRestrictedInfoEnabled(); } /* @@ -226,6 +229,60 @@ IsXLogLogicalInfoEnabled(void) return xlog_logical_info; } +/* + * Returns true if restricted logical WAL logging is enabled based on the shared memory + * status. + */ +bool +IsXLogRestrictedInfoEnabled(void) +{ + bool enabled; + + LWLockAcquire(LogicalDecodingControlLock, LW_SHARED); + enabled = LogicalDecodingCtl->xlog_restricted_info; + LWLockRelease(LogicalDecodingControlLock); + return enabled; +} + +/* + * Synchronously raise effective_wal_level to logical. + */ +void +EnsureRestrictedLogicalWAL(void) +{ + if (wal_level >= WAL_LEVEL_LOGICAL) + return; + Assert(wal_level >= WAL_LEVEL_REPLICA); + LWLockAcquire(LogicalDecodingControlLock, LW_EXCLUSIVE); + if (LogicalDecodingCtl->xlog_restricted_info) + { + LWLockRelease(LogicalDecodingControlLock); + return; + } + LogicalDecodingCtl->xlog_restricted_info = true; + LWLockRelease(LogicalDecodingControlLock); + WaitForProcSignalBarrier(EmitProcSignalBarrier(PROCSIGNAL_BARRIER_UPDATE_XLOG_LOGICAL_INFO)); +} + +/* + * Synchronously raise effective_wal_level to logical. + */ +void +EnsureFullLogicalWAL(void) +{ + if (wal_level >= WAL_LEVEL_LOGICAL) + return; + LWLockAcquire(LogicalDecodingControlLock, LW_EXCLUSIVE); + if (LogicalDecodingCtl->xlog_logical_info) + { + LWLockRelease(LogicalDecodingControlLock); + return; + } + LogicalDecodingCtl->xlog_logical_info = true; + LWLockRelease(LogicalDecodingControlLock); + WaitForProcSignalBarrier(EmitProcSignalBarrier(PROCSIGNAL_BARRIER_UPDATE_XLOG_LOGICAL_INFO)); +} + /* * Reset the local cache at end of the transaction. */ @@ -524,6 +581,8 @@ DisableLogicalDecoding(void) { bool in_recovery = RecoveryInProgress(); bool was_enabled; + bool has_logical_slots = false; + bool has_restricted_slots = false; LWLockAcquire(LogicalDecodingControlLock, LW_EXCLUSIVE); @@ -534,9 +593,15 @@ DisableLogicalDecoding(void) * (skip the slot check during recovery because the existing slots will be * invalidated after disabling logical decoding.) */ + if (!in_recovery) + { + has_logical_slots = CheckLogicalSlotExists(); + has_restricted_slots = CheckRestrictedLogicalSlotExists(); + } if ((!LogicalDecodingCtl->logical_decoding_enabled && - !LogicalDecodingCtl->xlog_logical_info) || - (!in_recovery && CheckLogicalSlotExists())) + !LogicalDecodingCtl->xlog_logical_info && + !LogicalDecodingCtl->xlog_restricted_info) || + (!in_recovery && CheckFullWalLogicalSlotExists())) { LogicalDecodingCtl->pending_disable = false; LWLockRelease(LogicalDecodingControlLock); @@ -557,14 +622,16 @@ DisableLogicalDecoding(void) * information WAL logging in order to ensure that no logical decoding * processes WAL records with insufficient information. */ - LogicalDecodingCtl->logical_decoding_enabled = false; + LogicalDecodingCtl->logical_decoding_enabled = has_logical_slots; /* Write the WAL to disable logical decoding on standbys too */ - if (!in_recovery && was_enabled) + if (!in_recovery && was_enabled && !has_logical_slots) write_logical_decoding_status_update_record(false); /* Now disable logical information WAL logging */ LogicalDecodingCtl->xlog_logical_info = false; + if (!has_restricted_slots) + LogicalDecodingCtl->xlog_restricted_info = false; LogicalDecodingCtl->pending_disable = false; END_CRIT_SECTION(); @@ -574,7 +641,7 @@ DisableLogicalDecoding(void) * the server log before its eventual "is enabled", making server log * diagnostics easy. */ - if (!in_recovery && was_enabled) + if (!in_recovery && was_enabled && !has_logical_slots) ereport(LOG, errmsg("logical decoding is disabled because there are no valid logical replication slots")); diff --git a/src/backend/replication/logical/meson.build b/src/backend/replication/logical/meson.build index 47a68b660d2..327a6f7e221 100644 --- a/src/backend/replication/logical/meson.build +++ b/src/backend/replication/logical/meson.build @@ -7,6 +7,7 @@ backend_sources += files( 'launcher.c', 'logical.c', 'logicalctl.c', + 'slotscope.c', 'logicalfuncs.c', 'message.c', 'origin.c', diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index 6aed6346366..cfbcd119900 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -103,6 +103,7 @@ #include "replication/logical.h" #include "replication/reorderbuffer.h" #include "replication/slot.h" +#include "replication/slotscope.h" #include "replication/snapbuild.h" /* just for SnapBuildSnapDecRefcount */ #include "storage/bufmgr.h" #include "storage/fd.h" @@ -2353,7 +2354,7 @@ ReorderBufferProcessTXN(ReorderBuffer *rb, ReorderBufferTXN *txn, relpathperm(change->data.tp.rlocator, MAIN_FORKNUM).str); - if (!RelationIsLogicallyLogged(relation)) + if (!RelationCanBeLogicallyLogged(relation)) goto change_done; /* @@ -2491,7 +2492,7 @@ ReorderBufferProcessTXN(ReorderBuffer *rb, ReorderBufferTXN *txn, if (!RelationIsValid(rel)) elog(ERROR, "could not open relation with OID %u", relid); - if (!RelationIsLogicallyLogged(rel)) + if (!RelationCanBeLogicallyLogged(rel)) continue; relations[nrelations++] = rel; diff --git a/src/backend/replication/logical/slotscope.c b/src/backend/replication/logical/slotscope.c new file mode 100644 index 00000000000..a2bae22f017 --- /dev/null +++ b/src/backend/replication/logical/slotscope.c @@ -0,0 +1,1423 @@ +/*------------------------------------------------------------------------- + * + * slotscope.c + * Publication-backed relation scopes for restricted logical slots. + * + * A restricted logical replication slot writes logical tuple WAL only for + * relations that may be needed by the slot's publications. The slot stores + * its immutable publication OIDs in a sidecar file, while + * pg_restricted_slot_relation maintains the current physical mappings from + * slot incarnations to relations. pg_class.relhasrestrictedslots provides + * the fast relation-level decision used during WAL insertion. + * + * The writer-side mapping is deliberately conservative. Stale mappings may + * cause unnecessary logical WAL to be written until they are cleaned up, but + * a required mapping must never be missing. This mapping does not determine + * which changes are emitted by an output plugin; pgoutput continues to apply + * the current publication definitions when decoding. + * + * Publication expansion, schema membership, partition attachment, and TOAST + * creation update the mappings transactionally. Obsolete mappings are + * removed lazily after their slot name and incarnation no longer identify a + * live restricted slot. + * + * Slot files and catalog mappings cannot be persisted atomically. Restricted + * slot creation therefore temporarily enables full logical WAL, installs the + * mappings and a completion marker in a transaction, and marks the slot ready + * after that transaction commits. After a restart, incomplete slot state is + * reconciled using the durable completion marker. + * + * + * Copyright (c) 2026, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/replication/logical/slotscope.c + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include + +#include "access/genam.h" +#include "access/heapam.h" +#include "access/table.h" +#include "access/xact.h" +#include "catalog/indexing.h" +#include "catalog/pg_class.h" +#include "catalog/pg_inherits.h" +#include "catalog/pg_publication.h" +#include "catalog/pg_restricted_slot_relation.h" +#include "common/file_utils.h" +#include "common/pg_prng.h" +#include "miscadmin.h" +#include "replication/logicalctl.h" +#include "replication/slot.h" +#include "replication/slotscope.h" +#include "storage/fd.h" +#include "storage/lmgr.h" +#include "utils/fmgroids.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" +#include "utils/memutils.h" +#include "utils/rel.h" +#include "utils/syscache.h" + +#define SLOT_PUBLICATIONS_MAGIC 0x51C0B11EU +#define SLOT_PUBLICATIONS_VERSION 1 +#define SLOT_PUBLICATIONS_FILE "publications" +#define SLOT_SCOPE_CHANGE_LOCK_SUBID 1 + +typedef struct LogicalSlotPublicationsOnDisk +{ + uint32 magic; + pg_crc32c checksum; + uint32 version; + uint32 npublications; + Oid publications[FLEXIBLE_ARRAY_MEMBER]; +} LogicalSlotPublicationsOnDisk; + +static List *read_publications_file(ReplicationSlot *slot); +static List *publication_relation_closure(List *publications, + bool conditional); +static List *restricted_slots_for_publication(Oid pubid); +static void restricted_relation_add(const char *slotname, Oid relid, + bool conditional); +static void restricted_ready_marker_add(const char *slotname, + uint64 incarnation); +static List *pending_ready_slots; +static bool callbacks_registered; + +typedef struct PendingReadySlot +{ + ReplicationSlot *slot; + NameData name; + bool finalize_at_xact_end; +} PendingReadySlot; + +typedef struct ReconcileSlot +{ + NameData name; + uint64 incarnation; +} ReconcileSlot; + +/* + * Acquire the database-wide lock that serializes restricted-slot scope + * operations. DDL paths release this lock explicitly after updating scope + * mappings, while initialization and maintenance may retain it until + * transaction end. + */ +static void +lock_database_scope(void) +{ + LockDatabaseObject(RestrictedSlotRelationRelationId, MyDatabaseId, 0, + ExclusiveLock); +} + +/* + * Release a database scope lock acquired for a single DDL scope operation. + * The transaction-level scope-change marker, if any, remains held until + * transaction end. + */ +static void +unlock_database_scope(void) +{ + UnlockDatabaseObject(RestrictedSlotRelationRelationId, MyDatabaseId, 0, + ExclusiveLock); +} + +/* + * Mark the current transaction as containing an uncommitted scope-changing + * DDL operation. + * + * DDL transactions acquire this marker in ShareLock mode and retain it until + * transaction end. Restricted-slot initialization conditionally requests an + * ExclusiveLock on the same lock tag, preventing it from constructing an + * initial scope while relevant catalog changes remain uncommitted. + */ +static void +mark_database_scope_change(void) +{ + LockDatabaseObject(RestrictedSlotRelationRelationId, MyDatabaseId, + SLOT_SCOPE_CHANGE_LOCK_SUBID, ShareLock); +} + +/* + * Ensure that no other transaction has an uncommitted scope-changing DDL + * operation. + * + * Acquire the transaction-level scope-change marker conditionally in + * ExclusiveLock mode. Fail with a retryable error instead of waiting, since + * the conflicting DDL may already hold relation locks needed by slot + * initialization. + */ +static void +check_no_database_scope_change(void) +{ + if (!ConditionalLockDatabaseObject(RestrictedSlotRelationRelationId, + MyDatabaseId, + SLOT_SCOPE_CHANGE_LOCK_SUBID, + ExclusiveLock)) + ereport(ERROR, + (errcode(ERRCODE_LOCK_NOT_AVAILABLE), + errmsg("could not initialize restricted logical replication slot due to concurrent activity"), + errhint("Retry creating the replication slot."))); +} + +/* + * Acquire ShareRowExclusiveLock on a relation while constructing or updating + * a restricted slot's physical scope. + * + * When conditional is true, fail with a retryable error rather than waiting. + * Slot initialization uses conditional locking because it already holds the + * database scope lock and waiting for DDL-held relation locks could deadlock. + * DDL paths use normal blocking locks before acquiring the scope lock. + */ +static void +lock_scope_relation(Oid relid, bool conditional) +{ + if (conditional) + { + if (ConditionalLockRelationOid(relid, ShareRowExclusiveLock)) + return; + ereport(ERROR, + (errcode(ERRCODE_LOCK_NOT_AVAILABLE), + errmsg("could not initialize restricted logical replication slot due to concurrent activity"), + errhint("Retry creating the replication slot."))); + } + LockRelationOid(relid, ShareRowExclusiveLock); +} + +/* + * Complete pending restricted-slot initializations at transaction end. + * + * The initial relation mappings and readiness marker are transactional, but + * replication-slot state is not. After commit, mark each pending slot ready + * at the end of the commit record and optionally release a slot whose caller + * delegated finalization to this callback. Request asynchronous removal of + * the temporary full-WAL requirement once the restricted mappings are + * committed. + * + * On abort, release slots delegated to this callback as ephemeral so that + * their incomplete on-disk state is removed. Other callers retain + * responsibility for releasing their slots. + */ +static void +slot_scope_xact_callback(XactEvent event, void *arg) +{ + if (event == XACT_EVENT_COMMIT) + { + foreach_ptr(PendingReadySlot, pending, pending_ready_slots) + { + ReplicationSlot *slot = pending->slot; + + if (slot->in_use && + strcmp(NameStr(slot->data.name), NameStr(pending->name)) == 0) + { + SpinLockAcquire(&slot->mutex); + slot->data.restricted_scope_ready = true; + slot->data.restricted_scope_ready_lsn = XactLastCommitEnd; + slot->just_dirtied = true; + slot->dirty = true; + SpinLockRelease(&slot->mutex); + if (pending->finalize_at_xact_end && MyReplicationSlot == slot) + ReplicationSlotRelease(); + } + } + if (pending_ready_slots != NIL) + RequestDisableLogicalDecoding(); + } + else if (event == XACT_EVENT_ABORT) + { + foreach_ptr(PendingReadySlot, pending, pending_ready_slots) + { + ReplicationSlot *slot = pending->slot; + + if (!pending->finalize_at_xact_end || MyReplicationSlot != slot) + continue; + SpinLockAcquire(&slot->mutex); + slot->data.persistency = RS_EPHEMERAL; + SpinLockRelease(&slot->mutex); + ReplicationSlotRelease(); + } + } + if (event == XACT_EVENT_COMMIT || event == XACT_EVENT_ABORT) + { + list_free_deep(pending_ready_slots); + pending_ready_slots = NIL; + } +} + +/* Register the transaction callback used by restricted-slot initialization. */ +void +LogicalSlotScopeInitialize(void) +{ + if (!callbacks_registered) + { + RegisterXactCallback(slot_scope_xact_callback, NULL); + callbacks_registered = true; + } +} + +/* + * Atomically replace a restricted slot's publication side file. + * + * The file stores the immutable publication identities associated with the + * slot. Write and fsync a temporary file, then durably rename it over the + * previous file. The slot I/O lock serializes this operation with readers + * and other slot-file operations. + */ +static void +write_publications_file(ReplicationSlot *slot, List *publications) +{ + LogicalSlotPublicationsOnDisk *ondisk; + char path[MAXPGPATH]; + char tmppath[MAXPGPATH]; + Size size; + int fd; + int i = 0; + + size = offsetof(LogicalSlotPublicationsOnDisk, publications) + + list_length(publications) * sizeof(Oid); + ondisk = palloc0(size); + ondisk->magic = SLOT_PUBLICATIONS_MAGIC; + ondisk->version = SLOT_PUBLICATIONS_VERSION; + ondisk->npublications = list_length(publications); + foreach_oid(pubid, publications) + ondisk->publications[i++] = pubid; + INIT_CRC32C(ondisk->checksum); + COMP_CRC32C(ondisk->checksum, + (char *) ondisk + offsetof(LogicalSlotPublicationsOnDisk, version), + size - offsetof(LogicalSlotPublicationsOnDisk, version)); + FIN_CRC32C(ondisk->checksum); + + snprintf(path, sizeof(path), "%s/%s/%s", PG_REPLSLOT_DIR, + NameStr(slot->data.name), SLOT_PUBLICATIONS_FILE); + snprintf(tmppath, sizeof(tmppath), "%s.tmp", path); + LWLockAcquire(&slot->io_in_progress_lock, LW_EXCLUSIVE); + if (unlink(tmppath) < 0 && errno != ENOENT) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not remove file \"%s\": %m", tmppath))); + fd = OpenTransientFile(tmppath, O_CREAT | O_EXCL | O_WRONLY | PG_BINARY); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create file \"%s\": %m", tmppath))); + if (write(fd, ondisk, size) != size || pg_fsync(fd) != 0 || + CloseTransientFile(fd) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write file \"%s\": %m", tmppath))); + (void) durable_rename(tmppath, path, ERROR); + LWLockRelease(&slot->io_in_progress_lock); + pfree(ondisk); +} + +/* + * Read and validate a restricted slot's publication side file. + * + * Verify the file header, size, version, and checksum before returning its + * publication OIDs. The slot I/O lock prevents the file from being replaced + * while it is being read. Unrestricted slots have no publication file and + * return an empty list. + */ +static List * +read_publications_file(ReplicationSlot *slot) +{ + LogicalSlotPublicationsOnDisk *ondisk; + pg_crc32c checksum; + struct stat st; + char path[MAXPGPATH]; + List *result = NIL; + Size expected; + int fd; + + if (slot->data.unrestricted) + return NIL; + snprintf(path, sizeof(path), "%s/%s/%s", PG_REPLSLOT_DIR, + NameStr(slot->data.name), SLOT_PUBLICATIONS_FILE); + LWLockAcquire(&slot->io_in_progress_lock, LW_SHARED); + fd = OpenTransientFile(path, O_RDONLY | PG_BINARY); + if (fd < 0 || fstat(fd, &st) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open restricted slot publication file \"%s\": %m", path))); + if (st.st_size < offsetof(LogicalSlotPublicationsOnDisk, publications)) + ereport(ERROR, + (errmsg("restricted slot publication file \"%s\" is too small", path))); + ondisk = palloc(st.st_size); + if (read(fd, ondisk, st.st_size) != st.st_size || + CloseTransientFile(fd) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read restricted slot publication file \"%s\": %m", path))); + LWLockRelease(&slot->io_in_progress_lock); + expected = offsetof(LogicalSlotPublicationsOnDisk, publications) + + ondisk->npublications * sizeof(Oid); + if (ondisk->magic != SLOT_PUBLICATIONS_MAGIC || + ondisk->version != SLOT_PUBLICATIONS_VERSION || expected != st.st_size) + ereport(ERROR, + (errmsg("invalid restricted slot publication file \"%s\"", path))); + INIT_CRC32C(checksum); + COMP_CRC32C(checksum, + (char *) ondisk + offsetof(LogicalSlotPublicationsOnDisk, version), + st.st_size - offsetof(LogicalSlotPublicationsOnDisk, version)); + FIN_CRC32C(checksum); + if (!EQ_CRC32C(checksum, ondisk->checksum)) + ereport(ERROR, + (errmsg("checksum mismatch for restricted slot publication file \"%s\"", path))); + for (uint32 i = 0; i < ondisk->npublications; i++) + result = lappend_oid(result, ondisk->publications[i]); + pfree(ondisk); + return result; +} + +/* Return the publication OIDs stored for a restricted slot. */ +List * +LogicalSlotScopeGetPublications(ReplicationSlot *slot) +{ + return read_publications_file(slot); +} + +/* + * Build and lock the current physical relation closure of a publication set. + * + * Expand explicit and schema publication members to include partition + * descendants, then add each relation's current TOAST relation. Ignore + * publications that have been dropped, reject FOR ALL TABLES publications, + * and remove duplicate relation OIDs. + * + * Acquire ShareRowExclusiveLock on every base relation so no writer can cross + * scope activation using stale relation metadata. When conditional is true, + * fail instead of waiting for a conflicting relation lock; this mode is used + * during restricted-slot initialization to avoid deadlocks with concurrent + * DDL. + */ +static List * +publication_relation_closure(List *publications, bool conditional) +{ + List *relations = NIL; + List *base; + + foreach_oid(pubid, publications) + { + Publication *pub; + List *pubrels; + + if (!SearchSysCacheExists1(PUBLICATIONOID, ObjectIdGetDatum(pubid))) + continue; + pub = GetPublication(pubid); + if (pub->alltables) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("publication \"%s\" is defined FOR ALL TABLES", pub->name), + errhint("Create an unrestricted logical replication slot instead."))); + pubrels = GetIncludedPublicationRelations(pubid, PUBLICATION_PART_ALL); + pubrels = list_concat(pubrels, + GetAllSchemaPublicationRelations(pubid, PUBLICATION_PART_ALL)); + relations = list_concat(relations, pubrels); + } + list_sort(relations, list_oid_cmp); + list_deduplicate_oid(relations); + + base = list_copy(relations); + foreach_oid(relid, base) + { + Relation rel; + + lock_scope_relation(relid, conditional); + rel = table_open(relid, NoLock); + + if (OidIsValid(rel->rd_rel->reltoastrelid)) + relations = lappend_oid(relations, rel->rd_rel->reltoastrelid); + table_close(rel, NoLock); + } + list_free(base); + + list_sort(relations, list_oid_cmp); + list_deduplicate_oid(relations); + return relations; +} + +/* Set pg_class.relhasrestrictedslots for a mapped relation. */ +static void +set_relation_restricted_flag(Oid relid) +{ + Relation classrel; + HeapTuple tuple; + Form_pg_class classform; + + tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); + if (!HeapTupleIsValid(tuple)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_TABLE), + errmsg("relation with OID %u does not exist", relid))); + + if (((Form_pg_class) GETSTRUCT(tuple))->relhasrestrictedslots) + { + ReleaseSysCache(tuple); + return; + } + ReleaseSysCache(tuple); + + /* + * Fetch and check the tuple again while holding the relation lock + * acquired by restricted_relation_add(), because its value might have + * changed since the fast-path check. + */ + classrel = table_open(RelationRelationId, RowExclusiveLock); + tuple = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(relid)); + if (!HeapTupleIsValid(tuple)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_TABLE), + errmsg("relation with OID %u does not exist", relid))); + + classform = (Form_pg_class) GETSTRUCT(tuple); + if (!classform->relhasrestrictedslots) + { + classform->relhasrestrictedslots = true; + CatalogTupleUpdate(classrel, &tuple->t_self, tuple); + CommandCounterIncrement(); + } + + heap_freetuple(tuple); + table_close(classrel, RowExclusiveLock); +} + +/* Look up the current incarnation of a named logical replication slot. */ +static uint64 +restricted_slot_incarnation(const char *slotname) +{ + uint64 incarnation = 0; + + LWLockAcquire(ReplicationSlotAllocationLock, LW_SHARED); + for (int i = 0; i < max_replication_slots + max_repack_replication_slots; i++) + { + ReplicationSlot *slot = &ReplicationSlotCtl->replication_slots[i]; + ReplicationSlotPersistentData data; + + if (!slot->in_use) + continue; + SpinLockAcquire(&slot->mutex); + data = slot->data; + SpinLockRelease(&slot->mutex); + if (data.database != InvalidOid && + strcmp(NameStr(data.name), slotname) == 0) + { + incarnation = data.restricted_scope_incarnation; + break; + } + } + LWLockRelease(ReplicationSlotAllocationLock); + return incarnation; +} + +/* + * Add a relation mapping for the current incarnation of a restricted slot. + * + * If the named slot no longer exists, do nothing. Otherwise, lock the + * relation against concurrent DML, insert the incarnation-qualified mapping + * if it is absent, and set pg_class.relhasrestrictedslots. + * + * The relation lock prevents a writer from crossing transaction commit with + * a stale false value for relhasrestrictedslots. The database-object lock + * serializes mapping changes with cleanup and other scope maintenance. + */ +static void +restricted_relation_add(const char *slotname, Oid relid, bool conditional) +{ + Relation maprel; + ScanKeyData keys[3]; + SysScanDesc scan; + HeapTuple tuple; + NameData keyname; + uint64 incarnation = restricted_slot_incarnation(slotname); + + if (incarnation == 0) + return; + + /* Keep DML out until the new flag and mapping become visible at commit. */ + lock_scope_relation(relid, conditional); + maprel = table_open(RestrictedSlotRelationRelationId, RowExclusiveLock); + namestrcpy(&keyname, slotname); + ScanKeyInit(&keys[0], Anum_pg_restricted_slot_relation_rsrslotname, + BTEqualStrategyNumber, F_NAMEEQ, NameGetDatum(&keyname)); + ScanKeyInit(&keys[1], Anum_pg_restricted_slot_relation_rsrrelid, + BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); + ScanKeyInit(&keys[2], Anum_pg_restricted_slot_relation_rsrincarnation, + BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum((int64) incarnation)); + scan = systable_beginscan(maprel, RestrictedSlotRelationSlotRelIndexId, + true, NULL, 3, keys); + tuple = systable_getnext(scan); + if (!HeapTupleIsValid(tuple)) + { + Datum values[Natts_pg_restricted_slot_relation]; + bool nulls[Natts_pg_restricted_slot_relation] = {false}; + NameData name; + + MemSet(values, 0, sizeof(values)); + namestrcpy(&name, slotname); + values[Anum_pg_restricted_slot_relation_rsrslotname - 1] = NameGetDatum(&name); + values[Anum_pg_restricted_slot_relation_rsrrelid - 1] = ObjectIdGetDatum(relid); + values[Anum_pg_restricted_slot_relation_rsrincarnation - 1] = + Int64GetDatum((int64) incarnation); + tuple = heap_form_tuple(RelationGetDescr(maprel), values, nulls); + CatalogTupleInsert(maprel, tuple); + heap_freetuple(tuple); + CommandCounterIncrement(); + } + systable_endscan(scan); + table_close(maprel, RowExclusiveLock); + set_relation_restricted_flag(relid); +} + +/* + * Add the transactional readiness marker for a restricted-slot incarnation. + * + * The marker uses InvalidOid as its relation OID and is inserted in the same + * transaction as the initial relation mappings. After a crash, its presence + * proves that initialization committed, including when the publications had + * no relation members. Its absence means that a not-ready slot must be + * treated as incomplete. + */ +static void +restricted_ready_marker_add(const char *slotname, uint64 incarnation) +{ + Relation maprel; + ScanKeyData keys[3]; + SysScanDesc scan; + HeapTuple tuple; + NameData keyname; + + maprel = table_open(RestrictedSlotRelationRelationId, RowExclusiveLock); + namestrcpy(&keyname, slotname); + ScanKeyInit(&keys[0], Anum_pg_restricted_slot_relation_rsrslotname, + BTEqualStrategyNumber, F_NAMEEQ, NameGetDatum(&keyname)); + ScanKeyInit(&keys[1], Anum_pg_restricted_slot_relation_rsrrelid, + BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(InvalidOid)); + ScanKeyInit(&keys[2], Anum_pg_restricted_slot_relation_rsrincarnation, + BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum((int64) incarnation)); + scan = systable_beginscan(maprel, RestrictedSlotRelationSlotRelIndexId, + true, NULL, 3, keys); + tuple = systable_getnext(scan); + if (!HeapTupleIsValid(tuple)) + { + Datum values[Natts_pg_restricted_slot_relation]; + bool nulls[Natts_pg_restricted_slot_relation] = {false}; + NameData name; + + MemSet(values, 0, sizeof(values)); + namestrcpy(&name, slotname); + values[Anum_pg_restricted_slot_relation_rsrslotname - 1] = + NameGetDatum(&name); + values[Anum_pg_restricted_slot_relation_rsrrelid - 1] = + ObjectIdGetDatum(InvalidOid); + values[Anum_pg_restricted_slot_relation_rsrincarnation - 1] = + Int64GetDatum((int64) incarnation); + tuple = heap_form_tuple(RelationGetDescr(maprel), values, nulls); + CatalogTupleInsert(maprel, tuple); + heap_freetuple(tuple); + } + systable_endscan(scan); + table_close(maprel, RowExclusiveLock); +} + +/* Return the names of restricted slots currently mapped to a relation. */ +static List * +restricted_slots_for_relation(Oid relid) +{ + Relation maprel; + ScanKeyData key; + SysScanDesc scan; + HeapTuple tuple; + List *result = NIL; + + maprel = table_open(RestrictedSlotRelationRelationId, AccessShareLock); + ScanKeyInit(&key, Anum_pg_restricted_slot_relation_rsrrelid, + BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); + scan = systable_beginscan(maprel, RestrictedSlotRelationRelIndexId, + true, NULL, 1, &key); + while (HeapTupleIsValid(tuple = systable_getnext(scan))) + { + Form_pg_restricted_slot_relation form = + (Form_pg_restricted_slot_relation) GETSTRUCT(tuple); + + result = lappend(result, pstrdup(NameStr(form->rsrslotname))); + } + systable_endscan(scan); + table_close(maprel, AccessShareLock); + return result; +} + +/* + * Return the names of valid restricted slots in the current database whose + * stored publication set contains pubid. + * + * Copy the slot names while holding the replication-slot allocation lock, but + * do not retain that lock across relation locking or mapping catalog changes. + * Include not-yet-ready restricted slots so concurrent publication expansion + * cannot be missed during slot initialization. + */ +static List * +restricted_slots_for_publication(Oid pubid) +{ + List *slotnames = NIL; + + LWLockAcquire(ReplicationSlotAllocationLock, LW_SHARED); + for (int i = 0; i < max_replication_slots + max_repack_replication_slots; i++) + { + ReplicationSlot *slot = &ReplicationSlotCtl->replication_slots[i]; + ReplicationSlotPersistentData data; + List *publications; + + if (!slot->in_use) + continue; + SpinLockAcquire(&slot->mutex); + data = slot->data; + SpinLockRelease(&slot->mutex); + if (data.database != MyDatabaseId || data.unrestricted || + data.invalidated != RS_INVAL_NONE) + continue; + publications = read_publications_file(slot); + if (list_member_oid(publications, pubid)) + slotnames = lappend(slotnames, pstrdup(NameStr(data.name))); + list_free(publications); + } + LWLockRelease(ReplicationSlotAllocationLock); + return slotnames; +} + +/* + * Build and lock the current physical closure of a list of relation roots. + * + * Discover every root and partition descendant under AccessShareLock, add + * their current TOAST relations, and sort and deduplicate the resulting OIDs. + * Acquire ShareRowExclusiveLock in OID order to prevent concurrent writers + * from crossing the transactional installation of scope mappings and to + * avoid inconsistent relation-lock ordering between concurrent operations. + */ +static List * +relation_physical_closure(List *roots) +{ + List *result = NIL; + List *base; + List *sorted_roots = list_copy(roots); + + list_sort(sorted_roots, list_oid_cmp); + list_deduplicate_oid(sorted_roots); + foreach_oid(root, sorted_roots) + { + List *descendants = find_all_inheritors(root, + AccessShareLock, NULL); + + result = list_concat(result, descendants); + } + list_free(sorted_roots); + list_sort(result, list_oid_cmp); + list_deduplicate_oid(result); + + base = list_copy(result); + foreach_oid(relid, base) + { + Relation rel = table_open(relid, NoLock); + + if (OidIsValid(rel->rd_rel->reltoastrelid)) + result = lappend_oid(result, rel->rd_rel->reltoastrelid); + table_close(rel, NoLock); + } + list_free(base); + + list_sort(result, list_oid_cmp); + list_deduplicate_oid(result); + foreach_oid(relid, result) + LockRelationOid(relid, ShareRowExclusiveLock); + return result; +} + +/* Add physical relations to a named restricted slot's scope. */ +static void +logical_slot_scope_add_relations(const char *slotname, List *relations, + bool conditional) +{ + foreach_oid(relid, relations) + restricted_relation_add(slotname, relid, conditional); +} + +/* + * Add newly published relations to restricted slots using a publication. + * + * Find valid restricted slots in the current database whose stored + * publication set contains pubid. Expand the supplied relation roots to + * their partition descendants and TOAST relations, then add the resulting + * mappings to each matching slot. + * + * The mappings and pg_class flags are changed in the caller's transaction, + * so they become visible atomically with the publication membership change. + * Relation locks prevent concurrent DML from missing logical tuple WAL while + * that change is being committed. + */ +void +LogicalSlotScopePublicationAddRelations(Oid pubid, List *relations) +{ + List *slotnames; + List *closure; + + /* + * To avoid unconditionally doing relation_physical_closure, check if + * there are any restricted slots for the publication first. If not, we + * can skip the closure and locking entirely. This is important for + * performance, as relation_physical_closure can be expensive for large + * relation sets. + */ + mark_database_scope_change(); + lock_database_scope(); + slotnames = restricted_slots_for_publication(pubid); + unlock_database_scope(); + if (slotnames == NIL) + return; + list_free_deep(slotnames); + + closure = relation_physical_closure(relations); + lock_database_scope(); + /* Recheck after taking relation locks and reacquiring serialization. */ + slotnames = restricted_slots_for_publication(pubid); + if (slotnames == NIL) + { + list_free(closure); + unlock_database_scope(); + return; + } + EnsureRestrictedLogicalWAL(); + foreach_ptr(char, slotname, slotnames) + logical_slot_scope_add_relations(slotname, closure, false); + list_free(closure); + list_free_deep(slotnames); + unlock_database_scope(); +} + +/* + * Propagate an owner's restricted-slot mappings to a new TOAST relation. + * + * Add the TOAST relation to every restricted slot currently mapped to its + * owning relation. The mapping and relhasrestrictedslots flag are changed + * in the transaction that creates the TOAST relation, so subsequent TOAST + * writes cannot become visible without the required logical WAL. + */ +void +LogicalSlotScopeNoteToastCreation(Oid owner, Oid toastrelid) +{ + List *slotnames; + + if (IsBootstrapProcessingMode()) + return; + mark_database_scope_change(); + lock_database_scope(); + slotnames = restricted_slots_for_relation(owner); + foreach_ptr(char, slotname, slotnames) + restricted_relation_add(slotname, toastrelid, false); + list_free_deep(slotnames); + unlock_database_scope(); +} + +/* + * Remove all restricted-slot mappings for a relation being dropped. + * + * Delete mappings for every slot incarnation in the same transaction that + * drops the relation. There is no need to clear relhasrestrictedslots + * because the relation's pg_class row is also being removed. + */ +void +LogicalSlotScopeRelationDrop(Oid relid) +{ + Relation maprel; + ScanKeyData key; + SysScanDesc scan; + HeapTuple tuple; + + mark_database_scope_change(); + lock_database_scope(); + maprel = table_open(RestrictedSlotRelationRelationId, RowExclusiveLock); + ScanKeyInit(&key, Anum_pg_restricted_slot_relation_rsrrelid, + BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); + scan = systable_beginscan(maprel, RestrictedSlotRelationRelIndexId, + true, NULL, 1, &key); + while (HeapTupleIsValid(tuple = systable_getnext(scan))) + CatalogTupleDelete(maprel, &tuple->t_self); + systable_endscan(scan); + table_close(maprel, RowExclusiveLock); + unlock_database_scope(); +} + +/* Return whether any restricted-slot mapping remains for a relation. */ +static bool +relation_has_mapping(Oid relid) +{ + Relation maprel; + ScanKeyData key; + SysScanDesc scan; + bool result; + + maprel = table_open(RestrictedSlotRelationRelationId, AccessShareLock); + ScanKeyInit(&key, Anum_pg_restricted_slot_relation_rsrrelid, + BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); + scan = systable_beginscan(maprel, RestrictedSlotRelationRelIndexId, + true, NULL, 1, &key); + result = HeapTupleIsValid(systable_getnext(scan)); + systable_endscan(scan); + table_close(maprel, AccessShareLock); + return result; +} + +/* Check for a committed readiness marker matching a slot incarnation. */ +static bool +restricted_ready_marker_exists(const char *slotname, uint64 incarnation) +{ + Relation maprel; + ScanKeyData keys[3]; + SysScanDesc scan; + NameData name; + bool result; + + maprel = table_open(RestrictedSlotRelationRelationId, AccessShareLock); + namestrcpy(&name, slotname); + ScanKeyInit(&keys[0], Anum_pg_restricted_slot_relation_rsrslotname, + BTEqualStrategyNumber, F_NAMEEQ, NameGetDatum(&name)); + ScanKeyInit(&keys[1], Anum_pg_restricted_slot_relation_rsrrelid, + BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(InvalidOid)); + ScanKeyInit(&keys[2], Anum_pg_restricted_slot_relation_rsrincarnation, + BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum((int64) incarnation)); + scan = systable_beginscan(maprel, RestrictedSlotRelationSlotRelIndexId, + true, NULL, 3, keys); + result = HeapTupleIsValid(systable_getnext(scan)); + systable_endscan(scan); + table_close(maprel, AccessShareLock); + return result; +} + +/* + * Reconcile incomplete restricted slots for the current database. + * + * Slot state and transactional relation mappings cannot be persisted + * atomically. After a crash, a slot may therefore remain not ready even + * though its mapping transaction committed. + * + * For each inactive, valid restricted slot that is not ready, look for the + * readiness marker matching its name and incarnation. If the marker exists, + * mark the slot ready and use the current flush position as a conservative + * mapping-durability boundary. If the marker does not exist, drop the slot + * as an aborted or incomplete creation. + * + * Slot state is rechecked after acquiring each slot because it may have + * changed since the initial scan. + */ +void +LogicalSlotScopeReconcileDatabase(void) +{ + List *slots = NIL; + bool found = false; + + Assert(IsTransactionState()); + + /* Avoid catalog locking on connections with nothing to reconcile. */ + LWLockAcquire(ReplicationSlotAllocationLock, LW_SHARED); + for (int i = 0; i < max_replication_slots + max_repack_replication_slots; i++) + { + ReplicationSlot *slot = &ReplicationSlotCtl->replication_slots[i]; + ReplicationSlotPersistentData data; + bool active; + + if (!slot->in_use) + continue; + SpinLockAcquire(&slot->mutex); + data = slot->data; + active = slot->active_proc != INVALID_PROC_NUMBER; + SpinLockRelease(&slot->mutex); + if (data.database != MyDatabaseId || data.unrestricted || + data.restricted_scope_ready || + data.invalidated != RS_INVAL_NONE || active) + continue; + found = true; + break; + } + LWLockRelease(ReplicationSlotAllocationLock); + if (!found) + return; + + lock_database_scope(); + LWLockAcquire(ReplicationSlotAllocationLock, LW_SHARED); + for (int i = 0; i < max_replication_slots + max_repack_replication_slots; i++) + { + ReplicationSlot *slot = &ReplicationSlotCtl->replication_slots[i]; + ReplicationSlotPersistentData data; + ReconcileSlot *item; + bool active; + + if (!slot->in_use) + continue; + SpinLockAcquire(&slot->mutex); + data = slot->data; + active = slot->active_proc != INVALID_PROC_NUMBER; + SpinLockRelease(&slot->mutex); + if (data.database != MyDatabaseId || data.unrestricted || + data.restricted_scope_ready || + data.invalidated != RS_INVAL_NONE || active) + continue; + item = palloc(sizeof(*item)); + item->name = data.name; + item->incarnation = data.restricted_scope_incarnation; + slots = lappend(slots, item); + } + LWLockRelease(ReplicationSlotAllocationLock); + + foreach_ptr(ReconcileSlot, item, slots) + { + if (restricted_ready_marker_exists(NameStr(item->name), item->incarnation)) + { + bool reconciled = false; + ReplicationSlotPersistentData data; + + if (!ReplicationSlotConditionalAcquire(NameStr(item->name), false)) + continue; + SpinLockAcquire(&MyReplicationSlot->mutex); + data = MyReplicationSlot->data; + if (data.database == MyDatabaseId && !data.unrestricted && + !data.restricted_scope_ready && + data.invalidated == RS_INVAL_NONE && + data.restricted_scope_incarnation == item->incarnation) + { + MyReplicationSlot->data.restricted_scope_ready = true; + MyReplicationSlot->data.restricted_scope_ready_lsn = GetFlushRecPtr(NULL); + MyReplicationSlot->just_dirtied = true; + MyReplicationSlot->dirty = true; + reconciled = true; + } + SpinLockRelease(&MyReplicationSlot->mutex); + ReplicationSlotRelease(); + if (reconciled) + { + EnableLogicalDecoding(); + EnsureRestrictedLogicalWAL(); + RequestDisableLogicalDecoding(); + } + } + else + { + ReplicationSlotPersistentData data; + + if (!ReplicationSlotConditionalAcquire(NameStr(item->name), false)) + continue; + SpinLockAcquire(&MyReplicationSlot->mutex); + data = MyReplicationSlot->data; + SpinLockRelease(&MyReplicationSlot->mutex); + if (data.database == MyDatabaseId && !data.unrestricted && + !data.restricted_scope_ready && + data.restricted_scope_incarnation == item->incarnation) + ReplicationSlotDropAcquired(true); + else + ReplicationSlotRelease(); + } + } + list_free_deep(slots); +} + +/* Test whether a list contains a matching slot name and incarnation. */ +static bool +slot_identity_list_contains(List *slots, const char *slotname, + uint64 incarnation) +{ + foreach_ptr(ReconcileSlot, candidate, slots) + if (strcmp(NameStr(candidate->name), slotname) == 0 && + candidate->incarnation == incarnation) + return true; + return false; +} + +/* Clear relhasrestrictedslots after a relation loses its last mapping. */ +static void +clear_relation_restricted_flag(Oid relid) +{ + Relation classrel; + HeapTuple tuple; + + /* A stale true flag is safe; maintenance must not wait behind DDL. */ + if (!ConditionalLockRelationOid(relid, ShareRowExclusiveLock)) + return; + if (relation_has_mapping(relid)) + return; + classrel = table_open(RelationRelationId, RowExclusiveLock); + tuple = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(relid)); + if (HeapTupleIsValid(tuple)) + { + Form_pg_class form = (Form_pg_class) GETSTRUCT(tuple); + + if (form->relhasrestrictedslots) + { + form->relhasrestrictedslots = false; + CatalogTupleUpdate(classrel, &tuple->t_self, tuple); + } + heap_freetuple(tuple); + } + table_close(classrel, RowExclusiveLock); +} + +/* + * Remove mappings belonging to obsolete restricted-slot incarnations. + * + * Build the set of valid restricted slots in the current database, identified + * by name and incarnation, and delete mappings whose owner is no longer in + * that set. After deleting mappings, clear relhasrestrictedslots only for + * relations that have no remaining mapping. + * + * Required mapping additions are always performed synchronously by the + * transaction that expands a scope. Cleanup must not attempt to reconstruct + * missing mappings after logical WAL could already have been omitted. + * Mappings made obsolete by publication contraction may remain indefinitely, + * since they cause only conservative extra WAL. + */ +void +LogicalSlotScopeCleanup(void) +{ + List *slots = NIL; + List *changed_relations = NIL; + Relation maprel; + TableScanDesc scan; + HeapTuple tuple; + + lock_database_scope(); + LWLockAcquire(ReplicationSlotAllocationLock, LW_SHARED); + for (int i = 0; i < max_replication_slots + max_repack_replication_slots; i++) + { + ReplicationSlot *slot = &ReplicationSlotCtl->replication_slots[i]; + ReplicationSlotPersistentData data; + ReconcileSlot *item; + + if (!slot->in_use) + continue; + SpinLockAcquire(&slot->mutex); + data = slot->data; + SpinLockRelease(&slot->mutex); + if (data.database != MyDatabaseId || data.unrestricted || + data.invalidated != RS_INVAL_NONE) + continue; + item = palloc(sizeof(*item)); + item->name = data.name; + item->incarnation = data.restricted_scope_incarnation; + slots = lappend(slots, item); + } + LWLockRelease(ReplicationSlotAllocationLock); + + /* + * Required additions are synchronous in their originating transaction; + * maintenance must never try to repair them after WAL could be lost. Keep + * this pass bounded to removing mappings for slots that no longer exist. + * Publication contraction can safely leave conservative entries. + */ + maprel = table_open(RestrictedSlotRelationRelationId, RowExclusiveLock); + scan = table_beginscan_catalog(maprel, 0, NULL); + while (HeapTupleIsValid(tuple = heap_getnext(scan, ForwardScanDirection))) + { + Form_pg_restricted_slot_relation form = + (Form_pg_restricted_slot_relation) GETSTRUCT(tuple); + + if (!slot_identity_list_contains(slots, NameStr(form->rsrslotname), + (uint64) form->rsrincarnation)) + { + if (OidIsValid(form->rsrrelid)) + changed_relations = list_append_unique_oid(changed_relations, + form->rsrrelid); + CatalogTupleDelete(maprel, &tuple->t_self); + } + } + table_endscan(scan); + table_close(maprel, RowExclusiveLock); + CommandCounterIncrement(); + foreach_oid(relid, changed_relations) + clear_relation_restricted_flag(relid); + list_free(changed_relations); + list_free_deep(slots); + RequestDisableLogicalDecoding(); +} + +/* Resolve publication names and prepare a restricted slot for mappings. */ +void +LogicalSlotScopePrepareFromPublications(ReplicationSlot *slot, List *pubnames) +{ + List *publications = NIL; + + Assert(IsTransactionState()); + Assert(pubnames != NIL); + foreach_ptr(char, pubname, pubnames) + { + Publication *pub = GetPublicationByName(pubname, false); + + if (pub->alltables) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("publication \"%s\" is defined FOR ALL TABLES", pubname), + errhint("Create an unrestricted logical replication slot instead."))); + publications = lappend_oid(publications, pub->oid); + } + list_sort(publications, list_oid_cmp); + list_deduplicate_oid(publications); + LogicalSlotScopePrepareFromPublicationOids(slot, publications); + list_free(publications); +} + +/* + * Prepare a slot for transactional restricted-scope initialization. + * + * Assign a new nonzero incarnation, mark the slot not ready, and persist its + * publication identities. Enable logical decoding and temporarily require + * full logical WAL so that concurrent changes cannot be missed before the + * initial relation mappings commit. Also enable restricted logical WAL for + * the steady state after the temporary full-WAL requirement is removed. + */ +void +LogicalSlotScopePrepareFromPublicationOids(ReplicationSlot *slot, + List *publications) +{ + uint64 incarnation; + + Assert(IsTransactionState()); + Assert(publications != NIL); + lock_database_scope(); + check_no_database_scope_change(); + do + { + incarnation = pg_prng_uint64(&pg_global_prng_state); + } while (incarnation == 0); + + /* + * Install the publication file before making the restricted slot visible + * to concurrent scope-expansion hooks. Such hooks may then safely + * include a not-yet-ready slot. + */ + write_publications_file(slot, publications); + + /* Enable logical decoding and WAL */ + EnsureLogicalDecodingEnabled(); + + /* Temporarily raise effective_wal_level to logical */ + EnsureFullLogicalWAL(); + + /* + * Raise restricted_wal_level to logical, but before + * LogicalSlotScopeFinishCreate, the slot is not yet ready. + */ + EnsureRestrictedLogicalWAL(); + + SpinLockAcquire(&slot->mutex); + slot->data.unrestricted = false; + slot->data.restricted_scope_ready = false; + slot->data.restricted_scope_incarnation = incarnation; + slot->data.restricted_scope_ready_lsn = InvalidXLogRecPtr; + SpinLockRelease(&slot->mutex); + ReplicationSlotMarkDirty(); +} + +/* + * Install the initial mappings for a prepared restricted slot. + * + * Re-read and expand the stored publications, add mappings for their current + * physical relation closure, and insert the readiness marker in the same + * transaction. Queue the slot for transaction-end finalization; it must not + * be marked ready until that transaction commits. + * + * If finalize_at_xact_end is true, the transaction callback also releases + * the slot after commit or abort. + */ +void +LogicalSlotScopeFinishCreate(ReplicationSlot *slot, bool finalize_at_xact_end) +{ + List *publications; + List *relations; + MemoryContext oldcontext; + PendingReadySlot *pending; + + Assert(IsTransactionState()); + Assert(!slot->data.unrestricted); + + lock_database_scope(); + check_no_database_scope_change(); + publications = read_publications_file(slot); + relations = publication_relation_closure(publications, true); + logical_slot_scope_add_relations(NameStr(slot->data.name), relations, true); + restricted_ready_marker_add(NameStr(slot->data.name), + slot->data.restricted_scope_incarnation); + ReplicationSlotMarkDirty(); + oldcontext = MemoryContextSwitchTo(TopTransactionContext); + pending = palloc(sizeof(*pending)); + pending->slot = slot; + pending->name = slot->data.name; + pending->finalize_at_xact_end = finalize_at_xact_end; + /* pending_ready_slots will be processed by the transaction callback */ + pending_ready_slots = lappend(pending_ready_slots, pending); + MemoryContextSwitchTo(oldcontext); + list_free(relations); + list_free(publications); +} + +/* + * Restore the scope metadata of a synchronized restricted slot. + * + * Install the publication identities and upstream slot incarnation, mark the + * slot ready at the supplied mapping-durability boundary, and enable + * restricted logical WAL. The caller must ensure that WAL through ready_lsn + * has been flushed locally, so the corresponding transactional catalog + * mappings are available before the synchronized slot becomes usable. + */ +void +LogicalSlotScopeRestorePublications(ReplicationSlot *slot, List *publications, + uint64 incarnation, XLogRecPtr ready_lsn) +{ + write_publications_file(slot, publications); + SpinLockAcquire(&slot->mutex); + slot->data.unrestricted = false; + slot->data.restricted_scope_ready = true; + slot->data.restricted_scope_incarnation = incarnation; + slot->data.restricted_scope_ready_lsn = ready_lsn; + SpinLockRelease(&slot->mutex); + EnableLogicalDecoding(); + EnsureRestrictedLogicalWAL(); + ReplicationSlotMarkDirty(); +} + +/* Configure the writer-WAL requirement for an unrestricted slot. */ +void +LogicalSlotScopeConfigureUnrestricted(ReplicationSlot *slot) +{ + SpinLockAcquire(&slot->mutex); + slot->data.unrestricted = true; + slot->data.restricted_scope_ready = true; + slot->data.restricted_scope_ready_lsn = InvalidXLogRecPtr; + SpinLockRelease(&slot->mutex); + EnsureLogicalDecodingEnabled(); +} + +/* + * Restore writer-side WAL requirements for a logical slot loaded from disk. + * + * A ready restricted slot requires restricted logical WAL. An incomplete + * restricted slot temporarily requires full logical WAL until database-level + * reconciliation either completes or drops it. Unrestricted slots require + * no scope-specific restoration here. + */ +void +LogicalSlotScopeRestore(ReplicationSlot *slot) +{ + if (!SlotIsLogical(slot)) + return; + if (!slot->data.unrestricted && slot->data.restricted_scope_ready) + { + EnableLogicalDecoding(); + EnsureRestrictedLogicalWAL(); + } + else if (!slot->data.unrestricted) + { + EnableLogicalDecoding(); + EnsureFullLogicalWAL(); + } +} + +/* + * Propagate restricted-slot mappings across a new partition hierarchy link. + * + * For every restricted slot mapped to the parent, add mappings for the child, + * its partition descendants, and their TOAST relations. These changes occur + * in the transaction that creates the hierarchy link, so the new members + * require logical WAL as soon as the link becomes visible. + * + * Mapping removal after detach is deliberately deferred, since a stale + * mapping causes only conservative extra WAL. + */ +void +CheckLogicalSlotScopeHierarchyChange(Oid childrelid, Oid parentrelid) +{ + List *slotnames; + List *roots; + List *closure; + + mark_database_scope_change(); + lock_database_scope(); + slotnames = restricted_slots_for_relation(parentrelid); + unlock_database_scope(); + if (slotnames == NIL) + { + return; + } + list_free_deep(slotnames); + + roots = list_make1_oid(childrelid); + closure = relation_physical_closure(roots); + lock_database_scope(); + /* Recheck after taking relation locks and reacquiring serialization. */ + slotnames = restricted_slots_for_relation(parentrelid); + if (slotnames == NIL) + { + list_free(closure); + list_free(roots); + unlock_database_scope(); + return; + } + EnsureRestrictedLogicalWAL(); + foreach_ptr(char, slotname, slotnames) + logical_slot_scope_add_relations(slotname, closure, false); + list_free(closure); + list_free(roots); + list_free_deep(slotnames); + unlock_database_scope(); +} + +/* + * Return whether changes to a relation require logical tuple information. + * + * Relations that cannot participate in logical decoding never require it. + * Full logical WAL covers every eligible relation. In restricted mode, use + * the relcache copy of pg_class.relhasrestrictedslots, avoiding catalog + * access in the WAL insertion path. + */ +bool +RelationNeedsLogicalTupleWAL(Relation relation) +{ + if (!RelationCanBeLogicallyLogged(relation)) + return false; + if (XLogFullLogicalInfoActive()) + return true; + if (!XLogRestrictedInfoActive()) + return false; + return relation->rd_rel->relhasrestrictedslots; +} + +/* + * Return whether a slot permits all requested publications. + * + * Unrestricted slots permit any publication. For a restricted slot, resolve + * each requested publication name to its current OID and require that OID to + * appear in the immutable publication set stored by the slot. This validates + * publication identity only; normal pgoutput processing determines current + * relation membership. + */ +bool +LogicalSlotScopePublicationsContain(ReplicationSlot *slot, List *pubnames) +{ + List *stored; + bool result = true; + + if (slot->data.unrestricted) + return true; + stored = read_publications_file(slot); + foreach_ptr(char, pubname, pubnames) + { + Publication *pub = GetPublicationByName(pubname, false); + + if (!list_member_oid(stored, pub->oid)) + { + result = false; + break; + } + } + list_free(stored); + return result; +} diff --git a/src/backend/replication/logical/slotsync.c b/src/backend/replication/logical/slotsync.c index c0403893e23..852fb77c846 100644 --- a/src/backend/replication/logical/slotsync.c +++ b/src/backend/replication/logical/slotsync.c @@ -68,6 +68,8 @@ #include "postmaster/interrupt.h" #include "replication/logical.h" #include "replication/logicalctl.h" +#include "replication/slot.h" +#include "replication/slotscope.h" #include "replication/slotsync.h" #include "replication/snapbuild.h" #include "storage/ipc.h" @@ -76,6 +78,7 @@ #include "storage/procarray.h" #include "storage/subsystems.h" #include "tcop/tcopprot.h" +#include "utils/array.h" #include "utils/builtins.h" #include "utils/memutils.h" #include "utils/pg_lsn.h" @@ -170,6 +173,11 @@ typedef struct RemoteSlot char *database; bool two_phase; bool failover; + bool unrestricted; + bool restricted_scope_ready; + uint64 restricted_scope_incarnation; + XLogRecPtr restricted_scope_ready_lsn; + List *publications; XLogRecPtr restart_lsn; XLogRecPtr confirmed_lsn; XLogRecPtr two_phase_at; @@ -233,7 +241,8 @@ update_local_synced_slot(RemoteSlot *remote_slot, Oid remote_dbid) * Make sure that concerned WAL is received and flushed before syncing * slot to target lsn received from the primary server. */ - if (remote_slot->confirmed_lsn > latestFlushPtr) + if (remote_slot->confirmed_lsn > latestFlushPtr || + remote_slot->restricted_scope_ready_lsn > latestFlushPtr) { update_slotsync_skip_stats(SS_SKIP_WAL_NOT_FLUSHED); @@ -786,7 +795,36 @@ synchronize_one_slot(RemoteSlot *remote_slot, Oid remote_dbid, bool slot_updated = false; /* Search for the named slot */ - if ((slot = SearchNamedReplicationSlot(remote_slot->name, true))) + slot = SearchNamedReplicationSlot(remote_slot->name, true); + if (slot != NULL) + { + ReplicationSlotPersistentData data; + + SpinLockAcquire(&slot->mutex); + data = slot->data; + SpinLockRelease(&slot->mutex); + if (data.synced && + (data.unrestricted != remote_slot->unrestricted || + (!data.unrestricted && + data.restricted_scope_incarnation != + remote_slot->restricted_scope_incarnation))) + { + ReplicationSlotAcquire(remote_slot->name, true, false); + ReplicationSlotDropAcquired(false); + slot = NULL; + slot_updated = true; + } + } + + /* Its transactional relation mappings may not have committed yet. */ + if (!remote_slot->unrestricted && !remote_slot->restricted_scope_ready) + { + if (slot_persistence_pending) + *slot_persistence_pending = true; + return slot_updated; + } + + if (slot != NULL) { bool synced; @@ -912,6 +950,14 @@ synchronize_one_slot(RemoteSlot *remote_slot, Oid remote_dbid, slot->data.plugin = plugin_name; SpinLockRelease(&slot->mutex); + if (remote_slot->unrestricted) + LogicalSlotScopeConfigureUnrestricted(slot); + else + LogicalSlotScopeRestorePublications(slot, + remote_slot->publications, + remote_slot->restricted_scope_incarnation, + remote_slot->restricted_scope_ready_lsn); + reserve_wal_for_local_slot(remote_slot->restart_lsn); LWLockAcquire(ReplicationSlotControlLock, LW_EXCLUSIVE); @@ -948,9 +994,10 @@ synchronize_one_slot(RemoteSlot *remote_slot, Oid remote_dbid, static List * fetch_remote_slots(WalReceiverConn *wrconn, List *slot_names) { -#define SLOTSYNC_COLUMN_COUNT 10 +#define SLOTSYNC_COLUMN_COUNT 15 Oid slotRow[SLOTSYNC_COLUMN_COUNT] = {TEXTOID, TEXTOID, LSNOID, - LSNOID, XIDOID, BOOLOID, LSNOID, BOOLOID, TEXTOID, TEXTOID}; + LSNOID, XIDOID, BOOLOID, LSNOID, BOOLOID, TEXTOID, TEXTOID, BOOLOID, + OIDARRAYOID, BOOLOID, INT8OID, LSNOID}; WalRcvExecResult *res; TupleTableSlot *tupslot; @@ -962,7 +1009,10 @@ fetch_remote_slots(WalReceiverConn *wrconn, List *slot_names) "SELECT slot_name, plugin, confirmed_flush_lsn," " restart_lsn, catalog_xmin, two_phase," " two_phase_at, failover," - " database, invalidation_reason" + " database, invalidation_reason, unrestricted," + " publication_oids, restricted_scope_ready," + " restricted_scope_incarnation," + " restricted_scope_ready_lsn" " FROM pg_catalog.pg_replication_slots" " WHERE failover and NOT temporary"); @@ -1044,6 +1094,35 @@ fetch_remote_slots(WalReceiverConn *wrconn, List *slot_names) remote_slot->invalidated = isnull ? RS_INVAL_NONE : GetSlotInvalidationCause(TextDatumGetCString(d)); + remote_slot->unrestricted = DatumGetBool(slot_getattr(tupslot, ++col, + &isnull)); + Assert(!isnull); + + d = slot_getattr(tupslot, ++col, &isnull); + if (!isnull) + { + Datum *elems; + int nelems; + + deconstruct_array_builtin(DatumGetArrayTypeP(d), OIDOID, + &elems, NULL, &nelems); + for (int i = 0; i < nelems; i++) + remote_slot->publications = lappend_oid(remote_slot->publications, + DatumGetObjectId(elems[i])); + } + + remote_slot->restricted_scope_ready = + DatumGetBool(slot_getattr(tupslot, ++col, &isnull)); + Assert(!isnull); + + d = slot_getattr(tupslot, ++col, &isnull); + remote_slot->restricted_scope_incarnation = + isnull ? 0 : (uint64) DatumGetInt64(d); + + d = slot_getattr(tupslot, ++col, &isnull); + remote_slot->restricted_scope_ready_lsn = + isnull ? InvalidXLogRecPtr : DatumGetLSN(d); + /* Sanity check */ Assert(col == SLOTSYNC_COLUMN_COUNT); @@ -1058,9 +1137,11 @@ fetch_remote_slots(WalReceiverConn *wrconn, List *slot_names) * pg_replication_slots view, then we can avoid fetching RS_EPHEMERAL * slots in the first place. */ - if ((!XLogRecPtrIsValid(remote_slot->restart_lsn) || - !XLogRecPtrIsValid(remote_slot->confirmed_lsn) || - !TransactionIdIsValid(remote_slot->catalog_xmin)) && + if (((!remote_slot->unrestricted && + !remote_slot->restricted_scope_ready) || + (!XLogRecPtrIsValid(remote_slot->restart_lsn) || + !XLogRecPtrIsValid(remote_slot->confirmed_lsn) || + !TransactionIdIsValid(remote_slot->catalog_xmin))) && remote_slot->invalidated == RS_INVAL_NONE) pfree(remote_slot); else diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index e5101997cd3..bc9f9d545f9 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -1423,6 +1423,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos) walrcv_create_slot(LogRepWorkerWalRcvConn, slotname, false /* permanent */ , false /* two_phase */ , MySubscription->failover, + MySubscription->unrestricted, MySubscription->publications, CRS_USE_SNAPSHOT, origin_startpos); /* diff --git a/src/backend/replication/pgoutput/pgoutput.c b/src/backend/replication/pgoutput/pgoutput.c index 484ffbe2cee..e013f18edd9 100644 --- a/src/backend/replication/pgoutput/pgoutput.c +++ b/src/backend/replication/pgoutput/pgoutput.c @@ -27,6 +27,8 @@ #include "replication/logicalproto.h" #include "replication/origin.h" #include "replication/pgoutput.h" +#include "replication/slot.h" +#include "replication/slotscope.h" #include "rewrite/rewriteHandler.h" #include "utils/builtins.h" #include "utils/inval.h" @@ -86,6 +88,7 @@ static void pgoutput_stream_prepare_txn(LogicalDecodingContext *ctx, static bool publications_valid; static List *LoadPublications(List *pubnames); +static void ValidateSlotScope(List *pubnames); static void publication_invalidation_cb(Datum arg, SysCacheIdentifier cacheid, uint32 hashvalue); static void send_repl_origin(LogicalDecodingContext *ctx, @@ -492,6 +495,20 @@ pgoutput_startup(LogicalDecodingContext *ctx, OutputPluginOptions *opt, /* Parse the params and ERROR if we see any we don't recognize */ parse_output_parameters(ctx->output_plugin_options, data); + /* + * A walsender invokes output-plugin startup outside a transaction, + * whereas SQL decoding functions already have one. Publication lookup + * needs a transaction environment in either case. + */ + if (!IsTransactionState()) + { + StartTransactionCommand(); + ValidateSlotScope(data->publication_names); + CommitTransactionCommand(); + } + else + ValidateSlotScope(data->publication_names); + /* Check if we support requested protocol */ if (data->protocol_version > LOGICALREP_PROTO_MAX_VERSION_NUM) ereport(ERROR, @@ -1805,6 +1822,8 @@ LoadPublications(List *pubnames) List *result = NIL; ListCell *lc; + ValidateSlotScope(pubnames); + foreach(lc, pubnames) { char *pubname = (char *) lfirst(lc); @@ -1823,6 +1842,24 @@ LoadPublications(List *pubnames) return result; } +/* A restricted slot may use any subset of its durable publication set. */ +static void +ValidateSlotScope(List *pubnames) +{ + if (MyReplicationSlot->data.unrestricted) + return; + if (!MyReplicationSlot->data.restricted_scope_ready) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("restricted replication slot \"%s\" is not ready", + NameStr(MyReplicationSlot->data.name)))); + if (!LogicalSlotScopePublicationsContain(MyReplicationSlot, pubnames)) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("requested publications are not contained in replication slot \"%s\"", + NameStr(MyReplicationSlot->data.name)))); +} + /* * Publication syscache invalidation callback. * diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index 63ce6d27885..a0dd9d2cfe3 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -43,13 +43,15 @@ #include "access/xlog_internal.h" #include "access/xlogrecovery.h" #include "common/file_utils.h" +#include "common/pg_prng.h" #include "common/string.h" #include "miscadmin.h" #include "pgstat.h" #include "postmaster/interrupt.h" #include "replication/logicallauncher.h" -#include "replication/slotsync.h" #include "replication/slot.h" +#include "replication/slotscope.h" +#include "replication/slotsync.h" #include "replication/walsender_private.h" #include "storage/fd.h" #include "storage/ipc.h" @@ -141,7 +143,8 @@ StaticAssertDecl(lengthof(SlotInvalidationCauses) == (RS_INVAL_MAX_CAUSES + 1), sizeof(ReplicationSlotOnDisk) - ReplicationSlotOnDiskConstantSize #define SLOT_MAGIC 0x1051CA1 /* format identifier */ -#define SLOT_VERSION 5 /* version for new files */ +#define SLOT_MIN_COMPATIBLE_VERSION 6 +#define SLOT_VERSION 6 /* version for new files */ /* Control array for replication slot management */ ReplicationSlotCtlData *ReplicationSlotCtl = NULL; @@ -239,6 +242,7 @@ ReplicationSlotsShmemInit(void *arg) void ReplicationSlotInitialize(void) { + LogicalSlotScopeInitialize(); before_shmem_exit(ReplicationSlotShmemExit, 0); } @@ -383,7 +387,12 @@ ReplicationSlotCreate(const char *name, bool db_specific, int startpoint, endpoint; - Assert(MyReplicationSlot == NULL); + if (MyReplicationSlot != NULL) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot create another replication slot while replication slot \"%s\" is acquired", + NameStr(MyReplicationSlot->data.name)), + errhint("Finish the transaction that is initializing the restricted slot first."))); /* * The logical launcher or pg_upgrade may create or migrate an internal @@ -482,6 +491,8 @@ ReplicationSlotCreate(const char *name, bool db_specific, slot->data.two_phase_at = InvalidXLogRecPtr; slot->data.failover = failover; slot->data.synced = synced; + slot->data.unrestricted = true; + slot->data.restricted_scope_ready = true; /* and then data only present in shared memory */ slot->just_dirtied = false; @@ -627,14 +638,20 @@ ReplicationSlotName(int index, Name name) * be invalid. It should always be set to true, except when we are temporarily * acquiring the slot and don't intend to change it. */ -void -ReplicationSlotAcquire(const char *name, bool nowait, bool error_if_invalid) +static bool +ReplicationSlotAcquireInternal(const char *name, bool nowait, + bool error_if_invalid, bool conditional) { ReplicationSlot *s; ProcNumber active_proc; int active_pid; Assert(name != NULL); + if (MyReplicationSlot != NULL) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot acquire replication slot \"%s\" while replication slot \"%s\" is acquired", + name, NameStr(MyReplicationSlot->data.name)))); retry: Assert(MyReplicationSlot == NULL); @@ -646,6 +663,8 @@ retry: if (s == NULL || !s->in_use) { LWLockRelease(ReplicationSlotControlLock); + if (conditional) + return false; ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), @@ -659,10 +678,17 @@ retry: * due to an error, and a backend process attempts to reuse the slot. */ if (!IsLogicalLauncher() && IsSlotForConflictCheck(name)) + { + if (conditional) + { + LWLockRelease(ReplicationSlotControlLock); + return false; + } ereport(ERROR, errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("cannot acquire replication slot \"%s\"", name), errdetail("The slot is reserved for conflict detection and can only be acquired by logical replication launcher.")); + } /* * This is the slot we want; check if it's active under some other @@ -714,6 +740,9 @@ retry: goto retry; } + if (conditional) + return false; + ereport(ERROR, (errcode(ERRCODE_OBJECT_IN_USE), errmsg("replication slot \"%s\" is active for PID %d", @@ -759,6 +788,19 @@ retry: : errmsg("acquired physical replication slot \"%s\"", NameStr(s->data.name))); } + return true; +} + +void +ReplicationSlotAcquire(const char *name, bool nowait, bool error_if_invalid) +{ + (void) ReplicationSlotAcquireInternal(name, nowait, error_if_invalid, false); +} + +bool +ReplicationSlotConditionalAcquire(const char *name, bool error_if_invalid) +{ + return ReplicationSlotAcquireInternal(name, true, error_if_invalid, true); } /* @@ -1091,6 +1133,7 @@ ReplicationSlotDropPtr(ReplicationSlot *slot) fsync_fname(tmppath, true); fsync_fname(PG_REPLSLOT_DIR, true); END_CRIT_SECTION(); + } else { @@ -1516,14 +1559,12 @@ void ReplicationSlotsDropDBSlots(Oid dboid) { int i; - bool found_valid_logicalslot; bool dropped = false; if (max_replication_slots + max_repack_replication_slots <= 0) return; restart: - found_valid_logicalslot = false; LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); for (i = 0; i < max_replication_slots + max_repack_replication_slots; i++) { @@ -1541,14 +1582,6 @@ restart: if (!SlotIsLogical(s)) continue; - /* - * Check logical slots on other databases too so we can disable - * logical decoding only if no slots in the cluster. - */ - SpinLockAcquire(&s->mutex); - found_valid_logicalslot |= (s->data.invalidated == RS_INVAL_NONE); - SpinLockRelease(&s->mutex); - /* not our database, skip */ if (s->data.database != dboid) continue; @@ -1610,7 +1643,7 @@ restart: } LWLockRelease(ReplicationSlotControlLock); - if (dropped && !found_valid_logicalslot) + if (dropped) RequestDisableLogicalDecoding(); } @@ -1655,6 +1688,47 @@ CheckLogicalSlotExists(void) return found; } +static bool +check_logical_slot_mode(bool restricted) +{ + bool found = false; + + LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); + for (int i = 0; i < max_replication_slots + max_repack_replication_slots; i++) + { + ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i]; + bool match; + + if (!s->in_use || SlotIsPhysical(s)) + continue; + SpinLockAcquire(&s->mutex); + match = s->data.invalidated == RS_INVAL_NONE && + (restricted ? (!s->data.unrestricted && + s->data.restricted_scope_ready) : + (s->data.unrestricted || !s->data.restricted_scope_ready)); + SpinLockRelease(&s->mutex); + if (match) + { + found = true; + break; + } + } + LWLockRelease(ReplicationSlotControlLock); + return found; +} + +bool +CheckRestrictedLogicalSlotExists(void) +{ + return check_logical_slot_mode(true); +} + +bool +CheckFullWalLogicalSlotExists(void) +{ + return check_logical_slot_mode(false); +} + /* * Check whether the server's configuration supports using replication * slots. @@ -2695,6 +2769,7 @@ RestoreSlotFromDisk(const char *name) TimestampTz now = 0; /* no need to lock here, no concurrent access allowed yet */ + memset(&cp, 0, sizeof(cp)); /* delete temp file if it exists */ sprintf(slotdir, "%s/%s", PG_REPLSLOT_DIR, name); @@ -2763,7 +2838,8 @@ RestoreSlotFromDisk(const char *name) path, cp.magic, SLOT_MAGIC))); /* verify version */ - if (cp.version != SLOT_VERSION) + if (cp.version > SLOT_VERSION || + cp.version < SLOT_MIN_COMPATIBLE_VERSION) ereport(PANIC, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("replication slot file \"%s\" has unsupported version %u", @@ -2915,6 +2991,7 @@ RestoreSlotFromDisk(const char *name) now = GetCurrentTimestamp(); ReplicationSlotSetInactiveSince(slot, now, false); + LogicalSlotScopeRestore(slot); restored = true; break; diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index fdeb6a23d7b..6c361e60c0b 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -13,16 +13,24 @@ #include "postgres.h" #include "access/htup_details.h" +#include "access/table.h" #include "access/xlog_internal.h" #include "access/xlogrecovery.h" #include "access/xlogutils.h" +#include "catalog/pg_class.h" +#include "catalog/pg_publication.h" #include "funcapi.h" +#include "nodes/pg_list.h" #include "replication/logical.h" #include "replication/slot.h" +#include "replication/slotscope.h" #include "replication/slotsync.h" #include "storage/proc.h" +#include "storage/lmgr.h" +#include "utils/array.h" #include "utils/builtins.h" #include "utils/guc.h" +#include "utils/lsyscache.h" #include "utils/pg_lsn.h" /* @@ -48,8 +56,6 @@ static void create_physical_replication_slot(char *name, bool immediately_reserve, bool temporary, XLogRecPtr restart_lsn) { - Assert(!MyReplicationSlot); - /* acquire replication slot, this will check for conflicting names */ ReplicationSlotCreate(name, false, temporary ? RS_TEMPORARY : RS_PERSISTENT, false, @@ -126,28 +132,11 @@ pg_create_physical_replication_slot(PG_FUNCTION_ARGS) * caller's responsibility to ensure it's set to something sensible. */ static void -create_logical_replication_slot(char *name, char *plugin, - bool temporary, bool two_phase, - bool failover, - XLogRecPtr restart_lsn, - bool find_startpoint) +initialize_logical_replication_slot(char *plugin, XLogRecPtr restart_lsn, + bool find_startpoint) { LogicalDecodingContext *ctx = NULL; - Assert(!MyReplicationSlot); - - /* - * Acquire a logical decoding slot, this will check for conflicting names. - * Initially create persistent slot as ephemeral - that allows us to - * nicely handle errors during initialization because it'll get dropped if - * this transaction fails. We'll make it persistent at the end. Temporary - * slots can be created as temporary from beginning as they get dropped on - * error as well. - */ - ReplicationSlotCreate(name, true, - temporary ? RS_TEMPORARY : RS_EPHEMERAL, two_phase, - false, failover, false); - /* * Ensure the logical decoding is enabled before initializing the logical * decoding context. @@ -191,23 +180,86 @@ create_logical_replication_slot(char *name, char *plugin, FreeDecodingContext(ctx); } +static void +create_logical_replication_slot(char *name, char *plugin, + bool temporary, bool two_phase, + bool failover, bool unrestricted, + List *publications, + XLogRecPtr restart_lsn, + bool find_startpoint) +{ + ReplicationSlotCreate(name, true, + temporary ? RS_TEMPORARY : RS_EPHEMERAL, two_phase, + false, failover, false); + + if (unrestricted) + { + LogicalSlotScopeConfigureUnrestricted(MyReplicationSlot); + initialize_logical_replication_slot(plugin, restart_lsn, find_startpoint); + } + else + { + LogicalSlotScopePrepareFromPublications(MyReplicationSlot, publications); + initialize_logical_replication_slot(plugin, restart_lsn, find_startpoint); + LogicalSlotScopeFinishCreate(MyReplicationSlot, true); + } +} + +static void +create_logical_replication_slot_from_publications(char *name, char *plugin, + bool temporary, bool two_phase, + bool failover, bool unrestricted, + List *publications, + XLogRecPtr restart_lsn, + bool find_startpoint) +{ + ReplicationSlotCreate(name, true, + temporary ? RS_TEMPORARY : RS_EPHEMERAL, two_phase, + false, failover, false); + if (unrestricted) + { + LogicalSlotScopeConfigureUnrestricted(MyReplicationSlot); + initialize_logical_replication_slot(plugin, restart_lsn, find_startpoint); + } + else + { + LogicalSlotScopePrepareFromPublicationOids(MyReplicationSlot, publications); + initialize_logical_replication_slot(plugin, restart_lsn, find_startpoint); + LogicalSlotScopeFinishCreate(MyReplicationSlot, true); + } +} + /* * SQL function for creating a new logical replication slot. */ Datum pg_create_logical_replication_slot(PG_FUNCTION_ARGS) { - Name name = PG_GETARG_NAME(0); - Name plugin = PG_GETARG_NAME(1); - bool temporary = PG_GETARG_BOOL(2); - bool two_phase = PG_GETARG_BOOL(3); - bool failover = PG_GETARG_BOOL(4); + Name name; + Name plugin; + bool temporary; + bool two_phase; + bool failover; + bool unrestricted; + bool publications_specified = false; + List *publications = NIL; Datum result; TupleDesc tupdesc; HeapTuple tuple; Datum values[2]; bool nulls[2]; + /* Preserve the former strict behavior for the original arguments. */ + for (int i = 0; i < 5; i++) + if (PG_ARGISNULL(i)) + PG_RETURN_NULL(); + + name = PG_GETARG_NAME(0); + plugin = PG_GETARG_NAME(1); + temporary = PG_GETARG_BOOL(2); + two_phase = PG_GETARG_BOOL(3); + failover = PG_GETARG_BOOL(4); + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) elog(ERROR, "return type must be a row type"); @@ -215,13 +267,48 @@ pg_create_logical_replication_slot(PG_FUNCTION_ARGS) CheckLogicalDecodingRequirements(false); + if (PG_NARGS() > 5 && !PG_ARGISNULL(5)) + { + ArrayType *array = PG_GETARG_ARRAYTYPE_P(5); + Datum *elems; + bool *nulls; + int nelems; + + publications_specified = true; + + deconstruct_array_builtin(array, TEXTOID, &elems, &nulls, &nelems); + for (int i = 0; i < nelems; i++) + { + if (nulls[i]) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("publications must not contain null values"))); + publications = lappend(publications, TextDatumGetCString(elems[i])); + } + if (publications == NIL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("publications must not be empty"))); + } + unrestricted = !publications_specified; + if (!unrestricted && strcmp(NameStr(*plugin), "pgoutput") != 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("restricted logical replication slots currently require pgoutput"))); + if (!unrestricted && IsTransactionBlock()) + ereport(ERROR, + (errcode(ERRCODE_ACTIVE_SQL_TRANSACTION), + errmsg("restricted logical replication slots cannot be created inside a transaction block"))); + create_logical_replication_slot(NameStr(*name), NameStr(*plugin), temporary, two_phase, failover, + unrestricted, publications, InvalidXLogRecPtr, true); + list_free(publications); values[0] = NameGetDatum(&MyReplicationSlot->data.name); values[1] = LSNGetDatum(MyReplicationSlot->data.confirmed_flush); @@ -231,10 +318,15 @@ pg_create_logical_replication_slot(PG_FUNCTION_ARGS) tuple = heap_form_tuple(tupdesc, values, nulls); result = HeapTupleGetDatum(tuple); - /* ok, slot is now fully created, mark it as persistent if needed */ + /* + * Restricted creation is finalized by its transaction callback, after the + * mapping catalog changes commit. Keep that slot acquired and ephemeral + * until then so abort can remove it safely. + */ if (!temporary) ReplicationSlotPersist(); - ReplicationSlotRelease(); + if (unrestricted) + ReplicationSlotRelease(); PG_RETURN_DATUM(result); } @@ -264,7 +356,7 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) Datum pg_get_replication_slots(PG_FUNCTION_ARGS) { -#define PG_GET_REPLICATION_SLOTS_COLS 21 +#define PG_GET_REPLICATION_SLOTS_COLS 26 ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; XLogRecPtr currlsn; int slotno; @@ -279,6 +371,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS) currlsn = GetXLogWriteRecPtr(); + LWLockAcquire(ReplicationSlotAllocationLock, LW_SHARED); LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); for (slotno = 0; slotno < max_replication_slots + max_repack_replication_slots; slotno++) { @@ -477,6 +570,52 @@ pg_get_replication_slots(PG_FUNCTION_ARGS) else values[i++] = CStringGetTextDatum(SlotSyncSkipReasonNames[slot_contents.slotsync_skip_reason]); + if (SlotIsPhysical(&slot_contents)) + nulls[i++] = true; + else + values[i++] = BoolGetDatum(slot_contents.data.unrestricted); + + if (SlotIsPhysical(&slot_contents) || slot_contents.data.unrestricted) + nulls[i++] = true; + else + { + List *publications; + Datum *oids; + int j = 0; + + /* + * ReplicationSlotAllocationLock keeps the slot directory stable. + * Do not hold ReplicationSlotControlLock across sidecar file I/O. + */ + LWLockRelease(ReplicationSlotControlLock); + publications = LogicalSlotScopeGetPublications(slot); + LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); + oids = palloc_array(Datum, list_length(publications)); + + foreach_oid(pubid, publications) + oids[j++] = ObjectIdGetDatum(pubid); + values[i++] = PointerGetDatum(construct_array_builtin(oids, j, OIDOID)); + list_free(publications); + pfree(oids); + } + + if (SlotIsPhysical(&slot_contents)) + nulls[i++] = true; + else + values[i++] = BoolGetDatum(slot_contents.data.restricted_scope_ready); + + if (SlotIsPhysical(&slot_contents) || slot_contents.data.unrestricted) + nulls[i++] = true; + else + values[i++] = Int64GetDatum( + (int64) slot_contents.data.restricted_scope_incarnation); + + if (SlotIsPhysical(&slot_contents) || slot_contents.data.unrestricted || + !slot_contents.data.restricted_scope_ready) + nulls[i++] = true; + else + values[i++] = LSNGetDatum(slot_contents.data.restricted_scope_ready_lsn); + Assert(i == PG_GET_REPLICATION_SLOTS_COLS); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, @@ -484,6 +623,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS) } LWLockRelease(ReplicationSlotControlLock); + LWLockRelease(ReplicationSlotAllocationLock); return (Datum) 0; } @@ -553,8 +693,6 @@ pg_replication_slot_advance(PG_FUNCTION_ARGS) HeapTuple tuple; Datum result; - Assert(!MyReplicationSlot); - CheckSlotPermissions(); if (!XLogRecPtrIsValid(moveto)) @@ -651,6 +789,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) Datum result; TupleDesc tupdesc; HeapTuple tuple; + List *copy_publications = NIL; if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) elog(ERROR, "return type must be a row type"); @@ -662,6 +801,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) else CheckSlotRequirements(false); + LWLockAcquire(ReplicationSlotAllocationLock, LW_SHARED); LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); /* @@ -691,6 +831,12 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) } LWLockRelease(ReplicationSlotControlLock); + if (src != NULL && SlotIsLogical(&first_slot_contents) && + !first_slot_contents.data.unrestricted) + { + copy_publications = LogicalSlotScopeGetPublications(src); + } + LWLockRelease(ReplicationSlotAllocationLock); if (src == NULL) ereport(ERROR, @@ -725,6 +871,12 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) errmsg("cannot copy invalidated replication slot \"%s\"", NameStr(*src_name))); + if (logical_slot && !first_slot_contents.data.unrestricted && + IsTransactionBlock()) + ereport(ERROR, + (errcode(ERRCODE_ACTIVE_SQL_TRANSACTION), + errmsg("restricted logical replication slots cannot be copied inside a transaction block"))); + /* Overwrite params from optional arguments */ if (PG_NARGS() >= 3) temporary = PG_GETARG_BOOL(2); @@ -755,13 +907,15 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) * on the promoted standby because the slot retains the restart_lsn * and confirmed_flush_lsn that are much later than expected. */ - create_logical_replication_slot(NameStr(*dst_name), - plugin, - temporary, - false, - false, - src_restart_lsn, - false); + create_logical_replication_slot_from_publications(NameStr(*dst_name), + plugin, + temporary, + false, + false, + first_slot_contents.data.unrestricted, + copy_publications, + src_restart_lsn, + false); } else create_physical_replication_slot(NameStr(*dst_name), @@ -885,7 +1039,9 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) tuple = heap_form_tuple(tupdesc, values, nulls); result = HeapTupleGetDatum(tuple); - ReplicationSlotRelease(); + if (!logical_slot || first_slot_contents.data.unrestricted) + ReplicationSlotRelease(); + list_free(copy_publications); PG_RETURN_DATUM(result); } diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index 61dc6a5588b..07120cf0ee2 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -441,7 +441,8 @@ WalReceiverMain(const void *startup_data, size_t startup_data_len) "pg_walreceiver_%lld", (long long int) walrcv_get_backend_pid(wrconn)); - walrcv_create_slot(wrconn, slotname, true, false, false, 0, NULL); + walrcv_create_slot(wrconn, slotname, true, false, false, true, NIL, + 0, NULL); SpinLockAcquire(&walrcv->mutex); strlcpy(walrcv->slotname, slotname, NAMEDATALEN); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index c65dd324325..15a259c748a 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -54,6 +54,7 @@ #include "access/timeline.h" #include "access/transam.h" #include "access/twophase.h" +#include "access/table.h" #include "access/xact.h" #include "access/xlog_internal.h" #include "access/xlogreader.h" @@ -62,6 +63,7 @@ #include "backup/basebackup.h" #include "backup/basebackup_incremental.h" #include "catalog/pg_authid.h" +#include "catalog/pg_publication.h" #include "catalog/pg_type.h" #include "commands/defrem.h" #include "funcapi.h" @@ -76,6 +78,7 @@ #include "replication/logical.h" #include "replication/slotsync.h" #include "replication/slot.h" +#include "replication/slotscope.h" #include "replication/snapbuild.h" #include "replication/syncrep.h" #include "replication/walreceiver.h" @@ -85,6 +88,7 @@ #include "storage/aio_subsys.h" #include "storage/fd.h" #include "storage/ipc.h" +#include "storage/lmgr.h" #include "storage/pmsignal.h" #include "storage/proc.h" #include "storage/procarray.h" @@ -101,6 +105,7 @@ #include "utils/ps_status.h" #include "utils/timeout.h" #include "utils/timestamp.h" +#include "utils/varlena.h" #include "utils/wait_event.h" /* Minimum interval used by walsender for stats flushes, in ms */ @@ -1184,17 +1189,20 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req /* * Process extra options given to CREATE_REPLICATION_SLOT. */ + static void parseCreateReplSlotOptions(CreateReplicationSlotCmd *cmd, bool *reserve_wal, CRSSnapshotAction *snapshot_action, - bool *two_phase, bool *failover) + bool *two_phase, bool *failover, + bool *unrestricted, List **publications) { ListCell *lc; bool snapshot_action_given = false; bool reserve_wal_given = false; bool two_phase_given = false; bool failover_given = false; + bool publication_names_given = false; /* Parse options */ foreach(lc, cmd->options) @@ -1253,9 +1261,28 @@ parseCreateReplSlotOptions(CreateReplicationSlotCmd *cmd, failover_given = true; *failover = defGetBoolean(defel); } + else if (strcmp(defel->defname, "publication_names") == 0) + { + if (publication_names_given || cmd->kind != REPLICATION_KIND_LOGICAL) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + publication_names_given = true; + if (!SplitIdentifierString(pstrdup(defGetString(defel)), ',', + publications)) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("invalid publication name list"))); + } else elog(ERROR, "unrecognized option: %s", defel->defname); } + + if (publication_names_given && *publications == NIL) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("publication_names must not be empty"))); + *unrestricted = !publication_names_given; } /* @@ -1270,6 +1297,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) bool reserve_wal = false; bool two_phase = false; bool failover = false; + bool unrestricted = true; + List *publications = NIL; CRSSnapshotAction snapshot_action = CRS_EXPORT_SNAPSHOT; DestReceiver *dest; TupOutputState *tstate; @@ -1277,10 +1306,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) Datum values[4]; bool nulls[4] = {0}; - Assert(!MyReplicationSlot); - parseCreateReplSlotOptions(cmd, &reserve_wal, &snapshot_action, &two_phase, - &failover); + &failover, &unrestricted, &publications); if (cmd->kind == REPLICATION_KIND_PHYSICAL) { @@ -1305,6 +1332,10 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) bool need_full_snapshot = false; Assert(cmd->kind == REPLICATION_KIND_LOGICAL); + if (!unrestricted && strcmp(cmd->plugin, "pgoutput") != 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("restricted logical replication slots currently require pgoutput"))); CheckLogicalDecodingRequirements(false); @@ -1371,6 +1402,19 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) * Ensure the logical decoding is enabled before initializing the * logical decoding context. */ + if (!unrestricted) + { + /* Publication catalog access needs a transaction. */ + if (snapshot_action != CRS_USE_SNAPSHOT) + StartTransactionCommand(); + LogicalSlotScopePrepareFromPublications(MyReplicationSlot, + publications); + if (snapshot_action != CRS_USE_SNAPSHOT) + CommitTransactionCommand(); + } + else + LogicalSlotScopeConfigureUnrestricted(MyReplicationSlot); + list_free(publications); EnsureLogicalDecodingEnabled(); /* See the comment in create_logical_replication_slot() */ @@ -1397,6 +1441,20 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) /* build initial snapshot, might take a while */ DecodingContextFindStartpoint(ctx); + /* + * Install transactional relation mappings only after logical decoding + * has established the slot's start point. CreateInitDecodingContext() + * rejects a transaction that has already performed catalog writes. + */ + if (!unrestricted) + { + if (snapshot_action != CRS_USE_SNAPSHOT) + StartTransactionCommand(); + LogicalSlotScopeFinishCreate(MyReplicationSlot, false); + if (snapshot_action != CRS_USE_SNAPSHOT) + CommitTransactionCommand(); + } + /* * Export or use the snapshot if we've been asked to do so. * @@ -1537,8 +1595,6 @@ StartLogicalReplication(StartReplicationCmd *cmd) /* make sure that our requirements are still fulfilled */ CheckLogicalDecodingRequirements(false); - Assert(!MyReplicationSlot); - ReplicationSlotAcquire(cmd->slotname, true, true); /* diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c index 3d8c9bdebd5..8695829900d 100644 --- a/src/backend/utils/init/postinit.c +++ b/src/backend/utils/init/postinit.c @@ -42,6 +42,7 @@ #include "postmaster/autovacuum.h" #include "postmaster/postmaster.h" #include "replication/slot.h" +#include "replication/slotscope.h" #include "replication/slotsync.h" #include "replication/walsender.h" #include "storage/aio_subsys.h" @@ -1239,6 +1240,9 @@ InitPostgres(const char *in_dbname, Oid dboid, CheckMyDatabase(dbname, am_superuser, (flags & INIT_PG_OVERRIDE_ALLOW_CONNS) != 0); + if (!bootstrap) + LogicalSlotScopeReconcileDatabase(); + /* * Now process any command-line switches and any additional GUC variable * settings passed in the startup packet. We couldn't do this before diff --git a/src/backend/utils/misc/guc_parameters.dat b/src/backend/utils/misc/guc_parameters.dat index 3c5e16ad1e7..d20a6452669 100644 --- a/src/backend/utils/misc/guc_parameters.dat +++ b/src/backend/utils/misc/guc_parameters.dat @@ -2579,6 +2579,15 @@ assign_hook => 'assign_restrict_nonsystem_relation_kind', }, +{ name => 'restricted_wal_level', type => 'enum', context => 'PGC_INTERNAL', group => 'PRESET_OPTIONS', + short_desc => 'Shows the WAL level required by restricted logical slots.', + flags => 'GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE', + variable => 'restricted_wal_level', + boot_val => 'WAL_LEVEL_REPLICA', + options => 'wal_level_options', + show_hook => 'show_restricted_wal_level', +}, + # Not for general use --- used by SET ROLE { name => 'role', type => 'string', context => 'PGC_USERSET', group => 'UNGROUPED', short_desc => 'Sets the current role.', diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index c6d9b2a6f89..6f6e2a879f5 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -649,6 +649,7 @@ static int shared_memory_size_in_huge_pages; static int wal_block_size; static int num_os_semaphores; static int effective_wal_level = WAL_LEVEL_REPLICA; +static int restricted_wal_level = WAL_LEVEL_REPLICA; static bool integer_datetimes; #ifdef USE_ASSERT_CHECKING diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index feed88f9854..18beaf963b3 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -5136,6 +5136,7 @@ getSubscriptions(Archive *fout) int i_suboriginremotelsn; int i_subenabled; int i_subfailover; + int i_subunrestricted; int i_subretaindeadtuples; int i_submaxretention; int i, @@ -5215,6 +5216,13 @@ getSubscriptions(Archive *fout) appendPQExpBufferStr(query, " false AS subfailover,\n"); + if (fout->remoteVersion >= 200000) + appendPQExpBufferStr(query, + " s.subunrestricted,\n"); + else + appendPQExpBufferStr(query, + " true AS subunrestricted,\n"); + if (fout->remoteVersion >= 190000) appendPQExpBufferStr(query, " s.subretaindeadtuples,\n"); @@ -5277,6 +5285,7 @@ getSubscriptions(Archive *fout) i_subpasswordrequired = PQfnumber(res, "subpasswordrequired"); i_subrunasowner = PQfnumber(res, "subrunasowner"); i_subfailover = PQfnumber(res, "subfailover"); + i_subunrestricted = PQfnumber(res, "subunrestricted"); i_subretaindeadtuples = PQfnumber(res, "subretaindeadtuples"); i_submaxretention = PQfnumber(res, "submaxretention"); i_subservername = PQfnumber(res, "subservername"); @@ -5318,6 +5327,8 @@ getSubscriptions(Archive *fout) (strcmp(PQgetvalue(res, i, i_subrunasowner), "t") == 0); subinfo[i].subfailover = (strcmp(PQgetvalue(res, i, i_subfailover), "t") == 0); + subinfo[i].subunrestricted = + (strcmp(PQgetvalue(res, i, i_subunrestricted), "t") == 0); subinfo[i].subretaindeadtuples = (strcmp(PQgetvalue(res, i, i_subretaindeadtuples), "t") == 0); subinfo[i].submaxretention = @@ -5593,6 +5604,9 @@ dumpSubscription(Archive *fout, const SubscriptionInfo *subinfo) if (subinfo->subfailover) appendPQExpBufferStr(query, ", failover = true"); + if (!subinfo->subunrestricted) + appendPQExpBufferStr(query, ", unrestricted_slot = false"); + if (subinfo->subretaindeadtuples) appendPQExpBufferStr(query, ", retain_dead_tuples = true"); diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index e6eefa98460..7b01224b8ab 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -720,6 +720,7 @@ typedef struct _SubscriptionInfo bool subpasswordrequired; bool subrunasowner; bool subfailover; + bool subunrestricted; bool subretaindeadtuples; int submaxretention; char *subservername; diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index 9258948b583..78adb4d2afa 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -3317,9 +3317,10 @@ my %tests = ( create_order => 50, create_sql => 'CREATE SUBSCRIPTION sub3 CONNECTION \'dbname=doesnotexist\' PUBLICATION pub1 - WITH (connect = false, origin = any, streaming = on);', + WITH (connect = false, origin = any, streaming = on, + unrestricted_slot = false);', regexp => qr/^ - \QCREATE SUBSCRIPTION sub3 CONNECTION 'dbname=doesnotexist' PUBLICATION pub1 WITH (connect = false, slot_name = 'sub3', streaming = on);\E + \QCREATE SUBSCRIPTION sub3 CONNECTION 'dbname=doesnotexist' PUBLICATION pub1 WITH (connect = false, slot_name = 'sub3', streaming = on, unrestricted_slot = false);\E /xm, like => { %full_runs, section_post_data => 1, }, unlike => { diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 338d68d7424..8141ea52506 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -110,6 +110,7 @@ typedef enum RecoveryState extern PGDLLIMPORT int wal_level; extern PGDLLIMPORT bool XLogLogicalInfo; +extern PGDLLIMPORT bool XLogRestrictedInfo; /* Is WAL archiving enabled (always or only while server is running normally)? */ #define XLogArchivingActive() \ @@ -147,8 +148,12 @@ extern PGDLLIMPORT bool XLogLogicalInfo; * change until an XID is assigned to the transaction. In other words, it * ensures that the same result is returned within an XID-assigned transaction. */ -#define XLogLogicalInfoActive() \ +#define XLogFullLogicalInfoActive() \ (wal_level >= WAL_LEVEL_LOGICAL || XLogLogicalInfo) +#define XLogRestrictedInfoActive() \ + (wal_level >= WAL_LEVEL_LOGICAL || XLogRestrictedInfo) +#define XLogLogicalInfoActive() \ + (XLogFullLogicalInfoActive() || XLogRestrictedInfoActive()) #ifdef WAL_DEBUG extern PGDLLIMPORT bool XLOG_DEBUG; diff --git a/src/include/catalog/Makefile b/src/include/catalog/Makefile index bab57372b88..41c613aa5b2 100644 --- a/src/include/catalog/Makefile +++ b/src/include/catalog/Makefile @@ -80,6 +80,7 @@ CATALOG_HEADERS := \ pg_publication.h \ pg_publication_namespace.h \ pg_publication_rel.h \ + pg_restricted_slot_relation.h \ pg_subscription.h \ pg_subscription_rel.h \ pg_propgraph_element.h \ diff --git a/src/include/catalog/meson.build b/src/include/catalog/meson.build index fa836e4ee25..7dd2e5a7af8 100644 --- a/src/include/catalog/meson.build +++ b/src/include/catalog/meson.build @@ -67,6 +67,7 @@ catalog_headers = [ 'pg_publication.h', 'pg_publication_namespace.h', 'pg_publication_rel.h', + 'pg_restricted_slot_relation.h', 'pg_subscription.h', 'pg_subscription_rel.h', 'pg_propgraph_element.h', diff --git a/src/include/catalog/pg_class.h b/src/include/catalog/pg_class.h index c4af599dc90..1484221a278 100644 --- a/src/include/catalog/pg_class.h +++ b/src/include/catalog/pg_class.h @@ -124,6 +124,9 @@ CATALOG(pg_class,1259,RelationRelationId) BKI_BOOTSTRAP BKI_ROWTYPE_OID(83,Relat /* is relation a partition? */ bool relispartition BKI_DEFAULT(f); + /* has conservative mappings for restricted logical slots */ + bool relhasrestrictedslots BKI_DEFAULT(f); + /* link to original rel during table rewrite; otherwise 0 */ Oid relrewrite BKI_DEFAULT(0) BKI_LOOKUP_OPT(pg_class); diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 66c3c9a04cf..1d5252fb101 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11733,18 +11733,18 @@ proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f', proretset => 't', provolatile => 's', prorettype => 'record', proargtypes => '', - proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,pg_lsn,timestamptz,bool,text,bool,bool,text}', - proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', - proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,two_phase_at,inactive_since,conflicting,invalidation_reason,failover,synced,slotsync_skip_reason}', + proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,pg_lsn,timestamptz,bool,text,bool,bool,text,bool,_oid,bool,int8,pg_lsn}', + proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,two_phase_at,inactive_since,conflicting,invalidation_reason,failover,synced,slotsync_skip_reason,unrestricted,publication_oids,restricted_scope_ready,restricted_scope_incarnation,restricted_scope_ready_lsn}', prosrc => 'pg_get_replication_slots' }, { oid => '3786', descr => 'set up a logical replication slot', proname => 'pg_create_logical_replication_slot', provolatile => 'v', - proparallel => 'u', prorettype => 'record', - proargtypes => 'name name bool bool bool', - proallargtypes => '{name,name,bool,bool,bool,name,pg_lsn}', - proargmodes => '{i,i,i,i,i,o,o}', - proargnames => '{slot_name,plugin,temporary,twophase,failover,slot_name,lsn}', - proargdefaults => '{false,false,false}', + proparallel => 'u', proisstrict => 'f', prorettype => 'record', + proargtypes => 'name name bool bool bool _text', + proallargtypes => '{name,name,bool,bool,bool,_text,name,pg_lsn}', + proargmodes => '{i,i,i,i,i,i,o,o}', + proargnames => '{slot_name,plugin,temporary,twophase,failover,publications,slot_name,lsn}', + proargdefaults => '{false,false,false,NULL}', prosrc => 'pg_create_logical_replication_slot' }, { oid => '4222', descr => 'copy a logical replication slot, changing temporality and plugin', diff --git a/src/include/catalog/pg_restricted_slot_relation.h b/src/include/catalog/pg_restricted_slot_relation.h new file mode 100644 index 00000000000..9e0bd547251 --- /dev/null +++ b/src/include/catalog/pg_restricted_slot_relation.h @@ -0,0 +1,36 @@ +/*------------------------------------------------------------------------- + * + * pg_restricted_slot_relation.h + * conservative relation mappings for restricted logical slots + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/catalog/pg_restricted_slot_relation.h + * + *------------------------------------------------------------------------- + */ +#ifndef PG_RESTRICTED_SLOT_RELATION_H +#define PG_RESTRICTED_SLOT_RELATION_H + +#include "catalog/genbki.h" +#include "catalog/pg_restricted_slot_relation_d.h" /* IWYU pragma: export */ + +BEGIN_CATALOG_STRUCT + +CATALOG(pg_restricted_slot_relation,8050,RestrictedSlotRelationRelationId) +{ + NameData rsrslotname; + Oid rsrrelid BKI_LOOKUP_OPT(pg_class); + int64 rsrincarnation; +} FormData_pg_restricted_slot_relation; + +END_CATALOG_STRUCT + +typedef FormData_pg_restricted_slot_relation *Form_pg_restricted_slot_relation; + +DECLARE_UNIQUE_INDEX_PKEY(pg_restricted_slot_relation_slot_rel_index, 8051, RestrictedSlotRelationSlotRelIndexId, pg_restricted_slot_relation, btree(rsrslotname name_ops, rsrrelid oid_ops, rsrincarnation int8_ops)); +DECLARE_INDEX(pg_restricted_slot_relation_rel_index, 8052, RestrictedSlotRelationRelIndexId, pg_restricted_slot_relation, btree(rsrrelid oid_ops)); +DECLARE_INDEX(pg_restricted_slot_relation_slot_index, 8053, RestrictedSlotRelationSlotIndexId, pg_restricted_slot_relation, btree(rsrslotname name_ops)); + +#endif /* PG_RESTRICTED_SLOT_RELATION_H */ diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h index d2781a0b837..fde1b81eaf8 100644 --- a/src/include/catalog/pg_subscription.h +++ b/src/include/catalog/pg_subscription.h @@ -80,6 +80,9 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW * slots) in the upstream database are enabled * to be synchronized to the standbys. */ + bool subunrestricted BKI_DEFAULT(t); /* True if the associated + * replication slots are unrestricted. */ + bool subretaindeadtuples; /* True if dead tuples useful for * conflict detection are retained */ @@ -163,6 +166,8 @@ typedef struct Subscription * (i.e. the main slot and the table sync * slots) in the upstream database are enabled * to be synchronized to the standbys. */ + bool unrestricted; /* True if the associated replication slots are + * unrestricted. */ bool retaindeadtuples; /* True if dead tuples useful for conflict * detection are retained */ int32 maxretention; /* The maximum duration (in milliseconds) for diff --git a/src/include/replication/logicalctl.h b/src/include/replication/logicalctl.h index 0bc1302f130..23fdfe78cfb 100644 --- a/src/include/replication/logicalctl.h +++ b/src/include/replication/logicalctl.h @@ -19,8 +19,11 @@ extern void InitializeProcessXLogLogicalInfo(void); extern bool ProcessBarrierUpdateXLogLogicalInfo(void); extern bool IsLogicalDecodingEnabled(void); extern bool IsXLogLogicalInfoEnabled(void); +extern bool IsXLogRestrictedInfoEnabled(void); extern void AtEOXact_LogicalCtl(void); extern void EnsureLogicalDecodingEnabled(void); +extern void EnsureFullLogicalWAL(void); +extern void EnsureRestrictedLogicalWAL(void); extern void EnableLogicalDecoding(void); extern void RequestDisableLogicalDecoding(void); extern void DisableLogicalDecodingIfNecessary(void); diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index 9b29444cbca..49bf1ee6a1b 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -159,6 +159,18 @@ typedef struct ReplicationSlotPersistentData * for logical slots on the primary server. */ bool failover; + + /* May this logical slot decode every eligible relation? */ + bool unrestricted; + + /* Are the transactional writer mappings for this restricted slot ready? */ + bool restricted_scope_ready; + + /* Identifies this restricted slot creation across name reuse. */ + uint64 restricted_scope_incarnation; + + /* WAL boundary after the transactional scope mappings committed. */ + XLogRecPtr restricted_scope_ready_lsn; } ReplicationSlotPersistentData; /* @@ -341,6 +353,8 @@ extern void ReplicationSlotAlter(const char *name, const bool *failover, extern void ReplicationSlotAcquire(const char *name, bool nowait, bool error_if_invalid); +extern bool ReplicationSlotConditionalAcquire(const char *name, + bool error_if_invalid); extern void ReplicationSlotRelease(void); extern void ReplicationSlotCleanup(bool synced_only); extern void ReplicationSlotSave(void); @@ -360,6 +374,8 @@ extern void ReplicationSlotsComputeRequiredLSN(void); extern XLogRecPtr ReplicationSlotsComputeLogicalRestartLSN(void); extern bool ReplicationSlotsCountDBSlots(Oid dboid, int *nslots, int *nactive); extern bool CheckLogicalSlotExists(void); +extern bool CheckRestrictedLogicalSlotExists(void); +extern bool CheckFullWalLogicalSlotExists(void); extern void ReplicationSlotsDropDBSlots(Oid dboid); extern bool InvalidateObsoleteReplicationSlots(uint32 possible_causes, XLogSegNo oldestSegno, diff --git a/src/include/replication/slotscope.h b/src/include/replication/slotscope.h new file mode 100644 index 00000000000..eae5531844b --- /dev/null +++ b/src/include/replication/slotscope.h @@ -0,0 +1,49 @@ +/*------------------------------------------------------------------------- + * + * slotscope.h + * Logical replication slot scope management. + * + * + * Portions Copyright (c) 2026, PostgreSQL Global Development Group + * + * src/include/replication/slotscope.h + * + *------------------------------------------------------------------------- + */ +#ifndef SLOTSCOPE_H +#define SLOTSCOPE_H + +#include "nodes/pg_list.h" + +struct RelationData; +struct ReplicationSlot; + +extern void LogicalSlotScopeInitialize(void); +extern void LogicalSlotScopeReconcileDatabase(void); +extern void LogicalSlotScopeConfigureUnrestricted(struct ReplicationSlot *slot); + +/* Caller must provide a transaction environment. */ +extern void LogicalSlotScopePrepareFromPublications( + struct ReplicationSlot *slot, List *pubnames); +extern void LogicalSlotScopePrepareFromPublicationOids( + struct ReplicationSlot *slot, List *publications); +extern void LogicalSlotScopeFinishCreate(struct ReplicationSlot *slot, + bool finalize_at_xact_end); +extern void LogicalSlotScopeRestorePublications(struct ReplicationSlot *slot, + List *publications, + uint64 incarnation, + XLogRecPtr ready_lsn); +extern void LogicalSlotScopeRestore(struct ReplicationSlot *slot); +extern List *LogicalSlotScopeGetPublications(struct ReplicationSlot *slot); +extern void LogicalSlotScopePublicationAddRelations(Oid pubid, List *relations); +extern void LogicalSlotScopeNoteToastCreation(Oid owner, Oid toastrelid); +extern void LogicalSlotScopeRelationDrop(Oid relid); +extern void LogicalSlotScopeCleanup(void); +extern bool LogicalSlotScopePublicationsContain(struct ReplicationSlot *slot, + List *pubnames); + +extern void CheckLogicalSlotScopeHierarchyChange(Oid childrelid, + Oid parentrelid); +extern bool RelationNeedsLogicalTupleWAL(struct RelationData *relation); + +#endif /* SLOTSCOPE_H */ diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h index 5cfe90f9989..515a9df9a6d 100644 --- a/src/include/replication/walreceiver.h +++ b/src/include/replication/walreceiver.h @@ -367,12 +367,14 @@ typedef void (*walrcv_send_fn) (WalReceiverConn *conn, * slot, or NULL for a physical slot. */ typedef char *(*walrcv_create_slot_fn) (WalReceiverConn *conn, - const char *slotname, - bool temporary, - bool two_phase, - bool failover, - CRSSnapshotAction snapshot_action, - XLogRecPtr *lsn); + const char *slotname, + bool temporary, + bool two_phase, + bool failover, + bool unrestricted, + List *publications, + CRSSnapshotAction snapshot_action, + XLogRecPtr *lsn); /* * walrcv_alter_slot_fn @@ -460,8 +462,8 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions; WalReceiverFunctions->walrcv_receive(conn, buffer, wait_fd) #define walrcv_send(conn, buffer, nbytes) \ WalReceiverFunctions->walrcv_send(conn, buffer, nbytes) -#define walrcv_create_slot(conn, slotname, temporary, two_phase, failover, snapshot_action, lsn) \ - WalReceiverFunctions->walrcv_create_slot(conn, slotname, temporary, two_phase, failover, snapshot_action, lsn) +#define walrcv_create_slot(conn, slotname, temporary, two_phase, failover, unrestricted, publications, snapshot_action, lsn) \ + WalReceiverFunctions->walrcv_create_slot(conn, slotname, temporary, two_phase, failover, unrestricted, publications, snapshot_action, lsn) #define walrcv_alter_slot(conn, slotname, failover, two_phase) \ WalReceiverFunctions->walrcv_alter_slot(conn, slotname, failover, two_phase) #define walrcv_get_backend_pid(conn) \ diff --git a/src/include/utils/guc_hooks.h b/src/include/utils/guc_hooks.h index 6a76f8d5ed6..8d26a0e7e51 100644 --- a/src/include/utils/guc_hooks.h +++ b/src/include/utils/guc_hooks.h @@ -62,6 +62,7 @@ extern void assign_default_text_search_config(const char *newval, void *extra); extern bool check_default_with_oids(bool *newval, void **extra, GucSource source); extern const char *show_effective_wal_level(void); +extern const char *show_restricted_wal_level(void); extern bool check_huge_page_size(int *newval, void **extra, GucSource source); extern void assign_io_method(int newval, void *extra); extern bool check_io_max_concurrency(int *newval, void **extra, GucSource source); diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h index 41ab4586c6b..1834ef448fd 100644 --- a/src/include/utils/rel.h +++ b/src/include/utils/rel.h @@ -709,8 +709,9 @@ RelationCloseSmgr(Relation relation) /* * RelationIsLogicallyLogged - * True if we need to log enough information to extract the data from the - * WAL stream. + * True if the relation is eligible for logical logging. This does not + * consider whether logical WAL is active or whether any replication slot + * requires changes for this particular relation. * * We don't log information for unlogged tables (since they don't WAL log * anyway), for foreign tables (since they don't WAL log, either), @@ -719,12 +720,24 @@ RelationCloseSmgr(Relation relation) * log information for user defined catalog tables since they presumably are * interesting to the user... */ -#define RelationIsLogicallyLogged(relation) \ - (XLogLogicalInfoActive() && \ - RelationNeedsWAL(relation) && \ - (relation)->rd_rel->relkind != RELKIND_FOREIGN_TABLE && \ +#define RelationCanBeLogicallyLogged(relation) \ + (RelationNeedsWAL(relation) && \ + (relation)->rd_rel->relkind != RELKIND_FOREIGN_TABLE && \ !IsCatalogRelation(relation)) +/* + * RelationIsLogicallyLogged + * True if we need to log enough information to extract the data from the + * WAL stream. + * + * This has included the check for RelationCanBeLogicallyLogged(), and also + * checks whether logical WAL is required for this relation. + */ +#define RelationIsLogicallyLogged(relation) \ + RelationNeedsLogicalTupleWAL(relation) + +extern bool RelationNeedsLogicalTupleWAL(Relation relation); + /* routines in utils/cache/relcache.c */ extern void RelationIncrementReferenceCount(Relation rel); extern void RelationDecrementReferenceCount(Relation rel); diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build index 39ec8c4946d..5162561bd27 100644 --- a/src/test/recovery/meson.build +++ b/src/test/recovery/meson.build @@ -64,6 +64,7 @@ tests += { 't/053_standby_login_event_trigger.pl', 't/054_unlogged_sequence_promotion.pl', 't/055_cascade_reconnect.pl', + 't/056_logical_slot_scope.pl', ], }, } diff --git a/src/test/recovery/t/040_standby_failover_slots_sync.pl b/src/test/recovery/t/040_standby_failover_slots_sync.pl index f8922aaa1a2..90afd993b40 100644 --- a/src/test/recovery/t/040_standby_failover_slots_sync.pl +++ b/src/test/recovery/t/040_standby_failover_slots_sync.pl @@ -210,6 +210,153 @@ is( $standby1->safe_psql( "t", 'synchronized slot has got its own inactive_since'); +################################################## +# Test synchronization and replacement of a restricted failover slot. +################################################## + +$primary->safe_psql( + 'postgres', q{ + CREATE TABLE restricted_sync_tab1 (a int); + CREATE TABLE restricted_sync_tab2 (a int); + CREATE PUBLICATION restricted_sync_pub1 FOR TABLE restricted_sync_tab1; + CREATE PUBLICATION restricted_sync_pub2 FOR TABLE restricted_sync_tab2; +}); +$primary->safe_psql( + 'postgres', q{ + SELECT pg_create_logical_replication_slot( + 'restricted_sync_slot', 'pgoutput', failover => true, + publications => ARRAY['restricted_sync_pub1']); +}); + +my $restricted_incarnation1 = $primary->safe_psql( + 'postgres', q{ + SELECT restricted_scope_incarnation + FROM pg_replication_slots + WHERE slot_name = 'restricted_sync_slot'; +}); +my $restricted_ready_lsn1 = $primary->safe_psql( + 'postgres', q{ + SELECT restricted_scope_ready_lsn + FROM pg_replication_slots + WHERE slot_name = 'restricted_sync_slot'; +}); + +$primary->wait_for_replay_catchup($standby1); +$standby1->safe_psql('postgres', "SELECT pg_sync_replication_slots();"); + +# A ready restricted failover slot should retain its scope metadata on standby. +is( $standby1->safe_psql( + 'postgres', qq{ + SELECT synced AND NOT temporary AND NOT unrestricted AND + restricted_scope_ready AND + restricted_scope_incarnation = $restricted_incarnation1 AND + restricted_scope_ready_lsn = '$restricted_ready_lsn1' AND + publication_oids = ARRAY[(SELECT oid FROM pg_publication + WHERE pubname = 'restricted_sync_pub1')] + FROM pg_replication_slots + WHERE slot_name = 'restricted_sync_slot'; + }), + "t", + 'restricted failover slot scope metadata is synchronized'); + +# Replayed mappings should enable selective WAL for the restricted relation. +is( $standby1->safe_psql( + 'postgres', q{ + SELECT c.relhasrestrictedslots AND EXISTS ( + SELECT 1 FROM pg_restricted_slot_relation AS m + WHERE m.rsrslotname = 'restricted_sync_slot' + AND m.rsrrelid = c.oid) + FROM pg_class AS c + WHERE c.oid = 'restricted_sync_tab1'::regclass; + }), + "t", + 'restricted failover slot relation mapping is available on standby'); + +$primary->safe_psql('postgres', + "SELECT pg_drop_replication_slot('restricted_sync_slot');"); +$primary->safe_psql( + 'postgres', q{ + SELECT pg_create_logical_replication_slot( + 'restricted_sync_slot', 'pgoutput', failover => true, + publications => ARRAY['restricted_sync_pub2']); +}); + +my $restricted_incarnation2 = $primary->safe_psql( + 'postgres', q{ + SELECT restricted_scope_incarnation + FROM pg_replication_slots + WHERE slot_name = 'restricted_sync_slot'; +}); +my $restricted_ready_lsn2 = $primary->safe_psql( + 'postgres', q{ + SELECT restricted_scope_ready_lsn + FROM pg_replication_slots + WHERE slot_name = 'restricted_sync_slot'; +}); + +$primary->wait_for_replay_catchup($standby1); +$standby1->safe_psql('postgres', "SELECT pg_sync_replication_slots();"); + +# A changed incarnation should replace the existing synchronized slot. +is( $standby1->safe_psql( + 'postgres', qq{ + SELECT restricted_scope_incarnation = $restricted_incarnation2 AND + restricted_scope_incarnation <> $restricted_incarnation1 AND + restricted_scope_ready_lsn = '$restricted_ready_lsn2' AND + publication_oids = ARRAY[(SELECT oid FROM pg_publication + WHERE pubname = 'restricted_sync_pub2')] + FROM pg_replication_slots + WHERE slot_name = 'restricted_sync_slot'; + }), + "t", + 'restricted synchronized slot is replaced after incarnation changes'); + +# Replacement should install mappings only for the new incarnation's scope. +is( $standby1->safe_psql( + 'postgres', qq{ + SELECT newrel.relhasrestrictedslots AND + NOT EXISTS ( + SELECT 1 FROM pg_restricted_slot_relation + WHERE rsrslotname = 'restricted_sync_slot' + AND rsrrelid = oldrel.oid + AND rsrincarnation = $restricted_incarnation2) AND + EXISTS ( + SELECT 1 FROM pg_restricted_slot_relation + WHERE rsrslotname = 'restricted_sync_slot' + AND rsrrelid = newrel.oid + AND rsrincarnation = $restricted_incarnation2) + FROM pg_class AS oldrel, pg_class AS newrel + WHERE oldrel.oid = 'restricted_sync_tab1'::regclass + AND newrel.oid = 'restricted_sync_tab2'::regclass; + }), + "t", + 'restricted synchronized slot replacement installs its new mappings'); + +$primary->safe_psql('postgres', + "SELECT pg_drop_replication_slot('restricted_sync_slot');"); +$primary->wait_for_replay_catchup($standby1); +$standby1->safe_psql('postgres', "SELECT pg_sync_replication_slots();"); + +# Dropping the remote restricted slot should remove its synchronized copy. +is( $standby1->safe_psql( + 'postgres', q{ + SELECT count(*) = 0 FROM pg_replication_slots + WHERE slot_name = 'restricted_sync_slot'; + }), + "t", + 'dropped restricted failover slot is removed from standby'); + +# Temporary restricted-slot relations should not remain in the FOR ALL TABLES +# publication used by later subscription tests. +$primary->safe_psql( + 'postgres', q{ + DROP PUBLICATION restricted_sync_pub1; + DROP PUBLICATION restricted_sync_pub2; + DROP TABLE restricted_sync_tab1; + DROP TABLE restricted_sync_tab2; +}); +$primary->wait_for_replay_catchup($standby1); + ################################################## # Test that the synchronized slot will be dropped if the corresponding remote # slot on the primary server has been dropped. diff --git a/src/test/recovery/t/056_logical_slot_scope.pl b/src/test/recovery/t/056_logical_slot_scope.pl new file mode 100644 index 00000000000..8583df9cbdb --- /dev/null +++ b/src/test/recovery/t/056_logical_slot_scope.pl @@ -0,0 +1,647 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('node'); +my ($ret, $stdout, $stderr); +$node->init(allows_streaming => 'logical'); +$node->append_conf('postgresql.conf', 'wal_level = replica'); +$node->append_conf('postgresql.conf', 'max_prepared_transactions = 10'); +$node->start; + +$node->safe_psql('postgres', q{ + CREATE TABLE orders (id integer PRIMARY KEY, payload text); + CREATE TABLE customers (id integer PRIMARY KEY, payload text); + CREATE PUBLICATION orders_pub FOR TABLE orders; + CREATE PUBLICATION customers_pub FOR TABLE customers; +}); + +# Specifying publications should create a restricted logical slot. +like($node->safe_psql('postgres', q{ + SELECT slot_name FROM pg_create_logical_replication_slot( + 'orders_slot', 'pgoutput', publications => ARRAY['orders_pub']); +}), qr/orders_slot/, 'a publication list creates a restricted slot'); + +# pg_replication_slots should expose the durable restricted-scope state. +is($node->safe_psql('postgres', q{ + SELECT NOT unrestricted AND restricted_scope_ready AND + (SELECT oid FROM pg_publication WHERE pubname = 'orders_pub') = + ANY (publication_oids) AND restricted_scope_incarnation <> 0 AND + restricted_scope_ready_lsn IS NOT NULL + FROM pg_replication_slots WHERE slot_name = 'orders_slot'; +}), 't', 'restricted slot properties are exposed'); + +# Slot creation should install the initial relation mapping and flag. +is($node->safe_psql('postgres', q{ + SELECT relhasrestrictedslots AND EXISTS ( + SELECT 1 FROM pg_restricted_slot_relation + WHERE rsrslotname = 'orders_slot' AND rsrrelid = 'orders'::regclass) + FROM pg_class WHERE oid = 'orders'::regclass; +}), 't', 'initial publication membership is installed'); + +# A restricted slot should enable selective, rather than full, WAL. +is($node->safe_psql('postgres', q{ + SELECT current_setting('effective_wal_level'), + current_setting('restricted_wal_level'); +}), "replica|logical", 'restricted slots enable only restricted logical WAL'); + +$node->stop('immediate'); +$node->start; +# Scope readiness and selective WAL should survive an immediate restart. +is($node->safe_psql('postgres', q{ + SELECT restricted_scope_ready AND + current_setting('restricted_wal_level') = 'logical' + FROM pg_replication_slots WHERE slot_name = 'orders_slot'; +}), 't', 'scope readiness is durable across an immediate restart'); + +# NULL original arguments should preserve the function's former strictness. +is($node->safe_psql('postgres', q{ + SELECT pg_create_logical_replication_slot(NULL, 'pgoutput') IS NULL AND + pg_create_logical_replication_slot('null_plugin', NULL) IS NULL AND + pg_create_logical_replication_slot( + 'null_temporary', 'pgoutput', NULL) IS NULL AND + pg_create_logical_replication_slot( + 'null_twophase', 'pgoutput', false, NULL) IS NULL AND + pg_create_logical_replication_slot( + 'null_failover', 'pgoutput', false, false, NULL) IS NULL; +}), 't', 'NULL original arguments retain strict behavior'); + +($ret, $stdout, $stderr) = $node->psql('postgres', q{ + SELECT pg_create_logical_replication_slot( + 'bad_plugin', 'test_decoding', publications => ARRAY['orders_pub']); +}); +# Restricted slots should reject output plugins other than pgoutput. +isnt($ret, 0, 'a restricted slot requires pgoutput'); +# The plugin restriction should produce the expected error message. +like($stderr, qr/restricted logical replication slots currently require pgoutput/, + 'wrong output plugin reports the restriction'); + +($ret, $stdout, $stderr) = $node->psql('postgres', q{ + BEGIN; + SELECT pg_create_logical_replication_slot( + 'in_xact_slot', 'pgoutput', publications => ARRAY['orders_pub']); +}); +# Restricted slot creation should be rejected in an explicit transaction. +isnt($ret, 0, 'restricted slot creation is rejected in a transaction block'); +# Explicit-transaction rejection should report the expected error. +like($stderr, qr/restricted logical replication slots cannot be created inside a transaction block/, + 'explicit transaction reports the creation restriction'); + +($ret, $stdout, $stderr) = $node->psql('postgres', q{ + BEGIN; + SAVEPOINT s; + SELECT pg_create_logical_replication_slot( + 'in_subxact_slot', 'pgoutput', publications => ARRAY['orders_pub']); +}); +# Restricted slot creation should also be rejected in a subtransaction. +isnt($ret, 0, 'restricted slot creation is rejected in a subtransaction'); +# Subtransaction rejection should report the expected error. +like($stderr, qr/restricted logical replication slots cannot be created inside a transaction block/, + 'subtransaction reports the creation restriction'); + +($ret, $stdout, $stderr) = $node->psql('postgres', q{ + BEGIN; + SELECT pg_copy_logical_replication_slot('orders_slot', 'copy_in_xact'); +}); +# Copying a restricted slot should be rejected in an explicit transaction. +isnt($ret, 0, 'restricted slot copying is rejected in a transaction block'); +# Restricted-slot copy rejection should report the expected error. +like($stderr, qr/restricted logical replication slots cannot be copied inside a transaction block/, + 'explicit transaction reports the copy restriction'); + +($ret, $stdout, $stderr) = $node->psql('postgres', q{ + SELECT count(*) FROM pg_logical_slot_peek_binary_changes( + 'orders_slot', NULL, NULL, + 'proto_version', '1', 'publication_names', 'customers_pub'); +}); +# Decoding should reject a publication absent from the fixed set. +isnt($ret, 0, 'pgoutput rejects an unstored publication'); +# Publication-set validation should produce the expected error. +like($stderr, qr/requested publications are not contained in replication slot/, + 'pgoutput reports publication-set containment failure'); + +($ret, $stdout, $stderr) = $node->psql('postgres', q{ + SELECT pg_create_logical_replication_slot( + 'null_publication_slot', 'pgoutput', + publications => ARRAY['orders_pub', NULL]); +}); +# A NULL publication name should be rejected as an invalid parameter. +isnt($ret, 0, 'NULL publication array element is rejected'); +like($stderr, qr/publications must not contain null values/, + 'NULL publication array element reports the expected error'); + +$node->safe_psql('postgres', q{ + CREATE SUBSCRIPTION restricted_test_sub + CONNECTION 'dbname=doesnotexist' PUBLICATION orders_pub, customers_pub + WITH (connect = false, slot_name = NONE, unrestricted_slot = false); +}); +($ret, $stdout, $stderr) = $node->psql('postgres', q{ + ALTER SUBSCRIPTION restricted_test_sub + SET PUBLICATION outside_pub WITH (refresh = false); +}); +# A restricted subscription should reject publication identity expansion. +isnt($ret, 0, 'restricted subscription publication expansion is rejected'); +like($stderr, + qr/cannot add publication "outside_pub" to restricted subscription "restricted_test_sub"/, + 'restricted subscription expansion reports the expected error'); +$node->safe_psql('postgres', q{ + ALTER SUBSCRIPTION restricted_test_sub + DROP PUBLICATION customers_pub WITH (refresh = false); +}); +($ret, $stdout, $stderr) = $node->psql('postgres', q{ + ALTER SUBSCRIPTION restricted_test_sub + ADD PUBLICATION customers_pub WITH (refresh = false); +}); +# A removed publication should not be added back to a restricted subscription. +isnt($ret, 0, 'restricted subscription ADD PUBLICATION is rejected'); +like($stderr, + qr/cannot add publications to restricted subscription "restricted_test_sub"/, + 'restricted subscription ADD PUBLICATION reports the expected error'); +$node->safe_psql('postgres', 'DROP SUBSCRIPTION restricted_test_sub'); + +$node->safe_psql('postgres', 'CREATE TABLE lock_target (id integer)'); +my $writer = $node->background_psql('postgres'); +$writer->query_safe('BEGIN'); +$writer->query_safe('INSERT INTO lock_target VALUES (1)'); +my $expander = $node->background_psql('postgres'); +$expander->query_until(qr/alter-start/, q{ + \echo alter-start + ALTER PUBLICATION orders_pub ADD TABLE lock_target; + \echo alter-done +}); +$node->poll_query_until('postgres', q{ + SELECT count(*) > 0 FROM pg_stat_activity + WHERE query LIKE 'ALTER PUBLICATION orders_pub ADD TABLE lock_target%' + AND wait_event_type = 'Lock'; +}, 't'); +# Publication expansion should wait for an existing relation writer. +pass('publication expansion waits for concurrent relation writers'); +$writer->query_safe('COMMIT'); +$writer->quit; +$expander->quit; +# The relation flag should be installed after the writer releases its lock. +is($node->safe_psql('postgres', q{ + SELECT relhasrestrictedslots FROM pg_class + WHERE oid = 'lock_target'::regclass; +}), 't', 'relation flag is visible after the writer interlock is released'); + +$node->safe_psql('postgres', 'ALTER PUBLICATION orders_pub ADD TABLE customers'); +# Publication expansion should install mapping state synchronously. +is($node->safe_psql('postgres', q{ + SELECT relhasrestrictedslots AND EXISTS ( + SELECT 1 FROM pg_restricted_slot_relation + WHERE rsrslotname = 'orders_slot' AND rsrrelid = 'customers'::regclass) + FROM pg_class WHERE oid = 'customers'::regclass; +}), 't', 'publication expansion synchronously installs membership'); + +$node->safe_psql('postgres', q{ + CREATE TABLE unused_scope_table (id integer); + CREATE PUBLICATION unused_scope_pub; +}); +my $unused_scope_writer = $node->background_psql('postgres'); +$unused_scope_writer->query_safe('BEGIN'); +$unused_scope_writer->query_safe( + 'INSERT INTO unused_scope_table VALUES (1)'); +# An unmatched publication should not take a stronger table lock. +is($node->safe_psql('postgres', q{ + SET statement_timeout = '1s'; + ALTER PUBLICATION unused_scope_pub ADD TABLE unused_scope_table; + SELECT 1; +}), '1', 'unmatched publication expansion avoids closure locking'); +$unused_scope_writer->query_safe('ROLLBACK'); +$unused_scope_writer->quit; + +$node->safe_psql('postgres', q{ + CREATE TABLE initializing_member (id integer); + CREATE TABLE concurrent_member (id integer); + CREATE PUBLICATION initializing_pub FOR TABLE initializing_member; +}); +$node->safe_psql('postgres', q{ + SELECT pg_create_logical_replication_slot( + 'initializing_source', 'pgoutput', + publications => ARRAY['initializing_pub']); +}); +my $initialization_expander = $node->background_psql('postgres'); +$initialization_expander->query_safe('BEGIN'); +$initialization_expander->query_safe( + 'ALTER PUBLICATION initializing_pub ADD TABLE concurrent_member'); +($ret, $stdout, $stderr) = $node->psql('postgres', q{ + SELECT pg_copy_logical_replication_slot( + 'initializing_source', 'initializing_slot'); +}); +# Copy initialization should fail instead of using a stale publication snapshot. +isnt($ret, 0, 'concurrent publication expansion rejects slot copying'); +like($stderr, + qr/could not initialize restricted logical replication slot due to concurrent activity/, + 'concurrent publication expansion reports the expected error'); + +$initialization_expander->query_safe('COMMIT'); +$initialization_expander->quit; +$node->safe_psql('postgres', q{ + SELECT pg_copy_logical_replication_slot( + 'initializing_source', 'initializing_slot'); +}); + +# Slot initialization should finish with the concurrent relation mapped. +is($node->safe_psql('postgres', q{ + SELECT s.restricted_scope_ready AND c.relhasrestrictedslots AND EXISTS ( + SELECT 1 FROM pg_restricted_slot_relation + WHERE rsrslotname = 'initializing_slot' + AND rsrrelid = c.oid) + FROM pg_replication_slots AS s, pg_class AS c + WHERE s.slot_name = 'initializing_slot' + AND c.oid = 'concurrent_member'::regclass; +}), 't', 'initial scope retains a concurrent publication expansion'); +$node->safe_psql('postgres', q{ + SELECT pg_drop_replication_slot('initializing_slot'); + SELECT pg_drop_replication_slot('initializing_source'); +}); + +$node->safe_psql('postgres', q{ + CREATE TABLE lock_order_root (id integer) PARTITION BY RANGE (id); + CREATE TABLE lock_order_child (LIKE lock_order_root); + CREATE PUBLICATION lock_order_pub FOR TABLE lock_order_root; +}); +$node->safe_psql('postgres', q{ + SELECT pg_create_logical_replication_slot( + 'lock_order_source', 'pgoutput', + publications => ARRAY['lock_order_pub']); +}); +$node->safe_psql('postgres', q{ + SELECT pg_create_logical_replication_slot( + 'lock_order_full_wal', 'pgoutput'); +}); +my $hierarchy_ddl = + $node->background_psql('postgres', on_error_stop => 0); +$hierarchy_ddl->query_safe('BEGIN'); +$hierarchy_ddl->query_safe( + 'LOCK TABLE lock_order_root IN ACCESS EXCLUSIVE MODE'); + +($ret, $stdout, $stderr) = $node->psql('postgres', q{ + SELECT pg_copy_logical_replication_slot( + 'lock_order_source', 'lock_order_slot'); +}); +# Slot copying should fail instead of waiting while holding the scope lock. +isnt($ret, 0, + 'restricted slot copying does not deadlock on a relation lock'); +# The lock-order failure should identify concurrent scope initialization activity. +like($stderr, + qr/could not initialize restricted logical replication slot due to concurrent activity/, + 'lock-order conflict reports the expected error'); +$hierarchy_ddl->query_safe('ROLLBACK'); +$hierarchy_ddl->quit; + +# Retrying the slot copy should succeed after the relation lock is released. +$node->safe_psql('postgres', q{ + SELECT pg_copy_logical_replication_slot( + 'lock_order_source', 'lock_order_slot'); +}); +$node->safe_psql('postgres', + "SELECT pg_drop_replication_slot('lock_order_full_wal')"); +$node->safe_psql('postgres', q{ + SELECT pg_drop_replication_slot('lock_order_slot'); + SELECT pg_drop_replication_slot('lock_order_source'); +}); + +$node->safe_psql('postgres', q{ + CREATE TABLE command_scope_a (id integer); + CREATE TABLE command_scope_b (id integer); + CREATE PUBLICATION command_scope_pub_a; + CREATE PUBLICATION command_scope_pub_b; +}); +my $scope_xact_a = $node->background_psql('postgres'); +my $scope_xact_b = $node->background_psql('postgres'); +$scope_xact_a->query_safe('BEGIN'); +$scope_xact_a->query_safe( + 'ALTER PUBLICATION command_scope_pub_a ADD TABLE command_scope_a'); +$scope_xact_b->query_safe('BEGIN'); +$scope_xact_b->query_safe( + 'ALTER PUBLICATION command_scope_pub_b ADD TABLE command_scope_b'); + +# A completed scope command should not retain the database scope lock. +pass('separate transactions can complete successive scope commands'); +$scope_xact_a->query_safe('ROLLBACK'); +$scope_xact_b->query_safe('ROLLBACK'); +$scope_xact_a->quit; +$scope_xact_b->quit; + +$node->safe_psql('postgres', q{ + CREATE SCHEMA published_schema; + CREATE SCHEMA unpublished_schema; + CREATE TABLE unpublished_schema.schema_move (id integer); + CREATE PUBLICATION schema_move_pub + FOR TABLES IN SCHEMA published_schema; +}); +$node->safe_psql('postgres', q{ + SELECT pg_create_logical_replication_slot( + 'schema_move_slot', 'pgoutput', + publications => ARRAY['schema_move_pub']); +}); +$node->safe_psql('postgres', q{ + ALTER TABLE unpublished_schema.schema_move SET SCHEMA published_schema; +}); + +# SET SCHEMA should map a table moved into a published schema. +is($node->safe_psql('postgres', q{ + SELECT c.relhasrestrictedslots AND EXISTS ( + SELECT 1 FROM pg_restricted_slot_relation + WHERE rsrslotname = 'schema_move_slot' + AND rsrrelid = c.oid) + FROM pg_class AS c + WHERE c.oid = 'published_schema.schema_move'::regclass; +}), 't', 'moving a table into a published schema installs its mapping'); + +$node->safe_psql('postgres', q{ + CREATE TABLE measurement (id integer, payload text) PARTITION BY RANGE (id); + CREATE TABLE measurement_old PARTITION OF measurement FOR VALUES FROM (0) TO (10); + ALTER PUBLICATION orders_pub ADD TABLE measurement; + CREATE TABLE measurement_new (LIKE measurement); + ALTER TABLE measurement ATTACH PARTITION measurement_new FOR VALUES FROM (10) TO (20); +}); +# Attaching a partition should propagate its parent's restricted mapping. +is($node->safe_psql('postgres', q{ + SELECT relhasrestrictedslots AND EXISTS ( + SELECT 1 FROM pg_restricted_slot_relation + WHERE rsrslotname = 'orders_slot' AND rsrrelid = 'measurement_new'::regclass) + FROM pg_class WHERE oid = 'measurement_new'::regclass; +}), 't', 'partition attachment propagates restricted membership'); + +$node->safe_psql('postgres', q{ + CREATE TABLE root_mode (id integer, payload text) PARTITION BY RANGE (id); + CREATE TABLE root_mode_old PARTITION OF root_mode + FOR VALUES FROM (0) TO (10); + CREATE PUBLICATION root_mode_pub FOR TABLE root_mode + WITH (publish_via_partition_root = true); +}); +$node->safe_psql('postgres', q{ + SELECT pg_create_logical_replication_slot( + 'root_mode_slot', 'pgoutput', publications => ARRAY['root_mode_pub']); +}); +$node->safe_psql('postgres', q{ + CREATE TABLE root_mode_new (LIKE root_mode); + BEGIN; + ALTER TABLE root_mode ATTACH PARTITION root_mode_new + FOR VALUES FROM (10) TO (20); + INSERT INTO root_mode VALUES (10, 'new partition'); + COMMIT; +}); +# The root, old leaf, and newly attached leaf should all be mapped. +is($node->safe_psql('postgres', q{ + SELECT bool_and(c.relhasrestrictedslots) AND count(*) = 3 + FROM pg_class AS c + JOIN pg_restricted_slot_relation AS m ON m.rsrrelid = c.oid + WHERE m.rsrslotname = 'root_mode_slot' + AND c.oid IN ('root_mode'::regclass, + 'root_mode_old'::regclass, + 'root_mode_new'::regclass); +}), 't', 'partition-root publication maps a newly attached partition'); + +# Table synchronization should handle an attached partition with both settings +# of publish_via_partition_root. In leaf mode, REFRESH PUBLICATION discovers +# a new subscription relation and copies its existing data. In root mode, the +# root is already a subscription relation, so refresh does not initiate another +# table synchronization for the newly attached leaf. +my $subscriber = PostgreSQL::Test::Cluster->new('subscriber'); +$subscriber->init(allows_streaming => 'logical'); +$subscriber->start; +my $publisher_connstr = $node->connstr . ' dbname=postgres'; + +$node->safe_psql('postgres', q{ + CREATE TABLE leaf_copy_root (id integer, payload text) + PARTITION BY RANGE (id); + CREATE TABLE leaf_copy_old PARTITION OF leaf_copy_root + FOR VALUES FROM (0) TO (10); + CREATE TABLE leaf_copy_new (LIKE leaf_copy_root); + INSERT INTO leaf_copy_new VALUES (10, 'before attach'); + CREATE PUBLICATION leaf_copy_pub FOR TABLE leaf_copy_root + WITH (publish_via_partition_root = false); + + CREATE TABLE root_copy_root (id integer, payload text) + PARTITION BY RANGE (id); + CREATE TABLE root_copy_old PARTITION OF root_copy_root + FOR VALUES FROM (0) TO (10); + CREATE TABLE root_copy_new (LIKE root_copy_root); + INSERT INTO root_copy_new VALUES (10, 'before attach'); + CREATE PUBLICATION root_copy_pub FOR TABLE root_copy_root + WITH (publish_via_partition_root = true); +}); +$subscriber->safe_psql('postgres', q{ + CREATE TABLE leaf_copy_root (id integer, payload text) + PARTITION BY RANGE (id); + CREATE TABLE leaf_copy_old PARTITION OF leaf_copy_root + FOR VALUES FROM (0) TO (10); + CREATE TABLE leaf_copy_new PARTITION OF leaf_copy_root + FOR VALUES FROM (10) TO (20); + + CREATE TABLE root_copy_root (id integer, payload text) + PARTITION BY RANGE (id); + CREATE TABLE root_copy_old PARTITION OF root_copy_root + FOR VALUES FROM (0) TO (10); + CREATE TABLE root_copy_new PARTITION OF root_copy_root + FOR VALUES FROM (10) TO (20); +}); +$subscriber->safe_psql('postgres', qq{ + CREATE SUBSCRIPTION leaf_copy_sub CONNECTION '$publisher_connstr' + PUBLICATION leaf_copy_pub + WITH (copy_data = true, unrestricted_slot = false); + CREATE SUBSCRIPTION root_copy_sub CONNECTION '$publisher_connstr' + PUBLICATION root_copy_pub + WITH (copy_data = true, unrestricted_slot = false); +}); +$subscriber->wait_for_subscription_sync($node, 'leaf_copy_sub'); +$subscriber->wait_for_subscription_sync($node, 'root_copy_sub'); + +$node->safe_psql('postgres', q{ + ALTER TABLE leaf_copy_root ATTACH PARTITION leaf_copy_new + FOR VALUES FROM (10) TO (20); + ALTER TABLE root_copy_root ATTACH PARTITION root_copy_new + FOR VALUES FROM (10) TO (20); +}); +$subscriber->safe_psql('postgres', q{ + ALTER SUBSCRIPTION leaf_copy_sub REFRESH PUBLICATION + WITH (copy_data = true); + ALTER SUBSCRIPTION root_copy_sub REFRESH PUBLICATION + WITH (copy_data = true); +}); +$subscriber->wait_for_subscription_sync($node, 'leaf_copy_sub'); +$subscriber->wait_for_subscription_sync($node, 'root_copy_sub'); + +# Leaf-mode refresh should copy data that predates partition attachment. +is($subscriber->safe_psql('postgres', q{ + SELECT string_agg(payload, ',' ORDER BY id) FROM leaf_copy_root; +}), 'before attach', + 'leaf publication copies a row that existed before partition attachment'); +# Root-mode refresh should not resynchronize an already known root table. +is($subscriber->safe_psql('postgres', q{ + SELECT count(*) FROM root_copy_root; +}), '0', + 'partition-root publication does not recopy its existing root relation'); + +$node->safe_psql('postgres', q{ + INSERT INTO leaf_copy_root VALUES (11, 'after attach'); + INSERT INTO root_copy_root VALUES (11, 'after attach'); +}); +$node->wait_for_catchup('leaf_copy_sub'); +$node->wait_for_catchup('root_copy_sub'); + +# Leaf mode should stream changes made after partition attachment. +is($subscriber->safe_psql('postgres', q{ + SELECT string_agg(payload, ',' ORDER BY id) FROM leaf_copy_root; +}), 'before attach,after attach', + 'leaf publication streams rows inserted after partition attachment'); +# Root mode should stream changes made after partition attachment. +is($subscriber->safe_psql('postgres', q{ + SELECT string_agg(payload, ',' ORDER BY id) FROM root_copy_root; +}), 'after attach', + 'partition-root publication streams rows inserted after attachment'); + +$subscriber->safe_psql('postgres', + 'ALTER SUBSCRIPTION root_copy_sub DISABLE'); +$node->poll_query_until('postgres', q{ + SELECT NOT active FROM pg_replication_slots + WHERE slot_name = 'root_copy_sub'; +}, 't'); +$subscriber->safe_psql('postgres', q{ + CREATE TABLE root_history_part PARTITION OF root_copy_root + FOR VALUES FROM (20) TO (30); +}); +$node->safe_psql('postgres', q{ + CREATE TABLE root_history_part (LIKE root_copy_root); + SELECT pg_create_logical_replication_slot( + 'history_cover_slot', 'test_decoding'); + INSERT INTO root_history_part VALUES (20, 'existing before attach'); + INSERT INTO root_history_part VALUES (21, 'before attach'); + ALTER TABLE root_copy_root ATTACH PARTITION root_history_part + FOR VALUES FROM (20) TO (30); + INSERT INTO root_history_part VALUES (22, 'after attach'); + ALTER TABLE root_copy_root DETACH PARTITION root_history_part; + INSERT INTO root_history_part VALUES (23, 'after detach'); + ALTER TABLE root_copy_root ATTACH PARTITION root_history_part + FOR VALUES FROM (20) TO (30); + INSERT INTO root_history_part VALUES (24, 'after reattach'); +}); +$subscriber->safe_psql('postgres', + 'ALTER SUBSCRIPTION root_copy_sub ENABLE'); +$node->wait_for_catchup('root_copy_sub'); + +# Delayed decoding should use partition membership at each change's WAL point. +is($subscriber->safe_psql('postgres', q{ + SELECT string_agg(id || ':' || payload, ',' ORDER BY id) + FROM root_copy_root WHERE id >= 20; +}), '22:after attach,24:after reattach', + 'delayed decoding follows attach and detach boundaries'); + +$node->safe_psql('postgres', + "SELECT pg_drop_replication_slot('history_cover_slot')"); + +$subscriber->safe_psql('postgres', + 'ALTER SUBSCRIPTION leaf_copy_sub DISABLE'); +$node->safe_psql('postgres', + q{INSERT INTO leaf_copy_root VALUES (12, 'while disabled')}); + +# A disabled subscription should not apply newly generated changes. +is($subscriber->safe_psql('postgres', q{ + SELECT string_agg(payload, ',' ORDER BY id) FROM leaf_copy_root; +}), 'before attach,after attach', + 'a disabled subscription does not apply new changes'); + +$subscriber->safe_psql('postgres', + 'ALTER SUBSCRIPTION leaf_copy_sub ENABLE'); +$node->wait_for_catchup('leaf_copy_sub'); + +# Enabling a subscription should replay scoped WAL written while disabled. +is($subscriber->safe_psql('postgres', q{ + SELECT string_agg(payload, ',' ORDER BY id) FROM leaf_copy_root; +}), 'before attach,after attach,while disabled', + 'enabling a subscription replays changes generated while disabled'); + +$node->safe_psql('postgres', q{ + CREATE TABLE speculative_unscoped ( + id integer PRIMARY KEY, payload text); + BEGIN; + INSERT INTO speculative_unscoped VALUES (1, repeat('x', 100000)) + ON CONFLICT DO NOTHING; + INSERT INTO leaf_copy_root VALUES (13, 'after speculative insert'); + COMMIT; +}); +$node->wait_for_catchup('leaf_copy_sub'); + +# An unscoped speculative insert should not prevent decoding a scoped change. +is($subscriber->safe_psql('postgres', q{ + SELECT payload FROM leaf_copy_root WHERE id = 13; +}), 'after speculative insert', + 'decoding preserves unscoped speculative insertion ordering'); + +$node->safe_psql('postgres', q{ + ALTER TABLE measurement DROP COLUMN payload; + CREATE TABLE measurement_after_drop (LIKE measurement); + ALTER TABLE measurement ATTACH PARTITION measurement_after_drop DEFAULT; +}); +# Dropped columns should not lose mappings needed by later partitions. +is($node->safe_psql('postgres', q{ + SELECT EXISTS ( + SELECT 1 FROM pg_restricted_slot_relation + WHERE rsrslotname = 'orders_slot' + AND rsrrelid = 'measurement_after_drop'::regclass); +}), 't', 'dropping a column preserves mappings used by later attachment'); + +$node->safe_psql('postgres', q{ + CREATE TABLE toast_later (id integer); + ALTER PUBLICATION orders_pub ADD TABLE toast_later; + BEGIN; + ALTER TABLE toast_later ADD COLUMN payload text; + INSERT INTO toast_later VALUES (1, repeat('x', 100000)); + COMMIT; +}); +# TOAST creation should propagate mapping before same-transaction DML. +is($node->safe_psql('postgres', q{ + SELECT toast.relhasrestrictedslots AND EXISTS ( + SELECT 1 FROM pg_restricted_slot_relation + WHERE rsrslotname = 'orders_slot' AND rsrrelid = owner.reltoastrelid) + FROM pg_class owner JOIN pg_class toast ON toast.oid = owner.reltoastrelid + WHERE owner.oid = 'toast_later'::regclass; +}), 't', 'TOAST creation propagates membership before same-transaction DML'); + +$node->restart; +# Restricted metadata and selective WAL should remain active after restart. +is($node->safe_psql('postgres', q{ + SELECT NOT unrestricted AND restricted_scope_ready AND + current_setting('restricted_wal_level') = 'logical' + FROM pg_replication_slots WHERE slot_name = 'orders_slot'; +}), 't', 'restricted slot metadata and WAL level survive restart'); + +$node->safe_psql('postgres', q{ + SELECT pg_create_logical_replication_slot('general_slot', 'test_decoding'); +}); +# Omitting publications should retain ordinary unrestricted-slot behavior. +is($node->safe_psql('postgres', q{ + SELECT unrestricted AND publication_oids IS NULL + FROM pg_replication_slots WHERE slot_name = 'general_slot'; +}), 't', 'an omitted publication list creates an unrestricted slot'); + +$node->safe_psql('postgres', q{ + SELECT pg_create_logical_replication_slot( + 'reuse_slot', 'pgoutput', publications => ARRAY['orders_pub']); +}); +my $old_incarnation = $node->safe_psql('postgres', q{ + SELECT restricted_scope_incarnation FROM pg_replication_slots + WHERE slot_name = 'reuse_slot'; +}); +$node->safe_psql('postgres', q{ + SELECT pg_drop_replication_slot('reuse_slot'); + SELECT pg_create_logical_replication_slot( + 'reuse_slot', 'pgoutput', publications => ARRAY['orders_pub']); +}); +my $new_incarnation = $node->safe_psql('postgres', q{ + SELECT restricted_scope_incarnation FROM pg_replication_slots + WHERE slot_name = 'reuse_slot'; +}); +# A recreated slot name should not reuse the old scope incarnation. +isnt($new_incarnation, $old_incarnation, + 'slot name reuse receives a distinct durable incarnation'); + +done_testing(); diff --git a/src/test/regress/expected/oidjoins.out b/src/test/regress/expected/oidjoins.out index d64169b7bf0..b971651c8ec 100644 --- a/src/test/regress/expected/oidjoins.out +++ b/src/test/regress/expected/oidjoins.out @@ -268,6 +268,7 @@ NOTICE: checking pg_publication_namespace {pnpubid} => pg_publication {oid} NOTICE: checking pg_publication_namespace {pnnspid} => pg_namespace {oid} NOTICE: checking pg_publication_rel {prpubid} => pg_publication {oid} NOTICE: checking pg_publication_rel {prrelid} => pg_class {oid} +NOTICE: checking pg_restricted_slot_relation {rsrrelid} => pg_class {oid} NOTICE: checking pg_subscription {subdbid} => pg_database {oid} NOTICE: checking pg_subscription {subowner} => pg_authid {oid} NOTICE: checking pg_subscription {subserver} => pg_foreign_server {oid} diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 1a29d46213e..f9e9d66ca10 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1510,8 +1510,13 @@ pg_replication_slots| SELECT l.slot_name, l.invalidation_reason, l.failover, l.synced, - l.slotsync_skip_reason - FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, two_phase_at, inactive_since, conflicting, invalidation_reason, failover, synced, slotsync_skip_reason) + l.slotsync_skip_reason, + l.unrestricted, + l.publication_oids, + l.restricted_scope_ready, + l.restricted_scope_incarnation, + l.restricted_scope_ready_lsn + FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, two_phase_at, inactive_since, conflicting, invalidation_reason, failover, synced, slotsync_skip_reason, unrestricted, publication_oids, restricted_scope_ready, restricted_scope_incarnation, restricted_scope_ready_lsn) LEFT JOIN pg_database d ON ((l.datoid = d.oid))); pg_roles| SELECT pg_authid.rolname, pg_authid.rolsuper, diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 298a3d586e7..b2fd3c4fa4c 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1698,6 +1698,10 @@ LogicalDecodeStreamTruncateCB LogicalDecodeTruncateCB LogicalDecodingContext LogicalDecodingCtlData +LogicalSlotScopeCtlData +LogicalSlotScopeEntry +LogicalSlotScopeKey +LogicalSlotScopeOnDisk LogicalErrorCallbackState LogicalOutputPluginInit LogicalOutputPluginWriterPrepareWrite -- 2.50.1 (Apple Git-155)