Subject: [PATCH] Check domain constraints for logical replication NULLs Logical replication passes non-NULL values through the subscriber type's input function, but stores explicit NULLs without running domain checks. Pass NULLs through type input when constructing an INSERT or UPDATE row. Keep search tuples and conflict-reporting tuples exempt from the new NULL checks. Old replica identity key tuples contain NULL placeholders for non-key attributes, so checking those would break valid DELETEs and key updates. Preserve unchanged TOAST values and subscriber-only defaults. Add regression coverage for domain NOT NULL and CHECK constraints in text and binary modes, including transaction rollback and valid key changes. Base commit: 11591f273e6a26fb55728b410538fa2627b19b28 --- src/backend/replication/logical/worker.c | 52 ++++++----- src/test/subscription/t/003_constraints.pl | 145 +++++++++++++++++++++++++++++ 2 files changed, 173 insertions(+), 24 deletions(-) diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 95044620e68bd5994950292f98aa6a80785aa938..80f4a528ec9a8526438bbb7aa3ba11c07542f8e3 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -1034,17 +1034,22 @@ slot_fill_defaults(LogicalRepRelMapEntry *rel, EState *estate, * Store tuple data into slot. * * Incoming data can be either text or binary format. + * + * When check_nulls is true, pass explicit NULLs through the type input + * function, too, so that domain constraints are checked. Leave this false + * for search tuples and conflict reporting: old key tuples can contain NULL + * placeholders for columns that are not part of the replica identity. */ static void slot_store_data(TupleTableSlot *slot, LogicalRepRelMapEntry *rel, - LogicalRepTupleData *tupleData) + LogicalRepTupleData *tupleData, bool check_nulls) { int natts = slot->tts_tupleDescriptor->natts; int i; ExecClearTuple(slot); - /* Call the "in" function for each non-dropped, non-null attribute */ + /* Call the input function for each supplied attribute. */ Assert(natts == rel->attrmap->maplen); for (i = 0; i < natts; i++) { @@ -1068,16 +1073,19 @@ slot_store_data(TupleTableSlot *slot, LogicalRepRelMapEntry *rel, /* Set attnum for error callback */ remote_ctx.remote_attnum = remoteattnum; - if (tupleData->colstatus[remoteattnum] == LOGICALREP_COLUMN_TEXT) + if (tupleData->colstatus[remoteattnum] == LOGICALREP_COLUMN_TEXT || + (check_nulls && + tupleData->colstatus[remoteattnum] == LOGICALREP_COLUMN_NULL)) { Oid typinput; Oid typioparam; + bool isnull = tupleData->colstatus[remoteattnum] == LOGICALREP_COLUMN_NULL; getTypeInputInfo(att->atttypid, &typinput, &typioparam); slot->tts_values[i] = - OidInputFunctionCall(typinput, colvalue->data, + OidInputFunctionCall(typinput, isnull ? NULL : colvalue->data, typioparam, att->atttypmod); - slot->tts_isnull[i] = false; + slot->tts_isnull[i] = isnull; } else if (tupleData->colstatus[remoteattnum] == LOGICALREP_COLUMN_BINARY) { @@ -1106,9 +1114,8 @@ slot_store_data(TupleTableSlot *slot, LogicalRepRelMapEntry *rel, else { /* - * NULL value from remote. (We don't expect to see - * LOGICALREP_COLUMN_UNCHANGED here, but if we do, treat it as - * NULL.) + * An unchecked NULL, or an unchanged TOAST value used in a + * search tuple or for conflict reporting. */ slot->tts_values[i] = (Datum) 0; slot->tts_isnull[i] = true; @@ -1189,16 +1196,19 @@ slot_modify_data(TupleTableSlot *slot, TupleTableSlot *srcslot, /* Set attnum for error callback */ remote_ctx.remote_attnum = remoteattnum; - if (tupleData->colstatus[remoteattnum] == LOGICALREP_COLUMN_TEXT) + /* Pass explicit NULLs through input to check domain constraints. */ + if (tupleData->colstatus[remoteattnum] == LOGICALREP_COLUMN_TEXT || + tupleData->colstatus[remoteattnum] == LOGICALREP_COLUMN_NULL) { Oid typinput; Oid typioparam; + bool isnull = tupleData->colstatus[remoteattnum] == LOGICALREP_COLUMN_NULL; getTypeInputInfo(att->atttypid, &typinput, &typioparam); slot->tts_values[i] = - OidInputFunctionCall(typinput, colvalue->data, + OidInputFunctionCall(typinput, isnull ? NULL : colvalue->data, typioparam, att->atttypmod); - slot->tts_isnull[i] = false; + slot->tts_isnull[i] = isnull; } else if (tupleData->colstatus[remoteattnum] == LOGICALREP_COLUMN_BINARY) { @@ -1224,12 +1234,6 @@ slot_modify_data(TupleTableSlot *slot, TupleTableSlot *srcslot, remoteattnum + 1))); slot->tts_isnull[i] = false; } - else - { - /* must be LOGICALREP_COLUMN_NULL */ - slot->tts_values[i] = (Datum) 0; - slot->tts_isnull[i] = true; - } /* Reset attnum for error callback */ remote_ctx.remote_attnum = -1; @@ -2726,7 +2730,7 @@ apply_handle_insert(StringInfo s) /* Process and store remote tuple in the slot */ oldctx = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate)); - slot_store_data(remoteslot, rel, &newtup); + slot_store_data(remoteslot, rel, &newtup, true); slot_fill_defaults(rel, estate, remoteslot); MemoryContextSwitchTo(oldctx); @@ -2919,7 +2923,7 @@ apply_handle_update(StringInfo s) /* Build the search tuple. */ oldctx = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate)); slot_store_data(remoteslot, rel, - has_oldtup ? &oldtup : &newtup); + has_oldtup ? &oldtup : &newtup, false); MemoryContextSwitchTo(oldctx); /* For a partitioned table, apply update to correct partition. */ @@ -2991,7 +2995,7 @@ apply_handle_update_internal(ApplyExecutionData *edata, /* Store the new tuple for conflict reporting */ newslot = table_slot_create(localrel, &estate->es_tupleTable); - slot_store_data(newslot, relmapentry, newtup); + slot_store_data(newslot, relmapentry, newtup, false); conflicttuple.slot = localslot; @@ -3033,7 +3037,7 @@ apply_handle_update_internal(ApplyExecutionData *edata, type = CT_UPDATE_MISSING; /* Store the new tuple for conflict reporting */ - slot_store_data(newslot, relmapentry, newtup); + slot_store_data(newslot, relmapentry, newtup, false); /* * The tuple to be updated could not be found or was deleted. Do @@ -3112,7 +3116,7 @@ apply_handle_delete(StringInfo s) /* Build the search tuple. */ oldctx = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate)); - slot_store_data(remoteslot, rel, &oldtup); + slot_store_data(remoteslot, rel, &oldtup, false); MemoryContextSwitchTo(oldctx); /* For a partitioned table, apply delete to correct partition. */ @@ -3553,7 +3557,7 @@ apply_handle_tuple_routing(ApplyExecutionData *edata, type = CT_UPDATE_MISSING; /* Store the new tuple for conflict reporting */ - slot_store_data(newslot, part_entry, newtup); + slot_store_data(newslot, part_entry, newtup, false); /* * The tuple to be updated could not be found or was @@ -3579,7 +3583,7 @@ apply_handle_tuple_routing(ApplyExecutionData *edata, /* Store the new tuple for conflict reporting */ newslot = table_slot_create(partrel, &estate->es_tupleTable); - slot_store_data(newslot, part_entry, newtup); + slot_store_data(newslot, part_entry, newtup, false); conflicttuple.slot = localslot; diff --git a/src/test/subscription/t/003_constraints.pl b/src/test/subscription/t/003_constraints.pl index 237e113013bae9d4cc50773cc3c9c2735c193d8a..2d2665d09ee78af9a98630c00b808c01a9329b72 100644 --- a/src/test/subscription/t/003_constraints.pl +++ b/src/test/subscription/t/003_constraints.pl @@ -135,6 +135,151 @@ $result = $node_subscriber->safe_psql('postgres', is($result, qq(2|1|2), 'check column trigger applied even on update for other column'); +# Domain constraints must also be checked for incoming NULLs. Use different +# types on the two nodes so that invalid values can originate on the publisher. +$node_subscriber->safe_psql('postgres', 'DROP SUBSCRIPTION tap_sub'); +$node_publisher->safe_psql( + 'postgres', qq{ + CREATE TABLE domain_test ( + id integer PRIMARY KEY, quantity integer, nullable integer, + plain integer, payload text); + ALTER TABLE domain_test ALTER payload SET STORAGE EXTERNAL; + CREATE PUBLICATION domain_pub FOR TABLE domain_test; +}); +$node_subscriber->safe_psql( + 'postgres', qq{ + CREATE DOMAIN required_int AS integer NOT NULL + CONSTRAINT required_int_positive CHECK (VALUE > 0); + CREATE DOMAIN checked_int AS integer + CONSTRAINT checked_int_nonnull CHECK (VALUE IS NOT NULL) + CONSTRAINT checked_int_positive CHECK (VALUE > 0); + CREATE DOMAIN nullable_int AS integer CHECK (VALUE > 0); + CREATE DOMAIN required_text AS text NOT NULL; + CREATE TABLE domain_test ( + id integer PRIMARY KEY, quantity required_int, nullable nullable_int, + plain integer, payload required_text, + local_only required_int DEFAULT 99); + ALTER TABLE domain_test ALTER payload SET STORAGE EXTERNAL; +}); + +my $domain_query = qq{ + SELECT id, quantity, nullable IS NULL, plain IS NULL, + length(payload), local_only FROM domain_test ORDER BY id +}; + +for my $binary ('false', 'true') +{ + for my $domain ('required_int', 'checked_int') + { + for my $operation ('insert', 'update', 'update_key', 'check') + { + my $testname = "binary=$binary, $domain, $operation"; + + # Start each case with a fresh subscription, so a failed transaction + # does not prevent subsequent cases from making progress. + $node_publisher->safe_psql( + 'postgres', qq{ + TRUNCATE domain_test; + INSERT INTO domain_test VALUES + (1, 10, NULL, NULL, repeat('abcdefghijklmnopqrstuvwxyz', 1000)); + }); + $node_subscriber->safe_psql( + 'postgres', qq{ + TRUNCATE domain_test; + ALTER TABLE domain_test ALTER quantity TYPE $domain + USING quantity::integer; + }); + $node_subscriber->safe_psql( + 'postgres', qq{ + CREATE SUBSCRIPTION domain_sub CONNECTION '$publisher_connstr' + PUBLICATION domain_pub + WITH (binary = $binary, disable_on_error = true) + }); + $node_subscriber->wait_for_subscription_sync($node_publisher, + 'domain_sub'); + is($node_subscriber->safe_psql('postgres', $domain_query), + '1|10|t|t|26000|99', "$testname: initial copy"); + + if ($operation eq 'insert') + { + # NULL remains valid for ordinary columns and nullable domains. + # The subscriber-only domain column must receive its default. + $node_publisher->safe_psql( + 'postgres', qq{ + INSERT INTO domain_test VALUES + (2, 10, NULL, NULL, repeat('abcdefghijklmnopqrstuvwxyz', 1000)); + }); + $node_publisher->wait_for_catchup('domain_sub'); + is($node_subscriber->safe_psql('postgres', $domain_query), + "1|10|t|t|26000|99\n2|10|t|t|26000|99", + "$testname: NULLs and subscriber-only default on insert"); + + # A key update or delete sends NULL placeholders for non-key + # columns in the old key tuple. They must not be checked against + # the domains. An unchanged TOAST value is not a NULL either. + $node_publisher->safe_psql( + 'postgres', qq{ + UPDATE domain_test SET id = 11, nullable = 1, plain = 1 WHERE id = 1; + DELETE FROM domain_test WHERE id = 2; + }); + $node_publisher->wait_for_catchup('domain_sub'); + is($node_subscriber->safe_psql('postgres', $domain_query), + '11|10|f|f|26000|99', + "$testname: key update, delete and unchanged TOAST"); + + $node_publisher->safe_psql( + 'postgres', qq{ + UPDATE domain_test SET nullable = NULL, plain = NULL WHERE id = 11; + UPDATE domain_test SET id = 1 WHERE id = 11; + }); + $node_publisher->wait_for_catchup('domain_sub'); + is($node_subscriber->safe_psql('postgres', $domain_query), + '1|10|t|t|26000|99', + "$testname: nullable columns and unchanged TOAST on update"); + } + + my %changes = ( + insert => "INSERT INTO domain_test VALUES (2, NULL, NULL, NULL, 'payload')", + update => 'UPDATE domain_test SET quantity = NULL WHERE id = 1', + update_key => 'UPDATE domain_test SET id = 2, quantity = NULL WHERE id = 1', + check => 'UPDATE domain_test SET quantity = -1 WHERE id = 1'); + my $error = $operation eq 'check' + ? qq(value for domain public.$domain violates check constraint "${domain}_positive") + : $domain eq 'required_int' + ? 'domain public.required_int does not allow null values' + : 'value for domain public.checked_int violates check constraint "checked_int_nonnull"'; + my $message = $operation eq 'insert' ? 'INSERT' : 'UPDATE'; + my $offset = -s $node_subscriber->logfile; + + # The preceding valid update must roll back along with the invalid + # change. Updating the key exercises replacement of the new tuple + # after looking up the row with the old key tuple. + $node_publisher->safe_psql( + 'postgres', qq{ + BEGIN; + UPDATE domain_test SET plain = 7 WHERE id = 1; + $changes{$operation}; + COMMIT; + }); + ok($node_subscriber->poll_query_until( + 'postgres', qq{ + SELECT NOT subenabled FROM pg_subscription + WHERE subname = 'domain_sub' + }), + "$testname: domain violation disables subscription"); + my $log = slurp_file($node_subscriber->logfile, $offset); + like($log, qr/\Q$error\E/, "$testname: domain error"); + like($log, + qr/CONTEXT:.*message type "$message".*relation "public.domain_test" column "quantity"/, + "$testname: error identifies the target column"); + is($node_subscriber->safe_psql('postgres', $domain_query), + '1|10|t|t|26000|99', "$testname: transaction rolled back"); + $node_subscriber->safe_psql('postgres', + 'DROP SUBSCRIPTION domain_sub'); + } + } +} + $node_subscriber->stop('fast'); $node_publisher->stop('fast');