From 9bf41134dd9029bff2c30eecb308cd1e6f48358d Mon Sep 17 00:00:00 2001 From: Nadav Shatz Date: Wed, 11 Nov 2026 14:01:24 +0200 Subject: [PATCH v2] Fix dml_adaptive write tracking with savepoints. dml_adaptive() treated ROLLBACK TO SAVEPOINT like ROLLBACK: it set is_in_transaction to false and discarded the tracked writes, although the transaction was still open. This had several consequences: - dml_adaptive_global: a write made before the savepoint was committed by the following COMMIT, but was not recorded, so a read of the table from another session could be load balanced to a standby that had not replayed it yet. - Statements after ROLLBACK TO were handled as autocommit statements. With dml_adaptive_global their writes were recorded when each statement completed rather than at COMMIT, so the record could expire before the transaction became visible. - dml_adaptive and dml_adaptive_global: a read inside the same transaction after ROLLBACK TO could be load balanced to a standby, which cannot see the transaction's uncommitted writes. Fix this by tracking savepoints. SAVEPOINT records the lengths of the table name and table OID write lists, which are append-only. ROLLBACK TO truncates both lists back to those lengths, forgetting exactly the writes the subtransaction undid, and keeps the savepoint. RELEASE drops the savepoint and keeps the writes. Savepoint names may be reused; like PostgreSQL the innermost one is used. An unknown savepoint changes nothing, which errs on the side of reading from the primary. Only COMMIT and ROLLBACK end the transaction now. Also, COMMIT of a transaction that has already failed does not raise an error: PostgreSQL rolls it back and returns the ROLLBACK command tag. dml_adaptive_global recorded the transaction's writes in that case. The writes are now recorded only after the command tag has been read, and not if a COMMIT returned a tag other than COMMIT. The tag is used only when it comes from the primary. In streaming and logical replication mode with the extended query protocol the writes are still recorded when Execute is forwarded, as before, since the tag is not known yet; that only errs on the side of reading from the primary. Add regression tests to 032.dml_adaptive_load_balance and 043.track_table_mutation. Reported-by: Taiki Koshino Author: Nadav Shatz --- src/context/pool_query_context.c | 199 ++++++++++++++++- src/context/pool_session_context.c | 2 + src/include/context/pool_session_context.h | 22 ++ src/include/protocol/pool_proto_modules.h | 1 + src/protocol/CommandComplete.c | 139 ++++++++---- src/protocol/pool_proto_modules.c | 6 + .../032.dml_adaptive_load_balance/test.sh | 71 ++++++ .../tests/043.track_table_mutation/test.sh | 203 ++++++++++++++++++ src/tools/pgindent/typedefs.list | 1 + 9 files changed, 591 insertions(+), 53 deletions(-) diff --git a/src/context/pool_query_context.c b/src/context/pool_query_context.c index c16cef7da..aded4e601 100644 --- a/src/context/pool_query_context.c +++ b/src/context/pool_query_context.c @@ -1971,6 +1971,174 @@ add_object_into_temp_write_list(Node *node, void *context) return raw_expression_tree_walker(node, add_object_into_temp_write_list, context); } +/* + * Free the savepoints in transaction_savepoint_list from position + * new_length onwards and truncate the list to new_length entries. + */ +static void +dml_adaptive_truncate_savepoints(POOL_SESSION_CONTEXT *session_context, + int new_length) +{ + List *savepoints = session_context->transaction_savepoint_list; + int i; + + for (i = new_length; i < list_length(savepoints); i++) + { + DmlAdaptiveSavepoint *sp = (DmlAdaptiveSavepoint *) list_nth(savepoints, i); + + pfree(sp->name); + pfree(sp); + } + + if (new_length <= 0) + { + list_free(savepoints); + session_context->transaction_savepoint_list = NIL; + } + else + session_context->transaction_savepoint_list = + list_truncate(savepoints, new_length); +} + +/* + * Forget all savepoints of the current transaction. + */ +void +dml_adaptive_free_savepoints(POOL_SESSION_CONTEXT *session_context) +{ + if (session_context->transaction_savepoint_list != NIL) + dml_adaptive_truncate_savepoints(session_context, 0); +} + +/* + * Return the position of the innermost savepoint called "name", or -1 if + * there is none. Savepoint names may be reused; like PostgreSQL we pick + * the most recently established one. + */ +static int +dml_adaptive_find_savepoint(POOL_SESSION_CONTEXT *session_context, + const char *name) +{ + List *savepoints = session_context->transaction_savepoint_list; + int i; + + if (name == NULL) + return -1; + + for (i = list_length(savepoints) - 1; i >= 0; i--) + { + DmlAdaptiveSavepoint *sp = (DmlAdaptiveSavepoint *) list_nth(savepoints, i); + + if (strcmp(sp->name, name) == 0) + return i; + } + return -1; +} + +/* + * Handle SAVEPOINT, ROLLBACK TO SAVEPOINT and RELEASE SAVEPOINT for + * dml_adaptive and dml_adaptive_global. Called with the session memory + * context current. + * + * SAVEPOINT records how long the two write lists are. Both lists are + * append-only, so on ROLLBACK TO truncating them back to the recorded + * lengths forgets exactly the writes that the subtransaction undid, while + * writes made before the savepoint are kept. ROLLBACK TO keeps the + * savepoint itself (it can be rolled back to again) but destroys every + * savepoint established after it. RELEASE destroys the savepoint and every + * savepoint established after it, but its writes become part of the + * enclosing (sub)transaction, so the write lists are left alone. + * + * If the savepoint is not known (e.g. it was established before Pgpool-II + * started tracking, or the name is misspelled and the backend will raise an + * error) nothing is changed. Keeping writes is always the safe direction: + * at worst a read is sent to the primary unnecessarily. + */ +static void +dml_adaptive_savepoint_command(POOL_SESSION_CONTEXT *session_context, + TransactionStmt *stmt) +{ + DmlAdaptiveSavepoint *sp; + int pos; + + if (!session_context->is_in_transaction) + return; + + switch (stmt->kind) + { + case TRANS_STMT_SAVEPOINT: + if (stmt->savepoint_name == NULL) + break; + sp = palloc(sizeof(DmlAdaptiveSavepoint)); + sp->name = pstrdup(stmt->savepoint_name); + sp->write_list_length = + list_length(session_context->transaction_temp_write_list); + sp->write_oid_list_length = + list_length(session_context->transaction_temp_write_oid_list); + session_context->transaction_savepoint_list = + lappend(session_context->transaction_savepoint_list, sp); + break; + + case TRANS_STMT_ROLLBACK_TO: + pos = dml_adaptive_find_savepoint(session_context, stmt->savepoint_name); + if (pos < 0) + break; + sp = (DmlAdaptiveSavepoint *) + list_nth(session_context->transaction_savepoint_list, pos); + + /* Forget table names written after the savepoint */ + if (list_length(session_context->transaction_temp_write_list) > sp->write_list_length) + { + List *names = session_context->transaction_temp_write_list; + int i; + + for (i = sp->write_list_length; i < list_length(names); i++) + pfree(list_nth(names, i)); + if (sp->write_list_length == 0) + { + list_free(names); + names = NIL; + } + else + names = list_truncate(names, sp->write_list_length); + session_context->transaction_temp_write_list = names; + } + + /* Forget table OIDs written after the savepoint */ + if (list_length(session_context->transaction_temp_write_oid_list) > sp->write_oid_list_length) + { + if (sp->write_oid_list_length == 0) + { + list_free(session_context->transaction_temp_write_oid_list); + session_context->transaction_temp_write_oid_list = NIL; + } + else + session_context->transaction_temp_write_oid_list = + list_truncate(session_context->transaction_temp_write_oid_list, + sp->write_oid_list_length); + } + + ereport(DEBUG1, + (errmsg("dml_adaptive: rolled back to savepoint \"%s\", %d table name(s) and %d table oid(s) remain", + sp->name, + list_length(session_context->transaction_temp_write_list), + list_length(session_context->transaction_temp_write_oid_list)))); + + /* The savepoint survives ROLLBACK TO; later ones do not */ + dml_adaptive_truncate_savepoints(session_context, pos + 1); + break; + + case TRANS_STMT_RELEASE: + pos = dml_adaptive_find_savepoint(session_context, stmt->savepoint_name); + if (pos >= 0) + dml_adaptive_truncate_savepoints(session_context, pos); + break; + + default: + break; + } +} + /* * dml adaptive. * @@ -2000,10 +2168,25 @@ dml_adaptive(Node *node, char *query) list_free(session_context->transaction_temp_write_oid_list); session_context->transaction_temp_write_oid_list = NIL; session_context->transaction_temp_write_dboid = 0; + + dml_adaptive_free_savepoints(session_context); } - else if (is_commit_or_rollback_query(node)) + + /* + * SAVEPOINT, ROLLBACK TO and RELEASE do not end the transaction. + * In particular ROLLBACK TO must not be treated like ROLLBACK: + * writes made before the savepoint are still part of the + * transaction and will be committed by a later COMMIT. + */ + else if (is_savepoint_query(node)) + { + dml_adaptive_savepoint_command(session_context, + (TransactionStmt *) node); + } + else if (is_commit_query(node) || is_rollback_query(node)) { session_context->is_in_transaction = false; + dml_adaptive_free_savepoints(session_context); /* The name list is only used during the transaction. */ if (session_context->transaction_temp_write_list != NIL) @@ -2018,9 +2201,11 @@ dml_adaptive(Node *node, char *query) * still fail (e.g. a deferred constraint violation), rolling * back the writes -- which must then not be recorded. We * therefore keep transaction_temp_write_oid_list intact on - * COMMIT and let handle_query_context() flush it on success - * (a failed COMMIT leaves it to be discarded by the next - * BEGIN). + * COMMIT and let pool_track_table_mutation_flush_writes() + * flush it on success (a failed COMMIT leaves it to be + * discarded by the next BEGIN). A COMMIT of an already + * failed transaction "succeeds" with a ROLLBACK command tag; + * that function checks the tag too. * * In every other case (plain dml_adaptive, or ROLLBACK) * discard it now. @@ -2053,9 +2238,9 @@ dml_adaptive(Node *node, char *query) * and the backend connection is idle so do_query (used by the * relcache lookups) is safe. The OIDs are marked in shared * memory only after the backend confirms success, in - * handle_query_context(); resolving them there would issue - * do_query into the in-flight statement/COMMIT response and hang - * the session. + * pool_track_table_mutation_flush_writes(); resolving them there + * would issue do_query into the in-flight statement/COMMIT + * response and hang the session. */ if (pool_config->disable_load_balance_on_write == DLBOW_DML_ADAPTIVE_GLOBAL) { diff --git a/src/context/pool_session_context.c b/src/context/pool_session_context.c index 8efb2633d..941798155 100644 --- a/src/context/pool_session_context.c +++ b/src/context/pool_session_context.c @@ -643,6 +643,7 @@ dml_adaptive_init(void) session_context->transaction_temp_write_list = NIL; session_context->transaction_temp_write_oid_list = NIL; session_context->transaction_temp_write_dboid = 0; + session_context->transaction_savepoint_list = NIL; } } @@ -657,6 +658,7 @@ dml_adaptive_destroy(void) list_free_deep(session_context->transaction_temp_write_list); if (session_context->transaction_temp_write_oid_list != NIL) list_free(session_context->transaction_temp_write_oid_list); + dml_adaptive_free_savepoints(session_context); } } diff --git a/src/include/context/pool_session_context.h b/src/include/context/pool_session_context.h index f798c7da3..671273784 100644 --- a/src/include/context/pool_session_context.h +++ b/src/include/context/pool_session_context.h @@ -174,6 +174,18 @@ typedef struct POOL_TEMP_TABLE_STATE state; /* see above */ } POOL_TEMP_TABLE; +/* + * A savepoint tracked by dml_adaptive/dml_adaptive_global (see + * transaction_savepoint_list). + */ +typedef struct +{ + char *name; /* savepoint name */ + int write_list_length; /* length of transaction_temp_write_list */ + int write_oid_list_length; /* length of + * transaction_temp_write_oid_list */ +} DmlAdaptiveSavepoint; + typedef enum { @@ -359,6 +371,15 @@ typedef struct List *transaction_temp_write_oid_list; int transaction_temp_write_dboid; + /* + * For dml_adaptive and dml_adaptive_global: the savepoints established in + * the current explicit transaction, innermost last. Each element is a + * DmlAdaptiveSavepoint. Both write lists above are append-only, so + * remembering their lengths at SAVEPOINT time is enough for ROLLBACK TO + * to forget exactly the writes made after the savepoint. + */ + List *transaction_savepoint_list; + #ifdef NOT_USED /* Preferred "main" node id. Only used for SimpleForwardToFrontend. */ int preferred_main_node_id; @@ -452,6 +473,7 @@ extern bool pool_is_really_writing_transaction(void); extern void pool_unset_failed_transaction(void); extern void pool_set_failed_transaction(void); extern bool pool_is_failed_transaction(void); +extern void dml_adaptive_free_savepoints(POOL_SESSION_CONTEXT *session_context); extern void pool_unset_transaction_isolation(void); extern void pool_set_transaction_isolation(POOL_TRANSACTION_ISOLATION isolation_level); extern POOL_TRANSACTION_ISOLATION pool_get_transaction_isolation(void); diff --git a/src/include/protocol/pool_proto_modules.h b/src/include/protocol/pool_proto_modules.h index 9b73cc032..a092ed33c 100644 --- a/src/include/protocol/pool_proto_modules.h +++ b/src/include/protocol/pool_proto_modules.h @@ -111,6 +111,7 @@ extern POOL_STATUS ProcessBackendResponse(POOL_CONNECTION *frontend, int *state); extern void handle_query_context(POOL_CONNECTION_POOL *backend);; +extern void pool_track_table_mutation_flush_writes(const char *command_tag); extern void pool_emit_log_for_message_length_diff(int *length_array, char *name); diff --git a/src/protocol/CommandComplete.c b/src/protocol/CommandComplete.c index a4cd0eabe..fc2913cb4 100644 --- a/src/protocol/CommandComplete.c +++ b/src/protocol/CommandComplete.c @@ -60,6 +60,7 @@ CommandComplete(POOL_CONNECTION *frontend, POOL_CONNECTION_POOL *backend, bool c POOL_SESSION_CONTEXT *session_context; POOL_CONNECTION *con; bool discard_all = false; + bool query_context_handled = false; p1 = NULL; len1 = 0; @@ -78,7 +79,10 @@ CommandComplete(POOL_CONNECTION *frontend, POOL_CONNECTION_POOL *backend, bool c * Handle misc process which is necessary when query context exists. */ if (session_context->query_context != NULL && (!SL_MODE || (SL_MODE && !pool_is_doing_extended_query_message()))) + { handle_query_context(backend); + query_context_handled = true; + } /* * If operated in streaming replication mode and doing an extended query, @@ -150,6 +154,18 @@ CommandComplete(POOL_CONNECTION *frontend, POOL_CONNECTION_POOL *backend, bool c return POOL_END; p1 = palloc(len); memcpy(p1, p, len); + + /* + * Now that the command tag is known, record the tables written by + * dml_adaptive_global. The tag is the primary's only if the main + * node is the primary; otherwise pass NULL so that the tables are + * marked unconditionally. In streaming/logical replication mode with + * the extended query protocol this is done when Execute is forwarded + * instead (see Execute() in pool_proto_modules.c). + */ + if (query_context_handled) + pool_track_table_mutation_flush_writes(MAIN_NODE_ID == PRIMARY_NODE_ID ? + p1 : NULL); } /* @@ -307,6 +323,83 @@ CommandComplete(POOL_CONNECTION *frontend, POOL_CONNECTION_POOL *backend, bool c return POOL_CONTINUE; } +/* + * Track table writes for the dml_adaptive_global feature. Only meaningful + * in streaming replication mode (MAIN_REPLICA). + * + * The OIDs of the tables written (and their database OID) were resolved by + * dml_adaptive() at DML routing time, while the backend connection was idle, + * and stashed in transaction_temp_write_oid_list. We mark them in shared + * memory only now, once the backend has confirmed the statement or + * transaction succeeded -- a failed autocommit statement, or a failed + * COMMIT, produces an ErrorResponse and never reaches CommandComplete, and a + * ROLLBACK clears the list while routing, so in all those cases the writes + * are correctly not recorded. + * + * We get here with is_in_transaction == false both for an autocommit + * statement and for a COMMIT (dml_adaptive() clears the flag while routing + * the COMMIT, and keeps the OID list intact for the COMMIT only). An + * in-transaction DML still has the flag set here and is correctly skipped, + * its writes deferred to the COMMIT. + * + * command_tag is the command tag the primary returned, or NULL if it is not + * known. It matters for COMMIT: COMMIT of a transaction that has already + * failed (e.g. a statement raised an error and was not rolled back to a + * savepoint) does not raise an error. PostgreSQL rolls the transaction back + * and answers with a CommandComplete carrying the "ROLLBACK" tag, so nothing + * was committed and nothing must be marked. When the tag is unknown we mark + * the tables, which is the safe direction. + * + * Crucially this path does NO backend I/O: resolving OIDs here via do_query + * would inject a query while the statement/COMMIT response is still being + * read from the backend, desyncing the protocol and hanging the session. + * That is why resolution happens at routing time instead. + */ +void +pool_track_table_mutation_flush_writes(const char *command_tag) +{ + POOL_SESSION_CONTEXT *session_context; + Node *node; + int dboid; + + if (pool_config->disable_load_balance_on_write != + DLBOW_DML_ADAPTIVE_GLOBAL || !MAIN_REPLICA) + return; + + session_context = pool_get_session_context(false); + + if (session_context->query_context == NULL || + session_context->query_context->parse_tree == NULL) + return; + + if (session_context->is_in_transaction || + session_context->transaction_temp_write_oid_list == NIL) + return; + + node = session_context->query_context->parse_tree; + dboid = session_context->transaction_temp_write_dboid; + + if (command_tag != NULL && is_commit_query(node) && + strcmp(command_tag, "COMMIT") != 0) + { + ereport(DEBUG1, + (errmsg("track_table_mutation: COMMIT completed with command tag \"%s\", not marking tables as written", + command_tag))); + } + else if (dboid > 0) + { + foreach_oid(toid, session_context->transaction_temp_write_oid_list) + { + if (toid > 0) + pool_track_table_mutation_mark_table_written(toid, dboid); + } + } + + list_free(session_context->transaction_temp_write_oid_list); + session_context->transaction_temp_write_oid_list = NIL; + session_context->transaction_temp_write_dboid = 0; +} + /* * Handle misc process which is necessary when query context exists. */ @@ -326,52 +419,6 @@ handle_query_context(POOL_CONNECTION_POOL *backend) node = session_context->query_context->parse_tree; - /* - * Track table writes for the dml_adaptive_global feature. Only - * meaningful in streaming replication mode (MAIN_REPLICA). - * - * The OIDs of the tables written (and their database OID) were resolved - * by dml_adaptive() at DML routing time, while the backend connection was - * idle, and stashed in transaction_temp_write_oid_list. We mark them in - * shared memory only now, once the backend has confirmed the statement or - * transaction succeeded -- a failed autocommit statement, or a failed / - * rolled-back COMMIT, produces an ErrorResponse and never reaches - * CommandComplete, and a ROLLBACK clears the list while routing, so in - * all those cases the writes are correctly not recorded. - * - * We reach this point with is_in_transaction == false both for an - * autocommit statement and for a COMMIT (dml_adaptive() clears the flag - * while routing the COMMIT, and keeps the OID list intact for the COMMIT - * only). An in-transaction DML still has the flag set here and is - * correctly skipped, its writes deferred to the COMMIT. - * - * Crucially this path does NO backend I/O: resolving OIDs here via - * do_query would inject a query while the statement/COMMIT response is - * still being read from the backend, desyncing the protocol and hanging - * the session. That is why resolution happens at routing time instead. - */ - if (pool_config->disable_load_balance_on_write == - DLBOW_DML_ADAPTIVE_GLOBAL && - MAIN_REPLICA && - !session_context->is_in_transaction && - session_context->transaction_temp_write_oid_list != NIL) - { - int dboid = session_context->transaction_temp_write_dboid; - - if (dboid > 0) - { - foreach_oid(toid, session_context->transaction_temp_write_oid_list) - { - if (toid > 0) - pool_track_table_mutation_mark_table_written(toid, dboid); - } - } - - list_free(session_context->transaction_temp_write_oid_list); - session_context->transaction_temp_write_oid_list = NIL; - session_context->transaction_temp_write_dboid = 0; - } - if (IsA(node, PrepareStmt)) { if (session_context->uncompleted_message) diff --git a/src/protocol/pool_proto_modules.c b/src/protocol/pool_proto_modules.c index 5e29031d2..787c54b99 100644 --- a/src/protocol/pool_proto_modules.c +++ b/src/protocol/pool_proto_modules.c @@ -1262,6 +1262,12 @@ Execute(POOL_CONNECTION *frontend, POOL_CONNECTION_POOL *backend, /* Various take care at the transaction start */ handle_query_context(backend); + /* + * Record the tables written by dml_adaptive_global. The command tag + * is not known yet at this point. + */ + pool_track_table_mutation_flush_writes(NULL); + /* * Take care of "writing transaction" flag. */ diff --git a/src/test/regression/tests/032.dml_adaptive_load_balance/test.sh b/src/test/regression/tests/032.dml_adaptive_load_balance/test.sh index 5692249fc..895dc2414 100755 --- a/src/test/regression/tests/032.dml_adaptive_load_balance/test.sh +++ b/src/test/regression/tests/032.dml_adaptive_load_balance/test.sh @@ -142,6 +142,77 @@ EOF fi echo ok: T2 final read was load balanced. +# check_node QUERY_ID TABLE NODE: check that the marker SELECT went to NODE +check_node() +{ + fgrep "SELECT i, '$1' FROM $2;" log/pgpool.log |grep "DB node id: $3">/dev/null 2>&1 + if [ $? != 0 ];then + echo fail: "SELECT i, '$1' FROM $2;" was not sent to node $3. + ./shutdownall + exit 1 + fi + echo ok: "$1" was sent to node $3. +} + +# Savepoints: ROLLBACK TO must not end the transaction, and must forget +# only the writes made after the savepoint. +$PSQL test < /dev/null 2>&1 +CREATE TABLE t_sp1(i INTEGER); +CREATE TABLE t_sp2(i INTEGER); +CREATE TABLE t_sp3a(i INTEGER); +CREATE TABLE t_sp3b(i INTEGER); +CREATE TABLE t_sp3c(i INTEGER); +CREATE TABLE t_sp4(i INTEGER); +CREATE TABLE t_sp5(i INTEGER); +CREATE TABLE t_sp6(i INTEGER); +CREATE TABLE t_sp7(i INTEGER); +CREATE TABLE t_sp8(i INTEGER); +CREATE TABLE t_sp9(i INTEGER); +CREATE TABLE t_sp10(i INTEGER); +EOF + + # Wait for any TTL to expire + sleep 3 + + echo "=== Test 12: Write before SAVEPOINT survives ROLLBACK TO ===" + > log/pgpool.log + $PSQL test < log/pgpool.log + $PSQL test < log/pgpool.log + $PSQL test < log/pgpool.log + $PSQL test < log/pgpool.log + $PSQL test < log/pgpool.log + $PSQL test < log/pgpool.log + $PSQL test < log/pgpool.log + $PSQL test < log/pgpool.log + $PSQL test < log/pgpool.log + result=$($PSQL -X -q -t -A test <