From b74dd79c4d8653e434cb386e501e6a591a2f3a9a Mon Sep 17 00:00:00 2001 From: Matheus Alcantara Date: Mon, 15 Jun 2026 12:05:28 -0300 Subject: [PATCH v16 2/3] Add COPY FROM bulk-load callback to the FDW API Add a new optional FdwRoutine callback, ExecForeignBatchCopy, that lets a foreign data wrapper load a batch of tuples during COPY FROM using a more efficient bulk-load protocol (such as the COPY protocol for remote PostgreSQL servers) instead of inserting the rows one at a time. CopyFrom() invokes the callback from the multi-insert path, so it is used when COPY FROM targets a foreign table directly or is routed into a foreign-table partition and batching is enabled for that table (that is, GetForeignModifyBatchSize() reports a batch size greater than one). It is not used when the target has AFTER ROW triggers, because the transferred rows are not returned and so cannot feed those triggers; in that case, and whenever the callback is not provided, COPY FROM falls back to ExecForeignBatchInsert/ExecForeignInsert. Each ExecForeignBatchCopy call is expected to be self-contained: the FDW opens the bulk-load session, transfers all the given tuples, and closes it before returning. This keeps the remote connection idle between calls, so several foreign partitions can share one connection safely. Setup and teardown reuse the existing BeginForeignInsert and EndForeignInsert callbacks. Author: Matheus Alcantara Discussion: https://www.postgresql.org/message-id/DDIZJ217OUDK.2R5WE4OGL5PTY%40gmail.com --- doc/src/sgml/fdwhandler.sgml | 49 +++++++++++++++++++++ src/backend/commands/copyfrom.c | 77 ++++++++++++++++++++++++++++++--- src/include/foreign/fdwapi.h | 6 +++ 3 files changed, 127 insertions(+), 5 deletions(-) diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml index 0103fdacfdf..732a53947d8 100644 --- a/doc/src/sgml/fdwhandler.sgml +++ b/doc/src/sgml/fdwhandler.sgml @@ -908,6 +908,55 @@ EndForeignInsert(EState *estate, +void +ExecForeignBatchCopy(EState *estate, + ResultRelInfo *rinfo, + TupleTableSlot **slots, + int numSlots); + + + Insert a batch of tuples into a foreign table during a + COPY FROM command, using an optimized bulk-load + protocol (such as the COPY protocol for remote + PostgreSQL servers) instead of inserting the + tuples one at a time. + estate is global execution state for the query. + rinfo is the ResultRelInfo struct + describing the target foreign table. + slots is an array of tuple table slots containing the + tuples to be inserted; numSlots is the number of tuples + in the array. + + + + This callback is used only when a COPY FROM is executed + directly on a foreign table, or is routed into a foreign-table partition, + and batching is enabled for the table (that is, the + GetForeignModifyBatchSize callback reports a batch + size greater than one). It is not used when the target has + AFTER ROW triggers, because the transferred rows are not + returned to feed those triggers. In all other cases the core code falls + back to ExecForeignBatchInsert or + ExecForeignInsert. + + + + Each call is expected to be self-contained: the callback should open the + bulk-load session, transfer all numSlots tuples, and + close the session before returning. This leaves the remote connection idle + between calls, which allows several foreign partitions to share one + connection safely. + + + + If the ExecForeignBatchCopy pointer is set to + NULL, COPY FROM falls back to + inserting tuples through + ExecForeignInsert/ExecForeignBatchInsert. + + + + int IsForeignRelUpdatable(Relation rel); diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index 80a527ed4c6..58964c6126e 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -464,7 +464,6 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo, Assert(buffer->bistate == NULL); /* Ensure that the FDW supports batching and it's enabled */ - Assert(resultRelInfo->ri_FdwRoutine->ExecForeignBatchInsert); Assert(batch_size > 1); /* @@ -474,6 +473,40 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo, Assert(!cstate->relname_only); cstate->relname_only = true; + if (resultRelInfo->ri_FdwRoutine->ExecForeignBatchCopy != NULL && + (resultRelInfo->ri_TrigDesc == NULL || + !resultRelInfo->ri_TrigDesc->trig_insert_after_row)) + { + /* + * Send the buffered tuples to the FDW in batches of at most + * batch_size, as we do for ExecForeignBatchInsert. Each call is a + * self-contained COPY operation. + * + * COPY provides no RETURNING, so this path is only usable when + * there are no AFTER ROW triggers that would need the stored rows. + */ + while (sent < nused) + { + int size = (batch_size < nused - sent) ? batch_size : (nused - sent); + + resultRelInfo->ri_FdwRoutine->ExecForeignBatchCopy(estate, + resultRelInfo, + &slots[sent], + size); + + sent += size; + + /* Update the row counter and progress of the COPY command */ + *processed += size; + pgstat_progress_update_param(PROGRESS_COPY_TUPLES_PROCESSED, + *processed); + } + } + else + { + /* Ensure that the FDW supports batching and it's enabled */ + Assert(resultRelInfo->ri_FdwRoutine->ExecForeignBatchInsert); + while (sent < nused) { int size = (batch_size < nused - sent) ? batch_size : (nused - sent); @@ -525,6 +558,7 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo, pgstat_progress_update_param(PROGRESS_COPY_TUPLES_PROCESSED, *processed); } + } for (i = 0; i < nused; i++) ExecClearTuple(slots[i]); @@ -774,6 +808,34 @@ CopyMultiInsertInfoStore(CopyMultiInsertInfo *miinfo, ResultRelInfo *rri, miinfo->bufferedBytes += tuplen; } +/* + * Can this foreign result relation accept batches of tuples during COPY FROM, + * i.e. does CopyMultiInsertBufferFlush() have a usable path for it? + * + * The FDW can batch if it provides ExecForeignBatchInsert, or if it provides + * ExecForeignBatchCopy and there are no AFTER ROW triggers to feed (COPY has no + * RETURNING, so the stored rows are not available for them). These two + * callbacks are independent: an FDW may implement either or both. + * + * Callers must have already established that batching is enabled for the + * relation (ri_BatchSize > 1). + */ +static bool +CopyFromFdwCanBatch(ResultRelInfo *resultRelInfo) +{ + FdwRoutine *fdwroutine = resultRelInfo->ri_FdwRoutine; + + if (fdwroutine->ExecForeignBatchInsert != NULL) + return true; + + if (fdwroutine->ExecForeignBatchCopy != NULL && + (resultRelInfo->ri_TrigDesc == NULL || + !resultRelInfo->ri_TrigDesc->trig_insert_after_row)) + return true; + + return false; +} + /* * Copy FROM file to relation. */ @@ -951,7 +1013,8 @@ CopyFrom(CopyFromState cstate) */ if (resultRelInfo->ri_FdwRoutine != NULL && resultRelInfo->ri_FdwRoutine->GetForeignModifyBatchSize && - resultRelInfo->ri_FdwRoutine->ExecForeignBatchInsert) + (resultRelInfo->ri_FdwRoutine->ExecForeignBatchInsert || + resultRelInfo->ri_FdwRoutine->ExecForeignBatchCopy)) resultRelInfo->ri_BatchSize = resultRelInfo->ri_FdwRoutine->GetForeignModifyBatchSize(resultRelInfo); else @@ -1007,11 +1070,14 @@ CopyFrom(CopyFromState cstate) insertMethod = CIM_SINGLE; } else if (resultRelInfo->ri_FdwRoutine != NULL && - resultRelInfo->ri_BatchSize == 1) + (resultRelInfo->ri_BatchSize == 1 || + !CopyFromFdwCanBatch(resultRelInfo))) { /* * Can't support multi-inserts to a foreign table if the FDW does not - * support batching, or it's disabled for the server or foreign table. + * support batching, or it's disabled for the server or foreign table, + * or the FDW only offers COPY-based batching but the table has AFTER + * ROW triggers to feed. */ insertMethod = CIM_SINGLE; } @@ -1235,7 +1301,8 @@ CopyFrom(CopyFromState cstate) !has_before_insert_row_trig && !has_instead_insert_row_trig && (resultRelInfo->ri_FdwRoutine == NULL || - resultRelInfo->ri_BatchSize > 1); + (resultRelInfo->ri_BatchSize > 1 && + CopyFromFdwCanBatch(resultRelInfo))); /* Set the multi-insert buffer to use for this partition. */ if (leafpart_use_multi_insert) diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h index abf59a0d8ad..0439fa8b24e 100644 --- a/src/include/foreign/fdwapi.h +++ b/src/include/foreign/fdwapi.h @@ -113,6 +113,11 @@ typedef void (*BeginForeignInsert_function) (ModifyTableState *mtstate, typedef void (*EndForeignInsert_function) (EState *estate, ResultRelInfo *rinfo); +typedef void (*ExecForeignBatchCopy_function) (EState *estate, + ResultRelInfo *rinfo, + TupleTableSlot **slots, + int numSlots); + typedef int (*IsForeignRelUpdatable_function) (Relation rel); typedef bool (*PlanDirectModify_function) (PlannerInfo *root, @@ -241,6 +246,7 @@ typedef struct FdwRoutine EndForeignModify_function EndForeignModify; BeginForeignInsert_function BeginForeignInsert; EndForeignInsert_function EndForeignInsert; + ExecForeignBatchCopy_function ExecForeignBatchCopy; IsForeignRelUpdatable_function IsForeignRelUpdatable; PlanDirectModify_function PlanDirectModify; BeginDirectModify_function BeginDirectModify; -- 2.50.1 (Apple Git-155)