From 113aee5cda7f8ce53fb9fdb568461ae90c9bbeba Mon Sep 17 00:00:00 2001 From: TatsuyaKawata Date: Thu, 15 Oct 2026 23:15:44 +0900 Subject: [PATCH v5] Add memory/disk usage for Function Scan nodes in EXPLAIN Commit 1eff8279d4 added the infrastructure to report the maximum tuplestore memory or disk usage with EXPLAIN ANALYZE, and 95d6e9af07 and 40708acd65 extended it to WindowAgg, CTE Scan, Table Function Scan and Recursive Union nodes. Function Scan was never covered, even though a function in FROM always materializes its result into a tuplestore, and that tuplestore can silently spill to disk once work_mem is exceeded. Report the storage type and maximum storage used for Function Scan nodes too. A Function Scan has one tuplestore per function, so we report the peak of the total space used by the tuplestores that exist at the same time. The storage type is reported as disk if any tuplestore, of any function and in any rescan, spilled to disk. ExecMakeTableFunctionResult() returns a tuplestore that already holds the whole result set of the function, so the statistics are updated each time a function's tuplestore is created, including after a rescan with changed parameters. As is done for Material nodes, parallel workers copy their statistics into shared memory when they end, so that EXPLAIN ANALYZE can show them per worker. --- src/backend/commands/explain.c | 45 +++++++ src/backend/executor/execParallel.c | 19 +++ src/backend/executor/nodeFunctionscan.c | 154 ++++++++++++++++++++++ src/include/executor/instrument_node.h | 21 +++ src/include/executor/nodeFunctionscan.h | 8 ++ src/include/nodes/execnodes.h | 4 + src/test/regress/expected/explain.out | 43 +++++- src/test/regress/expected/planner_est.out | 4 + src/test/regress/sql/explain.sql | 8 ++ src/test/regress/sql/planner_est.sql | 4 + src/tools/pgindent/typedefs.list | 2 + 11 files changed, 309 insertions(+), 3 deletions(-) diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index 4b1ad4227ac..1b7a3956e71 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -130,6 +130,8 @@ static void show_hash_info(HashState *hashstate, ExplainState *es); static void show_material_info(MaterialState *mstate, ExplainState *es); static void show_windowagg_info(WindowAggState *winstate, ExplainState *es); static void show_ctescan_info(CteScanState *ctescanstate, ExplainState *es); +static void show_functionscan_info(FunctionScanState *fsstate, + ExplainState *es); static void show_table_func_scan_info(TableFuncScanState *tscanstate, ExplainState *es); static void show_recursive_union_info(RecursiveUnionState *rstate, @@ -2097,6 +2099,7 @@ ExplainNode(PlanState *planstate, List *ancestors, if (plan->qual) show_instrumentation_count("Rows Removed by Filter", 1, planstate, es); + show_functionscan_info(castNode(FunctionScanState, planstate), es); break; case T_TableFuncScan: if (es->verbose) @@ -3570,6 +3573,48 @@ show_ctescan_info(CteScanState *ctescanstate, ExplainState *es) show_storage_info(usedDisk, maxSpaceUsed, es); } +/* + * Show information on Function Scan node, storage method and maximum + * memory/disk space used. + */ +static void +show_functionscan_info(FunctionScanState *fsstate, ExplainState *es) +{ + if (!es->analyze) + return; + + /* + * Nothing to show for the leader if execution didn't get as far as + * creating any tuplestore. + */ + if (fsstate->stats.maxSpaceUsed > 0) + show_storage_info(fsstate->stats.usedDisk, + fsstate->stats.maxSpaceUsed, es); + + if (fsstate->shared_info == NULL) + return; + + /* Show details from parallel workers */ + for (int n = 0; n < fsstate->shared_info->num_workers; n++) + { + FunctionScanInstrumentation *si; + + si = &fsstate->shared_info->sinstrument[n]; + + /* Skip workers that didn't create a tuplestore */ + if (si->maxSpaceUsed == 0) + continue; + + if (es->workers_state) + ExplainOpenWorker(n, es); + + show_storage_info(si->usedDisk, si->maxSpaceUsed, es); + + if (es->workers_state) + ExplainCloseWorker(n, es); + } +} + /* * Show information on Table Function Scan node, storage method and maximum * memory/disk space used. diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c index 316dd3f67c6..2fe874fc5d4 100644 --- a/src/backend/executor/execParallel.c +++ b/src/backend/executor/execParallel.c @@ -31,6 +31,7 @@ #include "executor/nodeBitmapIndexscan.h" #include "executor/nodeCustom.h" #include "executor/nodeForeignscan.h" +#include "executor/nodeFunctionscan.h" #include "executor/nodeHash.h" #include "executor/nodeHashjoin.h" #include "executor/nodeIncrementalSort.h" @@ -343,6 +344,10 @@ ExecParallelEstimate(PlanState *planstate, ExecParallelEstimateContext *e) /* even when not parallel-aware, for EXPLAIN ANALYZE */ ExecMaterialEstimate((MaterialState *) planstate, e->pcxt); break; + case T_FunctionScanState: + /* even when not parallel-aware, for EXPLAIN ANALYZE */ + ExecFunctionScanEstimate((FunctionScanState *) planstate, e->pcxt); + break; default: break; } @@ -595,6 +600,11 @@ ExecParallelInitializeDSM(PlanState *planstate, /* even when not parallel-aware, for EXPLAIN ANALYZE */ ExecMaterialInitializeDSM((MaterialState *) planstate, d->pcxt); break; + case T_FunctionScanState: + /* even when not parallel-aware, for EXPLAIN ANALYZE */ + ExecFunctionScanInitializeDSM((FunctionScanState *) planstate, + d->pcxt); + break; default: break; } @@ -1084,6 +1094,7 @@ ExecParallelReInitializeDSM(PlanState *planstate, case T_IncrementalSortState: case T_MemoizeState: case T_MaterialState: + case T_FunctionScanState: /* these nodes have DSM state, but no reinitialization is required */ break; @@ -1168,6 +1179,9 @@ ExecParallelRetrieveInstrumentation(PlanState *planstate, case T_MaterialState: ExecMaterialRetrieveInstrumentation((MaterialState *) planstate); break; + case T_FunctionScanState: + ExecFunctionScanRetrieveInstrumentation((FunctionScanState *) planstate); + break; case T_BitmapHeapScanState: ExecBitmapHeapRetrieveInstrumentation((BitmapHeapScanState *) planstate); break; @@ -1501,6 +1515,11 @@ ExecParallelInitializeWorker(PlanState *planstate, ParallelWorkerContext *pwcxt) /* even when not parallel-aware, for EXPLAIN ANALYZE */ ExecMaterialInitializeWorker((MaterialState *) planstate, pwcxt); break; + case T_FunctionScanState: + /* even when not parallel-aware, for EXPLAIN ANALYZE */ + ExecFunctionScanInitializeWorker((FunctionScanState *) planstate, + pwcxt); + break; default: break; } diff --git a/src/backend/executor/nodeFunctionscan.c b/src/backend/executor/nodeFunctionscan.c index 1416f1f09ae..8b37a812fd0 100644 --- a/src/backend/executor/nodeFunctionscan.c +++ b/src/backend/executor/nodeFunctionscan.c @@ -41,9 +41,18 @@ typedef struct FunctionScanPerFuncState Tuplestorestate *tstore; /* holds the function result set */ int64 rowcount; /* # of rows in result set, -1 if not known */ TupleTableSlot *func_slot; /* function result slot (or NULL) */ + + /* + * Storage statistics of the current tuplestore, to report in EXPLAIN + * ANALYZE. Only meaningful while tstore is not NULL. + */ + bool usedDisk; + int64 spaceUsed; } FunctionScanPerFuncState; static TupleTableSlot *FunctionNext(FunctionScanState *node); +static void update_storage_stats(FunctionScanState *node, + FunctionScanPerFuncState *fs); /* ---------------------------------------------------------------- @@ -104,6 +113,10 @@ FunctionNext(FunctionScanState *node) * call is fast, so the overhead shouldn't be an issue. */ tuplestore_rescan(tstore); + + /* Collect its storage statistics for EXPLAIN ANALYZE */ + if (node->ss.ps.instrument) + update_storage_stats(node, &node->funcstates[0]); } /* @@ -163,6 +176,10 @@ FunctionNext(FunctionScanState *node) * call is fast, so the overhead shouldn't be an issue. */ tuplestore_rescan(fs->tstore); + + /* Collect its storage statistics for EXPLAIN ANALYZE */ + if (node->ss.ps.instrument) + update_storage_stats(node, fs); } /* @@ -358,6 +375,8 @@ ExecInitFunctionScan(FunctionScan *node, EState *estate, int eflags) */ fs->tstore = NULL; fs->rowcount = -1; + fs->usedDisk = false; + fs->spaceUsed = 0; /* * Now build a tupdesc showing the result type we expect from the @@ -520,6 +539,8 @@ ExecInitFunctionScan(FunctionScan *node, EState *estate, int eflags) "Table function arguments", ALLOCSET_DEFAULT_SIZES); + scanstate->shared_info = NULL; + return scanstate; } @@ -534,6 +555,23 @@ ExecEndFunctionScan(FunctionScanState *node) { int i; + /* + * When ending a parallel worker, copy the tuplestore usage details into + * shared memory so that they can be picked up by the main process to + * report in EXPLAIN ANALYZE. The Gather node above us may be rescanned, + * in which case a new set of workers will write to the same slots, so + * merge with any existing values rather than overwriting them. + */ + if (node->shared_info != NULL && IsParallelWorker()) + { + FunctionScanInstrumentation *si; + + Assert(ParallelWorkerNumber < node->shared_info->num_workers); + si = &node->shared_info->sinstrument[ParallelWorkerNumber]; + si->maxSpaceUsed = Max(si->maxSpaceUsed, node->stats.maxSpaceUsed); + si->usedDisk |= node->stats.usedDisk; + } + /* * Release slots and tuplestore resources */ @@ -549,6 +587,35 @@ ExecEndFunctionScan(FunctionScanState *node) } } +/* + * update_storage_stats + * Update the storage statistics to report in EXPLAIN ANALYZE with the + * function's tuplestore, which already holds the whole result set. + * + * A FunctionScan node has one tuplestore per function. We report the peak + * of the total space used by the tuplestores existing at the same time. The + * storage type is reported as disk if any tuplestore, of any function and in + * any rescan, spilled to disk. + */ +static void +update_storage_stats(FunctionScanState *node, FunctionScanPerFuncState *fs) +{ + int64 totalSpaceUsed = 0; + + tuplestore_get_stats(fs->tstore, &fs->usedDisk, &fs->spaceUsed); + + for (int i = 0; i < node->nfuncs; i++) + { + FunctionScanPerFuncState *f = &node->funcstates[i]; + + if (f->tstore != NULL) + totalSpaceUsed += f->spaceUsed; + } + + node->stats.maxSpaceUsed = Max(node->stats.maxSpaceUsed, totalSpaceUsed); + node->stats.usedDisk |= fs->usedDisk; +} + /* ---------------------------------------------------------------- * ExecReScanFunctionScan * @@ -614,3 +681,90 @@ ExecReScanFunctionScan(FunctionScanState *node) tuplestore_rescan(node->funcstates[i].tstore); } } + +/* ---------------------------------------------------------------- + * Parallel Query Support + * ---------------------------------------------------------------- + */ + +/* ---------------------------------------------------------------- + * ExecFunctionScanEstimate + * + * Estimate space required to propagate function scan statistics. + * ---------------------------------------------------------------- + */ +void +ExecFunctionScanEstimate(FunctionScanState *node, ParallelContext *pcxt) +{ + Size size; + + /* don't need this if not instrumenting or no workers */ + if (!node->ss.ps.instrument || pcxt->nworkers == 0) + return; + + size = mul_size(pcxt->nworkers, sizeof(FunctionScanInstrumentation)); + size = add_size(size, offsetof(SharedFunctionScanInfo, sinstrument)); + shm_toc_estimate_chunk(&pcxt->estimator, size); + shm_toc_estimate_keys(&pcxt->estimator, 1); +} + +/* ---------------------------------------------------------------- + * ExecFunctionScanInitializeDSM + * + * Initialize DSM space for function scan statistics. + * ---------------------------------------------------------------- + */ +void +ExecFunctionScanInitializeDSM(FunctionScanState *node, ParallelContext *pcxt) +{ + Size size; + + /* don't need this if not instrumenting or no workers */ + if (!node->ss.ps.instrument || pcxt->nworkers == 0) + return; + + size = offsetof(SharedFunctionScanInfo, sinstrument) + + pcxt->nworkers * sizeof(FunctionScanInstrumentation); + node->shared_info = shm_toc_allocate(pcxt->toc, size); + /* ensure any unfilled slots will contain zeroes */ + memset(node->shared_info, 0, size); + node->shared_info->num_workers = pcxt->nworkers; + shm_toc_insert(pcxt->toc, node->ss.ps.plan->plan_node_id, + node->shared_info); +} + +/* ---------------------------------------------------------------- + * ExecFunctionScanInitializeWorker + * + * Attach worker to DSM space for function scan statistics. + * ---------------------------------------------------------------- + */ +void +ExecFunctionScanInitializeWorker(FunctionScanState *node, + ParallelWorkerContext *pwcxt) +{ + node->shared_info = + shm_toc_lookup(pwcxt->toc, node->ss.ps.plan->plan_node_id, true); +} + +/* ---------------------------------------------------------------- + * ExecFunctionScanRetrieveInstrumentation + * + * Transfer function scan statistics from DSM to private memory. + * ---------------------------------------------------------------- + */ +void +ExecFunctionScanRetrieveInstrumentation(FunctionScanState *node) +{ + Size size; + SharedFunctionScanInfo *si; + + if (node->shared_info == NULL) + return; + + size = offsetof(SharedFunctionScanInfo, sinstrument) + + node->shared_info->num_workers * sizeof(FunctionScanInstrumentation); + si = palloc(size); + memcpy(si, node->shared_info, size); + node->shared_info = si; +} diff --git a/src/include/executor/instrument_node.h b/src/include/executor/instrument_node.h index 4759a628afe..40db5a70c01 100644 --- a/src/include/executor/instrument_node.h +++ b/src/include/executor/instrument_node.h @@ -194,6 +194,27 @@ typedef struct SharedMaterialInfo } SharedMaterialInfo; +/* --------------------- + * Instrumentation information for FunctionScan + * --------------------- + */ +typedef struct FunctionScanInstrumentation +{ + int64 maxSpaceUsed; /* peak memory/disk usage in bytes, or 0 if no + * tuplestore was created */ + bool usedDisk; /* did any tuplestore spill to disk? */ +} FunctionScanInstrumentation; + +/* + * Shared memory container for per-worker function scan information + */ +typedef struct SharedFunctionScanInfo +{ + int num_workers; + FunctionScanInstrumentation sinstrument[FLEXIBLE_ARRAY_MEMBER]; +} SharedFunctionScanInfo; + + /* --------------------- * Instrumentation information for Sorts. * --------------------- diff --git a/src/include/executor/nodeFunctionscan.h b/src/include/executor/nodeFunctionscan.h index e6a1a42d3f9..3cfd61e8b8d 100644 --- a/src/include/executor/nodeFunctionscan.h +++ b/src/include/executor/nodeFunctionscan.h @@ -14,10 +14,18 @@ #ifndef NODEFUNCTIONSCAN_H #define NODEFUNCTIONSCAN_H +#include "access/parallel.h" #include "nodes/execnodes.h" extern FunctionScanState *ExecInitFunctionScan(FunctionScan *node, EState *estate, int eflags); extern void ExecEndFunctionScan(FunctionScanState *node); extern void ExecReScanFunctionScan(FunctionScanState *node); +extern void ExecFunctionScanEstimate(FunctionScanState *node, + ParallelContext *pcxt); +extern void ExecFunctionScanInitializeDSM(FunctionScanState *node, + ParallelContext *pcxt); +extern void ExecFunctionScanInitializeWorker(FunctionScanState *node, + ParallelWorkerContext *pwcxt); +extern void ExecFunctionScanRetrieveInstrumentation(FunctionScanState *node); #endif /* NODEFUNCTIONSCAN_H */ diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index e3cea52e7df..4017d8b9c0f 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1924,6 +1924,8 @@ typedef struct SubqueryScanState * funcstates per-function execution states (private in * nodeFunctionscan.c) * argcontext memory context to evaluate function arguments in + * stats storage statistics to report in EXPLAIN ANALYZE + * shared_info statistics for parallel workers * ---------------- */ struct FunctionScanPerFuncState; @@ -1938,6 +1940,8 @@ typedef struct FunctionScanState int nfuncs; struct FunctionScanPerFuncState *funcstates; /* array of length nfuncs */ MemoryContext argcontext; + FunctionScanInstrumentation stats; + SharedFunctionScanInfo *shared_info; } FunctionScanState; /* ---------------- diff --git a/src/test/regress/expected/explain.out b/src/test/regress/expected/explain.out index 74a4d87801e..1603c806876 100644 --- a/src/test/regress/expected/explain.out +++ b/src/test/regress/expected/explain.out @@ -799,9 +799,10 @@ select explain_filter('explain (analyze,buffers off,costs off) select sum(n) ove Window: w1 AS () Storage: Memory Maximum Storage: NkB -> Function Scan on generate_series a (actual time=N.N..N.N rows=N.N loops=N) + Storage: Memory Maximum Storage: NkB Planning Time: N.N ms Execution Time: N.N ms -(6 rows) +(7 rows) -- Test tuplestore storage usage in Window aggregate (disk case) set work_mem to 64; @@ -812,9 +813,10 @@ select explain_filter('explain (analyze,buffers off,costs off) select sum(n) ove Window: w1 AS () Storage: Disk Maximum Storage: NkB -> Function Scan on generate_series a (actual time=N.N..N.N rows=N.N loops=N) + Storage: Disk Maximum Storage: NkB Planning Time: N.N ms Execution Time: N.N ms -(6 rows) +(7 rows) -- Test tuplestore storage usage in Window aggregate (memory and disk case, final result is disk) select explain_filter('explain (analyze,buffers off,costs off) select sum(n) over(partition by m) from (SELECT n < 3 as m, n from generate_series(1,2500) a(n))'); @@ -827,8 +829,43 @@ select explain_filter('explain (analyze,buffers off,costs off) select sum(n) ove Sort Key: ((a.n < N)) Sort Method: external merge Disk: NkB -> Function Scan on generate_series a (actual time=N.N..N.N rows=N.N loops=N) + Storage: Disk Maximum Storage: NkB Planning Time: N.N ms Execution Time: N.N ms -(9 rows) +(10 rows) + +-- Test tuplestore storage usage in Function Scan (memory case) +select explain_filter('explain (analyze,buffers off,costs off) select count(*) from generate_series(1,10) a(n)'); + explain_filter +---------------------------------------------------------------------------------- + Aggregate (actual time=N.N..N.N rows=N.N loops=N) + -> Function Scan on generate_series a (actual time=N.N..N.N rows=N.N loops=N) + Storage: Memory Maximum Storage: NkB + Planning Time: N.N ms + Execution Time: N.N ms +(5 rows) + +-- Test tuplestore storage usage in Function Scan (disk case) +select explain_filter('explain (analyze,buffers off,costs off) select count(*) from generate_series(1,2500) a(n)'); + explain_filter +---------------------------------------------------------------------------------- + Aggregate (actual time=N.N..N.N rows=N.N loops=N) + -> Function Scan on generate_series a (actual time=N.N..N.N rows=N.N loops=N) + Storage: Disk Maximum Storage: NkB + Planning Time: N.N ms + Execution Time: N.N ms +(5 rows) + +-- Test tuplestore storage usage in Function Scan with ROWS FROM, which uses +-- one tuplestore per function +select explain_filter('explain (analyze,buffers off,costs off) select count(*) from rows from (generate_series(1,2500), generate_series(1,2500)) a(n,m)'); + explain_filter +------------------------------------------------------------------ + Aggregate (actual time=N.N..N.N rows=N.N loops=N) + -> Function Scan on a (actual time=N.N..N.N rows=N.N loops=N) + Storage: Disk Maximum Storage: NkB + Planning Time: N.N ms + Execution Time: N.N ms +(5 rows) reset work_mem; diff --git a/src/test/regress/expected/planner_est.out b/src/test/regress/expected/planner_est.out index 236cb274a78..fbe2d5ff1a1 100644 --- a/src/test/regress/expected/planner_est.out +++ b/src/test/regress/expected/planner_est.out @@ -29,6 +29,10 @@ BEGIN EXECUTE format('explain (analyze %s, costs on, summary off, timing off, buffers off) %s', analyze_str, query) LOOP + -- Ignore Storage output because the size varies depending on + -- the platform + CONTINUE WHEN ln ~ '^\s*Storage: '; + IF hide_costs = true THEN ln := regexp_replace(ln, 'cost=\d+\.\d\d\.\.\d+\.\d\d', 'cost=N..N'); END IF; diff --git a/src/test/regress/sql/explain.sql b/src/test/regress/sql/explain.sql index 2f163c64bf6..cb18f495343 100644 --- a/src/test/regress/sql/explain.sql +++ b/src/test/regress/sql/explain.sql @@ -190,4 +190,12 @@ set work_mem to 64; select explain_filter('explain (analyze,buffers off,costs off) select sum(n) over() from generate_series(1,2500) a(n)'); -- Test tuplestore storage usage in Window aggregate (memory and disk case, final result is disk) select explain_filter('explain (analyze,buffers off,costs off) select sum(n) over(partition by m) from (SELECT n < 3 as m, n from generate_series(1,2500) a(n))'); + +-- Test tuplestore storage usage in Function Scan (memory case) +select explain_filter('explain (analyze,buffers off,costs off) select count(*) from generate_series(1,10) a(n)'); +-- Test tuplestore storage usage in Function Scan (disk case) +select explain_filter('explain (analyze,buffers off,costs off) select count(*) from generate_series(1,2500) a(n)'); +-- Test tuplestore storage usage in Function Scan with ROWS FROM, which uses +-- one tuplestore per function +select explain_filter('explain (analyze,buffers off,costs off) select count(*) from rows from (generate_series(1,2500), generate_series(1,2500)) a(n,m)'); reset work_mem; diff --git a/src/test/regress/sql/planner_est.sql b/src/test/regress/sql/planner_est.sql index 2b696a4e4e5..72aae8807ec 100644 --- a/src/test/regress/sql/planner_est.sql +++ b/src/test/regress/sql/planner_est.sql @@ -30,6 +30,10 @@ BEGIN EXECUTE format('explain (analyze %s, costs on, summary off, timing off, buffers off) %s', analyze_str, query) LOOP + -- Ignore Storage output because the size varies depending on + -- the platform + CONTINUE WHEN ln ~ '^\s*Storage: '; + IF hide_costs = true THEN ln := regexp_replace(ln, 'cost=\d+\.\d\d\.\.\d+\.\d\d', 'cost=N..N'); END IF; diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index a7a8dbdd23f..0f010a68157 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1038,6 +1038,7 @@ FunctionCallInfoBaseData FunctionParameter FunctionParameterMode FunctionScan +FunctionScanInstrumentation FunctionScanPerFuncState FunctionScanState FuzzyAttrMatchState @@ -2850,6 +2851,7 @@ SharedDependencyObjectType SharedDependencyType SharedExecutorInstrumentation SharedFileSet +SharedFunctionScanInfo SharedHashInfo SharedIncrementalSortInfo SharedIndexScanInstrumentation -- 2.34.1