From e9b68957c39fc8b07a1df860216b30c32049cfc8 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Fri, 6 Apr 2018 17:59:10 -0300 Subject: [PATCH v22 1/4] Provide infrastructure to allow partition pruning during execution The query planner supports eliminating partitions of a partitioned table during query planning. This has its limitations as it can only perform the elimination using clauses which can be evaluated during planning. Allowing this partition elimination to occur during execution allows the values of Params to be used for elimination too, thus opening the door for PREPAREd statements to have unneeded partitions pruned too. The infrastructure provided here permits the building of a data structure which is able to perform the translation of the matching partition IDs as is returned by the existing partition pruning code into the List index of a subpaths list, as exist in node types such as Append, MergeAppend and ModifyTable. This allows us to translate a list of clauses into a Bitmapset of all the subpath indexes which must be included to satisfy the clause list. This commit does not add support for any node types. Support for this will arrive in follow-up commits. --- src/backend/commands/explain.c | 51 +++-- src/backend/executor/execPartition.c | 419 +++++++++++++++++++++++++++++++++++ src/backend/nodes/copyfuncs.c | 20 ++ src/backend/nodes/outfuncs.c | 27 +++ src/backend/nodes/readfuncs.c | 19 ++ src/backend/partitioning/partprune.c | 256 +++++++++++++++++++++ src/include/executor/execPartition.h | 75 +++++++ src/include/nodes/nodes.h | 1 + src/include/nodes/primnodes.h | 23 ++ src/include/partitioning/partprune.h | 14 ++ 10 files changed, 887 insertions(+), 18 deletions(-) diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index 79f639d5e2..549622da93 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -118,8 +118,8 @@ static void ExplainModifyTarget(ModifyTable *plan, ExplainState *es); static void ExplainTargetRel(Plan *plan, Index rti, ExplainState *es); static void show_modifytable_info(ModifyTableState *mtstate, List *ancestors, ExplainState *es); -static void ExplainMemberNodes(List *plans, PlanState **planstates, - List *ancestors, ExplainState *es); +static void ExplainMemberNodes(PlanState **planstates, int nsubnodes, + int nplans, List *ancestors, ExplainState *es); static void ExplainSubPlans(List *plans, List *ancestors, const char *relationship, ExplainState *es); static void ExplainCustomChildren(CustomScanState *css, @@ -1811,28 +1811,33 @@ ExplainNode(PlanState *planstate, List *ancestors, switch (nodeTag(plan)) { case T_ModifyTable: - ExplainMemberNodes(((ModifyTable *) plan)->plans, - ((ModifyTableState *) planstate)->mt_plans, + ExplainMemberNodes(((ModifyTableState *) planstate)->mt_plans, + ((ModifyTableState *) planstate)->mt_nplans, + list_length(((ModifyTable *) plan)->plans), ancestors, es); break; case T_Append: - ExplainMemberNodes(((Append *) plan)->appendplans, - ((AppendState *) planstate)->appendplans, + ExplainMemberNodes(((AppendState *) planstate)->appendplans, + ((AppendState *) planstate)->as_nplans, + list_length(((Append *) plan)->appendplans), ancestors, es); break; case T_MergeAppend: - ExplainMemberNodes(((MergeAppend *) plan)->mergeplans, - ((MergeAppendState *) planstate)->mergeplans, + ExplainMemberNodes(((MergeAppendState *) planstate)->mergeplans, + ((MergeAppendState *) planstate)->ms_nplans, + list_length(((MergeAppend *) plan)->mergeplans), ancestors, es); break; case T_BitmapAnd: - ExplainMemberNodes(((BitmapAnd *) plan)->bitmapplans, - ((BitmapAndState *) planstate)->bitmapplans, + ExplainMemberNodes(((BitmapAndState *) planstate)->bitmapplans, + ((BitmapAndState *) planstate)->nplans, + list_length(((BitmapAnd *) plan)->bitmapplans), ancestors, es); break; case T_BitmapOr: - ExplainMemberNodes(((BitmapOr *) plan)->bitmapplans, - ((BitmapOrState *) planstate)->bitmapplans, + ExplainMemberNodes(((BitmapOrState *) planstate)->bitmapplans, + ((BitmapOrState *) planstate)->nplans, + list_length(((BitmapOr *) plan)->bitmapplans), ancestors, es); break; case T_SubqueryScan: @@ -3173,18 +3178,28 @@ show_modifytable_info(ModifyTableState *mtstate, List *ancestors, * * The ancestors list should already contain the immediate parent of these * plans. - * - * Note: we don't actually need to examine the Plan list members, but - * we need the list in order to determine the length of the PlanState array. +* +* nsubnodes indicates the number of items in the planstates array. +* nplans indicates the original number of subnodes in the Plan, some of these +* may have been pruned by the run-time pruning code. */ static void -ExplainMemberNodes(List *plans, PlanState **planstates, +ExplainMemberNodes(PlanState **planstates, int nsubnodes, int nplans, List *ancestors, ExplainState *es) { - int nplans = list_length(plans); int j; - for (j = 0; j < nplans; j++) + /* + * The number of subnodes being lower than the number of subplans that was + * specified in the plan means that some subnodes have been ignored per + * instruction for the partition pruning code during the executor + * initialization. To make this a bit less mysterious, we'll indicate + * here that this has happened. + */ + if (nsubnodes < nplans) + ExplainPropertyInteger("Subplans Pruned", NULL, nplans - nsubnodes, es); + + for (j = 0; j < nsubnodes; j++) ExplainNode(planstates[j], ancestors, "Member", NULL, es); } diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c index ad532773a3..c5330d3d00 100644 --- a/src/backend/executor/execPartition.c +++ b/src/backend/executor/execPartition.c @@ -39,6 +39,10 @@ static char *ExecBuildSlotPartitionKeyDescription(Relation rel, bool *isnull, int maxfieldlen); static List *adjust_partition_tlist(List *tlist, TupleConversionMap *map); +static void find_subplans_for_params_recurse(PartitionPruning *pprune, + PartitionRelPruning *partrelprune, + bool allparams, + Bitmapset **validsubplans); /* @@ -1272,3 +1276,418 @@ adjust_partition_tlist(List *tlist, TupleConversionMap *map) return new_tlist; } + +/*------------------------------------------------------------------------- + * Run-Time Partition Pruning Support. + * + * The following series of functions exist to support the removal of unneeded + * subnodes for queries against partitioned tables. The supporting functions + * here are designed to work with any node type which supports an arbitrary + * number of subnodes, e.g. Append, MergeAppend. + * + * Normally this pruning work is performed by the query planner's partition + * pruning code, however, the planner is limited to only being able to prune + * away unneeded partitions using quals which compare the partition key to a + * value which is known to be Const during planning. To allow the same + * pruning to be performed for values which are only determined during + * execution, we must make an additional pruning attempt during execution. + * + * Here we support pruning using both external and exec Params. The main + * difference between these that we need to concern ourselves with is the + * time when the values of the Params are known. External Param values are + * known at any time of execution, including executor startup, but exec Param + * values are only known when the executor is running. + * + * For external Params we may be able to prune away unneeded partitions + * during executor startup. This has the added benefit of not having to + * initialize the unneeded subnodes at all. This is useful as it can save + * quite a bit of effort during executor startup. + * + * For exec Params, we must delay pruning until the executor is running. + * + * Functions: + * + * ExecSetupPartitionPruning: + * This must be called by nodes before any partition pruning is + * attempted. Normally executor startup is a good time. This function + * creates the PartitionPruning details which are required by each + * of the two pruning functions, details include information about + * how to map the partition index details which are returned by the + * planner's partition prune function into subnode indexes. + * + * ExecFindInitialMatchingSubPlans: + * Returns indexes of matching subnodes utilizing only external Params + * to eliminate subnodes. The function must only be called during + * executor startup for the given node before the subnodes themselves + * are initialized. Subnodes which are found not to match by this + * function must not be included in the node's list of subnodes as this + * function performs a remap of the partition index to subplan index map + * and the newly created map provides indexes only for subnodes which + * remain after calling this function. + * + * ExecFindMatchingSubPlans: + * Returns indexes of matching subnodes utilizing all Params to eliminate + * subnodes which can't possibly contain matching tuples. This function + * can only be called while the executor is running. + *------------------------------------------------------------------------- + */ + +/* + * ExecSetupPartitionPruning + * Setup the required data structure which is required for calling + * ExecFindInitialMatchingSubPlans and ExecFindMatchingSubPlans. + * + * 'partitionpruneinfo' is a List of PartitionPruneInfos as generated by + * make_partition_pruneinfo. Here we build a PartitionPruneContext for each + * item in the List. These contexts can be re-used each time we re-evaulate + * which partitions match the pruning steps provided in each + * PartitionPruneInfo. + */ +PartitionPruning * +ExecSetupPartitionPruning(PlanState *planstate, List *partitionpruneinfo) +{ + PartitionRelPruning *partrelprunes; + PartitionPruning *pprune; + ListCell *lc; + int i; + + Assert(partitionpruneinfo != NIL); + + pprune = (PartitionPruning *) palloc(sizeof(PartitionPruning)); + partrelprunes = (PartitionRelPruning *) + palloc(sizeof(PartitionRelPruning) * + list_length(partitionpruneinfo)); + + /* + * The first item in the array contains the details for the query's target + * partition, so record that as the root of the partition hierarchy. + */ + pprune->partrelpruning = partrelprunes; + pprune->npartrelpruning = list_length(partitionpruneinfo); + pprune->extparams = NULL; + pprune->execparams = NULL; + + /* + * Create a sub memory context which we'll use when making calls to the + * query planner's function to determine which partitions will match. The + * planner is not too careful about freeing memory, so we'll ensure we + * call the function in this context to avoid any memory leaking in the + * executor's memory context. + */ + pprune->prune_context = AllocSetContextCreate(CurrentMemoryContext, + "Partition Prune", + ALLOCSET_DEFAULT_SIZES); + + i = 0; + foreach(lc, partitionpruneinfo) + { + PartitionPruneInfo *pinfo = (PartitionPruneInfo *) lfirst(lc); + PartitionRelPruning *partrelprune = &partrelprunes[i]; + PartitionPruneContext *context = &partrelprune->context; + PartitionDesc partdesc; + Relation rel; + PartitionKey partkey; + int partnatts; + + partrelprune->present_parts = bms_copy(pinfo->present_parts); + partrelprune->subnode_map = palloc(sizeof(int) * pinfo->nparts); + + /* + * We must make a copy of this rather than pointing directly to the + * plan's version as we may end up making modifications to it later. + */ + memcpy(partrelprune->subnode_map, pinfo->subnode_map, + sizeof(int) * pinfo->nparts); + + /* We can use the subpart_map verbatim, since we never modify it */ + partrelprune->subpart_map = pinfo->subpart_map; + + rel = relation_open(pinfo->reloid, NoLock); + + partkey = RelationGetPartitionKey(rel); + partdesc = RelationGetPartitionDesc(rel); + + context->strategy = partkey->strategy; + context->partnatts = partnatts = partkey->partnatts; + + context->partopcintype = partkey->partopcintype; + context->partopfamily = partkey->partopfamily; + context->partcollation = partkey->partcollation; + context->partsupfunc = partkey->partsupfunc; + context->nparts = pinfo->nparts; + context->boundinfo = partition_bounds_copy(partdesc->boundinfo, partkey); + + context->planstate = planstate; + context->safeparams = NULL; /* empty for now */ + + partrelprune->pruning_steps = pinfo->pruning_steps; + + partrelprune->extparams = bms_copy(pinfo->extparams); + partrelprune->allparams = bms_union(pinfo->extparams, + pinfo->execparams); + + /* + * Accumulate the paramids which match the partitioned keys of all + * partitioned tables. + */ + pprune->extparams = bms_add_members(pprune->extparams, + pinfo->extparams); + + pprune->execparams = bms_add_members(pprune->execparams, + pinfo->execparams); + + relation_close(rel, NoLock); + + i++; + } + + /* + * Cache the union of the paramids of both types. This saves having to + * recalculate it everytime we need to know what they are. + */ + pprune->allparams = bms_union(pprune->extparams, pprune->execparams); + + return pprune; +} + +/* + * ExecFindInitialMatchingSubPlans + * Determine which subset of subplan nodes we need to initialize based + * on the details stored in 'pprune'. Here we only determine the + * matching partitions using values known during plan startup, which is + * only external Params. Exec Params will be unknown at this time. We + * must delay pruning using exec Params until the actual executor run. + * + * It is expected that callers of this function do so only once during their + * init plan. The caller must only initialize the subnodes which are returned + * by this function. The remaining subnodes should be discarded. Once this + * function has been called, future calls to ExecFindMatchingSubPlans will + * return its matching subnode indexes assuming that the caller discarded + * the original non-matching subnodes. + * + * This function must only be called if 'pprune' has any extparams. + * + * 'nsubnodes' must be passed as the total number of unpruned subnodes. + */ +Bitmapset * +ExecFindInitialMatchingSubPlans(PartitionPruning *pprune, int nsubnodes) +{ + PartitionRelPruning *partrelprune; + MemoryContext oldcontext; + Bitmapset *result = NULL; + + /* + * Ensure there's actually external params, or we've not been called + * already. + */ + Assert(!bms_is_empty(pprune->extparams)); + + partrelprune = pprune->partrelpruning; + + /* + * Switch to a temp context to avoid leaking memory in the executor's + * memory context. + */ + oldcontext = MemoryContextSwitchTo(pprune->prune_context); + + /* Determine which subnodes match the external params */ + find_subplans_for_params_recurse(pprune, partrelprune, false, &result); + + MemoryContextSwitchTo(oldcontext); + + /* Move to the correct memory context */ + result = bms_copy(result); + + MemoryContextReset(pprune->prune_context); + + /* + * Record that partition pruning has been performed for external params. + * This partly also serves to ensure we never call this function twice + * with the same input and also so that ExecFindMatchingSubPlans is aware + * that pruning has already been performed for external Params. + */ + bms_free(pprune->extparams); + pprune->extparams = NULL; + + /* + * If any subnodes were pruned, we must re-sequence the subnode indexes so + * that ExecFindMatchingSubPlans properly returns the indexes from the + * subnodes which will remain after execution of this function. + */ + if (bms_num_members(result) < nsubnodes) + { + int *new_subnode_indexes; + int i; + int newidx; + + /* + * First we must build an array which we can use to adjust the + * existing subnode_map so that it contains the new subnode indexes. + */ + new_subnode_indexes = (int *) palloc(sizeof(int) * nsubnodes); + newidx = 0; + for (i = 0; i < nsubnodes; i++) + { + if (bms_is_member(i, result)) + new_subnode_indexes[i] = newidx++; + else + new_subnode_indexes[i] = -1; /* Newly pruned */ + } + + /* + * Now we can re-sequence each PartitionPruneInfo's subnode_map so + * that they point to the new index of the subnode. + */ + for (i = 0; i < pprune->npartrelpruning; i++) + { + PartitionRelPruning *partrelprune; + int nparts; + int j; + + partrelprune = &pprune->partrelpruning[i]; + nparts = partrelprune->context.nparts; + + /* + * We also need to reset the present_parts field so that it only + * contains partition indexes that we actually still have subnodes + * for. It seems easier to build a fresh one, rather than trying + * to update the existing one. + */ + bms_free(partrelprune->present_parts); + partrelprune->present_parts = NULL; + + for (j = 0; j < nparts; j++) + { + int oldidx = partrelprune->subnode_map[j]; + + /* + * If this partition existed as a subnode then change the old + * subnode index to the new subnode index. The new index may + * become -1 if the partition was pruned above, or it may just + * come earlier in the subnode list due to some subnodes being + * removed earlier in the list. + */ + if (oldidx >= 0) + { + partrelprune->subnode_map[j] = new_subnode_indexes[oldidx]; + + if (new_subnode_indexes[oldidx] >= 0) + partrelprune->present_parts = + bms_add_member(partrelprune->present_parts, j); + } + } + } + + pfree(new_subnode_indexes); + } + + return result; +} + +/* + * ExecFindMatchingSubPlans + * Determine which subplans match the the pruning steps detailed in + * 'pprune' for the current Param values. + * + * Here we utilize both external and exec Params for pruning. + */ +Bitmapset * +ExecFindMatchingSubPlans(PartitionPruning *pprune) +{ + PartitionRelPruning *partrelprune; + MemoryContext oldcontext; + Bitmapset *result = NULL; + + partrelprune = pprune->partrelpruning; + + /* + * Switch to a temp context to avoid leaking memory in the executor's + * memory context. + */ + oldcontext = MemoryContextSwitchTo(pprune->prune_context); + + find_subplans_for_params_recurse(pprune, partrelprune, true, &result); + + MemoryContextSwitchTo(oldcontext); + + /* Move to the correct memory context */ + result = bms_copy(result); + + MemoryContextReset(pprune->prune_context); + + return result; +} + +/* + * find_subplans_for_params_recurse + * Recursive worker function for ExecFindMatchingSubPlans and + * ExecFindInitialMatchingSubPlans + */ +static void +find_subplans_for_params_recurse(PartitionPruning *pprune, + PartitionRelPruning *partrelprune, + bool allparams, + Bitmapset **validsubplans) +{ + PartitionPruneContext *context = &partrelprune->context; + Bitmapset *partset; + Bitmapset *pruneparams; + int i; + + /* Guard against stack overflow due to overly deep partition hierarchy. */ + check_stack_depth(); + + /* + * Use only external params unless we've been asked to also use exec + * params too. + */ + if (allparams) + pruneparams = partrelprune->allparams; + else + pruneparams = partrelprune->extparams; + + /* + * We only need to determine the matching partitions if there are any + * params matching the partition key at this level. If there are no + * matching params, then we can simply return all subnodes which belong to + * this parent partition. The planner should have already determined + * these to be the minimum possible set. We must still recursively visit + * any subpartitioned tables as we may find their partition keys match + * some Params at their level. + */ + if (!bms_is_empty(pruneparams)) + { + context->safeparams = pruneparams; + partset = get_matching_partitions(context, + partrelprune->pruning_steps); + } + else + partset = partrelprune->present_parts; + + /* Translate partset into subnode indexes */ + i = -1; + while ((i = bms_next_member(partset, i)) >= 0) + { + if (partrelprune->subnode_map[i] >= 0) + *validsubplans = bms_add_member(*validsubplans, + partrelprune->subnode_map[i]); + else + { + int partidx = partrelprune->subpart_map[i]; + + if (partidx != -1) + find_subplans_for_params_recurse(pprune, + &pprune->partrelpruning[partidx], + allparams, validsubplans); + else + { + /* + * This could only happen if clauses used in planning where + * more restrictive than those used here, or if the maps are + * somehow corrupt. + */ + elog(ERROR, "partition missing from subplans"); + } + } + } +} diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 9287baaedc..38c5592d32 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -2182,6 +2182,23 @@ _copyPartitionPruneStepCombine(const PartitionPruneStepCombine *from) return newnode; } +static PartitionPruneInfo * +_copyPartitionPruneInfo(const PartitionPruneInfo *from) +{ + PartitionPruneInfo *newnode = makeNode(PartitionPruneInfo); + + COPY_SCALAR_FIELD(reloid); + COPY_NODE_FIELD(pruning_steps); + COPY_BITMAPSET_FIELD(present_parts); + COPY_SCALAR_FIELD(nparts); + COPY_POINTER_FIELD(subnode_map, from->nparts * sizeof(int)); + COPY_POINTER_FIELD(subpart_map, from->nparts * sizeof(int)); + COPY_BITMAPSET_FIELD(extparams); + COPY_BITMAPSET_FIELD(execparams); + + return newnode; +} + /* **************************************************************** * relation.h copy functions * @@ -5121,6 +5138,9 @@ copyObjectImpl(const void *from) case T_PlaceHolderInfo: retval = _copyPlaceHolderInfo(from); break; + case T_PartitionPruneInfo: + retval = _copyPartitionPruneInfo(from); + break; /* * VALUE NODES diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 03a91c3352..f58efcd729 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -1758,6 +1758,30 @@ _outMergeAction(StringInfo str, const MergeAction *node) WRITE_NODE_FIELD(targetList); } +static void +_outPartitionPruneInfo(StringInfo str, const PartitionPruneInfo *node) +{ + int i; + + WRITE_NODE_TYPE("PARTITIONPRUNEINFO"); + + WRITE_OID_FIELD(reloid); + WRITE_NODE_FIELD(pruning_steps); + WRITE_BITMAPSET_FIELD(present_parts); + WRITE_INT_FIELD(nparts); + + appendStringInfoString(str, " :subnode_map"); + for (i = 0; i < node->nparts; i++) + appendStringInfo(str, " %d", node->subnode_map[i]); + + appendStringInfoString(str, " :subpart_map"); + for (i = 0; i < node->nparts; i++) + appendStringInfo(str, " %d", node->subpart_map[i]); + + WRITE_BITMAPSET_FIELD(extparams); + WRITE_BITMAPSET_FIELD(execparams); +} + /***************************************************************************** * * Stuff from relation.h. @@ -3992,6 +4016,9 @@ outNode(StringInfo str, const void *obj) case T_PartitionPruneStepCombine: _outPartitionPruneStepCombine(str, obj); break; + case T_PartitionPruneInfo: + _outPartitionPruneInfo(str, obj); + break; case T_Path: _outPath(str, obj); break; diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index 2812dc9646..cf040adcfb 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -1373,6 +1373,23 @@ _readMergeAction(void) READ_DONE(); } +static PartitionPruneInfo * +_readPartitionPruneInfo(void) +{ + READ_LOCALS(PartitionPruneInfo); + + READ_OID_FIELD(reloid); + READ_NODE_FIELD(pruning_steps); + READ_BITMAPSET_FIELD(present_parts); + READ_INT_FIELD(nparts); + READ_INT_ARRAY(subnode_map, local_node->nparts); + READ_INT_ARRAY(subpart_map, local_node->nparts); + READ_BITMAPSET_FIELD(extparams); + READ_BITMAPSET_FIELD(execparams); + + READ_DONE(); +} + /* * Stuff from parsenodes.h. */ @@ -2645,6 +2662,8 @@ parseNodeString(void) return_value = _readPartitionPruneStepOp(); else if (MATCH("PARTITIONPRUNESTEPCOMBINE", 25)) return_value = _readPartitionPruneStepCombine(); + else if (MATCH("PARTITIONPRUNEINFO", 18)) + return_value = _readPartitionPruneInfo(); else if (MATCH("RTE", 3)) return_value = _readRangeTblEntry(); else if (MATCH("RANGETBLFUNCTION", 16)) diff --git a/src/backend/partitioning/partprune.c b/src/backend/partitioning/partprune.c index 959ee1643d..ff68dca1e9 100644 --- a/src/backend/partitioning/partprune.c +++ b/src/backend/partitioning/partprune.c @@ -21,10 +21,12 @@ #include "catalog/pg_operator.h" #include "catalog/pg_opfamily.h" #include "catalog/pg_type.h" +#include "executor/executor.h" #include "miscadmin.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "optimizer/clauses.h" +#include "optimizer/pathnode.h" #include "optimizer/planner.h" #include "optimizer/predtest.h" #include "optimizer/prep.h" @@ -135,6 +137,7 @@ static PruneStepResult *get_matching_list_bounds(PartitionPruneContext *context, static PruneStepResult *get_matching_range_bounds(PartitionPruneContext *context, StrategyNumber opstrategy, Datum *values, int nvalues, FmgrInfo *partsupfunc, Bitmapset *nullkeys); +static bool pull_partkey_params(PartitionPruneInfo *pinfo, List *steps); static PruneStepResult *perform_pruning_base_step(PartitionPruneContext *context, PartitionPruneStepOp *opstep); static PruneStepResult *perform_pruning_combine_step(PartitionPruneContext *context, @@ -145,6 +148,181 @@ static bool match_boolean_partition_clause(Oid partopfamily, Expr *clause, static bool partkey_datum_from_expr(PartitionPruneContext *context, Expr *expr, Datum *value); +/* + * make_partition_pruneinfo + * Builds List of PartitionPruneInfos, one for each 'partitioned_rels'. + * These can be used in the executor to allow additional partition + * pruning to take place. + * + * Here we generate partition pruning steps for 'prunequal' and also build a + * data stucture which allows mapping of partition indexes into 'subpaths' + * indexes. + * + * If no Params were found to match the partition key in any of the + * 'partitioned_rels', then we return NIL. In such a case run-time partition + * pruning would be useless. + */ +List * +make_partition_pruneinfo(PlannerInfo *root, List *partition_rels, + List *subpaths, List *prunequal) +{ + RelOptInfo *targetpart = NULL; + ListCell *lc; + List *pinfolist = NIL; + int *relid_subnode_map; + int *relid_subpart_map; + int i; + bool gotparam = false; + + /* + * Allocate two arrays to store the 1-based indexes of the 'subpaths' and + * 'partitioned_rels' by relid. + */ + relid_subnode_map = palloc0(sizeof(int) * root->simple_rel_array_size); + relid_subpart_map = palloc0(sizeof(int) * root->simple_rel_array_size); + + i = 1; + foreach(lc, subpaths) + { + Path *path = (Path *) lfirst(lc); + RelOptInfo *pathrel = path->parent; + + Assert(IS_SIMPLE_REL(pathrel)); + Assert(pathrel->relid < root->simple_rel_array_size); + + relid_subnode_map[pathrel->relid] = i++; + } + + /* Likewise for the partition_rels */ + i = 1; + foreach(lc, partition_rels) + { + Index rti = lfirst_int(lc); + + Assert(rti < root->simple_rel_array_size); + + relid_subpart_map[rti] = i++; + } + + /* We now build a PartitionPruneInfo for each partition_rels */ + foreach(lc, partition_rels) + { + Index rti = lfirst_int(lc); + RelOptInfo *subpart = find_base_rel(root, rti); + PartitionPruneInfo *pinfo; + RangeTblEntry *rte; + Bitmapset *present_parts; + int nparts = subpart->nparts; + int *subnode_map; + int *subpart_map; + List *partprunequal; + List *pruning_steps; + bool contradictory; + + /* + * The first item in the list is the target partitioned relation. The + * quals belong to this relation, so require no translation. + */ + if (!targetpart) + { + targetpart = subpart; + partprunequal = prunequal; + } + else + { + /* + * For sub-partitioned tables the columns may not be in the same + * order as the parent, so we must translate the prunequal to make + * it compatible with this relation. + */ + partprunequal = (List *) + adjust_appendrel_attrs_multilevel(root, + (Node *) prunequal, + subpart->relids, + targetpart->relids); + } + + pruning_steps = gen_partprune_steps(subpart, partprunequal, + &contradictory); + + if (contradictory) + { + /* + * This shouldn't happen as the planner should have detected this + * earlier. However, we do use additional quals from parameterized + * paths here. These do only compare Params to the partition key, + * so this shouldn't cause the discovery of any new qual + * contradictions that were not previously discovered as the Param + * values are unknown during planning. Anyway, we'd better do + * something sane here, so let's just disable run-time pruning. + */ + return NIL; + } + + subnode_map = (int *) palloc(nparts * sizeof(int)); + subpart_map = (int *) palloc(nparts * sizeof(int)); + present_parts = NULL; + + /* + * Loop over each partition of the partitioned rel and record the + * subpath index for each. Any partitions which are not present in + * the subpaths List will be set to -1, and any sub-partitioned table + * which is not present will also be set to -1. + */ + for (i = 0; i < nparts; i++) + { + RelOptInfo *partrel = subpart->part_rels[i]; + int subnodeidx = relid_subnode_map[partrel->relid] - 1; + int subpartidx = relid_subpart_map[partrel->relid] - 1; + + subnode_map[i] = subnodeidx; + subpart_map[i] = subpartidx; + + /* + * Record the indexes of all the partition indexes that we have + * subnodes or subparts for. This allows an optimization to skip + * attempting any run-time pruning when no Params are found + * matching the partition key at this level. + */ + if (subnodeidx >= 0 || subpartidx >= 0) + present_parts = bms_add_member(present_parts, i); + } + + rte = root->simple_rte_array[subpart->relid]; + + pinfo = makeNode(PartitionPruneInfo); + pinfo->reloid = rte->relid; + pinfo->pruning_steps = pruning_steps; + pinfo->present_parts = present_parts; + pinfo->nparts = nparts; + pinfo->extparams = NULL; + pinfo->execparams = NULL; + pinfo->subnode_map = subnode_map; + pinfo->subpart_map = subpart_map; + + /* + * Extract Params matching partition key and record if we got any. + * We'll not bother enabling run-time pruning if no params matched the + * partition key at any level of partitioning. + */ + gotparam |= pull_partkey_params(pinfo, pruning_steps); + + pinfolist = lappend(pinfolist, pinfo); + } + + pfree(relid_subnode_map); + pfree(relid_subpart_map); + + if (gotparam) + return pinfolist; + + /* + * If no Params were found to match the partition key on any of the + * partitioned relations then there's no point doing any run-time + * partition pruning. + */ + return NIL; +} /* * gen_partprune_steps @@ -240,6 +418,10 @@ prune_append_rel_partitions(RelOptInfo *rel) context.nparts = rel->nparts; context.boundinfo = rel->boundinfo; + /* Not valid when being called from the planner */ + context.planstate = NULL; + context.safeparams = NULL; + /* Actual pruning happens here. */ partindexes = get_matching_partitions(&context, pruning_steps); @@ -2474,6 +2656,57 @@ get_matching_range_bounds(PartitionPruneContext *context, } /* + * pull_partkey_params + * Loop through each pruning step and record each external and exec + * Params being compared to the partition keys. + */ +static bool +pull_partkey_params(PartitionPruneInfo *pinfo, List *steps) +{ + ListCell *lc; + bool gotone = false; + + foreach(lc, steps) + { + PartitionPruneStepOp *stepop = lfirst(lc); + ListCell *lc2; + + if (!IsA(stepop, PartitionPruneStepOp)) + continue; + + foreach(lc2, stepop->exprs) + { + Expr *expr = lfirst(lc2); + + if (IsA(expr, Param)) + { + Param *param = (Param *) expr; + + switch (param->paramkind) + { + case PARAM_EXTERN: + pinfo->extparams = bms_add_member(pinfo->extparams, + param->paramid); + break; + case PARAM_EXEC: + pinfo->execparams = bms_add_member(pinfo->execparams, + param->paramid); + break; + + default: + elog(ERROR, "unrecognized paramkind: %d", + (int) param->paramkind); + break; + } + gotone = true; + } + } + } + + return gotone; +} + +/* * perform_pruning_base_step * Determines the indexes of datums that satisfy conditions specified in * 'opstep'. @@ -2774,6 +3007,29 @@ partkey_datum_from_expr(PartitionPruneContext *context, *value = ((Const *) expr)->constvalue; return true; + case T_Param: + + /* + * When being called from the executor we may be able to evaluate + * the Param's value. + */ + if (context->planstate && + bms_is_member(((Param *) expr)->paramid, context->safeparams)) + { + ExprState *exprstate; + bool isNull; + + exprstate = ExecInitExpr(expr, context->planstate); + + *value = ExecEvalExprSwitchContext(exprstate, + context->planstate->ps_ExprContext, + &isNull); + if (isNull) + return false; + + return true; + } + default: break; } diff --git a/src/include/executor/execPartition.h b/src/include/executor/execPartition.h index 9f55f6409e..7ee4bc7365 100644 --- a/src/include/executor/execPartition.h +++ b/src/include/executor/execPartition.h @@ -17,6 +17,7 @@ #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "nodes/plannodes.h" +#include "partitioning/partprune.h" /*----------------------- * PartitionDispatch - information about one partitioned table in a partition @@ -108,6 +109,75 @@ typedef struct PartitionTupleRouting TupleTableSlot *root_tuple_slot; } PartitionTupleRouting; +/*----------------------- + * PartitionRelPruning - Encapsulates all information required to support + * elimination of partitions in node types which support arbitrary Lists of + * subplans. Information stored here allows the planner's partition pruning + * functions to be called and the return value of partition indexes translated + * into the subpath indexes of node types such as Append, thus allowing us to + * bypass certain subnodes when we have proofs that indicate that no tuple + * matching the 'pruning_steps' will be found within. + * + * subnode_map An array containing the subnode index which + * matches this partition index, or -1 if the + * subnode has been pruned already. + * subpart_map An array containing the offset into the + * 'partrelpruning' array in PartitionPruning, or + * -1 if there is no such element in that array. + * present_parts A Bitmapset of the partition index that we have + * subnodes mapped for. + * context Contains the context details required to call + * the partition pruning code. + * pruning_steps Contains a list of PartitionPruneStep used to + * perform the actual pruning. + * extparams Contains paramids of external params found + * matching partition keys in 'pruning_steps'. + * allparams As 'extparams' but also including exec params. + *----------------------- + */ +typedef struct PartitionRelPruning +{ + int *subnode_map; + int *subpart_map; + Bitmapset *present_parts; + PartitionPruneContext context; + List *pruning_steps; + Bitmapset *extparams; + Bitmapset *allparams; +} PartitionRelPruning; + +/*----------------------- + * PartitionPruning - Encapsulates a hierarchy of PartitionRelPruning + * structs and also stores all paramids which were found to match the + * partition keys of each partition. This struct can be attached to node + * types which support arbitrary Lists of subnodes containing partitions to + * allow subnodes to be eliminated due to the clauses being unable to match + * to any tuple that the subnode could possibly produce. + * + * partrelpruning Array of PartitionRelPruning for the node's target + * partitioned relation. First element contains the + * details for the target partitioned table. + * npartrelpruning Number of items in 'partrelpruning' array. + * prune_context A memory context which can be used to call the query + * planner's partition prune functions. + * extparams All PARAM_EXTERN paramids which were found to match a + * partition key in each of the contained + * PartitionRelPruning structs. + * execparams As above but for PARAM_EXEC. + * allparams Union of 'extparams' and 'execparams', saved to avoid + * recalculation. + *----------------------- + */ +typedef struct PartitionPruning +{ + PartitionRelPruning *partrelpruning; + int npartrelpruning; + MemoryContext prune_context; + Bitmapset *extparams; + Bitmapset *execparams; + Bitmapset *allparams; +} PartitionPruning; + extern PartitionTupleRouting *ExecSetupPartitionTupleRouting(ModifyTableState *mtstate, Relation rel); extern int ExecFindPartition(ResultRelInfo *resultRelInfo, @@ -127,5 +197,10 @@ extern HeapTuple ConvertPartitionTupleSlot(TupleConversionMap *map, TupleTableSlot *new_slot, TupleTableSlot **p_my_slot); extern void ExecCleanupTupleRouting(PartitionTupleRouting *proute); +extern PartitionPruning *ExecSetupPartitionPruning(PlanState *planstate, + List *partitionpruneinfo); +extern Bitmapset *ExecFindMatchingSubPlans(PartitionPruning *partprune); +extern Bitmapset *ExecFindInitialMatchingSubPlans(PartitionPruning *partprune, + int nsubnodes); #endif /* EXECPARTITION_H */ diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index 4fc2de7184..defdbae507 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -196,6 +196,7 @@ typedef enum NodeTag T_PartitionPruneStep, T_PartitionPruneStepOp, T_PartitionPruneStepCombine, + T_PartitionPruneInfo, /* * TAGS FOR EXPRESSION STATE NODES (execnodes.h) diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h index ff5c4ff8e4..f90aa7b2a1 100644 --- a/src/include/nodes/primnodes.h +++ b/src/include/nodes/primnodes.h @@ -1581,4 +1581,27 @@ typedef struct PartitionPruneStepCombine List *source_stepids; } PartitionPruneStepCombine; +/*---------- + * PartitionPruneInfo - Details required to allow the executor to prune + * partitions. + * + * Here we store mapping details to allow translation of a partitioned table's + * index into subnode indexes for node types which support arbitrary numbers + * of sub nodes, such as Append. + *---------- + */ +typedef struct PartitionPruneInfo +{ + NodeTag type; + Oid reloid; /* Oid of partition rel */ + List *pruning_steps; /* List of PartitionPruneStep */ + Bitmapset *present_parts; /* Indexes of all partitions which subnodes + * are present for. */ + int nparts; /* The length of the following two arrays */ + int *subnode_map; /* subnode index by partition id, or -1 */ + int *subpart_map; /* subpart index by partition id, or -1 */ + Bitmapset *extparams; /* All external paramids seen in prunesteps */ + Bitmapset *execparams; /* All exec paramids seen in prunesteps */ +} PartitionPruneInfo; + #endif /* PRIMNODES_H */ diff --git a/src/include/partitioning/partprune.h b/src/include/partitioning/partprune.h index 52fadc7caf..3e8cc76c3d 100644 --- a/src/include/partitioning/partprune.h +++ b/src/include/partitioning/partprune.h @@ -37,9 +37,23 @@ typedef struct PartitionPruneContext /* Partition boundary info */ PartitionBoundInfo boundinfo; + + /* + * Can be set when the context is used from the executor to allow params + * found matching the partition key to be evaulated. + */ + PlanState *planstate; + + /* + * Parameters that are safe to be used for partition pruning. execparams + * are not safe to use until after init plan. + */ + Bitmapset *safeparams; } PartitionPruneContext; +extern List *make_partition_pruneinfo(PlannerInfo *root, List *partition_rels, + List *subpaths, List *prunequal); extern Relids prune_append_rel_partitions(RelOptInfo *rel); extern Bitmapset *get_matching_partitions(PartitionPruneContext *context, List *pruning_steps); -- 2.11.0