From 79909d4bd195abe65fa6c9571cf9aea425fa54e1 Mon Sep 17 00:00:00 2001 From: Ashutosh Bapat Date: Thu, 26 Sep 2024 12:30:13 +0530 Subject: [PATCH 4/6] Avoid translating RestrictInfo repeatedly RestrictInfo for the child relations (including the child join relations) are obtained by translating RestrictInfo applicable to the parent rel. Since these translations are not tracked, the same RestrictInfo may get translated multiple times for the same parent and child pair. When using partitionwise join this can happen as many times as the number of possible join orders between the partitioned tables and as many times a parent path is reparameterized for a child if a clause is used in such a path. Repeated translations are avoided by saving them in RestrictInfo hash table and reusing as needed. Ashutosh Bapat --- src/backend/optimizer/path/joinrels.c | 7 +- src/backend/optimizer/util/pathnode.c | 113 +++++++++++++++++-- src/backend/optimizer/util/relnode.c | 7 +- src/backend/optimizer/util/restrictinfo.c | 125 +++++++++++++++++++++- src/include/optimizer/restrictinfo.h | 7 ++ 5 files changed, 242 insertions(+), 17 deletions(-) diff --git a/src/backend/optimizer/path/joinrels.c b/src/backend/optimizer/path/joinrels.c index 7db5e30eef8..2aebdf0078b 100644 --- a/src/backend/optimizer/path/joinrels.c +++ b/src/backend/optimizer/path/joinrels.c @@ -19,6 +19,7 @@ #include "optimizer/joininfo.h" #include "optimizer/pathnode.h" #include "optimizer/paths.h" +#include "optimizer/restrictinfo.h" #include "partitioning/partbounds.h" #include "utils/memutils.h" @@ -1651,10 +1652,8 @@ try_partitionwise_join(PlannerInfo *root, RelOptInfo *rel1, RelOptInfo *rel2, * Construct restrictions applicable to the child join from those * applicable to the parent join. */ - child_restrictlist = - (List *) adjust_appendrel_attrs(root, - (Node *) parent_restrictlist, - nappinfos, appinfos); + child_restrictlist = get_child_restrictinfos(root, parent_restrictlist, + nappinfos, appinfos); /* Find or construct the child join's RelOptInfo */ child_joinrel = joinrel->part_rels[cnt_parts]; diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index fc97bf6ee26..4cdfb865b39 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -26,6 +26,7 @@ #include "optimizer/pathnode.h" #include "optimizer/paths.h" #include "optimizer/planmain.h" +#include "optimizer/restrictinfo.h" #include "optimizer/tlist.h" #include "parser/parsetree.h" #include "utils/memutils.h" @@ -4184,6 +4185,78 @@ reparameterize_path(PlannerInfo *root, Path *path, return NULL; } +/* + * build_child_iclauses_multilevel + * Translate IndexClause list applicable to the given top parent relation + * to that applicable to the given child relation where the child relation + * may be several levels below the top parent in the partition hierarchy. + * + * This is similar to adjust_appendrel_attrs_multilevel() for IndexClause + * except that it uses translated RestrictInfos if already available. + */ +static List * +build_child_iclauses_multilevel(PlannerInfo *root, + List *parent_iclauselist, + RelOptInfo *childrel, + RelOptInfo *top_parent) +{ + List *child_iclauses = NIL; + + foreach_node(IndexClause, iclause, parent_iclauselist) + { + List *rinfo_list = NIL; + List *child_rinfo_list; + IndexClause *child_iclause = makeNode(IndexClause); + ListCell *lcp; + ListCell *lcc; + List *indexquals; + + memcpy(child_iclause, iclause, sizeof(IndexClause)); + + /* + * Collect RestrictInfos to be translated. That's all there's to + * translate in an IndexClause. + */ + rinfo_list = lappend(rinfo_list, iclause->rinfo); + rinfo_list = list_concat(rinfo_list, iclause->indexquals); + + child_rinfo_list = get_child_restrictinfos_multilevel(root, rinfo_list, + childrel, top_parent); + child_iclause->rinfo = linitial(child_rinfo_list); + child_iclause->indexquals = NIL; + indexquals = list_delete_first(child_rinfo_list); + + /* + * indexquals of parent indexclause may have commuted RestrictInfos. + * Commute the child indexquals accordingly. + */ + forboth(lcc, indexquals, lcp, iclause->indexquals) + { + RestrictInfo *child_rinfo = lfirst_node(RestrictInfo, lcc); + RestrictInfo *rinfo = lfirst_node(RestrictInfo, lcp); + Relids child_left_relids; + + child_left_relids = adjust_child_relids_multilevel(root, rinfo->left_relids, + childrel, top_parent); + if (!bms_equal(child_left_relids, child_rinfo->left_relids)) + { + OpExpr *clause = castNode(OpExpr, rinfo->clause); + + Assert(bms_equal(child_left_relids, child_rinfo->right_relids)); + + child_rinfo = commute_restrictinfo(child_rinfo, clause->opno); + } + + child_iclause->indexquals = lappend(child_iclause->indexquals, child_rinfo); + } + + list_free(rinfo_list); + child_iclauses = lappend(child_iclauses, child_iclause); + } + + return child_iclauses; +} + /* * reparameterize_path_by_child * Given a path parameterized by the parent of the given child relation, @@ -4286,7 +4359,10 @@ do { \ IndexPath *ipath = (IndexPath *) path; ADJUST_CHILD_ATTRS(ipath->indexinfo->indrestrictinfo); - ADJUST_CHILD_ATTRS(ipath->indexclauses); + ipath->indexclauses = + build_child_iclauses_multilevel(root, + ipath->indexclauses, + child_rel, child_rel->top_parent); new_path = (Path *) ipath; } break; @@ -4365,7 +4441,11 @@ do { \ REPARAMETERIZE_CHILD_PATH(jpath->outerjoinpath); REPARAMETERIZE_CHILD_PATH(jpath->innerjoinpath); - ADJUST_CHILD_ATTRS(jpath->joinrestrictinfo); + jpath->joinrestrictinfo = + get_child_restrictinfos_multilevel(root, + jpath->joinrestrictinfo, + child_rel, + child_rel->top_parent); new_path = (Path *) npath; } break; @@ -4377,8 +4457,16 @@ do { \ REPARAMETERIZE_CHILD_PATH(jpath->outerjoinpath); REPARAMETERIZE_CHILD_PATH(jpath->innerjoinpath); - ADJUST_CHILD_ATTRS(jpath->joinrestrictinfo); - ADJUST_CHILD_ATTRS(mpath->path_mergeclauses); + jpath->joinrestrictinfo = + get_child_restrictinfos_multilevel(root, + jpath->joinrestrictinfo, + child_rel, + child_rel->top_parent); + mpath->path_mergeclauses = + get_child_restrictinfos_multilevel(root, + mpath->path_mergeclauses, + child_rel, + child_rel->top_parent); new_path = (Path *) mpath; } break; @@ -4390,8 +4478,16 @@ do { \ REPARAMETERIZE_CHILD_PATH(jpath->outerjoinpath); REPARAMETERIZE_CHILD_PATH(jpath->innerjoinpath); - ADJUST_CHILD_ATTRS(jpath->joinrestrictinfo); - ADJUST_CHILD_ATTRS(hpath->path_hashclauses); + jpath->joinrestrictinfo = + get_child_restrictinfos_multilevel(root, + jpath->joinrestrictinfo, + child_rel, + child_rel->top_parent); + hpath->path_hashclauses = + get_child_restrictinfos_multilevel(root, + hpath->path_hashclauses, + child_rel, + child_rel->top_parent); new_path = (Path *) hpath; } break; @@ -4468,7 +4564,10 @@ do { \ new_ppi->ppi_req_outer = bms_copy(required_outer); new_ppi->ppi_rows = old_ppi->ppi_rows; new_ppi->ppi_clauses = old_ppi->ppi_clauses; - ADJUST_CHILD_ATTRS(new_ppi->ppi_clauses); + new_ppi->ppi_clauses = + get_child_restrictinfos_multilevel(root, + new_ppi->ppi_clauses, child_rel, + child_rel->top_parent); new_ppi->ppi_serials = bms_copy(old_ppi->ppi_serials); rel->ppilist = lappend(rel->ppilist, new_ppi); diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index d7266e4cdba..fb86728b712 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -961,10 +961,9 @@ build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel, nappinfos, appinfos); /* Construct joininfo list. */ - joinrel->joininfo = (List *) adjust_appendrel_attrs(root, - (Node *) parent_joinrel->joininfo, - nappinfos, - appinfos); + joinrel->joininfo = get_child_restrictinfos(root, + parent_joinrel->joininfo, + nappinfos, appinfos); /* * Lateral relids referred in child join will be same as that referred in diff --git a/src/backend/optimizer/util/restrictinfo.c b/src/backend/optimizer/util/restrictinfo.c index 7a6f73daf6d..bed27ccf226 100644 --- a/src/backend/optimizer/util/restrictinfo.c +++ b/src/backend/optimizer/util/restrictinfo.c @@ -14,12 +14,13 @@ */ #include "postgres.h" +#include "common/hashfn.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" +#include "optimizer/appendinfo.h" #include "optimizer/clauses.h" #include "optimizer/optimizer.h" #include "optimizer/restrictinfo.h" -#include "common/hashfn.h" static RestrictInfo *make_restrictinfo_internal(PlannerInfo *root, Expr *clause, @@ -788,7 +789,7 @@ get_child_rinfo_hash(PlannerInfo *root) * add_restrictinfo * Add the given RestrictInfo to the RestrictInfo hash table. */ -extern void +void add_restrictinfo(PlannerInfo *root, RestrictInfo *rinfo) { HTAB *rinfo_hash = get_child_rinfo_hash(root); @@ -800,6 +801,13 @@ add_restrictinfo(PlannerInfo *root, RestrictInfo *rinfo) key.required_relids = rinfo->required_relids; rinfo_entry = hash_search(rinfo_hash, &key, HASH_ENTER, &found); + /* + * If the given RestrictInfo is already present in the hash table, + * multiple instances of the same RestrictInfo may have been created. This + * function is a good place to flag that. While multiple instances of same + * RestrictInfo being created is not a correctness issue, they consume + * memory unnecessarily. + */ Assert(!found); rinfo_entry->rinfo = rinfo; } @@ -826,3 +834,116 @@ find_restrictinfo(PlannerInfo *root, int rinfo_serial, Relids required_relids) NULL); return (rinfo_entry ? rinfo_entry->rinfo : NULL); } + +/* + * get_child_restrictinfos + * Returns list of child RestrictInfos obtained by translating the given + * parent RestrictInfos according to the given AppendRelInfos. + * + * RestrictInfos applicable to a child relation are obtained by translating the + * corresponding RestrictInfos applicable to the parent relation. The same + * parent RestictInfo appears in restrictlist or joininfo list of many parent + * join relations and thus may get translated multiple times producing multiple + * instances of the same child RestrictInfo. In order to avoid that we store the + * translated child RestrictInfos in a hash table and reused. + * + * If a required translated RestrictInfo is available in the RestrictInfo hash + * table, the function includes the available child RestrictInfo to the result + * list. Otherwise, it translates the corresponding parent RestrictInfo and + * adds it to the RestrictInfo hash table and the result list. + */ +List * +get_child_restrictinfos(PlannerInfo *root, List *parent_restrictinfos, + int nappinfos, AppendRelInfo **appinfos) +{ + List *child_clauselist = NIL; + + foreach_node(RestrictInfo, parent_rinfo, parent_restrictinfos) + { + Relids child_req_relids; + RestrictInfo *child_rinfo = NULL; + + child_req_relids = adjust_child_relids(parent_rinfo->required_relids, + nappinfos, appinfos); + + if (bms_equal(child_req_relids, parent_rinfo->required_relids)) + { + /* + * If no relid was translated, child's RestrictInfo is same as + * that of parent. + */ + child_rinfo = parent_rinfo; + } + else + { + child_rinfo = find_restrictinfo(root, parent_rinfo->rinfo_serial, child_req_relids); + + /* + * This function may be called thousands of times when there are + * thousands of partitions involved. We won't require the + * translated child relids further. Hence free those to avoid + * accumulating huge amounts of memory. + */ + bms_free(child_req_relids); + } + + if (!child_rinfo) + { + child_rinfo = castNode(RestrictInfo, + adjust_appendrel_attrs(root, (Node *) parent_rinfo, + nappinfos, appinfos)); + add_restrictinfo(root, child_rinfo); + } + + child_clauselist = lappend(child_clauselist, child_rinfo); + } + + return child_clauselist; +} + +/* + * get_child_restrictinfos_multilevel + * Similar to get_child_restrictinfos() but for translations through multiple + * levels of partitioning hierarchy. + * + * This function is similar to adjust_appendrel_attrs_multilevel() except that + * this function makes use of RestrictInfo hash table. + */ +List * +get_child_restrictinfos_multilevel(PlannerInfo *root, List *parent_clauselist, + RelOptInfo *child_rel, RelOptInfo *top_parent) +{ + AppendRelInfo **appinfos; + int nappinfos; + List *tmp_clauselist = parent_clauselist; + List *child_clauselist; + + /* Recursively traverse up the partition hierarchy. */ + if (child_rel->parent != top_parent) + { + if (child_rel->parent) + { + tmp_clauselist = get_child_restrictinfos_multilevel(root, + parent_clauselist, + child_rel->parent, + top_parent); + } + else + elog(ERROR, "child_rel is not a child of top_parent"); + } + + appinfos = find_appinfos_by_relids(root, child_rel->relids, &nappinfos); + child_clauselist = get_child_restrictinfos(root, tmp_clauselist, + nappinfos, appinfos); + + /* + * This function will be called thousands of times, if there are thousands + * of partitions involved. Free temporary objects created in this function + * to avoid accumulating huge memory. + */ + pfree(appinfos); + if (tmp_clauselist != parent_clauselist) + list_free(tmp_clauselist); + + return child_clauselist; +} diff --git a/src/include/optimizer/restrictinfo.h b/src/include/optimizer/restrictinfo.h index d2594fc5de4..c0624ad8c8e 100644 --- a/src/include/optimizer/restrictinfo.h +++ b/src/include/optimizer/restrictinfo.h @@ -50,5 +50,12 @@ extern bool join_clause_is_movable_into(RestrictInfo *rinfo, extern RestrictInfo *find_restrictinfo(PlannerInfo *root, int rinfo_serial, Relids child_required_relids); extern void add_restrictinfo(PlannerInfo *root, RestrictInfo *child_rinfo); +extern List *get_child_restrictinfos(PlannerInfo *root, + List *parent_restrictinfos, + int nappinfos, AppendRelInfo **appinfos); +extern List *get_child_restrictinfos_multilevel(PlannerInfo *root, + List *parent_clauselist, + RelOptInfo *child_rel, + RelOptInfo *top_parent); #endif /* RESTRICTINFO_H */ -- 2.34.1