From ad5f5b59265762b3674068ee3656baec3ec66b70 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 13 Jun 2018 21:45:32 +0300 Subject: [PATCH 2/2] Remove unique self joins. --- src/backend/optimizer/plan/analyzejoins.c | 513 ++++++++++++++++++++++++++++++ src/backend/optimizer/plan/planmain.c | 5 + src/include/optimizer/planmain.h | 1 + src/test/regress/expected/join.out | 38 ++- src/test/regress/sql/join.sql | 16 + 5 files changed, 569 insertions(+), 4 deletions(-) diff --git a/src/backend/optimizer/plan/analyzejoins.c b/src/backend/optimizer/plan/analyzejoins.c index c03010c..f593a17 100644 --- a/src/backend/optimizer/plan/analyzejoins.c +++ b/src/backend/optimizer/plan/analyzejoins.c @@ -22,12 +22,15 @@ */ #include "postgres.h" +#include "catalog/pg_class.h" #include "nodes/nodeFuncs.h" #include "optimizer/clauses.h" #include "optimizer/joininfo.h" #include "optimizer/pathnode.h" #include "optimizer/paths.h" #include "optimizer/planmain.h" +#include "optimizer/predtest.h" +#include "optimizer/restrictinfo.h" #include "optimizer/tlist.h" #include "optimizer/var.h" #include "utils/lsyscache.h" @@ -1122,3 +1125,513 @@ is_innerrel_unique_for(PlannerInfo *root, /* Let rel_is_distinct_for() do the hard work */ return rel_is_distinct_for(root, innerrel, clause_list, unique_index); } + +static bool +set_varno_walker(Node *node, void *context) +{ + if (node == NULL) + return false; + + if (IsA(node, Var)) + ((Var *) node)->varno = (Index) (intptr_t) context; + + return expression_tree_walker(node, set_varno_walker, context); +} + +/* + * Replace varno with 'relid' for all Vars in the expression. + */ +static void +set_varno(Expr *expr, Index relid) +{ + set_varno_walker((Node *) expr, (void*) (intptr_t) relid); +} + +static bool +references_relation_walker(Node *node, void *context) +{ + if (node == NULL) + return false; + + if (IsA(node, Var)) + return ((Var *) node)->varno == (Index) (intptr_t) context; + + return expression_tree_walker(node, references_relation_walker, context); +} + +/* + * Check whether the expression has any Vars from the relation identified + * by 'relid'. For EquivalenceClass nodes, checks the equivalence members. + */ +static bool +references_relation(Node *node, Index relid) +{ + if (IsA(node, EquivalenceClass)) + { + EquivalenceClass *ec = (EquivalenceClass *) node; + ListCell *lc; + foreach (lc, ec->ec_members) + { + if (references_relation( + (Node *) ((EquivalenceMember *) lfirst(lc))->em_expr, + relid)) + { + return true; + } + } + return false; + } + return references_relation_walker(node, (void *) (intptr_t) relid); +} + +/* + * Substitute newId for oldId in relids. + */ +static void +change_relid(Relids *relids, Index oldId, Index newId) +{ + if (bms_is_member(oldId, *relids)) + *relids = bms_add_member(bms_del_member(*relids, oldId), newId); +} + +/* + * Remove a relation after we have proven that it participates only in an + * unneeded unique self join. + * + * The joinclauses list is destructively changed. + */ +static void +remove_self_join_rel(PlannerInfo *root, Relids joinrelids, List *joinclauses, + RelOptInfo *toKeep, RelOptInfo *toRemove) +{ + ListCell *prev, *cell, *next; + List *toAppend; + + /* + * Join clauses become restriction clauses, when Vars of the relation we + * remove are replaced with corresponding Vars of the one we keep. + * Restriction clauses from the removed relation are as well transferred + * to the remaining one. + */ + toAppend = list_concat(joinclauses, toRemove->baserestrictinfo); + + /* + * Append the filters from the removed relation to the remaining one. + */ + foreach(cell, toAppend) + { + RestrictInfo *oldRinfo = lfirst_node(RestrictInfo, cell); + RestrictInfo *newRinfo = NULL; + Expr *newClause = NULL; + + /* + * Do not add multiple clauses derived from the same equivalence class. + */ + if (is_redundant_derived_clause(oldRinfo, toKeep->baserestrictinfo)) + continue; + + /* + * Make a copy of the clause and replace the references to the removed + * relation with references to the remaining one. + */ + newClause = (Expr *) copyObject(oldRinfo->clause); + set_varno(newClause, toKeep->relid); + + /* + * After we have replaced the Vars, check that the resulting clause is + * not implied by the existing ones. + */ + if (!contain_mutable_functions((Node *) newClause) + && predicate_implied_by(list_make1(newClause), toKeep->baserestrictinfo, + /* weak = */ false )) + continue; /* provably implied by r1 */ + + /* + * If the clause has the form of "X=X", replace it with null test. + */ + if (oldRinfo->mergeopfamilies) + { + Assert(is_opclause(newClause)); + Expr *leftOp = (Expr *) get_leftop(newClause); + Expr *rightOp = (Expr *) get_rightop(newClause); + if (leftOp != NULL && equal(leftOp, rightOp)) + { + NullTest *test = makeNode(NullTest); + test->arg = leftOp; + test->nulltesttype = IS_NOT_NULL; + test->argisrow = false; + test->location = -1; + newClause = (Expr *) test; + } + } + + /* + * Finally, correct the relids of the old rinfo, and replace the clause + * with the one we just constructed, and append it to the remaining + * relation. + */ + newRinfo = copyObject(oldRinfo); + newRinfo->clause = newClause; + change_relid(&newRinfo->required_relids, toRemove->relid, toKeep->relid); + change_relid(&newRinfo->left_relids, toRemove->relid, toKeep->relid); + change_relid(&newRinfo->right_relids, toRemove->relid, toKeep->relid); + change_relid(&newRinfo->clause_relids, toRemove->relid, toKeep->relid); + toKeep->baserestrictinfo = lappend(toKeep->baserestrictinfo, newRinfo); + } + + /* + * Remove the relation from the planner data structures. + */ + remove_rel_from_query(root, toRemove->relid, joinrelids); + + /* + * The equivalence classes that reference the removed rel can't also + * reference anything other than the remaining rel, or else this + * optimization wouldn't work (see rel_used_above_join). + * + * This means we don't have any ECs that can generate join clauses, + * and only have the ECs that can generate restrictions. But the restrictions + * have been generated already, and we've just transferred them to the + * remaining relation from join clauses and from the restrictions of the + * removed relation. + * + * Therefore, we don't need anymore the ECs that reference the removed + * relation, and can delete them. + */ + prev = NULL; + cell = NULL; + next = list_head(root->eq_classes); + while (next) + { + prev = cell; + cell = next; + next = lnext(next); + + if (references_relation(lfirst(cell), toRemove->relid)) + { + root->eq_classes = list_delete_cell(root->eq_classes, cell, prev); + cell = prev; + } + } +} + +/* + * Test whether the relations are joined on the same unique column. + */ +static bool +is_unique_self_join(PlannerInfo *root, Relids joinrelids, RelOptInfo *outer, + RelOptInfo *inner, List *restrictlist) +{ + IndexOptInfo *outeridx = NULL; + IndexOptInfo *inneridx = NULL; + + innerrel_is_unique(root, joinrelids, inner->relids, + outer, JOIN_INNER, restrictlist, true, &outeridx); + if (!outeridx) + return false; + + innerrel_is_unique(root, joinrelids, outer->relids, + inner, JOIN_INNER, restrictlist, true, &inneridx); + if (!inneridx) + return false; + + /* We must have the same unique index for both relations. */ + if (outeridx->indexoid != inneridx->indexoid) + return false; + + /* A sanity check: this is the same index on the same relation. */ + Assert(root->simple_rte_array[outer->relid]->relid + == root->simple_rte_array[inner->relid]->relid); + + return true; +} + +/* + * Scratch space for the unique self join removal code. + */ +typedef struct +{ + /* Temporary array for relation ids. */ + Index *relids; + + /* + * Array of Relids, one for each relation, indexed by relation id. + * Each element is a set of relation ids with which this relation + * has a special join. + */ + Relids *special_join_rels; + + /* Bitmapset for join relids that is used to avoid reallocation. */ + Relids joinrelids; +} UsjScratch; + +/* + * Find and remove unique self joins in a group of base relations that have + * the same Oid. + * + * Returns IntList of the relids that were removed. + */ +static List * +remove_self_joins_one_group(PlannerInfo *root, Index *relids, int n, UsjScratch *scratch) +{ + Relids joinrelids = scratch->joinrelids; + List *result = NIL; + int i, o; + + if (n < 2) + return NIL; + + for (o = 0; o < n; o++) + { + RelOptInfo *outer; + + if (relids[o] == 0) + /* Already removed. */ + continue; + + outer = root->simple_rel_array[relids[o]]; + for (i = o + 1; i < n; i++) + { + RelOptInfo *inner; + List *restrictlist; + + if (relids[i] == 0) + /* Already removed. */ + continue; + + inner = root->simple_rel_array[relids[i]]; + + /* + * Unique semi and left joins are processed by reduce_unique_semijoins + * and remove_useless_left_joins respectively, so we don't have to + * deal with them here. For full and anti joins, this optimization does + * not apply. Therefore, we do nothing if these relations have + * a special join. + */ + if (bms_is_member(relids[i], scratch->special_join_rels[relids[o]])) + continue; + + /* Reuse joinrelids bitset to avoid reallocation. */ + joinrelids = bms_del_members(joinrelids, joinrelids); + + /* + * We only deal with base rels here, so their relids bitset + * contains only one member -- their relid. + */ + joinrelids = bms_add_member(joinrelids, relids[o]); + joinrelids = bms_add_member(joinrelids, relids[i]); + + /* Is it a unique self join? */ + restrictlist = build_joinrel_restrictlist(root, joinrelids, outer, + inner); + if (!is_unique_self_join(root, joinrelids, outer, inner, + restrictlist)) + continue; + + /* A relation can be removed if it is not referenced above the join. */ + if (rel_used_above_join(root, joinrelids, inner)) + { + if (rel_used_above_join(root, joinrelids, outer)) + /* + * Both relations are referenced above the join, can't remove + * either one. + */ + continue; + else + { + /* Remove outer. */ + remove_self_join_rel(root, joinrelids, restrictlist, + inner, outer); + result = lappend_int(result, relids[o]); + relids[o] = 0; + break; + } + } + else + { + /* Remove inner. */ + remove_self_join_rel(root, joinrelids, restrictlist, + outer, inner); + result = lappend_int(result, relids[i]); + relids[i] = 0; + } + } + } + + scratch->joinrelids = joinrelids; + return result; +} + +/* + * A qsort comparator to sort the relids by the relation Oid. + */ +static int +compare_rte(const Index *left, const Index *right, PlannerInfo *root) +{ + return root->simple_rte_array[*left]->relid + < root->simple_rte_array[*right]->relid; +} + +/* + * Find and remove unique self joins on a particular level of the join tree. + */ +static void +remove_self_joins_one_level(PlannerInfo *root, List **joinlist, UsjScratch *scratch) +{ + ListCell *lc; + List *relidsToRemove = NIL; + Oid groupOid; + int groupStart; + int i; + int n = 0; + Index *relid_ascending = scratch->relids; + + /* + * Collect the ids of base relations at this level of the join tree. + */ + foreach (lc, *joinlist) + { + RangeTblRef *ref = (RangeTblRef *) lfirst(lc); + if (!IsA(ref, RangeTblRef)) + continue; + + /* + * We only care about base relations from which we select something. + */ + if (root->simple_rte_array[ref->rtindex]->rtekind == RTE_RELATION + && root->simple_rte_array[ref->rtindex]->relkind == RELKIND_RELATION + && root->simple_rel_array[ref->rtindex] != NULL) + { + relid_ascending[n++] = ref->rtindex; + } + + /* + * Limit the number of joins we process to control the quadratic behavior. + */ + if (n > join_collapse_limit) + break; + } + + if (n < 2) + return; + + /* + * Find and process the groups of relations that have same Oid. + */ + qsort_arg(relid_ascending, n, sizeof(*relid_ascending), + (qsort_arg_comparator) compare_rte, root); + groupOid = root->simple_rte_array[relid_ascending[0]]->relid; + groupStart = 0; + for (i = 1; i < n; i++) + { + RangeTblEntry *rte = root->simple_rte_array[relid_ascending[i]]; + Assert(rte->relid != InvalidOid); + if (rte->relid != groupOid) + { + relidsToRemove = list_concat(relidsToRemove, + remove_self_joins_one_group(root, &relid_ascending[groupStart], + i - groupStart, scratch)); + groupOid = rte->relid; + groupStart = i; + } + } + Assert(groupOid != InvalidOid); + Assert(groupStart < n); + relidsToRemove = list_concat(relidsToRemove, + remove_self_joins_one_group(root, &relid_ascending[groupStart], + n - groupStart, scratch)); + + /* + * Delete the removed relations from joinlist. + */ + foreach(lc, relidsToRemove) + { + Index indexToRemove = lfirst_int(lc); + ListCell *prev = NULL, *next = NULL; + ListCell *lc2 = list_head(*joinlist); + while (lc2) + { + next = lnext(lc2); + if (castNode(RangeTblRef, lfirst(lc2))->rtindex == indexToRemove) + *joinlist = list_delete_cell(*joinlist, lc2, prev); + else + prev = lc2; + lc2 = next; + } + } + + return; +} + +/* + * Find and remove unique self joins on a single level of a join tree, and + * recurse to handle deeper levels. + */ +static void +remove_self_joins_recurse(PlannerInfo *root, List **joinlist, UsjScratch *scratch) +{ + ListCell *lc; + foreach (lc, *joinlist) + { + switch (((Node*) lfirst(lc))->type) + { + case T_List: + remove_self_joins_recurse(root, (List **) &lfirst(lc), scratch); + break; + case T_RangeTblRef: + break; + default: + Assert(false); + } + } + remove_self_joins_one_level(root, joinlist, scratch); +} + +/* + * Find out which relations have special joins to which. + */ +static void +find_special_joins(PlannerInfo *root, Relids *special_join_rels) +{ + ListCell *lc; + foreach(lc, root->join_info_list) + { + SpecialJoinInfo *info = (SpecialJoinInfo *) lfirst(lc); + int bit = -1; + while ((bit = bms_next_member(info->min_lefthand, bit)) >= 0) + { + RelOptInfo *rel = find_base_rel(root, bit); + special_join_rels[rel->relid] = + bms_add_members(special_join_rels[rel->relid], info->min_righthand); + } + + bit = -1; + while ((bit = bms_next_member(info->min_righthand, bit)) >= 0) + { + RelOptInfo *rel = find_base_rel(root, bit); + special_join_rels[rel->relid] = + bms_add_members(special_join_rels[rel->relid], info->min_lefthand); + } + } +} + +/* + * Find and remove unique self joins in the entire join tree. + * + * First, we cache some data that will be needed later. + * Then, for each jointree level, we group all the participating + * base relations by their relation Oid. For every pair of relations in a + * group, we try to remove the join they make. + */ +void +remove_useless_self_joins(PlannerInfo *root, List **joinlist) +{ + UsjScratch scratch; + + scratch.relids = palloc(root->simple_rel_array_size * sizeof(Index)); + scratch.special_join_rels = palloc0(root->simple_rel_array_size * sizeof(Relids)); + scratch.joinrelids = NULL; + + find_special_joins(root, scratch.special_join_rels); + remove_self_joins_recurse(root, joinlist, &scratch); +} diff --git a/src/backend/optimizer/plan/planmain.c b/src/backend/optimizer/plan/planmain.c index 8e4abbe..12c3868 100644 --- a/src/backend/optimizer/plan/planmain.c +++ b/src/backend/optimizer/plan/planmain.c @@ -199,6 +199,11 @@ query_planner(PlannerInfo *root, List *tlist, reduce_unique_semijoins(root); /* + * Remove self joins on a unique column. + */ + remove_useless_self_joins(root, &joinlist); + + /* * Now distribute "placeholders" to base rels as needed. This has to be * done after join removal because removal could change whether a * placeholder is evaluable at a base rel. diff --git a/src/include/optimizer/planmain.h b/src/include/optimizer/planmain.h index 4805f74..501bb9a 100644 --- a/src/include/optimizer/planmain.h +++ b/src/include/optimizer/planmain.h @@ -111,6 +111,7 @@ extern bool innerrel_is_unique(PlannerInfo *root, Relids joinrelids, Relids outerrelids, RelOptInfo *innerrel, JoinType jointype, List *restrictlist, bool force_cache, IndexOptInfo **unique_index); +extern void remove_useless_self_joins(PlannerInfo *root, List **jointree); /* * prototypes for plan/setrefs.c diff --git a/src/test/regress/expected/join.out b/src/test/regress/expected/join.out index dc6262b..05e9403 100644 --- a/src/test/regress/expected/join.out +++ b/src/test/regress/expected/join.out @@ -4307,11 +4307,13 @@ explain (costs off) select p.* from (parent p left join child c on (p.k = c.k)) join parent x on p.k = x.k where p.k = 1 and p.k = 2; - QUERY PLAN --------------------------- + QUERY PLAN +------------------------------------------------ Result - One-Time Filter: false -(2 rows) + One-Time Filter: (false AND false) + -> Index Scan using parent_pkey on parent p + Index Cond: (k = 1) +(4 rows) -- bug 5255: this is not optimizable by join removal begin; @@ -4427,6 +4429,34 @@ select * from ----+----+----+---- (0 rows) +-- test that semi- or inner self-joins on a unique column are removed +create table sj (a int unique, b int); +insert into sj values (1, null), (null, 2), (2, 1); +analyze sj; +select p.* from sj p, sj q where q.a = p.a and q.b = q.a - 1; + a | b +---+--- + 2 | 1 +(1 row) + +explain (costs off) +select p.* from sj p, sj q where q.a = p.a and q.b = q.a - 1; + QUERY PLAN +----------------------------------------------- + Seq Scan on sj p + Filter: ((a IS NOT NULL) AND (b = (a - 1))) +(2 rows) + +explain (costs off) +select * from sj p +where exists (select * from sj q + where q.a = p.a and q.b < 10); + QUERY PLAN +------------------------------------------ + Seq Scan on sj p + Filter: ((a IS NOT NULL) AND (b < 10)) +(2 rows) + -- -- Test hints given on incorrect column references are useful -- diff --git a/src/test/regress/sql/join.sql b/src/test/regress/sql/join.sql index d3ba2a1..7620981 100644 --- a/src/test/regress/sql/join.sql +++ b/src/test/regress/sql/join.sql @@ -1527,6 +1527,22 @@ select * from select * from int8_tbl x join (int4_tbl x cross join int4_tbl y(ff)) j on q1 = f1; -- ok +-- test that semi- or inner self-joins on a unique column are removed + +create table sj (a int unique, b int); +insert into sj values (1, null), (null, 2), (2, 1); +analyze sj; + +select p.* from sj p, sj q where q.a = p.a and q.b = q.a - 1; + +explain (costs off) +select p.* from sj p, sj q where q.a = p.a and q.b = q.a - 1; + +explain (costs off) +select * from sj p +where exists (select * from sj q + where q.a = p.a and q.b < 10); + -- -- Test hints given on incorrect column references are useful -- -- 2.7.4