commit a2e0c5f8ddcab95f5004ba6d54feef72c0dea876 Author: Tom Lane Date: Mon Oct 31 19:47:31 2022 -0400 Rewrite reduce_outer_joins' matching of Vars. My draft patch for adding nulling-rel marks to Vars broke the logic in reduce_outer_joins_pass2 that detects antijoins by matching upper-level "Var IS NULL" tests to strict join quals. The problem of course is that the upper Var is no longer equal() to the one in the join qual, since the former will now be marked as being nulled by the outer join. Now, this logic was always pretty brute-force: doing list_intersect on a list full of Vars isn't especially cheap. So let's fix it by creating a better-suited data structure, namely an array of per-RTE bitmaps of relevant Vars' attnos. (I wonder if there aren't other applications for an array-of-bitmaps data structure. But for now I just settled for writing enough primitives for the immediate problem.) Discussion: https://postgr.es/m/CAMbWs4-mvPPCJ1W6iK6dD5HiNwoJdi6mZp=-7mE8N9Sh+cd0tQ@mail.gmail.com diff --git a/src/backend/optimizer/prep/prepjointree.c b/src/backend/optimizer/prep/prepjointree.c index cf558264eb..b1d5105bcd 100644 --- a/src/backend/optimizer/prep/prepjointree.c +++ b/src/backend/optimizer/prep/prepjointree.c @@ -124,8 +124,8 @@ static void reduce_outer_joins_pass2(Node *jtnode, reduce_outer_joins_pass2_state *state2, PlannerInfo *root, Relids nonnullable_rels, - List *nonnullable_vars, - List *forced_null_vars); + VarAttnoSet *nonnullable_vars, + VarAttnoSet *forced_null_vars); static void report_reduced_full_join(reduce_outer_joins_pass2_state *state2, int rtindex, Relids relids); static Node *remove_useless_results_recurse(PlannerInfo *root, Node *jtnode, @@ -2640,7 +2640,7 @@ reduce_outer_joins(PlannerInfo *root) reduce_outer_joins_pass2((Node *) root->parse->jointree, state1, &state2, - root, NULL, NIL, NIL); + root, NULL, NULL, NULL); /* * If we successfully reduced the strength of any outer joins, we must @@ -2756,8 +2756,8 @@ reduce_outer_joins_pass1(Node *jtnode) * state2: where to accumulate info about successfully-reduced joins * root: toplevel planner state * nonnullable_rels: set of base relids forced non-null by upper quals - * nonnullable_vars: list of Vars forced non-null by upper quals - * forced_null_vars: list of Vars forced null by upper quals + * nonnullable_vars: set of Vars forced non-null by upper quals + * forced_null_vars: set of Vars forced null by upper quals * * Returns info in state2 about outer joins that were successfully simplified. * Joins that were fully reduced to inner joins are all added to @@ -2771,8 +2771,8 @@ reduce_outer_joins_pass2(Node *jtnode, reduce_outer_joins_pass2_state *state2, PlannerInfo *root, Relids nonnullable_rels, - List *nonnullable_vars, - List *forced_null_vars) + VarAttnoSet *nonnullable_vars, + VarAttnoSet *forced_null_vars) { /* * pass 2 should never descend as far as an empty subnode or base rel, @@ -2788,19 +2788,21 @@ reduce_outer_joins_pass2(Node *jtnode, ListCell *l; ListCell *s; Relids pass_nonnullable_rels; - List *pass_nonnullable_vars; - List *pass_forced_null_vars; + VarAttnoSet *pass_nonnullable_vars; + VarAttnoSet *pass_forced_null_vars; /* Scan quals to see if we can add any constraints */ pass_nonnullable_rels = find_nonnullable_rels(f->quals); pass_nonnullable_rels = bms_add_members(pass_nonnullable_rels, nonnullable_rels); - pass_nonnullable_vars = find_nonnullable_vars(f->quals); - pass_nonnullable_vars = list_concat(pass_nonnullable_vars, - nonnullable_vars); - pass_forced_null_vars = find_forced_null_vars(f->quals); - pass_forced_null_vars = list_concat(pass_forced_null_vars, - forced_null_vars); + pass_nonnullable_vars = make_empty_varattnoset(list_length(root->parse->rtable)); + + find_nonnullable_vars(f->quals, pass_nonnullable_vars); + varattnoset_add_members(pass_nonnullable_vars, nonnullable_vars); + pass_forced_null_vars = make_empty_varattnoset(list_length(root->parse->rtable)); + find_forced_null_vars(f->quals, pass_forced_null_vars); + varattnoset_add_members(pass_forced_null_vars, + forced_null_vars); /* And recurse --- but only into interesting subtrees */ Assert(list_length(f->fromlist) == list_length(state1->sub_states)); forboth(l, f->fromlist, s, state1->sub_states) @@ -2824,8 +2826,7 @@ reduce_outer_joins_pass2(Node *jtnode, JoinType jointype = j->jointype; reduce_outer_joins_pass1_state *left_state = linitial(state1->sub_states); reduce_outer_joins_pass1_state *right_state = lsecond(state1->sub_states); - List *local_nonnullable_vars = NIL; - bool computed_local_nonnullable_vars = false; + VarAttnoSet *local_nonnullable_vars = NULL; /* Can we simplify this join? */ switch (jointype) @@ -2910,21 +2911,19 @@ reduce_outer_joins_pass2(Node *jtnode, */ if (jointype == JOIN_LEFT) { - List *overlap; + Relids overlap; - local_nonnullable_vars = find_nonnullable_vars(j->quals); - computed_local_nonnullable_vars = true; + local_nonnullable_vars = make_empty_varattnoset(list_length(root->parse->rtable)); + find_nonnullable_vars(j->quals, local_nonnullable_vars); /* * It's not sufficient to check whether local_nonnullable_vars and * forced_null_vars overlap: we need to know if the overlap * includes any RHS variables. */ - overlap = list_intersection(local_nonnullable_vars, - forced_null_vars); - if (overlap != NIL && - bms_overlap(pull_varnos(root, (Node *) overlap), - right_state->relids)) + overlap = varattnoset_intersect_relids(local_nonnullable_vars, + forced_null_vars); + if (bms_overlap(overlap, right_state->relids)) jointype = JOIN_ANTI; } @@ -2949,10 +2948,10 @@ reduce_outer_joins_pass2(Node *jtnode, if (left_state->contains_outer || right_state->contains_outer) { Relids local_nonnullable_rels; - List *local_forced_null_vars; + VarAttnoSet *local_forced_null_vars; Relids pass_nonnullable_rels; - List *pass_nonnullable_vars; - List *pass_forced_null_vars; + VarAttnoSet *pass_nonnullable_vars; + VarAttnoSet *pass_forced_null_vars; /* * If this join is (now) inner, we can add any constraints its @@ -2978,25 +2977,30 @@ reduce_outer_joins_pass2(Node *jtnode, if (jointype != JOIN_FULL) { local_nonnullable_rels = find_nonnullable_rels(j->quals); - if (!computed_local_nonnullable_vars) - local_nonnullable_vars = find_nonnullable_vars(j->quals); - local_forced_null_vars = find_forced_null_vars(j->quals); + if (!local_nonnullable_vars) + { + local_nonnullable_vars = make_empty_varattnoset(list_length(root->parse->rtable)); + find_nonnullable_vars(j->quals, local_nonnullable_vars); + } + local_forced_null_vars = make_empty_varattnoset(list_length(root->parse->rtable)); + + find_forced_null_vars(j->quals, local_forced_null_vars); if (jointype == JOIN_INNER || jointype == JOIN_SEMI) { /* OK to merge upper and local constraints */ local_nonnullable_rels = bms_add_members(local_nonnullable_rels, nonnullable_rels); - local_nonnullable_vars = list_concat(local_nonnullable_vars, - nonnullable_vars); - local_forced_null_vars = list_concat(local_forced_null_vars, - forced_null_vars); + varattnoset_add_members(local_nonnullable_vars, + nonnullable_vars); + varattnoset_add_members(local_forced_null_vars, + forced_null_vars); } } else { /* no use in calculating these */ local_nonnullable_rels = NULL; - local_forced_null_vars = NIL; + local_forced_null_vars = NULL; } if (left_state->contains_outer) @@ -3019,8 +3023,8 @@ reduce_outer_joins_pass2(Node *jtnode, { /* no constraints pass through JOIN_FULL */ pass_nonnullable_rels = NULL; - pass_nonnullable_vars = NIL; - pass_forced_null_vars = NIL; + pass_nonnullable_vars = NULL; + pass_forced_null_vars = NULL; } reduce_outer_joins_pass2(j->larg, left_state, state2, root, @@ -3042,8 +3046,8 @@ reduce_outer_joins_pass2(Node *jtnode, { /* no constraints pass through JOIN_FULL */ pass_nonnullable_rels = NULL; - pass_nonnullable_vars = NIL; - pass_forced_null_vars = NIL; + pass_nonnullable_vars = NULL; + pass_forced_null_vars = NULL; } reduce_outer_joins_pass2(j->rarg, right_state, state2, root, diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c index f9913ce3b5..3972e10804 100644 --- a/src/backend/optimizer/util/clauses.c +++ b/src/backend/optimizer/util/clauses.c @@ -105,7 +105,8 @@ static bool contain_context_dependent_node(Node *clause); static bool contain_context_dependent_node_walker(Node *node, int *flags); static bool contain_leaked_vars_walker(Node *node, void *context); static Relids find_nonnullable_rels_walker(Node *node, bool top_level); -static List *find_nonnullable_vars_walker(Node *node, bool top_level); +static void find_nonnullable_vars_walker(Node *node, VarAttnoSet *attnos, + bool top_level); static bool is_strict_saop(ScalarArrayOpExpr *expr, bool falseOK); static bool convert_saop_to_hashed_saop_walker(Node *node, void *context); static Node *eval_const_expressions_mutator(Node *node, @@ -1308,6 +1309,85 @@ contain_leaked_vars_walker(Node *node, void *context) context); } + +/* + * make_empty_varattnoset + * Create an empty VarAttnoSet structure for use with following routines. + * + * The maximum varno we expect to deal with is rangetable_length. + */ +VarAttnoSet * +make_empty_varattnoset(int rangetable_length) +{ + VarAttnoSet *result; + + /* palloc0 is sufficient to initialize all the bitmapsets to empty */ + result = (VarAttnoSet *) + palloc0(offsetof(VarAttnoSet, varattnos) + + (rangetable_length + 1) * sizeof(Bitmapset *)); + result->max_varno = rangetable_length; + return result; +} + +/* + * varattnoset_add_members + * Add all members of set b to set a. + * + * This is like bms_add_members, but for sets of bitmapsets. + * For convenience, we allow b (but not a) to be a NULL pointer. + */ +void +varattnoset_add_members(VarAttnoSet *a, const VarAttnoSet *b) +{ + if (b != NULL) + { + /* We don't really expect the max_varnos to differ, but allow b < a */ + Assert(b->max_varno <= a->max_varno); + for (int i = 1; i <= b->max_varno; i++) + a->varattnos[i] = bms_add_members(a->varattnos[i], + b->varattnos[i]); + } +} + +/* + * varattnoset_int_members + * Reduce set a to its intersection with set b. + * + * This is like bms_int_members, but for sets of bitmapsets. + */ +static void +varattnoset_int_members(VarAttnoSet *a, const VarAttnoSet *b) +{ + /* We don't really expect the max_varnos to differ, but allow a < b */ + Assert(a->max_varno <= b->max_varno); + for (int i = 1; i <= a->max_varno; i++) + a->varattnos[i] = bms_int_members(a->varattnos[i], + b->varattnos[i]); +} + +/* + * varattnoset_intersect_relids + * Identify the relations having common members in a and b. + * + * For convenience, we allow NULL inputs. + */ +Relids +varattnoset_intersect_relids(const VarAttnoSet *a, const VarAttnoSet *b) +{ + Relids result = NULL; + + if (a == NULL || b == NULL) + return NULL; + Assert(a->max_varno == b->max_varno); + for (int i = 1; i <= a->max_varno; i++) + { + if (bms_overlap(a->varattnos[i], b->varattnos[i])) + result = bms_add_member(result, i); + } + return result; +} + + /* * find_nonnullable_rels * Determine which base rels are forced nonnullable by given clause. @@ -1541,11 +1621,13 @@ find_nonnullable_rels_walker(Node *node, bool top_level) * find_nonnullable_vars * Determine which Vars are forced nonnullable by given clause. * - * Returns a list of all level-zero Vars that are referenced in the clause in + * Builds a set of all level-zero Vars that are referenced in the clause in * such a way that the clause cannot possibly return TRUE if any of these Vars * is NULL. (It is OK to err on the side of conservatism; hence the analysis * here is simplistic.) * + * Attnos of the identified Vars are added to a caller-supplied VarAttnoSet. + * * The semantics here are subtly different from contain_nonstrict_functions: * that function is concerned with NULL results from arbitrary expressions, * but here we assume that the input is a Boolean expression, and wish to @@ -1553,9 +1635,6 @@ find_nonnullable_rels_walker(Node *node, bool top_level) * the expression to have been AND/OR flattened and converted to implicit-AND * format. * - * The result is a palloc'd List, but we have not copied the member Var nodes. - * Also, we don't bother trying to eliminate duplicate entries. - * * top_level is true while scanning top-level AND/OR structure; here, showing * the result is either FALSE or NULL is good enough. top_level is false when * we have descended below a NOT or a strict function: now we must be able to @@ -1564,26 +1643,30 @@ find_nonnullable_rels_walker(Node *node, bool top_level) * We don't use expression_tree_walker here because we don't want to descend * through very many kinds of nodes; only the ones we can be sure are strict. */ -List * -find_nonnullable_vars(Node *clause) +void +find_nonnullable_vars(Node *clause, VarAttnoSet *attnos) { - return find_nonnullable_vars_walker(clause, true); + return find_nonnullable_vars_walker(clause, attnos, true); } -static List * -find_nonnullable_vars_walker(Node *node, bool top_level) +static void +find_nonnullable_vars_walker(Node *node, VarAttnoSet *attnos, bool top_level) { - List *result = NIL; ListCell *l; if (node == NULL) - return NIL; + return; if (IsA(node, Var)) { Var *var = (Var *) node; if (var->varlevelsup == 0) - result = list_make1(var); + { + Assert(var->varno > 0 && var->varno <= attnos->max_varno); + attnos->varattnos[var->varno] = + bms_add_member(attnos->varattnos[var->varno], + var->varattno - FirstLowInvalidHeapAttributeNumber); + } } else if (IsA(node, List)) { @@ -1598,9 +1681,7 @@ find_nonnullable_vars_walker(Node *node, bool top_level) */ foreach(l, (List *) node) { - result = list_concat(result, - find_nonnullable_vars_walker(lfirst(l), - top_level)); + find_nonnullable_vars_walker(lfirst(l), attnos, top_level); } } else if (IsA(node, FuncExpr)) @@ -1608,7 +1689,7 @@ find_nonnullable_vars_walker(Node *node, bool top_level) FuncExpr *expr = (FuncExpr *) node; if (func_strict(expr->funcid)) - result = find_nonnullable_vars_walker((Node *) expr->args, false); + find_nonnullable_vars_walker((Node *) expr->args, attnos, false); } else if (IsA(node, OpExpr)) { @@ -1616,14 +1697,14 @@ find_nonnullable_vars_walker(Node *node, bool top_level) set_opfuncid(expr); if (func_strict(expr->opfuncid)) - result = find_nonnullable_vars_walker((Node *) expr->args, false); + find_nonnullable_vars_walker((Node *) expr->args, attnos, false); } else if (IsA(node, ScalarArrayOpExpr)) { ScalarArrayOpExpr *expr = (ScalarArrayOpExpr *) node; if (is_strict_saop(expr, true)) - result = find_nonnullable_vars_walker((Node *) expr->args, false); + find_nonnullable_vars_walker((Node *) expr->args, attnos, false); } else if (IsA(node, BoolExpr)) { @@ -1632,11 +1713,16 @@ find_nonnullable_vars_walker(Node *node, bool top_level) switch (expr->boolop) { case AND_EXPR: - /* At top level we can just recurse (to the List case) */ + + /* + * At top level we can just recurse (to the List case), since + * the result should be the union of what we can prove in each + * arm. + */ if (top_level) { - result = find_nonnullable_vars_walker((Node *) expr->args, - top_level); + find_nonnullable_vars_walker((Node *) expr->args, attnos, + top_level); break; } @@ -1654,30 +1740,36 @@ find_nonnullable_vars_walker(Node *node, bool top_level) * OR is strict if all of its arms are, so we can take the * intersection of the sets of nonnullable vars for each arm. * This works for both values of top_level. + * + * The pfree's below miss cleaning up individual bitmapsets in + * each VarAttnoSet. Doesn't seem worth working harder. */ - foreach(l, expr->args) { - List *subresult; + VarAttnoSet *int_attnos = NULL; - subresult = find_nonnullable_vars_walker(lfirst(l), - top_level); - if (result == NIL) /* first subresult? */ - result = subresult; - else - result = list_intersection(result, subresult); - - /* - * If the intersection is empty, we can stop looking. This - * also justifies the test for first-subresult above. - */ - if (result == NIL) - break; + foreach(l, expr->args) + { + VarAttnoSet *sub_attnos; + + sub_attnos = make_empty_varattnoset(attnos->max_varno); + find_nonnullable_vars_walker(lfirst(l), sub_attnos, + top_level); + if (int_attnos == NULL) /* first subresult? */ + int_attnos = sub_attnos; + else + { + varattnoset_int_members(int_attnos, sub_attnos); + pfree(sub_attnos); + } + } + varattnoset_add_members(attnos, int_attnos); + pfree(int_attnos); } break; case NOT_EXPR: /* NOT will return null if its arg is null */ - result = find_nonnullable_vars_walker((Node *) expr->args, - false); + find_nonnullable_vars_walker((Node *) expr->args, attnos, + false); break; default: elog(ERROR, "unrecognized boolop: %d", (int) expr->boolop); @@ -1688,34 +1780,34 @@ find_nonnullable_vars_walker(Node *node, bool top_level) { RelabelType *expr = (RelabelType *) node; - result = find_nonnullable_vars_walker((Node *) expr->arg, top_level); + find_nonnullable_vars_walker((Node *) expr->arg, attnos, top_level); } else if (IsA(node, CoerceViaIO)) { /* not clear this is useful, but it can't hurt */ CoerceViaIO *expr = (CoerceViaIO *) node; - result = find_nonnullable_vars_walker((Node *) expr->arg, false); + find_nonnullable_vars_walker((Node *) expr->arg, attnos, false); } else if (IsA(node, ArrayCoerceExpr)) { /* ArrayCoerceExpr is strict at the array level; ignore elemexpr */ ArrayCoerceExpr *expr = (ArrayCoerceExpr *) node; - result = find_nonnullable_vars_walker((Node *) expr->arg, top_level); + find_nonnullable_vars_walker((Node *) expr->arg, attnos, top_level); } else if (IsA(node, ConvertRowtypeExpr)) { /* not clear this is useful, but it can't hurt */ ConvertRowtypeExpr *expr = (ConvertRowtypeExpr *) node; - result = find_nonnullable_vars_walker((Node *) expr->arg, top_level); + find_nonnullable_vars_walker((Node *) expr->arg, attnos, top_level); } else if (IsA(node, CollateExpr)) { CollateExpr *expr = (CollateExpr *) node; - result = find_nonnullable_vars_walker((Node *) expr->arg, top_level); + find_nonnullable_vars_walker((Node *) expr->arg, attnos, top_level); } else if (IsA(node, NullTest)) { @@ -1723,7 +1815,7 @@ find_nonnullable_vars_walker(Node *node, bool top_level) NullTest *expr = (NullTest *) node; if (top_level && expr->nulltesttype == IS_NOT_NULL && !expr->argisrow) - result = find_nonnullable_vars_walker((Node *) expr->arg, false); + find_nonnullable_vars_walker((Node *) expr->arg, attnos, false); } else if (IsA(node, BooleanTest)) { @@ -1734,15 +1826,14 @@ find_nonnullable_vars_walker(Node *node, bool top_level) (expr->booltesttype == IS_TRUE || expr->booltesttype == IS_FALSE || expr->booltesttype == IS_NOT_UNKNOWN)) - result = find_nonnullable_vars_walker((Node *) expr->arg, false); + find_nonnullable_vars_walker((Node *) expr->arg, attnos, false); } else if (IsA(node, PlaceHolderVar)) { PlaceHolderVar *phv = (PlaceHolderVar *) node; - result = find_nonnullable_vars_walker((Node *) phv->phexpr, top_level); + find_nonnullable_vars_walker((Node *) phv->phexpr, attnos, top_level); } - return result; } /* @@ -1754,23 +1845,25 @@ find_nonnullable_vars_walker(Node *node, bool top_level) * side of conservatism; hence the analysis here is simplistic. In fact, * we only detect simple "var IS NULL" tests at the top level.) * - * The result is a palloc'd List, but we have not copied the member Var nodes. - * Also, we don't bother trying to eliminate duplicate entries. + * As with find_nonnullable_vars, we add the varattnos of the identified Vars + * to a caller-provided VarAttnoSet. */ -List * -find_forced_null_vars(Node *node) +void +find_forced_null_vars(Node *node, VarAttnoSet *attnos) { - List *result = NIL; Var *var; ListCell *l; if (node == NULL) - return NIL; + return; /* Check single-clause cases using subroutine */ var = find_forced_null_var(node); if (var) { - result = list_make1(var); + Assert(var->varno > 0 && var->varno <= attnos->max_varno); + attnos->varattnos[var->varno] = + bms_add_member(attnos->varattnos[var->varno], + var->varattno - FirstLowInvalidHeapAttributeNumber); } /* Otherwise, handle AND-conditions */ else if (IsA(node, List)) @@ -1781,8 +1874,7 @@ find_forced_null_vars(Node *node) */ foreach(l, (List *) node) { - result = list_concat(result, - find_forced_null_vars(lfirst(l))); + find_forced_null_vars((Node *) lfirst(l), attnos); } } else if (IsA(node, BoolExpr)) @@ -1797,10 +1889,9 @@ find_forced_null_vars(Node *node) if (expr->boolop == AND_EXPR) { /* At top level we can just recurse (to the List case) */ - result = find_forced_null_vars((Node *) expr->args); + find_forced_null_vars((Node *) expr->args, attnos); } } - return result; } /* diff --git a/src/include/optimizer/clauses.h b/src/include/optimizer/clauses.h index ff242d1b6d..5466ada7ba 100644 --- a/src/include/optimizer/clauses.h +++ b/src/include/optimizer/clauses.h @@ -23,6 +23,14 @@ typedef struct List **windowFuncs; /* lists of WindowFuncs for each winref */ } WindowFuncLists; +/* Data structure to represent all level-zero Vars meeting some condition */ +typedef struct +{ + int max_varno; /* maximum index in varattnos[] */ + /* Attnos in these sets are offset by FirstLowInvalidHeapAttributeNumber */ + Bitmapset *varattnos[FLEXIBLE_ARRAY_MEMBER]; +} VarAttnoSet; + extern bool contain_agg_clause(Node *clause); extern bool contain_window_function(Node *clause); @@ -38,9 +46,14 @@ extern bool contain_nonstrict_functions(Node *clause); extern bool contain_exec_param(Node *clause, List *param_ids); extern bool contain_leaked_vars(Node *clause); +extern VarAttnoSet *make_empty_varattnoset(int rangetable_length); +extern void varattnoset_add_members(VarAttnoSet *a, const VarAttnoSet *b); +extern Relids varattnoset_intersect_relids(const VarAttnoSet *a, + const VarAttnoSet *b); + extern Relids find_nonnullable_rels(Node *clause); -extern List *find_nonnullable_vars(Node *clause); -extern List *find_forced_null_vars(Node *node); +extern void find_nonnullable_vars(Node *clause, VarAttnoSet *attnos); +extern void find_forced_null_vars(Node *node, VarAttnoSet *attnos); extern Var *find_forced_null_var(Node *node); extern bool is_pseudo_constant_clause(Node *clause);