From 9770fa835fb232dd8ab7a67e405442618f48b0ac Mon Sep 17 00:00:00 2001 From: Richard Guo Date: Thu, 18 Jun 2026 15:42:02 +0900 Subject: [PATCH v2] Prove a NOT IN's left-hand expressions non-nullable from quals When commit 383eb21eb taught the planner to convert "x NOT IN (SELECT y ...)" into an anti-join, it could prove the outer query's left-hand expressions non-nullable only from schema-level NOT NULL constraints and the outer-join-aware Var infrastructure. A NOT IN whose left-hand expression is a column that some qual forces non-null, but that lacks a NOT NULL constraint, was therefore left as a SubPlan filter rather than converted; for example "WHERE x IS NOT NULL AND x NOT IN (...)" or "WHERE x > 0 AND x NOT IN (...)". This patch proves such a left-hand Var non-nullable when a qual applied to the rows on which the NOT IN is evaluated forces it non-null. The usable quals are the WHERE/ON quals at or below the jointree node where the NOT IN is evaluated, restricted to rels that are not below the nullable side of any outer join. pull_up_sublinks_jointree_recurse accumulates them bottom-up as it traverses the jointree and hands them to convert_ANY_sublink_to_join, which checks them with find_nonnullable_vars. This is the outer-query counterpart of what the sub-select side already does through find_subquery_safe_quals and query_outputs_are_not_nullable. In theory a qual above the NOT IN's jointree node could also make the conversion safe in some cases. Since the conversion already requires the sub-select's outputs to be non-null, the anti-join keeps extra rows only where the left-hand side is NULL, so a filter above the join that discards them, such as "x IS NOT NULL" in a WHERE clause, suffices even though the left-hand side may be NULL where the anti-join runs. But that works only when the qual truly discards those rows; one in a higher outer join's ON clause does not, when the anti-join is on that join's preserved side. find_subquery_safe_quals applied to the whole outer jointree identifies exactly the right quals, the ones guaranteed to hold on the query's output, so it is the natural way to do this; that extension is left for the future. --- src/backend/optimizer/plan/subselect.c | 86 ++++++- src/backend/optimizer/prep/prepjointree.c | 181 +++++++++++--- src/include/optimizer/subselect.h | 16 +- src/test/regress/expected/subselect.out | 288 +++++++++++++++++++++- src/test/regress/sql/subselect.sql | 104 +++++++- src/tools/pgindent/typedefs.list | 1 + 6 files changed, 624 insertions(+), 52 deletions(-) diff --git a/src/backend/optimizer/plan/subselect.c b/src/backend/optimizer/plan/subselect.c index 6aa8971c95d..595a6ceac5e 100644 --- a/src/backend/optimizer/plan/subselect.c +++ b/src/backend/optimizer/plan/subselect.c @@ -23,6 +23,7 @@ #include "executor/nodeSubplan.h" #include "miscadmin.h" #include "nodes/makefuncs.h" +#include "nodes/multibitmapset.h" #include "nodes/nodeFuncs.h" #include "optimizer/clauses.h" #include "optimizer/cost.h" @@ -91,7 +92,8 @@ static bool contain_outer_selfref(Node *node); static bool contain_outer_selfref_walker(Node *node, Index *depth); static void inline_cte(PlannerInfo *root, CommonTableExpr *cte); static bool inline_cte_walker(Node *node, inline_cte_walker_context *context); -static bool sublink_testexpr_is_not_nullable(PlannerInfo *root, SubLink *sublink); +static bool sublink_testexpr_is_not_nullable(PlannerInfo *root, SubLink *sublink, + SafeQualsInfo *sqinfo); static bool simplify_EXISTS_query(PlannerInfo *root, Query *query); static Query *convert_EXISTS_to_ANY(PlannerInfo *root, Query *subselect, Node **testexpr, List **paramIds); @@ -1318,6 +1320,12 @@ convert_VALUES_to_ANY(PlannerInfo *root, Node *testexpr, Query *values) * The conversion must fail if the converted qual would reference any but * these parent-query relids. * + * sqinfo describes the qual clauses that are guaranteed to filter the + * rows on which the SubLink is evaluated. We only need it for the + * under_not case, where it may help prove that the left-hand expressions + * are non-nullable. It may be NULL, in which case no such proof is + * attempted. + * * On success, the returned JoinExpr has larg = NULL and rarg = the jointree * item representing the pulled-up subquery. The caller must set larg to * represent the relation(s) on the lefthand side of the new join, and insert @@ -1339,7 +1347,8 @@ convert_VALUES_to_ANY(PlannerInfo *root, Node *testexpr, Query *values) */ JoinExpr * convert_ANY_sublink_to_join(PlannerInfo *root, SubLink *sublink, - bool under_not, Relids available_rels) + bool under_not, Relids available_rels, + SafeQualsInfo *sqinfo) { JoinExpr *result; Query *parse = root->parse; @@ -1366,7 +1375,7 @@ convert_ANY_sublink_to_join(PlannerInfo *root, SubLink *sublink, * safe to convert NOT IN to an anti-join. */ if (under_not && - (!sublink_testexpr_is_not_nullable(root, sublink) || + (!sublink_testexpr_is_not_nullable(root, sublink, sqinfo) || !query_outputs_are_not_nullable(subselect))) return NULL; @@ -1471,6 +1480,15 @@ convert_ANY_sublink_to_join(PlannerInfo *root, SubLink *sublink, * behavior, ensuring the operator does not produce NULL results from non-null * inputs. * + * An outer expression is provably non-nullable if it is a column with a NOT + * NULL constraint, or more generally any expression that expr_is_nonnullable + * accepts. In addition, a plain Var can be proven non-nullable if some clause + * in sqinfo->quals forces it non-null. Those quals were collected by the + * caller from the jointree at or below the point where this NOT IN is + * evaluated, and are guaranteed to remove any NULL-valued rows before they can + * affect the result of the anti-join, so the NOT IN to anti-join conversion + * stays valid. + * * We handle the three standard parser representations for ANY sublinks: a * single OpExpr for single-column comparisons, a BoolExpr containing a list of * OpExprs for multi-column equality or inequality checks (where equality @@ -1482,7 +1500,8 @@ convert_ANY_sublink_to_join(PlannerInfo *root, SubLink *sublink, * side of conservatism: if we're not sure, it's okay to return FALSE. */ static bool -sublink_testexpr_is_not_nullable(PlannerInfo *root, SubLink *sublink) +sublink_testexpr_is_not_nullable(PlannerInfo *root, SubLink *sublink, + SafeQualsInfo *sqinfo) { Node *testexpr = sublink->testexpr; List *outer_exprs = NIL; @@ -1565,20 +1584,63 @@ sublink_testexpr_is_not_nullable(PlannerInfo *root, SubLink *sublink) /* Check that every outer expression is non-nullable */ foreach_ptr(Expr, expr, outer_exprs) { + /* + * Look through binary relabelings, since we know those don't + * introduce nulls. + */ + while (IsA(expr, RelabelType)) + expr = ((RelabelType *) expr)->arg; + /* * We have already collected relation-level not-null constraints for * the outer query, so we can consult the global hash table for * nullability information. */ - if (!expr_is_nonnullable(root, expr, NOTNULL_SOURCE_HASHTABLE)) - return false; + if (expr_is_nonnullable(root, expr, NOTNULL_SOURCE_HASHTABLE)) + continue; /* - * Note: It is possible to further prove non-nullability by examining - * the qual clauses available at or below the jointree node where this - * NOT IN clause is evaluated, but for the moment it doesn't seem - * worth the extra complication. + * For a plain Var, even if that didn't work, we can still prove it + * non-nullable if find_nonnullable_vars can find a "var IS NOT NULL" + * or similarly strict condition among sqinfo->quals. Those are the + * quals the caller determined are guaranteed to filter the rows on + * which this SubLink is evaluated. Compute the list of Vars they + * force non-null if we didn't already; the caller shares one + * SafeQualsInfo among all the SubLinks at a jointree node, so we do + * this once for all of them. + * + * Note that the quals must be run through flatten_join_alias_vars, + * just as the outer expressions were above, so that the Vars match + * up. + * + * Note also that we can only prove things about this query's own + * Vars. */ + if (sqinfo != NULL && sqinfo->quals != NIL && + IsA(expr, Var) && ((Var *) expr)->varlevelsup == 0) + { + Var *var = (Var *) expr; + + if (!sqinfo->computed) + { + List *flat_quals; + + flat_quals = (List *) + flatten_join_alias_vars(root, root->parse, + (Node *) sqinfo->quals); + sqinfo->nonnullable_vars = + find_nonnullable_vars((Node *) flat_quals); + sqinfo->computed = true; + } + + if (mbms_is_member(var->varno, + var->varattno - FirstLowInvalidHeapAttributeNumber, + sqinfo->nonnullable_vars)) + continue; + } + + /* We failed to prove this outer expression non-nullable */ + return false; } return true; @@ -1587,7 +1649,9 @@ sublink_testexpr_is_not_nullable(PlannerInfo *root, SubLink *sublink) /* * convert_EXISTS_sublink_to_join: try to convert an EXISTS SubLink to a join * - * The API of this function is identical to convert_ANY_sublink_to_join's. + * The API of this function is identical to convert_ANY_sublink_to_join's, + * except that it has no sqinfo parameter, since converting an EXISTS or NOT + * EXISTS never depends on proving an expression non-nullable. */ JoinExpr * convert_EXISTS_sublink_to_join(PlannerInfo *root, SubLink *sublink, diff --git a/src/backend/optimizer/prep/prepjointree.c b/src/backend/optimizer/prep/prepjointree.c index 72d39ba30fd..97ccd294bf4 100644 --- a/src/backend/optimizer/prep/prepjointree.c +++ b/src/backend/optimizer/prep/prepjointree.c @@ -109,10 +109,12 @@ static Query *expand_virtual_generated_columns(PlannerInfo *root, Query *parse, RangeTblEntry *rte, int rt_index, Relation relation); static Node *pull_up_sublinks_jointree_recurse(PlannerInfo *root, Node *jtnode, - Relids *relids); + Relids *relids, List **safe_quals); +static void init_safe_quals_info(SafeQualsInfo *sqinfo, List *quals); static Node *pull_up_sublinks_qual_recurse(PlannerInfo *root, Node *node, Node **jtlink1, Relids available_rels1, - Node **jtlink2, Relids available_rels2); + Node **jtlink2, Relids available_rels2, + SafeQualsInfo *sqinfo); static Node *pull_up_subqueries_recurse(PlannerInfo *root, Node *jtnode, JoinExpr *lowest_outer_join, AppendRelInfo *containing_appendrel); @@ -673,11 +675,12 @@ pull_up_sublinks(PlannerInfo *root) { Node *jtnode; Relids relids; + List *safe_quals; /* Begin recursion through the jointree */ jtnode = pull_up_sublinks_jointree_recurse(root, (Node *) root->parse->jointree, - &relids); + &relids, &safe_quals); /* * root->parse->jointree must always be a FromExpr, so insert a dummy one @@ -694,10 +697,14 @@ pull_up_sublinks(PlannerInfo *root) * * In addition to returning the possibly-modified jointree node, we return * a relids set of the contained rels into *relids. + * + * We also return into *safe_quals the WHERE/ON quals at or below this node + * that constrain rels not below the nullable side of any outer join, which + * can be used to prove an expression evaluated at our level non-nullable. */ static Node * pull_up_sublinks_jointree_recurse(PlannerInfo *root, Node *jtnode, - Relids *relids) + Relids *relids, List **safe_quals) { /* Since this function recurses, it could be driven to stack overflow. */ check_stack_depth(); @@ -705,12 +712,14 @@ pull_up_sublinks_jointree_recurse(PlannerInfo *root, Node *jtnode, if (jtnode == NULL) { *relids = NULL; + *safe_quals = NIL; } else if (IsA(jtnode, RangeTblRef)) { int varno = ((RangeTblRef *) jtnode)->rtindex; *relids = bms_make_singleton(varno); + *safe_quals = NIL; /* jtnode is returned unmodified */ } else if (IsA(jtnode, FromExpr)) @@ -718,30 +727,44 @@ pull_up_sublinks_jointree_recurse(PlannerInfo *root, Node *jtnode, FromExpr *f = (FromExpr *) jtnode; List *newfromlist = NIL; Relids frelids = NULL; + List *fsafequals = NIL; + SafeQualsInfo sqinfo; FromExpr *newf; Node *jtlink; ListCell *l; - /* First, recurse to process children and collect their relids */ + /* + * First, recurse to process children and collect their relids and + * safe quals. + */ foreach(l, f->fromlist) { Node *newchild; Relids childrelids; + List *childsafequals; newchild = pull_up_sublinks_jointree_recurse(root, lfirst(l), - &childrelids); + &childrelids, + &childsafequals); newfromlist = lappend(newfromlist, newchild); frelids = bms_join(frelids, childrelids); + fsafequals = list_concat(fsafequals, childsafequals); } + /* This level's WHERE quals are safe to use as well. */ + if (f->quals) + fsafequals = lappend(fsafequals, f->quals); + /* Build the replacement FromExpr; no quals yet */ newf = makeFromExpr(newfromlist, NULL); /* Set up a link representing the rebuilt jointree */ jtlink = (Node *) newf; /* Now process qual --- all children are available for use */ + init_safe_quals_info(&sqinfo, fsafequals); newf->quals = pull_up_sublinks_qual_recurse(root, f->quals, &jtlink, frelids, - NULL, NULL); + NULL, NULL, + &sqinfo); /* * Note that the result will be either newf, or a stack of JoinExprs @@ -753,6 +776,7 @@ pull_up_sublinks_jointree_recurse(PlannerInfo *root, Node *jtnode, * outputs anyway. */ *relids = frelids; + *safe_quals = fsafequals; jtnode = jtlink; } else if (IsA(jtnode, JoinExpr)) @@ -760,6 +784,10 @@ pull_up_sublinks_jointree_recurse(PlannerInfo *root, Node *jtnode, JoinExpr *j; Relids leftrelids; Relids rightrelids; + List *leftsafequals; + List *rightsafequals; + List *passquals; + SafeQualsInfo sqinfo; Node *jtlink; /* @@ -770,11 +798,13 @@ pull_up_sublinks_jointree_recurse(PlannerInfo *root, Node *jtnode, memcpy(j, jtnode, sizeof(JoinExpr)); jtlink = (Node *) j; - /* Recurse to process children and collect their relids */ + /* Recurse to process children and collect their relids and quals */ j->larg = pull_up_sublinks_jointree_recurse(root, j->larg, - &leftrelids); + &leftrelids, + &leftsafequals); j->rarg = pull_up_sublinks_jointree_recurse(root, j->rarg, - &rightrelids); + &rightrelids, + &rightsafequals); /* * Now process qual, showing appropriate child relids as available, @@ -791,26 +821,66 @@ pull_up_sublinks_jointree_recurse(PlannerInfo *root, Node *jtnode, switch (j->jointype) { case JOIN_INNER: + + /* + * For an inner join, both children's quals and the join's own + * quals constrain its output, so all of them are safe here + * and above. + */ + passquals = list_concat(leftsafequals, rightsafequals); + if (j->quals) + passquals = lappend(passquals, j->quals); + + init_safe_quals_info(&sqinfo, passquals); j->quals = pull_up_sublinks_qual_recurse(root, j->quals, &jtlink, bms_union(leftrelids, rightrelids), - NULL, NULL); + NULL, NULL, + &sqinfo); + + *safe_quals = passquals; break; case JOIN_LEFT: + + /* + * Above this join only the left child's quals stay safe; the + * join may null-extend the right side, so the right child's + * quals and the ON quals are not safe above us. But they are + * safe for this node's own quals, since a sublink pulled up + * here goes into the right side. + */ + passquals = rightsafequals; + if (j->quals) + passquals = lappend(passquals, j->quals); + + init_safe_quals_info(&sqinfo, passquals); j->quals = pull_up_sublinks_qual_recurse(root, j->quals, &j->rarg, rightrelids, - NULL, NULL); + NULL, NULL, + &sqinfo); + + *safe_quals = leftsafequals; break; case JOIN_FULL: /* can't do anything with full-join quals */ + *safe_quals = NIL; break; case JOIN_RIGHT: + /* Mirror of the JOIN_LEFT case */ + passquals = leftsafequals; + if (j->quals) + passquals = lappend(passquals, j->quals); + + init_safe_quals_info(&sqinfo, passquals); j->quals = pull_up_sublinks_qual_recurse(root, j->quals, &j->larg, leftrelids, - NULL, NULL); + NULL, NULL, + &sqinfo); + + *safe_quals = rightsafequals; break; default: elog(ERROR, "unrecognized join type: %d", @@ -837,6 +907,17 @@ pull_up_sublinks_jointree_recurse(PlannerInfo *root, Node *jtnode, return jtnode; } +/* + * Set up a jointree node's safe quals for convert_ANY_sublink_to_join. + */ +static void +init_safe_quals_info(SafeQualsInfo *sqinfo, List *quals) +{ + sqinfo->quals = quals; + sqinfo->nonnullable_vars = NIL; + sqinfo->computed = false; +} + /* * Recurse through top-level qual nodes for pull_up_sublinks() * @@ -850,12 +931,21 @@ pull_up_sublinks_jointree_recurse(PlannerInfo *root, Node *jtnode, * and/or jtlink2 in the order we encounter them. We rely on subsequent * optimization to rearrange the stack if appropriate. * + * sqinfo is what we hand to convert_ANY_sublink_to_join when trying to convert + * a NOT IN against available_rels1; see there for what it means. Those quals + * are known to apply to the rows at this node, so we pass it down unchanged + * through AND clauses (whose arms share that context), but pass NULL when + * recursing into a just-pulled-up SubLink's quals, or when converting against + * available_rels2, since those are evaluated on different rows that the quals + * say nothing about. + * * Returns the replacement qual node, or NULL if the qual should be removed. */ static Node * pull_up_sublinks_qual_recurse(PlannerInfo *root, Node *node, Node **jtlink1, Relids available_rels1, - Node **jtlink2, Relids available_rels2) + Node **jtlink2, Relids available_rels2, + SafeQualsInfo *sqinfo) { if (node == NULL) return NULL; @@ -864,6 +954,7 @@ pull_up_sublinks_qual_recurse(PlannerInfo *root, Node *node, SubLink *sublink = (SubLink *) node; JoinExpr *j; Relids child_rels; + List *child_safequals; /* Is it a convertible ANY or EXISTS clause? */ if (sublink->subLinkType == ANY_SUBLINK) @@ -882,7 +973,7 @@ pull_up_sublinks_qual_recurse(PlannerInfo *root, Node *node, } if ((j = convert_ANY_sublink_to_join(root, sublink, false, - available_rels1)) != NULL) + available_rels1, NULL)) != NULL) { /* Yes; insert the new join node into the join tree */ j->larg = *jtlink1; @@ -890,7 +981,8 @@ pull_up_sublinks_qual_recurse(PlannerInfo *root, Node *node, /* Recursively process pulled-up jointree nodes */ j->rarg = pull_up_sublinks_jointree_recurse(root, j->rarg, - &child_rels); + &child_rels, + &child_safequals); /* * Now recursively process the pulled-up quals. Any inserted @@ -902,13 +994,14 @@ pull_up_sublinks_qual_recurse(PlannerInfo *root, Node *node, &j->larg, available_rels1, &j->rarg, - child_rels); + child_rels, + NULL); /* Return NULL representing constant TRUE */ return NULL; } if (available_rels2 != NULL && (j = convert_ANY_sublink_to_join(root, sublink, false, - available_rels2)) != NULL) + available_rels2, NULL)) != NULL) { /* Yes; insert the new join node into the join tree */ j->larg = *jtlink2; @@ -916,7 +1009,8 @@ pull_up_sublinks_qual_recurse(PlannerInfo *root, Node *node, /* Recursively process pulled-up jointree nodes */ j->rarg = pull_up_sublinks_jointree_recurse(root, j->rarg, - &child_rels); + &child_rels, + &child_safequals); /* * Now recursively process the pulled-up quals. Any inserted @@ -928,7 +1022,8 @@ pull_up_sublinks_qual_recurse(PlannerInfo *root, Node *node, &j->larg, available_rels2, &j->rarg, - child_rels); + child_rels, + NULL); /* Return NULL representing constant TRUE */ return NULL; } @@ -944,7 +1039,8 @@ pull_up_sublinks_qual_recurse(PlannerInfo *root, Node *node, /* Recursively process pulled-up jointree nodes */ j->rarg = pull_up_sublinks_jointree_recurse(root, j->rarg, - &child_rels); + &child_rels, + &child_safequals); /* * Now recursively process the pulled-up quals. Any inserted @@ -956,7 +1052,8 @@ pull_up_sublinks_qual_recurse(PlannerInfo *root, Node *node, &j->larg, available_rels1, &j->rarg, - child_rels); + child_rels, + NULL); /* Return NULL representing constant TRUE */ return NULL; } @@ -970,7 +1067,8 @@ pull_up_sublinks_qual_recurse(PlannerInfo *root, Node *node, /* Recursively process pulled-up jointree nodes */ j->rarg = pull_up_sublinks_jointree_recurse(root, j->rarg, - &child_rels); + &child_rels, + &child_safequals); /* * Now recursively process the pulled-up quals. Any inserted @@ -982,7 +1080,8 @@ pull_up_sublinks_qual_recurse(PlannerInfo *root, Node *node, &j->larg, available_rels2, &j->rarg, - child_rels); + child_rels, + NULL); /* Return NULL representing constant TRUE */ return NULL; } @@ -996,13 +1095,15 @@ pull_up_sublinks_qual_recurse(PlannerInfo *root, Node *node, SubLink *sublink = (SubLink *) get_notclausearg((Expr *) node); JoinExpr *j; Relids child_rels; + List *child_safequals; if (sublink && IsA(sublink, SubLink)) { if (sublink->subLinkType == ANY_SUBLINK) { if ((j = convert_ANY_sublink_to_join(root, sublink, true, - available_rels1)) != NULL) + available_rels1, + sqinfo)) != NULL) { /* Yes; insert the new join node into the join tree */ j->larg = *jtlink1; @@ -1010,7 +1111,8 @@ pull_up_sublinks_qual_recurse(PlannerInfo *root, Node *node, /* Recursively process pulled-up jointree nodes */ j->rarg = pull_up_sublinks_jointree_recurse(root, j->rarg, - &child_rels); + &child_rels, + &child_safequals); /* * Now recursively process the pulled-up quals. Because @@ -1022,13 +1124,15 @@ pull_up_sublinks_qual_recurse(PlannerInfo *root, Node *node, j->quals, &j->rarg, child_rels, - NULL, NULL); + NULL, NULL, + NULL); /* Return NULL representing constant TRUE */ return NULL; } if (available_rels2 != NULL && (j = convert_ANY_sublink_to_join(root, sublink, true, - available_rels2)) != NULL) + available_rels2, + NULL)) != NULL) { /* Yes; insert the new join node into the join tree */ j->larg = *jtlink2; @@ -1036,7 +1140,8 @@ pull_up_sublinks_qual_recurse(PlannerInfo *root, Node *node, /* Recursively process pulled-up jointree nodes */ j->rarg = pull_up_sublinks_jointree_recurse(root, j->rarg, - &child_rels); + &child_rels, + &child_safequals); /* * Now recursively process the pulled-up quals. Because @@ -1048,7 +1153,8 @@ pull_up_sublinks_qual_recurse(PlannerInfo *root, Node *node, j->quals, &j->rarg, child_rels, - NULL, NULL); + NULL, NULL, + NULL); /* Return NULL representing constant TRUE */ return NULL; } @@ -1064,7 +1170,8 @@ pull_up_sublinks_qual_recurse(PlannerInfo *root, Node *node, /* Recursively process pulled-up jointree nodes */ j->rarg = pull_up_sublinks_jointree_recurse(root, j->rarg, - &child_rels); + &child_rels, + &child_safequals); /* * Now recursively process the pulled-up quals. Because @@ -1076,7 +1183,8 @@ pull_up_sublinks_qual_recurse(PlannerInfo *root, Node *node, j->quals, &j->rarg, child_rels, - NULL, NULL); + NULL, NULL, + NULL); /* Return NULL representing constant TRUE */ return NULL; } @@ -1090,7 +1198,8 @@ pull_up_sublinks_qual_recurse(PlannerInfo *root, Node *node, /* Recursively process pulled-up jointree nodes */ j->rarg = pull_up_sublinks_jointree_recurse(root, j->rarg, - &child_rels); + &child_rels, + &child_safequals); /* * Now recursively process the pulled-up quals. Because @@ -1102,7 +1211,8 @@ pull_up_sublinks_qual_recurse(PlannerInfo *root, Node *node, j->quals, &j->rarg, child_rels, - NULL, NULL); + NULL, NULL, + NULL); /* Return NULL representing constant TRUE */ return NULL; } @@ -1127,7 +1237,8 @@ pull_up_sublinks_qual_recurse(PlannerInfo *root, Node *node, jtlink1, available_rels1, jtlink2, - available_rels2); + available_rels2, + sqinfo); if (newclause) newclauses = lappend(newclauses, newclause); } diff --git a/src/include/optimizer/subselect.h b/src/include/optimizer/subselect.h index 4ecccf46bd3..4481b75b65e 100644 --- a/src/include/optimizer/subselect.h +++ b/src/include/optimizer/subselect.h @@ -16,6 +16,19 @@ #include "nodes/pathnodes.h" #include "nodes/plannodes.h" +/* + * The quals that are guaranteed to filter the rows on which the SubLinks at + * one jointree node are evaluated, and the Vars they force non-null. The + * latter is computed on first use and then shared by every SubLink at that + * node. + */ +typedef struct SafeQualsInfo +{ + List *quals; /* qual clauses, with implicit-AND semantics */ + List *nonnullable_vars; /* multibitmapset; valid if computed */ + bool computed; /* have we computed nonnullable_vars yet? */ +} SafeQualsInfo; + extern void SS_process_ctes(PlannerInfo *root); extern ScalarArrayOpExpr *convert_VALUES_to_ANY(PlannerInfo *root, Node *testexpr, @@ -23,7 +36,8 @@ extern ScalarArrayOpExpr *convert_VALUES_to_ANY(PlannerInfo *root, extern JoinExpr *convert_ANY_sublink_to_join(PlannerInfo *root, SubLink *sublink, bool under_not, - Relids available_rels); + Relids available_rels, + SafeQualsInfo *sqinfo); extern JoinExpr *convert_EXISTS_sublink_to_join(PlannerInfo *root, SubLink *sublink, bool under_not, diff --git a/src/test/regress/expected/subselect.out b/src/test/regress/expected/subselect.out index ef49b5756a7..cb7a78e22f8 100644 --- a/src/test/regress/expected/subselect.out +++ b/src/test/regress/expected/subselect.out @@ -3597,19 +3597,268 @@ WHERE id NOT IN (SELECT id FROM null_tab WHERE id IS NOT NULL); Filter: (id IS NOT NULL) (6 rows) --- No ANTI JOIN: outer side is nullable (we don't check outer query quals for now) +-- ANTI JOIN: outer side is forced non-nullable by an "IS NOT NULL" WHERE qual EXPLAIN (COSTS OFF) SELECT * FROM null_tab WHERE id IS NOT NULL AND id NOT IN (SELECT id FROM not_null_tab); - QUERY PLAN ---------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------- + Hash Anti Join + Hash Cond: (null_tab.id = not_null_tab.id) + -> Seq Scan on null_tab + Filter: (id IS NOT NULL) + -> Hash + -> Seq Scan on not_null_tab +(6 rows) + +-- ANTI JOIN: outer side is forced non-nullable by a strict WHERE qual +EXPLAIN (COSTS OFF) +SELECT * FROM null_tab +WHERE id > 0 + AND id NOT IN (SELECT id FROM not_null_tab); + QUERY PLAN +---------------------------------------------- + Hash Anti Join + Hash Cond: (null_tab.id = not_null_tab.id) + -> Seq Scan on null_tab + Filter: (id > 0) + -> Hash + -> Seq Scan on not_null_tab +(6 rows) + +-- ANTI JOIN: outer side is forced non-nullable by an inner join's ON qual +-- located below the NOT IN +EXPLAIN (COSTS OFF) +SELECT * FROM null_tab t1 +INNER JOIN not_null_tab t2 ON t1.id > 0 +WHERE t1.id NOT IN (SELECT id FROM not_null_tab); + QUERY PLAN +---------------------------------------------------- + Nested Loop + -> Seq Scan on not_null_tab t2 + -> Materialize + -> Hash Anti Join + Hash Cond: (t1.id = not_null_tab.id) + -> Seq Scan on null_tab t1 + Filter: (id > 0) + -> Hash + -> Seq Scan on not_null_tab +(9 rows) + +-- ANTI JOIN: both outer columns are forced non-nullable by WHERE quals +EXPLAIN (COSTS OFF) +SELECT * FROM null_tab +WHERE id IS NOT NULL AND val IS NOT NULL + AND (id, val) NOT IN (SELECT id, val FROM not_null_tab); + QUERY PLAN +--------------------------------------------------------------------------------------- + Merge Anti Join + Merge Cond: ((null_tab.id = not_null_tab.id) AND (null_tab.val = not_null_tab.val)) + -> Sort + Sort Key: null_tab.id, null_tab.val + -> Seq Scan on null_tab + Filter: ((id IS NOT NULL) AND (val IS NOT NULL)) + -> Sort + Sort Key: not_null_tab.id, not_null_tab.val + -> Seq Scan on not_null_tab +(9 rows) + +-- ANTI JOIN: outer side is forced non-nullable by the sibling ON qual +EXPLAIN (COSTS OFF) +SELECT * FROM not_null_tab t1 +LEFT JOIN null_tab t2 +ON t2.val > 0 AND t2.val NOT IN (SELECT id FROM not_null_tab); + QUERY PLAN +----------------------------------------------------- + Nested Loop Left Join + -> Seq Scan on not_null_tab t1 + -> Materialize + -> Hash Anti Join + Hash Cond: (t2.val = not_null_tab.id) + -> Seq Scan on null_tab t2 + Filter: (val > 0) + -> Hash + -> Seq Scan on not_null_tab +(9 rows) + +-- ANTI JOIN: outer side is forced non-nullable by the inner join's ON qual +EXPLAIN (COSTS OFF) +SELECT * FROM null_tab t1 LEFT JOIN + (null_tab t2 INNER JOIN null_tab t3 ON t2.id = t3.id) + ON t2.id NOT IN (SELECT id FROM not_null_tab); + QUERY PLAN +---------------------------------------------------------------- + Nested Loop Left Join + -> Seq Scan on null_tab t1 + -> Materialize + -> Merge Join + Merge Cond: (t2.id = t3.id) + -> Sort + Sort Key: t2.id + -> Hash Anti Join + Hash Cond: (t2.id = not_null_tab.id) + -> Seq Scan on null_tab t2 + -> Hash + -> Seq Scan on not_null_tab + -> Sort + Sort Key: t3.id + -> Seq Scan on null_tab t3 +(15 rows) + +-- No ANTI JOIN: nothing proves the outer side non-nullable +EXPLAIN (COSTS OFF) +SELECT * FROM null_tab t1 LEFT JOIN + (null_tab t2 INNER JOIN null_tab t3 ON t2.id = t3.id) + ON true +WHERE t2.id NOT IN (SELECT id FROM not_null_tab); + QUERY PLAN +------------------------------------------------------------- + Nested Loop Left Join + Filter: (NOT (ANY (t2.id = (hashed SubPlan any_1).col1))) + -> Seq Scan on null_tab t1 + -> Materialize + -> Merge Join + Merge Cond: (t2.id = t3.id) + -> Sort + Sort Key: t2.id + -> Seq Scan on null_tab t2 + -> Sort + Sort Key: t3.id + -> Seq Scan on null_tab t3 + SubPlan any_1 + -> Seq Scan on not_null_tab +(14 rows) + +-- No ANTI JOIN: the proving qual is below a FULL JOIN, whose output may be +-- null-extended on either side +EXPLAIN (COSTS OFF) +SELECT * FROM (null_tab t1 INNER JOIN null_tab t2 ON t1.id > 0) +FULL JOIN null_tab t3 ON true +WHERE t1.id NOT IN (SELECT id FROM not_null_tab); + QUERY PLAN +------------------------------------------------------------- + Merge Full Join + Filter: (NOT (ANY (t1.id = (hashed SubPlan any_1).col1))) + -> Nested Loop + -> Seq Scan on null_tab t2 + -> Materialize + -> Seq Scan on null_tab t1 + Filter: (id > 0) + -> Materialize + -> Seq Scan on null_tab t3 + SubPlan any_1 + -> Seq Scan on not_null_tab +(11 rows) + +-- No ANTI JOIN: the qual that would prove the outer side non-nullable is +-- under an OR +EXPLAIN (COSTS OFF) +SELECT * FROM null_tab +WHERE (id IS NOT NULL OR val = 1) + AND id NOT IN (SELECT id FROM not_null_tab); + QUERY PLAN +------------------------------------------------------------------------------------------------ Seq Scan on null_tab - Filter: ((id IS NOT NULL) AND (NOT (ANY (id = (hashed SubPlan any_1).col1)))) + Filter: (((id IS NOT NULL) OR (val = 1)) AND (NOT (ANY (id = (hashed SubPlan any_1).col1)))) SubPlan any_1 -> Seq Scan on not_null_tab (4 rows) +-- Verify the conversion preserves NOT IN semantics +CREATE TEMP TABLE qual_outer (a int, b int); +CREATE TEMP TABLE qual_inner (y int NOT NULL); +INSERT INTO qual_outer VALUES (1, NULL), (2, 5), (3, 7); +INSERT INTO qual_inner VALUES (5); +EXPLAIN (COSTS OFF) +SELECT * FROM qual_outer +WHERE b IS NOT NULL AND b NOT IN (SELECT y FROM qual_inner) +ORDER BY a; + QUERY PLAN +-------------------------------------------------- + Sort + Sort Key: qual_outer.a + -> Hash Anti Join + Hash Cond: (qual_outer.b = qual_inner.y) + -> Seq Scan on qual_outer + Filter: (b IS NOT NULL) + -> Hash + -> Seq Scan on qual_inner +(8 rows) + +SELECT * FROM qual_outer +WHERE b IS NOT NULL AND b NOT IN (SELECT y FROM qual_inner) +ORDER BY a; + a | b +---+--- + 3 | 7 +(1 row) + +-- Verify the conversion preserves semantics when the NOT IN is in a left +-- join's ON clause and a sibling ON qual supplies the non-nullability proof +EXPLAIN (COSTS OFF) +SELECT * FROM qual_outer t1 +LEFT JOIN qual_outer t2 +ON t1.b = t2.b AND t2.b NOT IN (SELECT y FROM qual_inner); + QUERY PLAN +------------------------------------------------ + Merge Right Join + Merge Cond: (t2.b = t1.b) + -> Sort + Sort Key: t2.b + -> Hash Anti Join + Hash Cond: (t2.b = qual_inner.y) + -> Seq Scan on qual_outer t2 + -> Hash + -> Seq Scan on qual_inner + -> Sort + Sort Key: t1.b + -> Seq Scan on qual_outer t1 +(12 rows) + +SELECT * FROM qual_outer t1 +LEFT JOIN qual_outer t2 +ON t1.b = t2.b AND t2.b NOT IN (SELECT y FROM qual_inner) +ORDER BY t1.a, t2.a; + a | b | a | b +---+---+---+--- + 1 | | | + 2 | 5 | | + 3 | 7 | 3 | 7 +(3 rows) + +-- The same for the mirrored RIGHT JOIN case +EXPLAIN (COSTS OFF) +SELECT * FROM qual_outer t1 +RIGHT JOIN qual_outer t2 +ON t2.b = t1.b AND t1.b NOT IN (SELECT y FROM qual_inner); + QUERY PLAN +------------------------------------------------ + Merge Right Join + Merge Cond: (t1.b = t2.b) + -> Sort + Sort Key: t1.b + -> Hash Anti Join + Hash Cond: (t1.b = qual_inner.y) + -> Seq Scan on qual_outer t1 + -> Hash + -> Seq Scan on qual_inner + -> Sort + Sort Key: t2.b + -> Seq Scan on qual_outer t2 +(12 rows) + +SELECT * FROM qual_outer t1 +RIGHT JOIN qual_outer t2 +ON t2.b = t1.b AND t1.b NOT IN (SELECT y FROM qual_inner) +ORDER BY t2.a, t1.a; + a | b | a | b +---+---+---+--- + | | 1 | + | | 2 | 5 + 3 | 7 | 3 | 7 +(3 rows) + -- ANTI JOIN: outer side is defined NOT NULL, inner side is defined NOT NULL -- and is not nulled by outer join EXPLAIN (COSTS OFF) @@ -4022,4 +4271,35 @@ WHERE COALESCE(t1.id, -1) NOT IN ----+----- (0 rows) +-- No ANTI JOIN: one of the outer query's expressions is an upper-level Var, so +-- the quals of the query containing the NOT IN tell us nothing about it. +INSERT INTO null_tab VALUES (NULL, 2); +EXPLAIN (COSTS OFF) +SELECT u.b, ss.val FROM qual_outer u, + LATERAL (SELECT t.val FROM null_tab t + WHERE t.val IS NOT NULL + AND (t.val, u.b) NOT IN (SELECT id, val FROM not_null_tab)) ss; + QUERY PLAN +------------------------------------------------------------------------------------------------------------ + Nested Loop + Join Filter: (NOT (ANY ((t.val = (hashed SubPlan any_1).col1) AND (u.b = (hashed SubPlan any_1).col2)))) + -> Seq Scan on qual_outer u + -> Materialize + -> Seq Scan on null_tab t + Filter: (val IS NOT NULL) + SubPlan any_1 + -> Seq Scan on not_null_tab +(8 rows) + +-- NOT IN with NULL on outer side should not return the u.b IS NULL row +SELECT u.b, ss.val FROM qual_outer u, + LATERAL (SELECT t.val FROM null_tab t + WHERE t.val IS NOT NULL + AND (t.val, u.b) NOT IN (SELECT id, val FROM not_null_tab)) ss; + b | val +---+----- + 5 | 2 + 7 | 2 +(2 rows) + ROLLBACK; diff --git a/src/test/regress/sql/subselect.sql b/src/test/regress/sql/subselect.sql index 0b18e0132aa..f64d9602d4a 100644 --- a/src/test/regress/sql/subselect.sql +++ b/src/test/regress/sql/subselect.sql @@ -1586,12 +1586,98 @@ EXPLAIN (COSTS OFF) SELECT * FROM not_null_tab WHERE id NOT IN (SELECT id FROM null_tab WHERE id IS NOT NULL); --- No ANTI JOIN: outer side is nullable (we don't check outer query quals for now) +-- ANTI JOIN: outer side is forced non-nullable by an "IS NOT NULL" WHERE qual EXPLAIN (COSTS OFF) SELECT * FROM null_tab WHERE id IS NOT NULL AND id NOT IN (SELECT id FROM not_null_tab); +-- ANTI JOIN: outer side is forced non-nullable by a strict WHERE qual +EXPLAIN (COSTS OFF) +SELECT * FROM null_tab +WHERE id > 0 + AND id NOT IN (SELECT id FROM not_null_tab); + +-- ANTI JOIN: outer side is forced non-nullable by an inner join's ON qual +-- located below the NOT IN +EXPLAIN (COSTS OFF) +SELECT * FROM null_tab t1 +INNER JOIN not_null_tab t2 ON t1.id > 0 +WHERE t1.id NOT IN (SELECT id FROM not_null_tab); + +-- ANTI JOIN: both outer columns are forced non-nullable by WHERE quals +EXPLAIN (COSTS OFF) +SELECT * FROM null_tab +WHERE id IS NOT NULL AND val IS NOT NULL + AND (id, val) NOT IN (SELECT id, val FROM not_null_tab); + +-- ANTI JOIN: outer side is forced non-nullable by the sibling ON qual +EXPLAIN (COSTS OFF) +SELECT * FROM not_null_tab t1 +LEFT JOIN null_tab t2 +ON t2.val > 0 AND t2.val NOT IN (SELECT id FROM not_null_tab); + +-- ANTI JOIN: outer side is forced non-nullable by the inner join's ON qual +EXPLAIN (COSTS OFF) +SELECT * FROM null_tab t1 LEFT JOIN + (null_tab t2 INNER JOIN null_tab t3 ON t2.id = t3.id) + ON t2.id NOT IN (SELECT id FROM not_null_tab); + +-- No ANTI JOIN: nothing proves the outer side non-nullable +EXPLAIN (COSTS OFF) +SELECT * FROM null_tab t1 LEFT JOIN + (null_tab t2 INNER JOIN null_tab t3 ON t2.id = t3.id) + ON true +WHERE t2.id NOT IN (SELECT id FROM not_null_tab); + +-- No ANTI JOIN: the proving qual is below a FULL JOIN, whose output may be +-- null-extended on either side +EXPLAIN (COSTS OFF) +SELECT * FROM (null_tab t1 INNER JOIN null_tab t2 ON t1.id > 0) +FULL JOIN null_tab t3 ON true +WHERE t1.id NOT IN (SELECT id FROM not_null_tab); + +-- No ANTI JOIN: the qual that would prove the outer side non-nullable is +-- under an OR +EXPLAIN (COSTS OFF) +SELECT * FROM null_tab +WHERE (id IS NOT NULL OR val = 1) + AND id NOT IN (SELECT id FROM not_null_tab); + +-- Verify the conversion preserves NOT IN semantics +CREATE TEMP TABLE qual_outer (a int, b int); +CREATE TEMP TABLE qual_inner (y int NOT NULL); +INSERT INTO qual_outer VALUES (1, NULL), (2, 5), (3, 7); +INSERT INTO qual_inner VALUES (5); +EXPLAIN (COSTS OFF) +SELECT * FROM qual_outer +WHERE b IS NOT NULL AND b NOT IN (SELECT y FROM qual_inner) +ORDER BY a; +SELECT * FROM qual_outer +WHERE b IS NOT NULL AND b NOT IN (SELECT y FROM qual_inner) +ORDER BY a; + +-- Verify the conversion preserves semantics when the NOT IN is in a left +-- join's ON clause and a sibling ON qual supplies the non-nullability proof +EXPLAIN (COSTS OFF) +SELECT * FROM qual_outer t1 +LEFT JOIN qual_outer t2 +ON t1.b = t2.b AND t2.b NOT IN (SELECT y FROM qual_inner); +SELECT * FROM qual_outer t1 +LEFT JOIN qual_outer t2 +ON t1.b = t2.b AND t2.b NOT IN (SELECT y FROM qual_inner) +ORDER BY t1.a, t2.a; + +-- The same for the mirrored RIGHT JOIN case +EXPLAIN (COSTS OFF) +SELECT * FROM qual_outer t1 +RIGHT JOIN qual_outer t2 +ON t2.b = t1.b AND t1.b NOT IN (SELECT y FROM qual_inner); +SELECT * FROM qual_outer t1 +RIGHT JOIN qual_outer t2 +ON t2.b = t1.b AND t1.b NOT IN (SELECT y FROM qual_inner) +ORDER BY t2.a, t1.a; + -- ANTI JOIN: outer side is defined NOT NULL, inner side is defined NOT NULL -- and is not nulled by outer join EXPLAIN (COSTS OFF) @@ -1766,4 +1852,20 @@ SELECT * FROM null_tab t1 WHERE COALESCE(t1.id, -1) NOT IN (SELECT t1.val FROM not_null_tab t2 WHERE t2.val IS NOT NULL); +-- No ANTI JOIN: one of the outer query's expressions is an upper-level Var, so +-- the quals of the query containing the NOT IN tell us nothing about it. +INSERT INTO null_tab VALUES (NULL, 2); + +EXPLAIN (COSTS OFF) +SELECT u.b, ss.val FROM qual_outer u, + LATERAL (SELECT t.val FROM null_tab t + WHERE t.val IS NOT NULL + AND (t.val, u.b) NOT IN (SELECT id, val FROM not_null_tab)) ss; + +-- NOT IN with NULL on outer side should not return the u.b IS NULL row +SELECT u.b, ss.val FROM qual_outer u, + LATERAL (SELECT t.val FROM null_tab t + WHERE t.val IS NOT NULL + AND (t.val, u.b) NOT IN (SELECT id, val FROM not_null_tab)) ss; + ROLLBACK; diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 85d989f395d..7c23dae3670 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2788,6 +2788,7 @@ STRLEN SV SYNCHRONIZATION_BARRIER SYSTEM_INFO +SafeQualsInfo SampleScan SampleScanGetSampleSize_function SampleScanState -- 2.37.1 (Apple Git-137.1)